Initial version of the API

pyurlabuse/LICENSE

@@ -0,0 +1,26 @@
+Copyright (c) 2013, 2014 Raphaël Vinot
+Copyright (c) 2013, 2014 CIRCL - Computer Incident Response Center Luxembourg
+                         (c/o smile, security made in Lëtzebuerg, Groupement
+                         d'Intérêt Economique)
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+       this list of conditions and the following disclaimer.
+    2. Redistributions in binary form must reproduce the above copyright notice,
+       this list of conditions and the following disclaimer in the documentation
+       and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+OF THE POSSIBILITY OF SUCH DAMAGE.

pyurlabuse/MANIFEST.in

@@ -0,0 +1 @@
+include README.md

pyurlabuse/README.md

@@ -0,0 +1,8 @@
+Client API for URL Abuse
+========================
+
+Client API to query CIRCL URL Abuse system.
+
+
+
+

pyurlabuse/pyurlabuse/__init__.py

@@ -0,0 +1 @@
+from api import PyURLAbuse

pyurlabuse/pyurlabuse/api.py

@@ -0,0 +1,137 @@
+#!/bin/python
+# -*- coding: utf-8 -*-
+
+import json
+import requests
+import time
+from base64 import urlsafe_b64encode
+
+
+class PyURLAbuse(object):
+
+    # def __init__(self, url='https://www.circl.lu/urlabuse/'):
+    def __init__(self, url='http://0.0.0.0:5100/'):
+        self.url = url
+
+        self.session = requests.Session()
+        self.session.headers.update({'content-type': 'application/json'})
+
+    def get_result(self, job_id):
+        response = self.session.get('{}_result/{}' .format(self.url, job_id))
+        if response.status_code == 202:
+            return None
+        else:
+            return response.json()
+
+    def _async(self, path, query):
+        response = self.session.post('{}{}' .format(self.url, path),
+                                     data=json.dumps(query))
+        return response.text
+
+    def start(self, q):
+        query = {'url': q}
+        return self._async('start', query)
+
+    def urls(self, q):
+        query = {'url': q}
+        return self._async('urls', query)
+
+    def resolve(self, q):
+        query = {'url': q}
+        return self._async('resolve', query)
+
+    def phishtank(self, q):
+        query = {'query': q}
+        return self._async('phishtank', query)
+
+    def virustotal(self, q):
+        query = {'query': q}
+        return self._async('virustotal_report', query)
+
+    def googlesafebrowsing(self, q):
+        query = {'query': q}
+        return self._async('googlesafebrowsing', query)
+
+    def urlquery(self, q):
+        query = {'query': q}
+        return self._async('urlquery', query)
+
+    def ticket(self, q):
+        query = {'query': q}
+        return self._async('ticket', query)
+
+    def whoismail(self, q):
+        query = {'query': q}
+        return self._async('whois', query)
+
+    def pdnscircl(self, q):
+        query = {'query': q}
+        return self._async('pdnscircl', query)
+
+    def bgpr(self, q):
+        query = {'query': q}
+        return self._async('bgpranking', query)
+
+    def sslcircl(self, q):
+        query = {'query': q}
+        return self._async('psslcircl', query)
+
+    def run_query(self, q):
+        cached = self.get_cache(q)
+        if len(cached[0][q]) > 0:
+            return {'info': 'Used cached content'}, cached
+        job_id = self.urls(q)
+        all_urls = None
+        while True:
+            all_urls = self.get_result(job_id)
+            if all_urls is None:
+                time.sleep(.5)
+            else:
+                break
+
+        res = {}
+        for u in all_urls:
+            res[u] = self.resolve(u)
+            self.phishtank(u)
+            self.virustotal(u)
+            self.googlesafebrowsing(u)
+            self.urlquery(u)
+            self.ticket(u)
+            self.whoismail(u)
+
+        waiting = True
+        done = []
+        while waiting:
+            waiting = False
+            for u, job_id in res.iteritems():
+                if job_id in done:
+                    continue
+                ips = self.get_result(job_id)
+                if ips is not None:
+                    done.append(job_id)
+                    v4, v6 = ips
+                    if v4 is not None:
+                        for ip in v4:
+                            self.phishtank(ip)
+                            self.bgpr(ip)
+                            self.urlquery(ip)
+                            self.pdnscircl(ip)
+                            self.sslcircl(ip)
+                            self.ticket(ip)
+                            self.whoismail(ip)
+                    if v6 is not None:
+                        for ip in v6:
+                            self.phishtank(ip)
+                            self.urlquery(ip)
+                            self.pdnscircl(ip)
+                            self.ticket(ip)
+                            self.whoismail(ip)
+                waiting = True
+                time.sleep(.5)
+        time.sleep(1)
+        return {'info': 'New query, all the details may not be available.'}, self.get_cache(q)
+
+    def get_cache(self, q):
+        q = urlsafe_b64encode(q)
+        response = self.session.get('{}get_cache/{}'.format(self.url, q))
+        return response.json()

pyurlabuse/setup.py

@@ -0,0 +1,26 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+from setuptools import setup
+
+setup(
+    name='pyurlabuse',
+    version='1.0',
+    author='Raphaël Vinot',
+    author_email='raphael.vinot@circl.lu',
+    maintainer='Raphaël Vinot',
+    url='https://github.com/CIRCL/url-abuse',
+    description='Python API for URL Abuse.',
+    long_description=open('README.md').read(),
+    packages=['pyurlabuse'],
+    classifiers=[
+        'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
+        'Development Status :: 3 - Alpha',
+        'Environment :: Console',
+        'Intended Audience :: Science/Research',
+        'Intended Audience :: Telecommunications Industry',
+        'Programming Language :: Python',
+        'Topic :: Security',
+        'Topic :: Internet',
+    ],
+    install_requires=['requests'],
+)