diff --git a/pyurlabuse/LICENSE b/pyurlabuse/LICENSE new file mode 100644 index 0000000..a13cf07 --- /dev/null +++ b/pyurlabuse/LICENSE @@ -0,0 +1,26 @@ +Copyright (c) 2013, 2014 Raphaël Vinot +Copyright (c) 2013, 2014 CIRCL - Computer Incident Response Center Luxembourg + (c/o smile, security made in Lëtzebuerg, Groupement + d'Intérêt Economique) + +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pyurlabuse/MANIFEST.in b/pyurlabuse/MANIFEST.in new file mode 100644 index 0000000..bb3ec5f --- /dev/null +++ b/pyurlabuse/MANIFEST.in @@ -0,0 +1 @@ +include README.md diff --git a/pyurlabuse/README.md b/pyurlabuse/README.md new file mode 100644 index 0000000..49410e0 --- /dev/null +++ b/pyurlabuse/README.md @@ -0,0 +1,8 @@ +Client API for URL Abuse +======================== + +Client API to query CIRCL URL Abuse system. + + + + diff --git a/pyurlabuse/pyurlabuse/__init__.py b/pyurlabuse/pyurlabuse/__init__.py new file mode 100644 index 0000000..dd3c1ba --- /dev/null +++ b/pyurlabuse/pyurlabuse/__init__.py @@ -0,0 +1 @@ +from api import PyURLAbuse diff --git a/pyurlabuse/pyurlabuse/api.py b/pyurlabuse/pyurlabuse/api.py new file mode 100644 index 0000000..55f4b16 --- /dev/null +++ b/pyurlabuse/pyurlabuse/api.py @@ -0,0 +1,137 @@ +#!/bin/python +# -*- coding: utf-8 -*- + +import json +import requests +import time +from base64 import urlsafe_b64encode + + +class PyURLAbuse(object): + + # def __init__(self, url='https://www.circl.lu/urlabuse/'): + def __init__(self, url='http://0.0.0.0:5100/'): + self.url = url + + self.session = requests.Session() + self.session.headers.update({'content-type': 'application/json'}) + + def get_result(self, job_id): + response = self.session.get('{}_result/{}' .format(self.url, job_id)) + if response.status_code == 202: + return None + else: + return response.json() + + def _async(self, path, query): + response = self.session.post('{}{}' .format(self.url, path), + data=json.dumps(query)) + return response.text + + def start(self, q): + query = {'url': q} + return self._async('start', query) + + def urls(self, q): + query = {'url': q} + return self._async('urls', query) + + def resolve(self, q): + query = {'url': q} + return self._async('resolve', query) + + def phishtank(self, q): + query = {'query': q} + return self._async('phishtank', query) + + def virustotal(self, q): + query = {'query': q} + return self._async('virustotal_report', query) + + def googlesafebrowsing(self, q): + query = {'query': q} + return self._async('googlesafebrowsing', query) + + def urlquery(self, q): + query = {'query': q} + return self._async('urlquery', query) + + def ticket(self, q): + query = {'query': q} + return self._async('ticket', query) + + def whoismail(self, q): + query = {'query': q} + return self._async('whois', query) + + def pdnscircl(self, q): + query = {'query': q} + return self._async('pdnscircl', query) + + def bgpr(self, q): + query = {'query': q} + return self._async('bgpranking', query) + + def sslcircl(self, q): + query = {'query': q} + return self._async('psslcircl', query) + + def run_query(self, q): + cached = self.get_cache(q) + if len(cached[0][q]) > 0: + return {'info': 'Used cached content'}, cached + job_id = self.urls(q) + all_urls = None + while True: + all_urls = self.get_result(job_id) + if all_urls is None: + time.sleep(.5) + else: + break + + res = {} + for u in all_urls: + res[u] = self.resolve(u) + self.phishtank(u) + self.virustotal(u) + self.googlesafebrowsing(u) + self.urlquery(u) + self.ticket(u) + self.whoismail(u) + + waiting = True + done = [] + while waiting: + waiting = False + for u, job_id in res.iteritems(): + if job_id in done: + continue + ips = self.get_result(job_id) + if ips is not None: + done.append(job_id) + v4, v6 = ips + if v4 is not None: + for ip in v4: + self.phishtank(ip) + self.bgpr(ip) + self.urlquery(ip) + self.pdnscircl(ip) + self.sslcircl(ip) + self.ticket(ip) + self.whoismail(ip) + if v6 is not None: + for ip in v6: + self.phishtank(ip) + self.urlquery(ip) + self.pdnscircl(ip) + self.ticket(ip) + self.whoismail(ip) + waiting = True + time.sleep(.5) + time.sleep(1) + return {'info': 'New query, all the details may not be available.'}, self.get_cache(q) + + def get_cache(self, q): + q = urlsafe_b64encode(q) + response = self.session.get('{}get_cache/{}'.format(self.url, q)) + return response.json() diff --git a/pyurlabuse/setup.py b/pyurlabuse/setup.py new file mode 100644 index 0000000..9ce95f6 --- /dev/null +++ b/pyurlabuse/setup.py @@ -0,0 +1,26 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +from setuptools import setup + +setup( + name='pyurlabuse', + version='1.0', + author='Raphaël Vinot', + author_email='raphael.vinot@circl.lu', + maintainer='Raphaël Vinot', + url='https://github.com/CIRCL/url-abuse', + description='Python API for URL Abuse.', + long_description=open('README.md').read(), + packages=['pyurlabuse'], + classifiers=[ + 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', + 'Development Status :: 3 - Alpha', + 'Environment :: Console', + 'Intended Audience :: Science/Research', + 'Intended Audience :: Telecommunications Industry', + 'Programming Language :: Python', + 'Topic :: Security', + 'Topic :: Internet', + ], + install_requires=['requests'], +)