CIRCL
/
url-abuse
mirror of https://github.com/CIRCL/url-abuse
2
0
Fork 0
Code Issues Releases Wiki Activity

Initial version of the API

travis
Raphaël Vinot 2015-03-11 18:15:56 +01:00
parent 52cefba027
commit 89658c709b
6 changed files with 199 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
pyurlabuse/LICENSE Normal file
View File

@ -0,0 +1,26 @@
Copyright (c) 2013, 2014 Raphaël Vinot
Copyright (c) 2013, 2014 CIRCL - Computer Incident Response Center Luxembourg
(c/o smile, security made in Lëtzebuerg, Groupement
d'Intérêt Economique)
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
OF THE POSSIBILITY OF SUCH DAMAGE.

1
pyurlabuse/MANIFEST.in Normal file
View File

@ -0,0 +1 @@
include README.md

8
pyurlabuse/README.md Normal file
View File

@ -0,0 +1,8 @@
Client API for URL Abuse
========================
Client API to query CIRCL URL Abuse system.

1
pyurlabuse/pyurlabuse/__init__.py Normal file
View File

@ -0,0 +1 @@
from api import PyURLAbuse

137
pyurlabuse/pyurlabuse/api.py Normal file
View File

@ -0,0 +1,137 @@
#!/bin/python
# -*- coding: utf-8 -*-
import json
import requests
import time
from base64 import urlsafe_b64encode
class PyURLAbuse(object):
# def __init__(self, url='https://www.circl.lu/urlabuse/'):
def __init__(self, url='http://0.0.0.0:5100/'):
self.url = url
self.session = requests.Session()
self.session.headers.update({'content-type': 'application/json'})
def get_result(self, job_id):
response = self.session.get('{}_result/{}' .format(self.url, job_id))
if response.status_code == 202:
return None
else:
return response.json()
def _async(self, path, query):
response = self.session.post('{}{}' .format(self.url, path),
data=json.dumps(query))
return response.text
def start(self, q):
query = {'url': q}
return self._async('start', query)
def urls(self, q):
query = {'url': q}
return self._async('urls', query)
def resolve(self, q):
query = {'url': q}
return self._async('resolve', query)
def phishtank(self, q):
query = {'query': q}
return self._async('phishtank', query)
def virustotal(self, q):
query = {'query': q}
return self._async('virustotal_report', query)
def googlesafebrowsing(self, q):
query = {'query': q}
return self._async('googlesafebrowsing', query)
def urlquery(self, q):
query = {'query': q}
return self._async('urlquery', query)
def ticket(self, q):
query = {'query': q}
return self._async('ticket', query)
def whoismail(self, q):
query = {'query': q}
return self._async('whois', query)
def pdnscircl(self, q):
query = {'query': q}
return self._async('pdnscircl', query)
def bgpr(self, q):
query = {'query': q}
return self._async('bgpranking', query)
def sslcircl(self, q):
query = {'query': q}
return self._async('psslcircl', query)
def run_query(self, q):
cached = self.get_cache(q)
if len(cached[0][q]) > 0:
return {'info': 'Used cached content'}, cached
job_id = self.urls(q)
all_urls = None
while True:
all_urls = self.get_result(job_id)
if all_urls is None:
time.sleep(.5)
else:
break
res = {}
for u in all_urls:
res[u] = self.resolve(u)
self.phishtank(u)
self.virustotal(u)
self.googlesafebrowsing(u)
self.urlquery(u)
self.ticket(u)
self.whoismail(u)
waiting = True
done = []
while waiting:
waiting = False
for u, job_id in res.iteritems():
if job_id in done:
continue
ips = self.get_result(job_id)
if ips is not None:
done.append(job_id)
v4, v6 = ips
if v4 is not None:
for ip in v4:
self.phishtank(ip)
self.bgpr(ip)
self.urlquery(ip)
self.pdnscircl(ip)
self.sslcircl(ip)
self.ticket(ip)
self.whoismail(ip)
if v6 is not None:
for ip in v6:
self.phishtank(ip)
self.urlquery(ip)
self.pdnscircl(ip)
self.ticket(ip)
self.whoismail(ip)
waiting = True
time.sleep(.5)
time.sleep(1)
return {'info': 'New query, all the details may not be available.'}, self.get_cache(q)
def get_cache(self, q):
q = urlsafe_b64encode(q)
response = self.session.get('{}get_cache/{}'.format(self.url, q))
return response.json()

26
pyurlabuse/setup.py Normal file
View File

@ -0,0 +1,26 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
from setuptools import setup
setup(
name='pyurlabuse',
version='1.0',
author='Raphaël Vinot',
author_email='raphael.vinot@circl.lu',
maintainer='Raphaël Vinot',
url='https://github.com/CIRCL/url-abuse',
description='Python API for URL Abuse.',
long_description=open('README.md').read(),
packages=['pyurlabuse'],
classifiers=[
'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
'Development Status :: 3 - Alpha',
'Environment :: Console',
'Intended Audience :: Science/Research',
'Intended Audience :: Telecommunications Industry',
'Programming Language :: Python',
'Topic :: Security',
'Topic :: Internet',
],
install_requires=['requests'],
)