From aaf8bcb4601a21db1007bcbd53808a8c969eb31b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 31 Jan 2019 17:29:20 +0100 Subject: [PATCH] new: Add permalink to lookyloo --- Pipfile | 1 + Pipfile.lock | 24 +++++++++++++++--------- bin/start_website.py | 2 +- urlabuse/urlabuse.py | 15 ++++++++++++++- website/web/__init__.py | 6 ++++++ website/web/static/main.js | 6 ++++++ website/web/templates/index.html | 2 +- 7 files changed, 44 insertions(+), 12 deletions(-) diff --git a/Pipfile b/Pipfile index aca258b..ca2a1b1 100644 --- a/Pipfile +++ b/Pipfile @@ -21,6 +21,7 @@ flask-wtf = "*" gunicorn = {extras = ["gevent"],version = "*"} pyurlabuse = {editable = true,path = "./client"} pyfaup = {editable = true,git = "https://github.com/stricaud/faup.git/",subdirectory = "src/lib/bindings/python/"} +pylookyloo = {editable = true,git = "https://github.com/CIRCL/lookyloo.git/",subdirectory = "client"} [requires] python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock index b7e5dd4..74f0386 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "21547085a908ef46ba32fe38cb88c8236645e1b778cbbeeb5d13c28e9055d858" + "sha256": "33ab0963bdc6c913e8d3ef25f35a88d726ef5a6fb882606e6cd5c0aa9f897415" }, "pipfile-spec": 6, "requires": { @@ -218,7 +218,7 @@ "pybgpranking": { "editable": true, "git": "https://github.com/D4-project/BGP-Ranking.git/", - "ref": "7a39941c235067127c98d8a3cdd2785882e1cbb1", + "ref": "7e698f87366e6f99b4d0d11852737db28e3ddc62", "subdirectory": "client" }, "pyeupi": { @@ -237,7 +237,13 @@ "pyipasnhistory": { "editable": true, "git": "https://github.com/D4-project/IPASN-History.git/", - "ref": "2fb329beedb610788045482f36ddff0798398a27", + "ref": "e846cd36fe1ed6b22f60890bba89f84e61b62e59", + "subdirectory": "client" + }, + "pylookyloo": { + "editable": true, + "git": "https://github.com/CIRCL/lookyloo.git/", + "ref": "608d8816a8ebc1a7fb379356344ee3167243eeb4", "subdirectory": "client" }, "pypdns": { @@ -267,11 +273,11 @@ }, "redis": { "hashes": [ - "sha256:2100750629beff143b6a200a2ea8e719fcf26420adabb81402895e144c5083cf", - "sha256:8e0bdd2de02e829b6225b25646f9fb9daffea99a252610d040409a6738541f0a" + "sha256:74c892041cba46078ae1ef845241548baa3bd3634f9a6f0f952f006eb1619c71", + "sha256:7ba8612bbfd966dea8c62322543fed0095da2834dbd5a7c124afbc617a156aa7" ], "index": "pypi", - "version": "==3.0.1" + "version": "==3.1.0" }, "requests": { "hashes": [ @@ -296,10 +302,10 @@ }, "soupsieve": { "hashes": [ - "sha256:10687fc53eeb3518e01a0ac84d3d711da623d3298a3039459d3f649927c4a270", - "sha256:b23a0d7da0247200fe83c67c34de9d7599ad404106367313d8e65e04174d0b4b" + "sha256:466910df7561796a60748826781ebe9a888f7a1668a636ae86783f44d10aae73", + "sha256:87db12ae79194f0ff9808d2b1641c4f031ae39ffa3cab6b907ea7c1e5e5ed445" ], - "version": "==1.7.2" + "version": "==1.7.3" }, "urllib3": { "hashes": [ diff --git a/bin/start_website.py b/bin/start_website.py index 0f673ab..db16673 100755 --- a/bin/start_website.py +++ b/bin/start_website.py @@ -8,7 +8,7 @@ if __name__ == '__main__': website_dir = get_homedir() / 'website' Popen([f'{website_dir}/3drparty.sh'], cwd=website_dir) try: - Popen(['gunicorn', '--worker-class', 'gevent', '-w', '10', '-b', '0.0.0.0:5100', 'web:app'], + Popen(['gunicorn', '--worker-class', 'gevent', '-w', '10', '-b', '0.0.0.0:5200', 'web:app'], cwd=website_dir).communicate() except KeyboardInterrupt: print('Stopping gunicorn.') diff --git a/urlabuse/urlabuse.py b/urlabuse/urlabuse.py index 98d0b2a..4d1d246 100644 --- a/urlabuse/urlabuse.py +++ b/urlabuse/urlabuse.py @@ -21,7 +21,7 @@ import logging from pypdns import PyPDNS from pyipasnhistory import IPASNHistory from pybgpranking import BGPRanking - +from pylookyloo import Lookyloo from pypssl import PyPSSL from pyeupi import PyEUPI @@ -471,6 +471,17 @@ class Query(): self._cache_set(ip, to_return, 'bgpranking') return to_return + def lookyloo(self, url): + cached = self._cache_get(url, 'lookyloo') + if cached is not None: + return cached + lookyloo = Lookyloo() + lookyloo_perma_url = lookyloo.enqueue(url) + if lookyloo_perma_url: + self._cache_set(url, lookyloo_perma_url, 'lookyloo') + return lookyloo_perma_url + return None + def _deserialize_cached(self, entry): to_return = {} redirects = [] @@ -537,6 +548,8 @@ class Query(): to_return += '\n{}\n'.format(url) if 'whois' in info: all_mails.update(info['whois']) + if 'lookyloo' in info: + to_return += '\tLookyloo permanent URL: {}\n'.format(info['lookyloo']) if 'vt' in info and len(info['vt']) == 4: if info['vt'][2] is not None: to_return += '\t{} out of {} positive detections in VT - {}\n'.format( diff --git a/website/web/__init__.py b/website/web/__init__.py index e37dcd5..4a62945 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -189,6 +189,12 @@ def run_query(): return enqueue('is_valid_url', {'url': url}) +@app.route('/lookyloo', methods=['POST']) +def lookyloo(): + data = request.get_json(force=True) + return enqueue('lookyloo', {'url': data["url"]}) + + @app.route('/urls', methods=['POST']) def urls(): data = request.get_json(force=True) diff --git a/website/web/static/main.js b/website/web/static/main.js index 300fd6e..c5e09f0 100644 --- a/website/web/static/main.js +++ b/website/web/static/main.js @@ -78,6 +78,11 @@ var userInput = $scope.input_url; + var lookyloo = function(jobID) { + $scope.poller(jobID, function(data){ + $scope.lookyloo_url = data; + }); + }; var check_validity = function(jobID) { $scope.poller(jobID, function(data){ @@ -91,6 +96,7 @@ }; $scope.query('start', {"url": userInput}, check_validity); + $scope.query('lookyloo', {"url": userInput}, lookyloo); }; $scope.submit_email = function() { diff --git a/website/web/templates/index.html b/website/web/templates/index.html index 0fcd570..dfd1572 100644 --- a/website/web/templates/index.html +++ b/website/web/templates/index.html @@ -42,6 +42,7 @@

Report

{{ query_url }}

+

See on Lookyloo

@@ -54,7 +55,6 @@

Send report to CIRCL
-