From 608d8816a8ebc1a7fb379356344ee3167243eeb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 30 Jan 2019 16:01:55 +0100 Subject: [PATCH] chg: Start the whole thing --- Pipfile | 1 + Pipfile.lock | 8 ++++++-- client/bin/lookyloo | 19 +++++++++++++++++++ client/pylookyloo/__init__.py | 1 + client/pylookyloo/api.py | 25 +++++++++++++++++++++++++ client/setup.py | 29 +++++++++++++++++++++++++++++ etc/systemd/system/lookyloo.service | 2 +- website/3rdparty.sh | 2 +- website/web/__init__.py | 11 +++++++---- 9 files changed, 90 insertions(+), 8 deletions(-) create mode 100755 client/bin/lookyloo create mode 100644 client/pylookyloo/__init__.py create mode 100644 client/pylookyloo/api.py create mode 100644 client/setup.py diff --git a/Pipfile b/Pipfile index f5ba562..c324e22 100644 --- a/Pipfile +++ b/Pipfile @@ -16,6 +16,7 @@ gunicorn = {extras = ["eventlet"],version = "*"} lookyloo = {editable = true,path = "."} cchardet = "*" redis = "*" +pylookyloo = {editable = true,path = "./client"} [requires] python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock index 674b0d7..abdb133 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "c3c9657b345f0168789235083c9309852a08bdcfec02df214e6d54f5927e9f20" + "sha256": "766c81240a1f3be46b1f7e845ee38ff7306eeb9a91ffa6aa6be204f51c582b7c" }, "pipfile-spec": 6, "requires": { @@ -403,6 +403,10 @@ ], "version": "==1.9.0" }, + "pylookyloo": { + "editable": true, + "path": "./client" + }, "pyopenssl": { "hashes": [ "sha256:aeca66338f6de19d1aa46ed634c3b9ae519a64b458f8468aec688e7e3c20f200", @@ -413,7 +417,7 @@ "pysanejs": { "editable": true, "git": "https://github.com/CIRCL/PySaneJS.git", - "ref": "9153b38c1819d93725aee70c8b0195d7e662f978" + "ref": "d4d9125e04eba1fc41dcd82302e346343d10b2a1" }, "queuelib": { "hashes": [ diff --git a/client/bin/lookyloo b/client/bin/lookyloo new file mode 100755 index 0000000..318a948 --- /dev/null +++ b/client/bin/lookyloo @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import argparse +from pylookyloo import Lookyloo + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Enqueue a URL on Lookyloo') + parser.add_argument('--url', type=str, help='URL of the instance.') + parser.add_argument('--query', required=True, help='URL to unqueue') + args = parser.parse_args() + + if args.url: + lookyloo = Lookyloo(args.url) + else: + lookyloo = Lookyloo() + + url = lookyloo.enqueue(args.query) + print(url) diff --git a/client/pylookyloo/__init__.py b/client/pylookyloo/__init__.py new file mode 100644 index 0000000..1f94739 --- /dev/null +++ b/client/pylookyloo/__init__.py @@ -0,0 +1 @@ +from .api import Lookyloo diff --git a/client/pylookyloo/api.py b/client/pylookyloo/api.py new file mode 100644 index 0000000..de5b4b4 --- /dev/null +++ b/client/pylookyloo/api.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import requests +import json + +from urllib.parse import urljoin + + +class Lookyloo(): + + def __init__(self, root_url: str='https://lookyloo.circl.lu/'): + self.root_url = root_url + if not self.root_url.endswith('/'): + self.root_url += '/' + self.session = requests.session() + + @property + def is_up(self): + r = self.session.head(self.root_url) + return r.status_code == 200 + + def enqueue(self, url: str): + response = self.session.post(urljoin(self.root_url, 'submit'), data=json.dumps({'url': url})) + return urljoin(self.root_url, f'tree/{response.text}') diff --git a/client/setup.py b/client/setup.py new file mode 100644 index 0000000..a022d46 --- /dev/null +++ b/client/setup.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from setuptools import setup + + +setup( + name='pylookyloo', + version='0.1', + author='Raphaël Vinot', + author_email='raphael.vinot@circl.lu', + maintainer='Raphaël Vinot', + url='https://github.com/CIRCL/lookyloo/client', + description='Python client for Lookyloo', + packages=['pylookyloo'], + scripts=['bin/lookyloo'], + install_requires=['requests'], + classifiers=[ + 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)', + 'Development Status :: 3 - Alpha', + 'Environment :: Console', + 'Operating System :: POSIX :: Linux', + 'Intended Audience :: Science/Research', + 'Intended Audience :: Telecommunications Industry', + 'Intended Audience :: Information Technology', + 'Programming Language :: Python :: 3', + 'Topic :: Security', + 'Topic :: Internet', + ] +) diff --git a/etc/systemd/system/lookyloo.service b/etc/systemd/system/lookyloo.service index c8cc04d..a807619 100644 --- a/etc/systemd/system/lookyloo.service +++ b/etc/systemd/system/lookyloo.service @@ -7,7 +7,7 @@ User=www-data Group=www-data WorkingDirectory=/home//lookyloo Environment="PATH=/home///bin" -ExecStart=/home///bin/start_website.py +ExecStart=start.py Environment=LOOKYLOO_HOME=/home//lookyloo [Install] diff --git a/website/3rdparty.sh b/website/3rdparty.sh index 76cbd62..5ed3625 100755 --- a/website/3rdparty.sh +++ b/website/3rdparty.sh @@ -5,7 +5,7 @@ set -x mkdir -p web/static/ -wget https://d3js.org/d3.v5.js -O web/static/d3.v5.js +wget https://d3js.org/d3.v5.min.js -O web/static/d3.v5.min.js FileSaver="5733e40e5af936eb3f48554cf6a8a7075d71d18a" diff --git a/website/web/__init__.py b/website/web/__init__.py index 6c526c1..389db7f 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -113,8 +113,9 @@ def urlnode_details(node_uuid): @app.route('/tree//image', methods=['GET']) def image(tree_uuid): - lookup_dirs = lookyloo.lookup_dirs - report_dir = lookup_dirs[tree_uuid] + report_dir = lookyloo.lookup_dirs.get(tree_uuid) + if not report_dir: + return Response('Not available.', mimetype='text/text') to_return = lookyloo.load_image(report_dir) return send_file(to_return, mimetype='image/png', as_attachment=True, attachment_filename='image.png') @@ -122,8 +123,10 @@ def image(tree_uuid): @app.route('/tree/', methods=['GET']) def tree(tree_uuid): - lookup_dirs = lookyloo.lookup_dirs - report_dir = lookup_dirs[tree_uuid] + report_dir = lookyloo.lookup_dirs.get(tree_uuid) + if not report_dir: + return redirect(url_for('index')) + tree_json, start_time, user_agent, root_url = load_tree(report_dir) return render_template('tree.html', tree_json=tree_json, start_time=start_time, user_agent=user_agent, root_url=root_url, tree_uuid=tree_uuid)