chg: Start the whole thing

Pipfile

@@ -16,6 +16,7 @@ gunicorn = {extras = ["eventlet"],version = "*"}
 lookyloo = {editable = true,path = "."}
 cchardet = "*"
 redis = "*"
+pylookyloo = {editable = true,path = "./client"}
 
 [requires]
 python_version = "3.6"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "c3c9657b345f0168789235083c9309852a08bdcfec02df214e6d54f5927e9f20"
+            "sha256": "766c81240a1f3be46b1f7e845ee38ff7306eeb9a91ffa6aa6be204f51c582b7c"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -403,6 +403,10 @@
             ],
             "version": "==1.9.0"
         },
+        "pylookyloo": {
+            "editable": true,
+            "path": "./client"
+        },
         "pyopenssl": {
             "hashes": [
                 "sha256:aeca66338f6de19d1aa46ed634c3b9ae519a64b458f8468aec688e7e3c20f200",
@@ -413,7 +417,7 @@
         "pysanejs": {
             "editable": true,
             "git": "https://github.com/CIRCL/PySaneJS.git",
-            "ref": "9153b38c1819d93725aee70c8b0195d7e662f978"
+            "ref": "d4d9125e04eba1fc41dcd82302e346343d10b2a1"
         },
         "queuelib": {
             "hashes": [

client/bin/lookyloo

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import argparse
+from pylookyloo import Lookyloo
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Enqueue a URL on Lookyloo')
+    parser.add_argument('--url', type=str, help='URL of the instance.')
+    parser.add_argument('--query', required=True, help='URL to unqueue')
+    args = parser.parse_args()
+
+    if args.url:
+        lookyloo = Lookyloo(args.url)
+    else:
+        lookyloo = Lookyloo()
+
+    url = lookyloo.enqueue(args.query)
+    print(url)

client/pylookyloo/__init__.py

@@ -0,0 +1 @@
+from .api import Lookyloo

client/pylookyloo/api.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import requests
+import json
+
+from urllib.parse import urljoin
+
+
+class Lookyloo():
+
+    def __init__(self, root_url: str='https://lookyloo.circl.lu/'):
+        self.root_url = root_url
+        if not self.root_url.endswith('/'):
+            self.root_url += '/'
+        self.session = requests.session()
+
+    @property
+    def is_up(self):
+        r = self.session.head(self.root_url)
+        return r.status_code == 200
+
+    def enqueue(self, url: str):
+        response = self.session.post(urljoin(self.root_url, 'submit'), data=json.dumps({'url': url}))
+        return urljoin(self.root_url, f'tree/{response.text}')

client/setup.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from setuptools import setup
+
+
+setup(
+    name='pylookyloo',
+    version='0.1',
+    author='Raphaël Vinot',
+    author_email='raphael.vinot@circl.lu',
+    maintainer='Raphaël Vinot',
+    url='https://github.com/CIRCL/lookyloo/client',
+    description='Python client for Lookyloo',
+    packages=['pylookyloo'],
+    scripts=['bin/lookyloo'],
+    install_requires=['requests'],
+    classifiers=[
+        'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)',
+        'Development Status :: 3 - Alpha',
+        'Environment :: Console',
+        'Operating System :: POSIX :: Linux',
+        'Intended Audience :: Science/Research',
+        'Intended Audience :: Telecommunications Industry',
+        'Intended Audience :: Information Technology',
+        'Programming Language :: Python :: 3',
+        'Topic :: Security',
+        'Topic :: Internet',
+    ]
+)

etc/systemd/system/lookyloo.service

@@ -7,7 +7,7 @@ User=www-data
 Group=www-data
 WorkingDirectory=/home/<CHANGE_ME>/lookyloo
 Environment="PATH=/home/<CHANGE_ME>/<MY_VIRTUALENV_PATH>/bin"
-ExecStart=/home/<CHANGE_ME>/<MY_VIRTUALENV_PATH>/bin/start_website.py
+ExecStart=start.py
 Environment=LOOKYLOO_HOME=/home/<CHANGE_ME>/lookyloo
 
 [Install]

website/3rdparty.sh

@@ -5,7 +5,7 @@ set -x
 
 mkdir -p web/static/
 
-wget https://d3js.org/d3.v5.js -O web/static/d3.v5.js
+wget https://d3js.org/d3.v5.min.js -O web/static/d3.v5.min.js
 
 FileSaver="5733e40e5af936eb3f48554cf6a8a7075d71d18a"
 

website/web/__init__.py

@@ -113,8 +113,9 @@ def urlnode_details(node_uuid):
 
 @app.route('/tree/<string:tree_uuid>/image', methods=['GET'])
 def image(tree_uuid):
-    lookup_dirs = lookyloo.lookup_dirs
+    report_dir = lookyloo.lookup_dirs.get(tree_uuid)
-    report_dir = lookup_dirs[tree_uuid]
+    if not report_dir:
+        return Response('Not available.', mimetype='text/text')
     to_return = lookyloo.load_image(report_dir)
     return send_file(to_return, mimetype='image/png',
                      as_attachment=True, attachment_filename='image.png')
@@ -122,8 +123,10 @@ def image(tree_uuid):
 
 @app.route('/tree/<string:tree_uuid>', methods=['GET'])
 def tree(tree_uuid):
-    lookup_dirs = lookyloo.lookup_dirs
+    report_dir = lookyloo.lookup_dirs.get(tree_uuid)
-    report_dir = lookup_dirs[tree_uuid]
+    if not report_dir:
+        return redirect(url_for('index'))
+
     tree_json, start_time, user_agent, root_url = load_tree(report_dir)
     return render_template('tree.html', tree_json=tree_json, start_time=start_time,
                            user_agent=user_agent, root_url=root_url, tree_uuid=tree_uuid)