chg: Improve popup, remove session.

lookyloo/lookyloo.py

@@ -18,7 +18,7 @@ from uuid import uuid4
 from zipfile import ZipFile
 
 from defang import refang  # type: ignore
-from har2tree import CrawledTree, Har2TreeError, HarFile
+from har2tree import CrawledTree, Har2TreeError, HarFile, HostNode, URLNode
 from redis import Redis
 from scrapysplashwrapper import crawl
 
@@ -84,6 +84,14 @@ class Lookyloo():
             sample_config = json.load(_c)
         return sample_config[entry]
 
+    def get_urlnode_from_tree(self, capture_dir: Path, node_uuid: str) -> URLNode:
+        ct = self._load_pickle(capture_dir / 'tree.pickle')
+        return ct.root_hartree.get_url_node_by_uuid(node_uuid)
+
+    def get_hostnode_from_tree(self, capture_dir: Path, node_uuid: str) -> HostNode:
+        ct = self._load_pickle(capture_dir / 'tree.pickle')
+        return ct.root_hartree.get_host_node_by_uuid(node_uuid)
+
     def get_statistics(self, capture_dir: Path) -> Dict[str, Any]:
         # We need the pickle
         ct = self._load_pickle(capture_dir / 'tree.pickle')
@@ -280,7 +288,7 @@ class Lookyloo():
                 ct = CrawledTree(har_files, uuid)
                 with pickle_file.open('wb') as _p:
                     pickle.dump(ct, _p)
-            return str(pickle_file), ct.to_json(), ct.start_time.isoformat(), ct.user_agent, ct.root_url, meta
+            return ct.to_json(), ct.start_time.isoformat(), ct.user_agent, ct.root_url, meta
         except Har2TreeError as e:
             raise NoValidHarFile(e.message)
 

lookyloo/modules.py

@@ -18,6 +18,11 @@ from pysanejs import SaneJS
 
 class SaneJavaScript():
 
+    skip_lookup: Dict[str, str] = {
+        "717ea0ff7f3f624c268eccb244e24ec1305ab21557abb3d6f1a7e183ff68a2d28f13d1d2af926c9ef6d1fb16dd8cbe34cd98cacf79091dddc7874dcee21ecfdc": "1*1px gif",
+        "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e": "Empty file"
+    }
+
     def __init__(self, config: Dict[str, Any]):
         if not ('enabled' in config or config['enabled']):
             self.available = False
@@ -44,12 +49,12 @@ class SaneJavaScript():
             with sanejs_unknowns.open() as f:
                 unknown_hashes = [line.strip() for line in f.readlines()]
 
-        if force:
+        to_return = {h: details for h, details in self.skip_lookup.items() if h in sha512}
-            to_lookup = hashes
+
-        else:
+        to_lookup = [h for h in hashes if h not in self.skip_lookup]
-            to_lookup = [h for h in sha512 if (h not in unknown_hashes
+        if not force:
-                                               and not (today_dir / h).exists())]
+            to_lookup = [h for h in to_lookup if (h not in unknown_hashes
-        to_return = {}
+                                                  and not (today_dir / h).exists())]
         for h in to_lookup:
             response = self.client.sha512(h)
             if 'error' in response:

website/web/__init__.py

@@ -1,15 +1,13 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-import json
-import pickle
 from zipfile import ZipFile, ZIP_DEFLATED
 from io import BytesIO
 import os
 from pathlib import Path
 from datetime import datetime, timedelta
 
-from flask import Flask, render_template, request, session, send_file, redirect, url_for, Response, flash
+from flask import Flask, render_template, request, send_file, redirect, url_for, Response, flash
 from flask_bootstrap import Bootstrap  # type: ignore
 from flask_httpauth import HTTPDigestAuth  # type: ignore
 
@@ -18,7 +16,7 @@ from lookyloo.lookyloo import Lookyloo
 from lookyloo.exceptions import NoValidHarFile
 from .proxied import ReverseProxied
 
-from typing import Tuple, Optional, Dict, Any
+from typing import Optional, Dict, Any
 
 import logging
 
@@ -79,14 +77,6 @@ def rebuild_tree(tree_uuid: str):
     return redirect(url_for('index'))
 
 
-# keep
-def load_tree(capture_dir: Path) -> Tuple[str, str, str, str, Dict[str, Any]]:
-    session.clear()
-    temp_file_name, tree_json, tree_time, tree_ua, tree_root_url, meta = lookyloo.load_tree(capture_dir)
-    session["tree"] = temp_file_name
-    return tree_json, tree_time, tree_ua, tree_root_url, meta
-
-
 @app.route('/submit', methods=['POST', 'GET'])
 def submit():
     to_query = request.get_json(force=True)
@@ -116,11 +106,10 @@ def scrape_web():
     return render_template('scrape.html', user_agents=user_agents)
 
 
-@app.route('/tree/hostname/<string:node_uuid>/text', methods=['GET'])
+@app.route('/tree/<string:tree_uuid>/hostname/<string:node_uuid>/text', methods=['GET'])
-def hostnode_details_text(node_uuid: str):
+def hostnode_details_text(tree_uuid: str, node_uuid: str):
-    with open(session["tree"], 'rb') as f:
+    capture_dir = lookyloo.lookup_capture_dir(tree_uuid)
-        ct = pickle.load(f)
+    hostnode = lookyloo.get_hostnode_from_tree(capture_dir, node_uuid)
-    hostnode = ct.root_hartree.get_host_node_by_uuid(node_uuid)
     urls = []
     for url in hostnode.urls:
         urls.append(url.name)
@@ -134,11 +123,10 @@ def hostnode_details_text(node_uuid: str):
                      as_attachment=True, attachment_filename='file.md')
 
 
-@app.route('/tree/hostname_popup/<string:node_uuid>', methods=['GET'])
+@app.route('/tree/<string:tree_uuid>/hostname_popup/<string:node_uuid>', methods=['GET'])
-def hostnode_popup(node_uuid: str):
+def hostnode_popup(tree_uuid: str, node_uuid: str):
-    with open(session["tree"], 'rb') as f:
+    capture_dir = lookyloo.lookup_capture_dir(tree_uuid)
-        ct = pickle.load(f)
+    hostnode = lookyloo.get_hostnode_from_tree(capture_dir, node_uuid)
-    hostnode = ct.root_hartree.get_host_node_by_uuid(node_uuid)
     table_keys = {
         'js': "/static/javascript.png",
         'exe': "/static/exe.png",
@@ -163,36 +151,21 @@ def hostnode_popup(node_uuid: str):
     for url in hostnode.urls:
         if lookyloo.sanejs.available and hasattr(url, 'body_hash') and url.body_hash in lookups:
             url.add_feature('sane_js_details', lookups[url.body_hash])
-            # TODO: Do something with it.
+            if lookups[url.body_hash] and isinstance(lookups[url.body_hash], list):
+                url.add_feature('sane_js_details_to_print', f'{" ".join(lookups[url.body_hash][0].split("|"))} and {len(lookups[url.body_hash])-1} other files')
         urls.append(url)
     return render_template('hostname_popup.html',
+                           tree_uuid=tree_uuid,
                            hostname_uuid=node_uuid,
                            hostname=hostnode.name,
                            urls=urls,
                            keys=table_keys)
 
 
-@app.route('/tree/hostname/<string:node_uuid>', methods=['GET'])
+@app.route('/tree/<string:tree_uuid>/url/<string:node_uuid>', methods=['GET'])
-def hostnode_details(node_uuid: str):
+def urlnode_details(tree_uuid: str, node_uuid: str):
-    with open(session["tree"], 'rb') as f:
+    capture_dir = lookyloo.lookup_capture_dir(tree_uuid)
-        ct = pickle.load(f)
+    urlnode = lookyloo.get_urlnode_from_tree(capture_dir, node_uuid)
-    hostnode = ct.root_hartree.get_host_node_by_uuid(node_uuid)
-    urls = []
-    for url in hostnode.urls:
-        if hasattr(url, 'body_hash'):
-            sane_js_r = lookyloo.sane_js_query(url.body_hash)
-            if sane_js_r.get('response'):
-                url.add_feature('sane_js_details', sane_js_r['response'])
-                print('######## SANEJS ##### ', url.sane_js_details)
-        urls.append(url.to_json())
-    return json.dumps(urls)
-
-
-@app.route('/tree/url/<string:node_uuid>', methods=['GET'])
-def urlnode_details(node_uuid: str):
-    with open(session["tree"], 'rb') as f:
-        ct = pickle.load(f)
-    urlnode = ct.root_hartree.get_url_node_by_uuid(node_uuid)
     to_return = BytesIO()
     got_content = False
     if hasattr(urlnode, 'body'):
@@ -338,7 +311,7 @@ def tree(tree_uuid: str):
             enable_mail_notification = True
         else:
             enable_mail_notification = False
-        tree_json, start_time, user_agent, root_url, meta = load_tree(capture_dir)
+        tree_json, start_time, user_agent, root_url, meta = lookyloo.load_tree(capture_dir)
         return render_template('tree.html', tree_json=tree_json, start_time=start_time,
                                user_agent=user_agent, root_url=root_url, tree_uuid=tree_uuid,
                                meta=meta, enable_mail_notification=enable_mail_notification)

website/web/static/tree.js

@@ -93,7 +93,7 @@ d3.selection.prototype.moveToBack = function() {
 };
 
 function hostnode_click_popup(d) {
-    window.open('/tree/hostname_popup/' + d.data.uuid, '_blank', 'width=700,height=500,left=200,top=100');
+    window.open('/tree/' + treeUUID + '/hostname_popup/' + d.data.uuid, '_blank', 'width=1024,height=768,left=200,top=100');
 };
 
 function ProcessChildMessage(message) {

website/web/templates/hostname_popup.html

@@ -14,7 +14,7 @@
   <center>
     <h3>{{ hostname }}</h3>
     <button type="button" class="btn btn-secondary" onclick="whereAmI()">Locate node on tree</button>
-    <a href="{{ url_for('hostnode_details_text', node_uuid=hostname_uuid) }}" class="btn btn-info" role="button">Get URLs as text</a>
+    <a href="{{ url_for('hostnode_details_text', tree_uuid=tree_uuid, node_uuid=hostname_uuid) }}" class="btn btn-info" role="button">Get URLs as text</a>
   </center>
   <p>Click on the URL to get the content of the response</p>
   <div class="table-responsive">
@@ -22,6 +22,7 @@
       <thead>
         <tr>
           <th>URL</th>
+          <th>Known file</th>
           {% for alt, path in keys.items() %}
           <th><img src="{{ path }}" alt="{{ alt }}" width="21" height="21"/></th>
           {% endfor %}
@@ -31,7 +32,14 @@
     {% for url in urls %}
       <tr>
         <td style="white-space:nowrap">
-          <a href="{{ url_for('urlnode_details', node_uuid=url.uuid) }}">{{ url.name }}</a>
+          <a href="{{ url_for('urlnode_details', tree_uuid=tree_uuid, node_uuid=url.uuid) }}">{{ url.name }}</a>
+        </td>
+        <td><br/>
+            {% if url.sane_js_details_to_print %}
+                {{ url.sane_js_details_to_print }}
+            {% else %}
+                {{ url.sane_js_details }}
+            {% endif %}
         </td>
         {% for key in keys.keys() %}
         <td><br/>

website/web/templates/tree.html

@@ -40,6 +40,7 @@
 {% block content %}
 {{super()}}
 <script>
+  var treeUUID = "{{ tree_uuid }}";
   var treeData = {{ tree_json | safe }};
 </script>