chg: Improve popup, make sanejs a module, cache

config/modules.json.sample

@@ -2,5 +2,8 @@
   "VirusTotal": {
     "apikey": "KEY",
     "autosubmit": false
+  },
+  "SaneJS": {
+    "enabled": true
   }
 }

lookyloo/lookyloo.py

@@ -19,13 +19,12 @@ from zipfile import ZipFile
 
 from defang import refang  # type: ignore
 from har2tree import CrawledTree, Har2TreeError, HarFile
-from pysanejs import SaneJS
 from redis import Redis
 from scrapysplashwrapper import crawl
 
 from .exceptions import NoValidHarFile
 from .helpers import get_homedir, get_socket_path, load_cookies, load_configs, safe_create_dir, get_email_template
-from .modules import VirusTotal
+from .modules import VirusTotal, SaneJavaScript
 
 
 class Lookyloo():
@@ -50,17 +49,14 @@ class Lookyloo():
                 self.vt = VirusTotal(self.configs['modules']['VirusTotal'])
                 if not self.vt.available:
                     self.logger.warning('Unable to setup the VirusTotal module')
+            if 'SaneJS' in self.configs['modules']:
+                self.sanejs = SaneJavaScript(self.configs['modules']['SaneJS'])
+                if not self.sanejs.available:
+                    self.logger.warning('Unable to setup the SaneJS module')
 
         if not self.redis.exists('cache_loaded'):
             self._init_existing_dumps()
 
-        # Try to reach sanejs
-        self.sanejs = SaneJS()
-        if not self.sanejs.is_up:
-            self.use_sane_js = False
-        else:
-            self.use_sane_js = True
-
     def rebuild_cache(self) -> None:
         self.redis.flushdb()
         self._init_existing_dumps()
@@ -312,11 +308,6 @@ class Lookyloo():
     def get_capture(self, capture_dir: Path) -> BytesIO:
         return self._get_raw(capture_dir)
 
-    def sane_js_query(self, sha512: str) -> Dict[str, Any]:
-        if self.use_sane_js:
-            return self.sanejs.sha512(sha512)
-        return {'response': []}
-
     def scrape(self, url: str, cookies_pseudofile: Optional[BufferedIOBase]=None,
                depth: int=1, listing: bool=True, user_agent: Optional[str]=None,
                perma_uuid: str=None, os: str=None, browser: str=None) -> Union[bool, str]:

lookyloo/modules.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from typing import Dict, Any, Optional
+from typing import Dict, Any, Optional, List, Union
 from datetime import date
 import hashlib
 import json
@@ -13,6 +13,65 @@ from .helpers import get_homedir
 from .exceptions import ConfigError
 
 import vt  # type: ignore
+from pysanejs import SaneJS
+
+
+class SaneJavaScript():
+
+    def __init__(self, config: Dict[str, Any]):
+        if not ('enabled' in config or config['enabled']):
+            self.available = False
+            return
+        self.client = SaneJS()
+        if not self.client.is_up:
+            self.available = False
+            return
+        self.available = True
+        self.storage_dir = get_homedir() / 'sanejs'
+        self.storage_dir.mkdir(parents=True, exist_ok=True)
+
+    def hashes_lookup(self, sha512: Union[List[str], str], force: bool=False) -> Optional[Dict[str, Any]]:
+        if isinstance(sha512, str):
+            hashes = [sha512]
+        else:
+            hashes = sha512
+
+        today_dir = self.storage_dir / date.today().isoformat()
+        today_dir.mkdir(parents=True, exist_ok=True)
+        sanejs_unknowns = today_dir / 'unknown'
+        unknown_hashes = []
+        if sanejs_unknowns.exists():
+            with sanejs_unknowns.open() as f:
+                unknown_hashes = [line.strip() for line in f.readlines()]
+
+        if force:
+            to_lookup = hashes
+        else:
+            to_lookup = [h for h in sha512 if (h not in unknown_hashes
+                                               and not (today_dir / h).exists())]
+        to_return = {}
+        for h in to_lookup:
+            response = self.client.sha512(h)
+            if 'error' in response:
+                # Server not ready
+                break
+            if 'response' in response and response['response']:
+                cached_path = today_dir / h
+                with cached_path.open('w') as f:
+                    json.dump(response['response'], f)
+                to_return[h] = response['response']
+            else:
+                unknown_hashes.append(h)
+
+        for h in hashes:
+            cached_path = today_dir / h
+            if h in unknown_hashes or h in to_return:
+                continue
+            elif cached_path.exists():
+                with cached_path.open() as f:
+                    to_return[h] = json.load(f)
+
+        return to_return
 
 
 class VirusTotal():

website/web/__init__.py

@@ -136,7 +136,40 @@ def hostnode_details_text(node_uuid: str):
 
 @app.route('/tree/hostname_popup/<string:node_uuid>', methods=['GET'])
 def hostnode_popup(node_uuid: str):
-    return render_template('hostname_popup.html', hostname_uuid=node_uuid)
+    with open(session["tree"], 'rb') as f:
+        ct = pickle.load(f)
+    hostnode = ct.root_hartree.get_host_node_by_uuid(node_uuid)
+    table_keys = {
+        'js': "/static/javascript.png",
+        'exe': "/static/exe.png",
+        'css': "/static/css.png",
+        'font': "/static/font.png",
+        'html': "/static/html.png",
+        'json': "/static/json.png",
+        'iframe': "/static/ifr.png",
+        'image': "/static/img.png",
+        'unknown_mimetype': "/static/wtf.png",
+        'video': "/static/video.png",
+        'request_cookie': "/static/cookie_read.png",
+        'response_cookie': "/static/cookie_received.png",
+        'redirect': "/static/redirect.png",
+        'redirect_to_nothing': "/static/cookie_in_url.png"
+    }
+
+    urls = []
+    if lookyloo.sanejs.available:
+        to_lookup = [url.body_hash for url in hostnode.urls if hasattr(url, 'body_hash')]
+        lookups = lookyloo.sanejs.hashes_lookup(to_lookup)
+    for url in hostnode.urls:
+        if lookyloo.sanejs.available and hasattr(url, 'body_hash') and url.body_hash in lookups:
+            url.add_feature('sane_js_details', lookups[url.body_hash])
+            # TODO: Do something with it.
+        urls.append(url)
+    return render_template('hostname_popup.html',
+                           hostname_uuid=node_uuid,
+                           hostname=hostnode.name,
+                           urls=urls,
+                           keys=table_keys)
 
 
 @app.route('/tree/hostname/<string:node_uuid>', methods=['GET'])

website/web/templates/hostname_popup.html

@@ -1,6 +1,6 @@
 {% extends "main.html" %}
 
-{% block title %}Details for hostname {% endblock %}
+{% block title %}Details for {{ hostname }} {% endblock %}
 
 {% block scripts %}
   <script>
@@ -11,5 +11,40 @@
 {% endblock %}
 
 {% block content %}
-  <button onclick="whereAmI()">Where am I</button>
+  <center>
+    <h3>{{ hostname }}</h3>
+    <button type="button" class="btn btn-secondary" onclick="whereAmI()">Locate node on tree</button>
+    <a href="{{ url_for('hostnode_details_text', node_uuid=hostname_uuid) }}" class="btn btn-info" role="button">Get URLs as text</a>
+  </center>
+  <p>Click on the URL to get the content of the response</p>
+  <div class="table-responsive">
+    <table id="table" class="table">
+      <thead>
+        <tr>
+          <th>URL</th>
+          {% for alt, path in keys.items() %}
+          <th><img src="{{ path }}" alt="{{ alt }}" width="21" height="21"/></th>
+          {% endfor %}
+        </tr>
+      </thead>
+    <tbody>
+    {% for url in urls %}
+      <tr>
+        <td style="white-space:nowrap">
+          <a href="{{ url_for('urlnode_details', node_uuid=url.uuid) }}">{{ url.name }}</a>
+        </td>
+        {% for key in keys.keys() %}
+        <td><br/>
+        {% if url[key] %}
+        X
+        {% else%}
+        -
+        {%endif%}
+        </td>
+        {% endfor %}
+      </tr>
+    {% endfor %}
+    </tbody>
+    </table>
+  </div>
 {% endblock %}

website/web/templates/statistics.html

@@ -4,7 +4,6 @@
     <dl class="row">
       <dt class="col-sm-3">{{ key }}</dt>
       <dd class="col-sm-3">{{ value }}</dd>
-    </center>
     </dl>
   {% endfor %}
 {% endif%}