diff --git a/config/modules.json.sample b/config/modules.json.sample index 79ee9449..8e2da922 100644 --- a/config/modules.json.sample +++ b/config/modules.json.sample @@ -2,5 +2,8 @@ "VirusTotal": { "apikey": "KEY", "autosubmit": false + }, + "SaneJS": { + "enabled": true } } diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index b27bd943..9850f8f7 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -19,13 +19,12 @@ from zipfile import ZipFile from defang import refang # type: ignore from har2tree import CrawledTree, Har2TreeError, HarFile -from pysanejs import SaneJS from redis import Redis from scrapysplashwrapper import crawl from .exceptions import NoValidHarFile from .helpers import get_homedir, get_socket_path, load_cookies, load_configs, safe_create_dir, get_email_template -from .modules import VirusTotal +from .modules import VirusTotal, SaneJavaScript class Lookyloo(): @@ -50,17 +49,14 @@ class Lookyloo(): self.vt = VirusTotal(self.configs['modules']['VirusTotal']) if not self.vt.available: self.logger.warning('Unable to setup the VirusTotal module') + if 'SaneJS' in self.configs['modules']: + self.sanejs = SaneJavaScript(self.configs['modules']['SaneJS']) + if not self.sanejs.available: + self.logger.warning('Unable to setup the SaneJS module') if not self.redis.exists('cache_loaded'): self._init_existing_dumps() - # Try to reach sanejs - self.sanejs = SaneJS() - if not self.sanejs.is_up: - self.use_sane_js = False - else: - self.use_sane_js = True - def rebuild_cache(self) -> None: self.redis.flushdb() self._init_existing_dumps() @@ -312,11 +308,6 @@ class Lookyloo(): def get_capture(self, capture_dir: Path) -> BytesIO: return self._get_raw(capture_dir) - def sane_js_query(self, sha512: str) -> Dict[str, Any]: - if self.use_sane_js: - return self.sanejs.sha512(sha512) - return {'response': []} - def scrape(self, url: str, cookies_pseudofile: Optional[BufferedIOBase]=None, depth: int=1, listing: bool=True, user_agent: Optional[str]=None, perma_uuid: str=None, os: str=None, browser: str=None) -> Union[bool, str]: diff --git a/lookyloo/modules.py b/lookyloo/modules.py index ed8fe0ca..19cd56e3 100644 --- a/lookyloo/modules.py +++ b/lookyloo/modules.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from typing import Dict, Any, Optional +from typing import Dict, Any, Optional, List, Union from datetime import date import hashlib import json @@ -13,6 +13,65 @@ from .helpers import get_homedir from .exceptions import ConfigError import vt # type: ignore +from pysanejs import SaneJS + + +class SaneJavaScript(): + + def __init__(self, config: Dict[str, Any]): + if not ('enabled' in config or config['enabled']): + self.available = False + return + self.client = SaneJS() + if not self.client.is_up: + self.available = False + return + self.available = True + self.storage_dir = get_homedir() / 'sanejs' + self.storage_dir.mkdir(parents=True, exist_ok=True) + + def hashes_lookup(self, sha512: Union[List[str], str], force: bool=False) -> Optional[Dict[str, Any]]: + if isinstance(sha512, str): + hashes = [sha512] + else: + hashes = sha512 + + today_dir = self.storage_dir / date.today().isoformat() + today_dir.mkdir(parents=True, exist_ok=True) + sanejs_unknowns = today_dir / 'unknown' + unknown_hashes = [] + if sanejs_unknowns.exists(): + with sanejs_unknowns.open() as f: + unknown_hashes = [line.strip() for line in f.readlines()] + + if force: + to_lookup = hashes + else: + to_lookup = [h for h in sha512 if (h not in unknown_hashes + and not (today_dir / h).exists())] + to_return = {} + for h in to_lookup: + response = self.client.sha512(h) + if 'error' in response: + # Server not ready + break + if 'response' in response and response['response']: + cached_path = today_dir / h + with cached_path.open('w') as f: + json.dump(response['response'], f) + to_return[h] = response['response'] + else: + unknown_hashes.append(h) + + for h in hashes: + cached_path = today_dir / h + if h in unknown_hashes or h in to_return: + continue + elif cached_path.exists(): + with cached_path.open() as f: + to_return[h] = json.load(f) + + return to_return class VirusTotal(): diff --git a/website/web/__init__.py b/website/web/__init__.py index 67743433..d171be9a 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -136,7 +136,40 @@ def hostnode_details_text(node_uuid: str): @app.route('/tree/hostname_popup/', methods=['GET']) def hostnode_popup(node_uuid: str): - return render_template('hostname_popup.html', hostname_uuid=node_uuid) + with open(session["tree"], 'rb') as f: + ct = pickle.load(f) + hostnode = ct.root_hartree.get_host_node_by_uuid(node_uuid) + table_keys = { + 'js': "/static/javascript.png", + 'exe': "/static/exe.png", + 'css': "/static/css.png", + 'font': "/static/font.png", + 'html': "/static/html.png", + 'json': "/static/json.png", + 'iframe': "/static/ifr.png", + 'image': "/static/img.png", + 'unknown_mimetype': "/static/wtf.png", + 'video': "/static/video.png", + 'request_cookie': "/static/cookie_read.png", + 'response_cookie': "/static/cookie_received.png", + 'redirect': "/static/redirect.png", + 'redirect_to_nothing': "/static/cookie_in_url.png" + } + + urls = [] + if lookyloo.sanejs.available: + to_lookup = [url.body_hash for url in hostnode.urls if hasattr(url, 'body_hash')] + lookups = lookyloo.sanejs.hashes_lookup(to_lookup) + for url in hostnode.urls: + if lookyloo.sanejs.available and hasattr(url, 'body_hash') and url.body_hash in lookups: + url.add_feature('sane_js_details', lookups[url.body_hash]) + # TODO: Do something with it. + urls.append(url) + return render_template('hostname_popup.html', + hostname_uuid=node_uuid, + hostname=hostnode.name, + urls=urls, + keys=table_keys) @app.route('/tree/hostname/', methods=['GET']) diff --git a/website/web/templates/hostname_popup.html b/website/web/templates/hostname_popup.html index 7158ac90..1fa166f0 100644 --- a/website/web/templates/hostname_popup.html +++ b/website/web/templates/hostname_popup.html @@ -1,6 +1,6 @@ {% extends "main.html" %} -{% block title %}Details for hostname {% endblock %} +{% block title %}Details for {{ hostname }} {% endblock %} {% block scripts %}