From 0751321e9bc65c20c7c594cfd4b077af5400f2d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 4 Jun 2020 18:23:36 +0200 Subject: [PATCH] new: Cookies investigation --- lookyloo/lookyloo.py | 62 +++++++++++++++- lookyloo/modules.py | 1 + poetry.lock | 22 +++--- website/web/__init__.py | 27 +------ website/web/templates/hostname_popup.html | 89 ++++++++++++++++------- 5 files changed, 137 insertions(+), 64 deletions(-) diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index d8c12009..6d786ffc 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -2,6 +2,8 @@ # -*- coding: utf-8 -*- import base64 +from collections import defaultdict + from datetime import datetime from email.message import EmailMessage from io import BufferedIOBase, BytesIO @@ -12,7 +14,7 @@ from pathlib import Path import pickle import smtplib import socket -from typing import Union, Dict, List, Tuple, Optional, Any, MutableMapping +from typing import Union, Dict, List, Tuple, Optional, Any, MutableMapping, Set from urllib.parse import urlsplit from uuid import uuid4 from zipfile import ZipFile @@ -423,3 +425,61 @@ class Lookyloo(): self._set_capture_cache(dirpath) return perma_uuid + + def get_hostnode_investigator(self, capture_dir: Path, node_uuid: str) -> Tuple[HostNode, List[Dict[str, Any]]]: + ct = self._load_pickle(capture_dir / 'tree.pickle') + if not ct: + raise MissingUUID(f'Unable to find {capture_dir}') + hostnode = ct.root_hartree.get_host_node_by_uuid(node_uuid) + if not hostnode: + raise MissingUUID(f'Unable to find UUID {node_uuid} in {capture_dir}') + + sanejs_lookups: Dict[str, List[str]] = {} + if hasattr(self, 'sanejs') and self.sanejs.available: + to_lookup = [url.body_hash for url in hostnode.urls if hasattr(url, 'body_hash')] + sanejs_lookups = self.sanejs.hashes_lookup(to_lookup) + + urls: List[Dict[str, Any]] = [] + for url in hostnode.urls: + # For the popup, we need: + # * https vs http + # * everything after the domain + # * the full URL + to_append: Dict[str, Any] = { + 'encrypted': url.name.startswith('https'), + 'url_path': url.name.split('/', 3)[-1], + 'url_object': url + } + + # Optional: SaneJS information + if hasattr(url, 'body_hash') and url.body_hash in sanejs_lookups: + if sanejs_lookups[url.body_hash]: + if isinstance(sanejs_lookups[url.body_hash], list): + libname, version, path = sanejs_lookups[url.body_hash][0].split("|") + other_files = len(sanejs_lookups[url.body_hash]) + to_append['sane_js'] = (libname, version, path, other_files) + else: + # Predefined generic file + to_append['sane_js'] = sanejs_lookups[url.body_hash] + + # Optional: Cookies sent to server in request -> map to nodes who set the cookie in response + if hasattr(url, 'cookies_sent'): + to_display: Dict[str, Set[Tuple[str, str]]] = defaultdict(set) + for cookie, contexts in url.cookies_sent.items(): + if not contexts: + # FIXME Locally created? + continue + for context in contexts: + to_display[cookie].add((context['setter'].hostname, context['setter'].hostnode_uuid)) + to_append['cookies_sent'] = to_display + + # Optional: Cookies received from server in response -> map to nodes who send the cookie in request + if hasattr(url, 'cookies_received'): + to_display = defaultdict(set) + for domain, c_received, is_3rd_party in url.cookies_received: + for url_node in ct.root_hartree.cookies_sent[c_received]: + to_display[c_received].add((url_node.hostname, url_node.hostnode_uuid)) + to_append['cookies_received'] = to_display + + urls.append(to_append) + return hostnode, urls diff --git a/lookyloo/modules.py b/lookyloo/modules.py index 26d77457..a6ae7d8f 100644 --- a/lookyloo/modules.py +++ b/lookyloo/modules.py @@ -49,6 +49,7 @@ class SaneJavaScript(): "16dd1560fdd43c3eee7bcf622d940be93e7e74dee90286da37992d69cea844130911b97f41c71f8287b54f00bd3a388191112f490470cf27c374d524f49ba516": "This is a 1*1 pixel GIF", "01211111688dc2007519ff56603fbe345d057337b911c829aaee97b8d02e7d885e7a2c2d51730f54a04aebc1821897c8041f15e216f1c973ed313087fa91a3fb": "This is a 1*1 pixel GIF", "71db01662075fac031dea18b2c766826c77dbab01400a8642cdc7059394841d5df9020076554c3beca6f808187d42e1a1acc98fad9a0e1ad32ae869145f53746": "This is a 1*1 pixel GIF", + "49b8daf1f5ba868bc8c6b224c787a75025ca36513ef8633d1d8f34e48ee0b578f466fcc104a7bed553404ddc5f9faff3fef5f894b31cd57f32245e550fad656a": "This is a 1*1 pixel GIF", # "": "This is a 1*1 pixel GIF", "f1c33e72643ce366fd578e3b5d393799e8c9ea27b180987826af43b4fc00b65a4eaae5e6426a23448956fee99e3108c6a86f32fb4896c156e24af0571a11c498": "This is a 1*1 pixel PNG", "dc7c40381b3d22919e32c1b700ccb77b1b0aea2690642d01c1ac802561e135c01d5a4d2a0ea18efc0ec3362e8c549814a10a23563f1f56bd62aee0ced7e2bd99": "This is a 1*1 pixel PNG", diff --git a/poetry.lock b/poetry.lock index 2ea57d19..1f6a8893 100644 --- a/poetry.lock +++ b/poetry.lock @@ -306,7 +306,7 @@ publicsuffix2 = "^2.20191221" six = "^1.14.0" [package.source] -reference = "888dab5e75b7da56a84c74e7e55c4cf1c79cbf0b" +reference = "6ee33972962351214413e07c33559691f1ae9b9d" type = "git" url = "https://github.com/viper-framework/har2tree.git" [[package]] @@ -739,7 +739,7 @@ description = "pytest: simple powerful testing with Python" name = "pytest" optional = false python-versions = ">=3.5" -version = "5.4.2" +version = "5.4.3" [package.dependencies] atomicwrites = ">=1.0" @@ -788,7 +788,7 @@ description = "Python client for Redis key-value store" name = "redis" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "3.5.2" +version = "3.5.3" [package.extras] hiredis = ["hiredis (>=0.1.3)"] @@ -1011,11 +1011,11 @@ six = ">=1.4.1" [[package]] category = "main" -description = "Measures number of Terminal column cells of wide-character codes" +description = "Measures the displayed width of unicode strings in a terminal" name = "wcwidth" optional = false python-versions = "*" -version = "0.1.9" +version = "0.2.3" [[package]] category = "main" @@ -1502,8 +1502,8 @@ pypydispatcher = [ ] pysanejs = [] pytest = [ - {file = "pytest-5.4.2-py3-none-any.whl", hash = "sha256:95c710d0a72d91c13fae35dce195633c929c3792f54125919847fdcdf7caa0d3"}, - {file = "pytest-5.4.2.tar.gz", hash = "sha256:eb2b5e935f6a019317e455b6da83dd8650ac9ffd2ee73a7b657a30873d67a698"}, + {file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"}, + {file = "pytest-5.4.3.tar.gz", hash = "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8"}, ] pytest-httpserver = [ {file = "pytest_httpserver-0.3.4-py3-none-any.whl", hash = "sha256:7feab352b2626d1a0ecdebffcac5e5875979f08ad7e621b2289980ce8f6ebc5b"}, @@ -1514,8 +1514,8 @@ queuelib = [ {file = "queuelib-1.5.0.tar.gz", hash = "sha256:42b413295551bdc24ed9376c1a2cd7d0b1b0fa4746b77b27ca2b797a276a1a17"}, ] redis = [ - {file = "redis-3.5.2-py2.py3-none-any.whl", hash = "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242"}, - {file = "redis-3.5.2.tar.gz", hash = "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251"}, + {file = "redis-3.5.3-py2.py3-none-any.whl", hash = "sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24"}, + {file = "redis-3.5.3.tar.gz", hash = "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2"}, ] requests = [ {file = "requests-2.23.0-py2.py3-none-any.whl", hash = "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee"}, @@ -1615,8 +1615,8 @@ w3lib = [ {file = "w3lib-1.22.0.tar.gz", hash = "sha256:0ad6d0203157d61149fd45aaed2e24f53902989c32fc1dccc2e2bfba371560df"}, ] wcwidth = [ - {file = "wcwidth-0.1.9-py2.py3-none-any.whl", hash = "sha256:cafe2186b3c009a04067022ce1dcd79cb38d8d65ee4f4791b8888d6599d1bbe1"}, - {file = "wcwidth-0.1.9.tar.gz", hash = "sha256:ee73862862a156bf77ff92b09034fc4825dd3af9cf81bc5b360668d425f3c5f1"}, + {file = "wcwidth-0.2.3-py2.py3-none-any.whl", hash = "sha256:980fbf4f3c196c0f329cdcd1e84c554d6a211f18e252e525a0cf4223154a41d6"}, + {file = "wcwidth-0.2.3.tar.gz", hash = "sha256:edbc2b718b4db6cdf393eefe3a420183947d6aa312505ce6754516f458ff8830"}, ] werkzeug = [ {file = "Werkzeug-1.0.1-py2.py3-none-any.whl", hash = "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43"}, diff --git a/website/web/__init__.py b/website/web/__init__.py index 37f3a844..9c3ab186 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -17,7 +17,7 @@ from lookyloo.lookyloo import Lookyloo from lookyloo.exceptions import NoValidHarFile from .proxied import ReverseProxied -from typing import Optional, Dict, Any, List, Tuple +from typing import Optional, Dict, Any import logging @@ -144,7 +144,7 @@ def hostnode_popup(tree_uuid: str, node_uuid: str): capture_dir = lookyloo.lookup_capture_dir(tree_uuid) if not capture_dir: return - hostnode = lookyloo.get_hostnode_from_tree(capture_dir, node_uuid) + keys_response = { 'js': "/static/javascript.png", 'exe': "/static/exe.png", @@ -164,29 +164,8 @@ def hostnode_popup(tree_uuid: str, node_uuid: str): 'request_cookie': "/static/cookie_read.png", } - sanejs_lookups: Dict[str, List[str]] = {} - if hasattr(lookyloo, 'sanejs') and lookyloo.sanejs.available: - to_lookup = [url.body_hash for url in hostnode.urls if hasattr(url, 'body_hash')] - sanejs_lookups = lookyloo.sanejs.hashes_lookup(to_lookup) + hostnode, urls = lookyloo.get_hostnode_investigator(capture_dir, node_uuid) - urls: List[Tuple[bool, str, str]] = [] - for url in hostnode.urls: - if hasattr(url, 'body_hash') and url.body_hash in sanejs_lookups: - url.add_feature('sane_js_details', sanejs_lookups[url.body_hash]) - if sanejs_lookups[url.body_hash]: - if isinstance(sanejs_lookups[url.body_hash], list): - libname, version, path = sanejs_lookups[url.body_hash][0].split("|") - other_files = len(sanejs_lookups[url.body_hash]) - url.add_feature('sane_js_details_to_print', (libname, version, path, other_files)) - else: - # Predefined generic file - url.add_feature('sane_js_details_to_print', sanejs_lookups[url.body_hash]) - - # For the popup, we need: - # * https vs http - # * everything after the domain - # * the full URL - urls.append((url.name.startswith('https'), url.name.split('/', 3)[-1], url)) return render_template('hostname_popup.html', tree_uuid=tree_uuid, hostname_uuid=node_uuid, diff --git a/website/web/templates/hostname_popup.html b/website/web/templates/hostname_popup.html index e18c4926..fb2e540c 100644 --- a/website/web/templates/hostname_popup.html +++ b/website/web/templates/hostname_popup.html @@ -5,8 +5,8 @@ {% block scripts %} {{ super() }}