CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

new: Cookies investigation

pull/79/head
Raphaël Vinot 2020-06-04 18:23:36 +02:00
parent af1028ca45
commit 0751321e9b
5 changed files with 137 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
lookyloo/lookyloo.py
View File

@ -2,6 +2,8 @@
# -*- coding: utf-8 -*-
import base64
from collections import defaultdict
from datetime import datetime
from email.message import EmailMessage
from io import BufferedIOBase, BytesIO
@ -12,7 +14,7 @@ from pathlib import Path
import pickle
import smtplib
import socket
from typing import Union, Dict, List, Tuple, Optional, Any, MutableMapping
from typing import Union, Dict, List, Tuple, Optional, Any, MutableMapping, Set
from urllib.parse import urlsplit
from uuid import uuid4
from zipfile import ZipFile
@ -423,3 +425,61 @@ class Lookyloo():
self._set_capture_cache(dirpath)
return perma_uuid
def get_hostnode_investigator(self, capture_dir: Path, node_uuid: str) -> Tuple[HostNode, List[Dict[str, Any]]]:
ct = self._load_pickle(capture_dir / 'tree.pickle')
if not ct:
raise MissingUUID(f'Unable to find {capture_dir}')
hostnode = ct.root_hartree.get_host_node_by_uuid(node_uuid)
if not hostnode:
raise MissingUUID(f'Unable to find UUID {node_uuid} in {capture_dir}')
sanejs_lookups: Dict[str, List[str]] = {}
if hasattr(self, 'sanejs') and self.sanejs.available:
to_lookup = [url.body_hash for url in hostnode.urls if hasattr(url, 'body_hash')]
sanejs_lookups = self.sanejs.hashes_lookup(to_lookup)
urls: List[Dict[str, Any]] = []
for url in hostnode.urls:
# For the popup, we need:
# * https vs http
# * everything after the domain
# * the full URL
to_append: Dict[str, Any] = {
'encrypted': url.name.startswith('https'),
'url_path': url.name.split('/', 3)[-1],
'url_object': url
}
# Optional: SaneJS information
if hasattr(url, 'body_hash') and url.body_hash in sanejs_lookups:
if sanejs_lookups[url.body_hash]:
if isinstance(sanejs_lookups[url.body_hash], list):
libname, version, path = sanejs_lookups[url.body_hash][0].split("|")
other_files = len(sanejs_lookups[url.body_hash])
to_append['sane_js'] = (libname, version, path, other_files)
else:
# Predefined generic file
to_append['sane_js'] = sanejs_lookups[url.body_hash]
# Optional: Cookies sent to server in request -> map to nodes who set the cookie in response
if hasattr(url, 'cookies_sent'):
to_display: Dict[str, Set[Tuple[str, str]]] = defaultdict(set)
for cookie, contexts in url.cookies_sent.items():
if not contexts:
# FIXME Locally created?
continue
for context in contexts:
to_display[cookie].add((context['setter'].hostname, context['setter'].hostnode_uuid))
to_append['cookies_sent'] = to_display
# Optional: Cookies received from server in response -> map to nodes who send the cookie in request
if hasattr(url, 'cookies_received'):
to_display = defaultdict(set)
for domain, c_received, is_3rd_party in url.cookies_received:
for url_node in ct.root_hartree.cookies_sent[c_received]:
to_display[c_received].add((url_node.hostname, url_node.hostnode_uuid))
to_append['cookies_received'] = to_display
urls.append(to_append)
return hostnode, urls

1
lookyloo/modules.py
View File

@ -49,6 +49,7 @@ class SaneJavaScript():
"16dd1560fdd43c3eee7bcf622d940be93e7e74dee90286da37992d69cea844130911b97f41c71f8287b54f00bd3a388191112f490470cf27c374d524f49ba516": "This is a 1*1 pixel GIF",
"01211111688dc2007519ff56603fbe345d057337b911c829aaee97b8d02e7d885e7a2c2d51730f54a04aebc1821897c8041f15e216f1c973ed313087fa91a3fb": "This is a 1*1 pixel GIF",
"71db01662075fac031dea18b2c766826c77dbab01400a8642cdc7059394841d5df9020076554c3beca6f808187d42e1a1acc98fad9a0e1ad32ae869145f53746": "This is a 1*1 pixel GIF",
"49b8daf1f5ba868bc8c6b224c787a75025ca36513ef8633d1d8f34e48ee0b578f466fcc104a7bed553404ddc5f9faff3fef5f894b31cd57f32245e550fad656a": "This is a 1*1 pixel GIF",
# "": "This is a 1*1 pixel GIF",
"f1c33e72643ce366fd578e3b5d393799e8c9ea27b180987826af43b4fc00b65a4eaae5e6426a23448956fee99e3108c6a86f32fb4896c156e24af0571a11c498": "This is a 1*1 pixel PNG",
"dc7c40381b3d22919e32c1b700ccb77b1b0aea2690642d01c1ac802561e135c01d5a4d2a0ea18efc0ec3362e8c549814a10a23563f1f56bd62aee0ced7e2bd99": "This is a 1*1 pixel PNG",

22
poetry.lock generated
View File

@ -306,7 +306,7 @@ publicsuffix2 = "^2.20191221"
six = "^1.14.0"
[package.source]
reference = "888dab5e75b7da56a84c74e7e55c4cf1c79cbf0b"
reference = "6ee33972962351214413e07c33559691f1ae9b9d"
type = "git"
url = "https://github.com/viper-framework/har2tree.git"
[[package]]
@ -739,7 +739,7 @@ description = "pytest: simple powerful testing with Python"
name = "pytest"
optional = false
python-versions = ">=3.5"
version = "5.4.2"
version = "5.4.3"
[package.dependencies]
atomicwrites = ">=1.0"
@ -788,7 +788,7 @@ description = "Python client for Redis key-value store"
name = "redis"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "3.5.2"
version = "3.5.3"
[package.extras]
hiredis = ["hiredis (>=0.1.3)"]
@ -1011,11 +1011,11 @@ six = ">=1.4.1"
[[package]]
category = "main"
description = "Measures number of Terminal column cells of wide-character codes"
description = "Measures the displayed width of unicode strings in a terminal"
name = "wcwidth"
optional = false
python-versions = "*"
version = "0.1.9"
version = "0.2.3"
[[package]]
category = "main"
@ -1502,8 +1502,8 @@ pypydispatcher = [
]
pysanejs = []
pytest = [
{file = "pytest-5.4.2-py3-none-any.whl", hash = "sha256:95c710d0a72d91c13fae35dce195633c929c3792f54125919847fdcdf7caa0d3"},
{file = "pytest-5.4.2.tar.gz", hash = "sha256:eb2b5e935f6a019317e455b6da83dd8650ac9ffd2ee73a7b657a30873d67a698"},
{file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"},
{file = "pytest-5.4.3.tar.gz", hash = "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8"},
]
pytest-httpserver = [
{file = "pytest_httpserver-0.3.4-py3-none-any.whl", hash = "sha256:7feab352b2626d1a0ecdebffcac5e5875979f08ad7e621b2289980ce8f6ebc5b"},
@ -1514,8 +1514,8 @@ queuelib = [
{file = "queuelib-1.5.0.tar.gz", hash = "sha256:42b413295551bdc24ed9376c1a2cd7d0b1b0fa4746b77b27ca2b797a276a1a17"},
]
redis = [
{file = "redis-3.5.2-py2.py3-none-any.whl", hash = "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242"},
{file = "redis-3.5.2.tar.gz", hash = "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251"},
{file = "redis-3.5.3-py2.py3-none-any.whl", hash = "sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24"},
{file = "redis-3.5.3.tar.gz", hash = "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2"},
]
requests = [
{file = "requests-2.23.0-py2.py3-none-any.whl", hash = "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee"},
@ -1615,8 +1615,8 @@ w3lib = [
{file = "w3lib-1.22.0.tar.gz", hash = "sha256:0ad6d0203157d61149fd45aaed2e24f53902989c32fc1dccc2e2bfba371560df"},
]
wcwidth = [
{file = "wcwidth-0.1.9-py2.py3-none-any.whl", hash = "sha256:cafe2186b3c009a04067022ce1dcd79cb38d8d65ee4f4791b8888d6599d1bbe1"},
{file = "wcwidth-0.1.9.tar.gz", hash = "sha256:ee73862862a156bf77ff92b09034fc4825dd3af9cf81bc5b360668d425f3c5f1"},
{file = "wcwidth-0.2.3-py2.py3-none-any.whl", hash = "sha256:980fbf4f3c196c0f329cdcd1e84c554d6a211f18e252e525a0cf4223154a41d6"},
{file = "wcwidth-0.2.3.tar.gz", hash = "sha256:edbc2b718b4db6cdf393eefe3a420183947d6aa312505ce6754516f458ff8830"},
]
werkzeug = [
{file = "Werkzeug-1.0.1-py2.py3-none-any.whl", hash = "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43"},

27
website/web/__init__.py
View File

@ -17,7 +17,7 @@ from lookyloo.lookyloo import Lookyloo
from lookyloo.exceptions import NoValidHarFile
from .proxied import ReverseProxied
from typing import Optional, Dict, Any, List, Tuple
from typing import Optional, Dict, Any
import logging
@ -144,7 +144,7 @@ def hostnode_popup(tree_uuid: str, node_uuid: str):
capture_dir = lookyloo.lookup_capture_dir(tree_uuid)
if not capture_dir:
return
hostnode = lookyloo.get_hostnode_from_tree(capture_dir, node_uuid)
keys_response = {
'js': "/static/javascript.png",
'exe': "/static/exe.png",
@ -164,29 +164,8 @@ def hostnode_popup(tree_uuid: str, node_uuid: str):
'request_cookie': "/static/cookie_read.png",
}
sanejs_lookups: Dict[str, List[str]] = {}
if hasattr(lookyloo, 'sanejs') and lookyloo.sanejs.available:
to_lookup = [url.body_hash for url in hostnode.urls if hasattr(url, 'body_hash')]
sanejs_lookups = lookyloo.sanejs.hashes_lookup(to_lookup)
hostnode, urls = lookyloo.get_hostnode_investigator(capture_dir, node_uuid)
urls: List[Tuple[bool, str, str]] = []
for url in hostnode.urls:
if hasattr(url, 'body_hash') and url.body_hash in sanejs_lookups:
url.add_feature('sane_js_details', sanejs_lookups[url.body_hash])
if sanejs_lookups[url.body_hash]:
if isinstance(sanejs_lookups[url.body_hash], list):
libname, version, path = sanejs_lookups[url.body_hash][0].split("|")
other_files = len(sanejs_lookups[url.body_hash])
url.add_feature('sane_js_details_to_print', (libname, version, path, other_files))
else:
# Predefined generic file
url.add_feature('sane_js_details_to_print', sanejs_lookups[url.body_hash])
# For the popup, we need:
# * https vs http
# * everything after the domain
# * the full URL
urls.append((url.name.startswith('https'), url.name.split('/', 3)[-1], url))
return render_template('hostname_popup.html',
tree_uuid=tree_uuid,
hostname_uuid=node_uuid,

89
website/web/templates/hostname_popup.html
View File

@ -5,8 +5,8 @@
{% block scripts %}
{{ super() }}
<script>
function whereAmI() {
window.opener.ProcessChildMessage("{{ hostname_uuid }}");
function whereAmI(hostname_uuid) {
window.opener.ProcessChildMessage(hostname_uuid);
};
</script>
<script>
@ -44,22 +44,22 @@
{% block content %}
<center>
<h3>{{ hostname }}</h3>
<button type="button" class="btn btn-info" onclick="whereAmI()">Locate node on tree</button>
<button type="button" class="btn btn-info" onclick="whereAmI('{{ hostname_uuid }}')">Locate node on tree</button>
<a href="{{ url_for('hostnode_details_text', tree_uuid=tree_uuid, node_uuid=hostname_uuid) }}" class="btn btn-info" role="button">Get URLs as text</a>
</center>
<p>Click on the URL to get the content of the response</p>
<ul class="list-group-flush">
{% for secure, path, url in urls %}
{% for url in urls %}
<li class="list-group-item">
<div class="h3">
<button type="button" class="btn btn-default btn-copy js-copy"
data-toggle="tooltip" data-placement="bottom" data-copy="{{url.name}}" title="Copy to clipboard">
data-toggle="tooltip" data-placement="bottom" data-copy="{{ url['url_object'].name }}" title="Copy to clipboard">
<svg class="bi bi-clipboard" width="1em" height="1em" viewBox="0 0 16 16" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" d="M4 1.5H3a2 2 0 0 0-2 2V14a2 2 0 0 0 2 2h10a2 2 0 0 0 2-2V3.5a2 2 0 0 0-2-2h-1v1h1a1 1 0 0 1 1 1V14a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V3.5a1 1 0 0 1 1-1h1v-1z"/>
<path fill-rule="evenodd" d="M9.5 1h-3a.5.5 0 0 0-.5.5v1a.5.5 0 0 0 .5.5h3a.5.5 0 0 0 .5-.5v-1a.5.5 0 0 0-.5-.5zm-3-1A1.5 1.5 0 0 0 5 1.5v1A1.5 1.5 0 0 0 6.5 4h3A1.5 1.5 0 0 0 11 2.5v-1A1.5 1.5 0 0 0 9.5 0h-3z"/>
</svg>
</button>
{% if secure %}
{% if url['encrypted'] %}
<svg class="bi bi-lock" width="1em" height="1em" viewBox="0 0 16 16" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" d="M11.5 8h-7a1 1 0 0 0-1 1v5a1 1 0 0 0 1 1h7a1 1 0 0 0 1-1V9a1 1 0 0 0-1-1zm-7-1a2 2 0 0 0-2 2v5a2 2 0 0 0 2 2h7a2 2 0 0 0 2-2V9a2 2 0 0 0-2-2h-7zm0-3a3.5 3.5 0 1 1 7 0v3h-1V4a2.5 2.5 0 0 0-5 0v3h-1V4z"/>
</svg>
@ -68,7 +68,7 @@
<path fill-rule="evenodd" d="M9.655 8H2.333c-.264 0-.398.068-.471.121a.73.73 0 0 0-.224.296 1.626 1.626 0 0 0-.138.59V14c0 .342.076.531.14.635.064.106.151.18.256.237a1.122 1.122 0 0 0 .436.127l.013.001h7.322c.264 0 .398-.068.471-.121a.73.73 0 0 0 .224-.296 1.627 1.627 0 0 0 .138-.59V9c0-.342-.076-.531-.14-.635a.658.658 0 0 0-.255-.237A1.122 1.122 0 0 0 9.655 8zm.012-1H2.333C.5 7 .5 9 .5 9v5c0 2 1.833 2 1.833 2h7.334c1.833 0 1.833-2 1.833-2V9c0-2-1.833-2-1.833-2zM8.5 4a3.5 3.5 0 1 1 7 0v3h-1V4a2.5 2.5 0 0 0-5 0v3h-1V4z"/>
</svg>
{%endif%}
... /{{ path }}
... /{{ url['url_path'] }}
</div>
<ul class="list-group">
<li class="list-group-item">
@ -76,9 +76,9 @@
<div>
{% for key, path in keys_response.items() %}
{% if url[key] %}
{% if url['url_object'][key] %}
{% if key == "response_cookie" %}
<a href="{{ url_for('urlnode_response_cookies', tree_uuid=tree_uuid, node_uuid=url.uuid) }}">
<a href="{{ url_for('urlnode_response_cookies', tree_uuid=tree_uuid, node_uuid=url['url_object'].uuid) }}">
<img src="{{ path }}" alt="{{ key }}" width="21" height="21"/>
</a>
{% else %}
@ -89,38 +89,55 @@
</div>
<div>
{% if not url.empty_response %}
<a href="{{ url_for('urlnode_details', tree_uuid=tree_uuid, node_uuid=url.uuid) }}">
{% if not url['url_object'].empty_response %}
<a href="{{ url_for('urlnode_details', tree_uuid=tree_uuid, node_uuid=url['url_object'].uuid) }}">
Download response body.
</a></br>
Body size: {{ sizeof_fmt(url.body.getbuffer().nbytes) }}
Body size: {{ sizeof_fmt(url['url_object'].body.getbuffer().nbytes) }}
{% else %}
Empty body.
{%endif%}
</div>
<div>
Status Code: {{ url.response['status'] }}
Status Code: {{ url['url_object'].response['status'] }}
</div>
{% if url.sane_js_details_to_print %}
{% if url['sane_js'] %}
<div>
{% if url.sane_js_details_to_print is string %}
{{ url.sane_js_details_to_print }}
{% if url['sane_js'] is string %}
{{ url['sane_js'] }}
{% else %}
This file is known as part of <b>{{ url.sane_js_details_to_print[0] }}</b>
version <b>{{ url.sane_js_details_to_print[1] }}</b>: <b>{{ url.sane_js_details_to_print[2] }}</b>.
{% if url.sane_js_details_to_print[3] > 1%}
It is also present in <b>{{ url.sane_js_details_to_print[3] -1 }}</b> other libraries.
This file is known as part of <b>{{ url['sane_js'][0] }}</b>
version <b>{{ url['sane_js'][1] }}</b>: <b>{{ url['sane_js'][2] }}</b>.
{% if url['sane_js'][3] > 1%}
It is also present in <b>{{ url['sane_js'][3] -1 }}</b> other libraries.
{%endif%}
{%endif%}
</div>
{% endif %}
{% if url.set_third_party_cookies %}
{% if url['url_object'].set_third_party_cookies %}
<div>This response contains 3rd party cookies.</div>
{% endif %}
{% if url['cookies_received'] %}
<p class="h6">List of cookies received in the response to that URL</p>
<ul class="list-group">
{% for cookie, details in url['cookies_received'].items() %}
<li class="list-group-item">{{ cookie }}
<ul class="list-group">
{% for hostname, hostnode_uuid in details %}
<li class="list-group-item"> {{ hostname }} -
<button type="button" class="btn btn-info" onclick="whereAmI('{{ hostnode_uuid }}')">Show on tree node sending this cookie</button>
</li>
{% endfor %}
</ul>
</li>
{% endfor %}
</ul>
</p>
{% endif %}
</li>
@ -128,9 +145,9 @@
<p class="h4">Request</p>
<div>
{% for key, path in keys_request.items() %}
{% if url[key] %}
{% if url['url_object'][key] %}
{% if key == "request_cookie" %}
<a href="{{ url_for('urlnode_request_cookies', tree_uuid=tree_uuid, node_uuid=url.uuid) }}">
<a href="{{ url_for('urlnode_request_cookies', tree_uuid=tree_uuid, node_uuid=url['url_object'].uuid) }}">
<img src="{{ path }}" alt="{{ key }}" width="21" height="21"/>
</a>
{% else %}
@ -139,14 +156,30 @@
{%endif%}
{% endfor %}
</div>
{% if url.posted_data %}
<a href="{{ url_for('urlnode_post_request', tree_uuid=tree_uuid, node_uuid=url.uuid) }}">
{% if url['url_object'].posted_data %}
<a href="{{ url_for('urlnode_post_request', tree_uuid=tree_uuid, node_uuid=url['url_object'].uuid) }}">
Download posted data
</a></br>
Posted data size: {{ sizeof_fmt(url.posted_data|length) }}
Posted data size: {{ sizeof_fmt(url['url_object'].posted_data|length) }}
{% endif %}
{% if url['cookies_sent'] %}
<p class="h6">List of cookies sent in the request to that URL</p>
<ul class="list-group">
{% for cookie, details in url['cookies_sent'].items() %}
<li class="list-group-item">{{ cookie }}
<ul class="list-group">
{% for hostname, hostnode_uuid in details %}
<li class="list-group-item"> {{ hostname }} -
<button type="button" class="btn btn-info" onclick="whereAmI('{{ hostnode_uuid }}')">Show on tree node setting this cookie</button>
</li>
{% endfor %}
</ul>
</li>
{% endfor %}
</ul>
</p>
{% endif %}
</li>
</ul>
</li>
{% endfor %}
</ul>