diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py
index 28623927..2974f82c 100644
--- a/lookyloo/lookyloo.py
+++ b/lookyloo/lookyloo.py
@@ -570,6 +570,18 @@ class Lookyloo():
break
return details, body_content
+ def get_all_body_hashes(self, capture_uuid: str, /) -> Dict[str, Dict[str, Union[URLNode, int]]]:
+ ct = self.get_crawled_tree(capture_uuid)
+ to_return: Dict[str, Dict[str, Union[URLNode, int]]] = defaultdict()
+ for node in ct.root_hartree.url_tree.traverse():
+ if node.empty_response or node.body_hash in to_return:
+ # If we have the same hash more than once, skip
+ continue
+ total_captures, details = self.indexing.get_body_hash_captures(node.body_hash, limit=-1)
+ # Note for future: mayeb get url, capture title, something better than just the hash to show to the user
+ to_return[node.body_hash] = {'node': node, 'total_captures': total_captures}
+ return to_return
+
def get_latest_url_capture(self, url: str, /) -> Optional[CaptureCache]:
'''Get the most recent capture with this URL'''
captures = self.sorted_capture_cache(self.indexing.get_captures_url(url))
diff --git a/poetry.lock b/poetry.lock
index 3a37e10d..12b20b45 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -403,7 +403,7 @@ tornado = ["tornado (>=0.2)"]
[[package]]
name = "har2tree"
-version = "1.13.2"
+version = "1.13.3"
description = "HTTP Archive (HAR) to ETE Toolkit generator"
category = "main"
optional = false
@@ -421,7 +421,7 @@ six = ">=1.16.0,<2.0.0"
w3lib = ">=1.22.0,<2.0.0"
[package.extras]
-docs = ["Sphinx (>=5.0.2,<6.0.0)"]
+docs = ["Sphinx (>=5.1.1,<6.0.0)"]
[[package]]
name = "hiredis"
@@ -544,7 +544,7 @@ i18n = ["Babel (>=2.7)"]
[[package]]
name = "jsonschema"
-version = "4.8.0"
+version = "4.9.0"
description = "An implementation of JSON Schema validation for Python"
category = "main"
optional = false
@@ -553,6 +553,7 @@ python-versions = ">=3.7"
[package.dependencies]
attrs = ">=17.4.0"
importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""}
+pkgutil-resolve-name = {version = ">=1.3.10", markers = "python_version < \"3.9\""}
pyrsistent = ">=0.14.0,<0.17.0 || >0.17.0,<0.17.1 || >0.17.1,<0.17.2 || >0.17.2"
[package.extras]
@@ -713,6 +714,14 @@ python-versions = ">=3.7"
docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-issues (>=3.0.1)", "sphinx-removed-in", "sphinxext-opengraph"]
tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"]
+[[package]]
+name = "pkgutil-resolve-name"
+version = "1.3.10"
+description = "Resolve a name to an object."
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
[[package]]
name = "playwright"
version = "1.24.0"
@@ -1192,7 +1201,7 @@ python-versions = "*"
[[package]]
name = "types-redis"
-version = "4.3.11"
+version = "4.3.13"
description = "Typing stubs for redis"
category = "dev"
optional = false
@@ -1200,7 +1209,7 @@ python-versions = "*"
[[package]]
name = "types-requests"
-version = "2.28.5"
+version = "2.28.6"
description = "Typing stubs for requests"
category = "dev"
optional = false
@@ -1219,7 +1228,7 @@ python-versions = "*"
[[package]]
name = "types-urllib3"
-version = "1.26.17"
+version = "1.26.20"
description = "Typing stubs for urllib3"
category = "dev"
optional = false
@@ -1391,7 +1400,7 @@ misp = ["python-magic", "pydeep2"]
[metadata]
lock-version = "1.1"
python-versions = ">=3.8,<3.11"
-content-hash = "73b3a7e37ff853a7edf8632744c175dc933552cfcb7b6fcd9c4bd5ebc1194801"
+content-hash = "44380fea67a49952b9e376628a50dcbe383929de3c6e174ce8b1c1b7c59f2525"
[metadata.files]
aiohttp = [
@@ -1753,8 +1762,8 @@ gunicorn = [
{file = "gunicorn-20.1.0.tar.gz", hash = "sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8"},
]
har2tree = [
- {file = "har2tree-1.13.2-py3-none-any.whl", hash = "sha256:e96db03800fe67b352c0c564c3f58ac6ce2e261fe64d15ec7f4e0b81c3a8edaa"},
- {file = "har2tree-1.13.2.tar.gz", hash = "sha256:c05c0952ae5c9af7d9d01c7362aa23c4060e8a48b8d6daa1f44ce70a1a8cfff3"},
+ {file = "har2tree-1.13.3-py3-none-any.whl", hash = "sha256:1cb921d3dfd9048244edcfc1aa01e5f0884248bf0cc39aafa18165eaed393db9"},
+ {file = "har2tree-1.13.3.tar.gz", hash = "sha256:7481696e9c4f2907d4df391027e4c4ee01b03f4a588a726dc1904035c0d959af"},
]
hiredis = [
{file = "hiredis-2.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b4c8b0bc5841e578d5fb32a16e0c305359b987b850a06964bd5a62739d688048"},
@@ -1828,8 +1837,8 @@ jinja2 = [
{file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"},
]
jsonschema = [
- {file = "jsonschema-4.8.0-py3-none-any.whl", hash = "sha256:58bb77251318cef5e1179e33dd6e7a008a3c6c638487ab4d943c2f370cc31a1a"},
- {file = "jsonschema-4.8.0.tar.gz", hash = "sha256:c1d410e379b210ba903bee6adf3fce6d5204cea4c2b622d63f914d2dbfef0993"},
+ {file = "jsonschema-4.9.0-py3-none-any.whl", hash = "sha256:5d0be0cd1b670438b71c3d3145b2abba1f9d197e3e91adc4c4bae4c0e114e252"},
+ {file = "jsonschema-4.9.0.tar.gz", hash = "sha256:df10e65c8f3687a48e93d0d348ce0ce5f897b5a28e9bbcbbe8f7c7eaf019e850"},
]
lief = [
{file = "lief-0.12.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:4fbbc9d520de87ac22210c62d22a9b088e5460f9a028741311e6f68ef8877ddd"},
@@ -2147,6 +2156,10 @@ pillow = [
{file = "Pillow-9.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:0030fdbd926fb85844b8b92e2f9449ba89607231d3dd597a21ae72dc7fe26927"},
{file = "Pillow-9.2.0.tar.gz", hash = "sha256:75e636fd3e0fb872693f23ccb8a5ff2cd578801251f3a4f6854c6a5d437d3c04"},
]
+pkgutil-resolve-name = [
+ {file = "pkgutil_resolve_name-1.3.10-py3-none-any.whl", hash = "sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e"},
+ {file = "pkgutil_resolve_name-1.3.10.tar.gz", hash = "sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174"},
+]
playwright = [
{file = "playwright-1.24.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:942c938a8fc8d5daa01cf4e2ca7dfd4cad9216e3e0af88992c083dd324f9210e"},
{file = "playwright-1.24.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fbc74dd91c0c00d282d340b6d08d22010279cbe84052afe0f0854dc1e3a51315"},
@@ -2425,20 +2438,20 @@ types-python-dateutil = [
{file = "types_python_dateutil-2.8.19-py3-none-any.whl", hash = "sha256:6284df1e4783d8fc6e587f0317a81333856b872a6669a282f8a325342bce7fa8"},
]
types-redis = [
- {file = "types-redis-4.3.11.tar.gz", hash = "sha256:83e5633f216c729fb39c5d3f71dd3b0080c651f75b5d4128733878512409c744"},
- {file = "types_redis-4.3.11-py3-none-any.whl", hash = "sha256:ba4612407a758353aa4fe5e1e4b804126e1160dbd6797840223958b1b5dda34d"},
+ {file = "types-redis-4.3.13.tar.gz", hash = "sha256:b8334a96a2f431521bfa72205b343129acdc5a646ffcfb304d80a1cd0deff548"},
+ {file = "types_redis-4.3.13-py3-none-any.whl", hash = "sha256:cc2209ecfab2ad6df1e3eec730c06f9b2dec77f4164eb86e04dad455a651b394"},
]
types-requests = [
- {file = "types-requests-2.28.5.tar.gz", hash = "sha256:ac618bfefcb3742eaf97c961e13e9e5a226e545eda4a3dbe293b898d40933ad1"},
- {file = "types_requests-2.28.5-py3-none-any.whl", hash = "sha256:98ab647ae88b5e2c41d6d20cfcb5117da1bea561110000b6fdeeea07b3e89877"},
+ {file = "types-requests-2.28.6.tar.gz", hash = "sha256:cf3383bbd79394bf051a0a9202d6831fa962f186f923c178f7c059e3424bd00e"},
+ {file = "types_requests-2.28.6-py3-none-any.whl", hash = "sha256:d8d7607419cd4b41a7b9497e15e8c0bad78d50df43c48ad25bc526a11518c3a9"},
]
types-setuptools = [
{file = "types-setuptools-63.2.2.tar.gz", hash = "sha256:a9aa0c01d5f3443cd544026d5ffc97b95ddadf731dab13419c393d43fd8617c0"},
{file = "types_setuptools-63.2.2-py3-none-any.whl", hash = "sha256:a370df7a1e0dc856af9d998234f6e2ab04f30f25b8e1410f6db65910979f6252"},
]
types-urllib3 = [
- {file = "types-urllib3-1.26.17.tar.gz", hash = "sha256:73fd274524c3fc7cd8cd9ceb0cb67ed99b45f9cb2831013e46d50c1451044800"},
- {file = "types_urllib3-1.26.17-py3-none-any.whl", hash = "sha256:0d027fcd27dbb3cb532453b4d977e05bc1e13aefd70519866af211b3003d895d"},
+ {file = "types-urllib3-1.26.20.tar.gz", hash = "sha256:1fb6e2af519a7216a19dd6be8cd2ee787b402a754ccb4a13ca1c0e5b202aea5a"},
+ {file = "types_urllib3-1.26.20-py3-none-any.whl", hash = "sha256:6249b6223226cb2012db3b4ff6945c9cb0e12ece9b24f5e29787c4f05028a979"},
]
types-werkzeug = [
{file = "types-Werkzeug-1.0.9.tar.gz", hash = "sha256:5cc269604c400133d452a40cee6397655f878fc460e03fde291b9e3a5eaa518c"},
diff --git a/pyproject.toml b/pyproject.toml
index 5c03fef3..59ec2edf 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -63,7 +63,7 @@ pyhashlookup = "^1.2.0"
lief = "^0.12.1"
ua-parser = "^0.15.0"
Flask-Login = "^0.6.2"
-har2tree = "^1.13.2"
+har2tree = "^1.13.3"
playwrightcapture = "^1.13.4"
passivetotal = "^2.5.9"
werkzeug = "2.1.2"
@@ -74,8 +74,8 @@ misp = ['python-magic', 'pydeep2']
[tool.poetry.dev-dependencies]
mypy = "^0.971"
ipython = "^8.4.0"
-types-redis = "^4.3.11"
-types-requests = "^2.28.5"
+types-redis = "^4.3.13"
+types-requests = "^2.28.6"
types-Flask = "^1.1.6"
types-pkg-resources = "^0.1.3"
types-Deprecated = "^1.2.9"
diff --git a/website/web/__init__.py b/website/web/__init__.py
index 9a3dc34c..5af59b16 100644
--- a/website/web/__init__.py
+++ b/website/web/__init__.py
@@ -8,7 +8,7 @@ import os
import time
from datetime import date, datetime, timedelta, timezone
from io import BytesIO, StringIO
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union, TypedDict
from urllib.parse import quote_plus, unquote_plus, urlparse
import flask_login # type: ignore
@@ -160,6 +160,38 @@ def get_sri(directory: str, filename: str) -> str:
app.jinja_env.globals.update(get_sri=get_sri)
+class Icon(TypedDict):
+ icon: str
+ tooltip: str
+
+
+def get_icon(icon_id: str) -> Optional[Icon]:
+ available_icons: Dict[str, Icon] = {
+ 'js': {'icon': "javascript.png", 'tooltip': 'The content of the response is a javascript'},
+ 'exe': {'icon': "exe.png", 'tooltip': 'The content of the response is an executable'},
+ 'css': {'icon': "css.png", 'tooltip': 'The content of the response is a CSS'},
+ 'font': {'icon': "font.png", 'tooltip': 'The content of the response is a font'},
+ 'html': {'icon': "html.png", 'tooltip': 'The content of the response is a HTML document'},
+ 'json': {'icon': "json.png", 'tooltip': 'The content of the response is a Json'},
+ 'text': {'icon': "json.png", 'tooltip': 'The content of the response is a text'}, # FIXME: Need new icon
+ 'iframe': {'icon': "ifr.png", 'tooltip': 'This content is loaded from an Iframe'},
+ 'image': {'icon': "img.png", 'tooltip': 'The content of the response is an image'},
+ 'unset_mimetype': {'icon': "wtf.png", 'tooltip': 'The type of content of the response is not set'},
+ 'octet-stream': {'icon': "wtf.png", 'tooltip': 'The type of content of the response is a binary blob'},
+ 'unknown_mimetype': {'icon': "wtf.png", 'tooltip': 'The type of content of the response is of an unknown type'},
+ 'video': {'icon': "video.png", 'tooltip': 'The content of the response is a video'},
+ 'livestream': {'icon': "video.png", 'tooltip': 'The content of the response is a livestream'},
+ 'response_cookie': {'icon': "cookie_received.png", 'tooltip': 'There are cookies in the response'},
+ 'request_cookie': {'icon': "cookie_read.png", 'tooltip': 'There are cookies in the request'},
+ 'redirect': {'icon': "redirect.png", 'tooltip': 'The request is redirected'},
+ 'redirect_to_nothing': {'icon': "cookie_in_url.png", 'tooltip': 'The request is redirected to an URL we do not have in the capture'}
+ }
+ return available_icons.get(icon_id)
+
+
+app.jinja_env.globals.update(get_icon=get_icon)
+
+
# ##### Generic/configuration methods #####
@app.after_request
@@ -201,30 +233,6 @@ def urls_hostnode(tree_uuid: str, node_uuid: str):
@app.route('/tree/ Request
File type | +Captures total | +Hash (sha512) | +
---|---|---|
+
+ Click to download." + {% endif %} + /> + + |
+ {{ info['total_captures'] }} | ++ {{body_hash}} + | +