CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

new: Flag to quickly say if two captures are different or not

pull/694/head
Raphaël Vinot 2023-05-11 15:07:37 +02:00
parent aa474e2b17
commit a504facf5e
4 changed files with 87 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
lookyloo/comparator.py
View File

@ -3,7 +3,7 @@
import fnmatch import fnmatch
import logging import logging
from typing import Dict, Any, Union, List, Optional, TypedDict, Set from typing import Dict, Any, Union, List, Optional, TypedDict, Set, Tuple
from har2tree import URLNode from har2tree import URLNode
@ -45,10 +45,11 @@ class Comparator():
to_return['ip_address'] = str(node.ip_address) to_return['ip_address'] = str(node.ip_address)
return to_return return to_return
def _compare_nodes(self, left: Dict[str, str], right: Dict[str, str], /) -> Dict[str, Any]: def _compare_nodes(self, left: Dict[str, str], right: Dict[str, str], /, different: bool) -> Tuple[bool, Dict[str, Any]]:
to_return = {} to_return = {}
# URL # URL
if left['url'] != right['url']: if left['url'] != right['url']:
different = True
to_return['url'] = {'message': 'The nodes have different URLs.', to_return['url'] = {'message': 'The nodes have different URLs.',
'details': [left['url'], right['url']]} 'details': [left['url'], right['url']]}
# Hostname # Hostname
@ -64,6 +65,7 @@ class Comparator():
# IP in HAR # IP in HAR
if left.get('ip_address') and right.get('ip_address'): if left.get('ip_address') and right.get('ip_address'):
if left['ip_address'] != right['ip_address']: if left['ip_address'] != right['ip_address']:
different = True
to_return['ip'] = {'message': 'The nodes load content from different IPs.', to_return['ip'] = {'message': 'The nodes load content from different IPs.',
'details': [left['ip_address'], right['ip_address']]} 'details': [left['ip_address'], right['ip_address']]}
else: else:
@ -71,7 +73,7 @@ class Comparator():
'details': left['ip_address']} 'details': left['ip_address']}
# IPs in hostnode + ASNs # IPs in hostnode + ASNs
return to_return return different, to_return
def get_comparables_capture(self, capture_uuid: str) -> Dict[str, Any]: def get_comparables_capture(self, capture_uuid: str) -> Dict[str, Any]:
if capture_uuid not in self._captures_index: if capture_uuid not in self._captures_index:
@ -88,12 +90,13 @@ class Comparator():
to_return['ressources'] = {(a.name, a.hostname) for a in capture.tree.root_hartree.rendered_node.traverse()} to_return['ressources'] = {(a.name, a.hostname) for a in capture.tree.root_hartree.rendered_node.traverse()}
return to_return return to_return
def compare_captures(self, capture_left: str, capture_right: str, /, *, settings: Optional[CompareSettings]=None) -> Dict[str, Any]: def compare_captures(self, capture_left: str, capture_right: str, /, *, settings: Optional[CompareSettings]=None) -> Tuple[bool, Dict[str, Any]]:
if capture_left not in self._captures_index: if capture_left not in self._captures_index:
raise MissingUUID(f'{capture_left} does not exists.') raise MissingUUID(f'{capture_left} does not exists.')
if capture_right not in self._captures_index: if capture_right not in self._captures_index:
raise MissingUUID(f'{capture_right} does not exists.') raise MissingUUID(f'{capture_right} does not exists.')
different: bool = False
to_return: Dict[str, Dict[str, Union[str, to_return: Dict[str, Dict[str, Union[str,
List[Union[str, Dict[str, Any]]], List[Union[str, Dict[str, Any]]],
Dict[str, Union[int, str, Dict[str, Union[int, str,
@ -102,6 +105,7 @@ class Comparator():
right = self.get_comparables_capture(capture_right) right = self.get_comparables_capture(capture_right)
# Compare initial URL (first entry in HAR) # Compare initial URL (first entry in HAR)
if left['root_url'] != right['root_url']: if left['root_url'] != right['root_url']:
different = True
to_return['root_url'] = {'message': 'The captures are for different URLs.', to_return['root_url'] = {'message': 'The captures are for different URLs.',
'details': [left['root_url'], right['root_url']]} 'details': [left['root_url'], right['root_url']]}
else: else:
@ -110,6 +114,7 @@ class Comparator():
# Compare landing page (URL in browser) # Compare landing page (URL in browser)
if left['final_url'] != right['final_url']: if left['final_url'] != right['final_url']:
different = True
to_return['final_url'] = {'message': 'The landing page is different.', to_return['final_url'] = {'message': 'The landing page is different.',
'details': [left['final_url'], right['final_url']]} 'details': [left['final_url'], right['final_url']]}
# => if different, check if the hostname is the same # => if different, check if the hostname is the same
@ -124,6 +129,7 @@ class Comparator():
'details': left['final_url']} 'details': left['final_url']}
if left['final_status_code'] != right['final_status_code']: if left['final_status_code'] != right['final_status_code']:
different = True
to_return['final_status_code'] = {'message': 'The status code of the rendered page is different.', to_return['final_status_code'] = {'message': 'The status code of the rendered page is different.',
'details': [left['final_status_code'], right['final_status_code']]} 'details': [left['final_status_code'], right['final_status_code']]}
else: else:
@ -132,6 +138,7 @@ class Comparator():
to_return['redirects'] = {'length': {}, 'nodes': []} to_return['redirects'] = {'length': {}, 'nodes': []}
if left['redirects']['length'] != right['redirects']['length']: if left['redirects']['length'] != right['redirects']['length']:
different = True
to_return['redirects']['length'] = {'message': 'The captures have a different amount of redirects', to_return['redirects']['length'] = {'message': 'The captures have a different amount of redirects',
'details': [left['redirects']['length'], right['redirects']['length']]} 'details': [left['redirects']['length'], right['redirects']['length']]}
else: else:
@ -141,7 +148,8 @@ class Comparator():
# Compare chain of redirects # Compare chain of redirects
for redirect_left, redirect_right in zip(right['redirects']['nodes'], left['redirects']['nodes']): for redirect_left, redirect_right in zip(right['redirects']['nodes'], left['redirects']['nodes']):
if isinstance(to_return['redirects']['nodes'], list): if isinstance(to_return['redirects']['nodes'], list):
to_return['redirects']['nodes'].append(self._compare_nodes(redirect_left, redirect_right)) different, node_compare = self._compare_nodes(redirect_left, redirect_right, different)
to_return['redirects']['nodes'].append(node_compare)
# Compare all ressources URLs # Compare all ressources URLs
to_return['ressources'] = {} to_return['ressources'] = {}
@ -167,12 +175,14 @@ class Comparator():
if present_in_both := ressources_left & ressources_right: if present_in_both := ressources_left & ressources_right:
to_return['ressources']['both'] = sorted(present_in_both) to_return['ressources']['both'] = sorted(present_in_both)
if present_left := ressources_left - ressources_right: if present_left := ressources_left - ressources_right:
different = True
to_return['ressources']['left'] = sorted(present_left) to_return['ressources']['left'] = sorted(present_left)
if present_right := ressources_right - ressources_left: if present_right := ressources_right - ressources_left:
different = True
to_return['ressources']['right'] = sorted(present_right) to_return['ressources']['right'] = sorted(present_right)
# IP/ASN checks - Note: there is the IP in the HAR, and the ones resolved manually - if the IP is different, but part of the list, it's cool # IP/ASN checks - Note: there is the IP in the HAR, and the ones resolved manually - if the IP is different, but part of the list, it's cool
# For each node up to the landing page # For each node up to the landing page
# Compare IPs # Compare IPs
# Compare ASNs # Compare ASNs
return to_return return different, to_return

122
poetry.lock generated
View File

@ -1629,38 +1629,38 @@ files = [
[[package]] [[package]]
name = "mypy" name = "mypy"
version = "1.2.0" version = "1.3.0"
description = "Optional static typing for Python" description = "Optional static typing for Python"
category = "dev" category = "dev"
optional = false optional = false
python-versions = ">=3.7" python-versions = ">=3.7"
files = [ files = [
{file = "mypy-1.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:701189408b460a2ff42b984e6bd45c3f41f0ac9f5f58b8873bbedc511900086d"}, {file = "mypy-1.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c1eb485cea53f4f5284e5baf92902cd0088b24984f4209e25981cc359d64448d"},
{file = "mypy-1.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fe91be1c51c90e2afe6827601ca14353bbf3953f343c2129fa1e247d55fd95ba"}, {file = "mypy-1.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4c99c3ecf223cf2952638da9cd82793d8f3c0c5fa8b6ae2b2d9ed1e1ff51ba85"},
{file = "mypy-1.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d26b513225ffd3eacece727f4387bdce6469192ef029ca9dd469940158bc89e"}, {file = "mypy-1.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:550a8b3a19bb6589679a7c3c31f64312e7ff482a816c96e0cecec9ad3a7564dd"},
{file = "mypy-1.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3a2d219775a120581a0ae8ca392b31f238d452729adbcb6892fa89688cb8306a"}, {file = "mypy-1.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cbc07246253b9e3d7d74c9ff948cd0fd7a71afcc2b77c7f0a59c26e9395cb152"},
{file = "mypy-1.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:2e93a8a553e0394b26c4ca683923b85a69f7ccdc0139e6acd1354cc884fe0128"}, {file = "mypy-1.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a22435632710a4fcf8acf86cbd0d69f68ac389a3892cb23fbad176d1cddaf228"},
{file = "mypy-1.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3efde4af6f2d3ccf58ae825495dbb8d74abd6d176ee686ce2ab19bd025273f41"}, {file = "mypy-1.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6e33bb8b2613614a33dff70565f4c803f889ebd2f859466e42b46e1df76018dd"},
{file = "mypy-1.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:695c45cea7e8abb6f088a34a6034b1d273122e5530aeebb9c09626cea6dca4cb"}, {file = "mypy-1.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7d23370d2a6b7a71dc65d1266f9a34e4cde9e8e21511322415db4b26f46f6b8c"},
{file = "mypy-1.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0e9464a0af6715852267bf29c9553e4555b61f5904a4fc538547a4d67617937"}, {file = "mypy-1.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:658fe7b674769a0770d4b26cb4d6f005e88a442fe82446f020be8e5f5efb2fae"},
{file = "mypy-1.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8293a216e902ac12779eb7a08f2bc39ec6c878d7c6025aa59464e0c4c16f7eb9"}, {file = "mypy-1.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6e42d29e324cdda61daaec2336c42512e59c7c375340bd202efa1fe0f7b8f8ca"},
{file = "mypy-1.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:f46af8d162f3d470d8ffc997aaf7a269996d205f9d746124a179d3abe05ac602"}, {file = "mypy-1.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:d0b6c62206e04061e27009481cb0ec966f7d6172b5b936f3ead3d74f29fe3dcf"},
{file = "mypy-1.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:031fc69c9a7e12bcc5660b74122ed84b3f1c505e762cc4296884096c6d8ee140"}, {file = "mypy-1.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:76ec771e2342f1b558c36d49900dfe81d140361dd0d2df6cd71b3db1be155409"},
{file = "mypy-1.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:390bc685ec209ada4e9d35068ac6988c60160b2b703072d2850457b62499e336"}, {file = "mypy-1.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebc95f8386314272bbc817026f8ce8f4f0d2ef7ae44f947c4664efac9adec929"},
{file = "mypy-1.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4b41412df69ec06ab141808d12e0bf2823717b1c363bd77b4c0820feaa37249e"}, {file = "mypy-1.3.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:faff86aa10c1aa4a10e1a301de160f3d8fc8703b88c7e98de46b531ff1276a9a"},
{file = "mypy-1.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:4e4a682b3f2489d218751981639cffc4e281d548f9d517addfd5a2917ac78119"}, {file = "mypy-1.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:8c5979d0deb27e0f4479bee18ea0f83732a893e81b78e62e2dda3e7e518c92ee"},
{file = "mypy-1.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a197ad3a774f8e74f21e428f0de7f60ad26a8d23437b69638aac2764d1e06a6a"}, {file = "mypy-1.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c5d2cc54175bab47011b09688b418db71403aefad07cbcd62d44010543fc143f"},
{file = "mypy-1.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c9a084bce1061e55cdc0493a2ad890375af359c766b8ac311ac8120d3a472950"}, {file = "mypy-1.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:87df44954c31d86df96c8bd6e80dfcd773473e877ac6176a8e29898bfb3501cb"},
{file = "mypy-1.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eaeaa0888b7f3ccb7bcd40b50497ca30923dba14f385bde4af78fac713d6d6f6"}, {file = "mypy-1.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:473117e310febe632ddf10e745a355714e771ffe534f06db40702775056614c4"},
{file = "mypy-1.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bea55fc25b96c53affab852ad94bf111a3083bc1d8b0c76a61dd101d8a388cf5"}, {file = "mypy-1.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:74bc9b6e0e79808bf8678d7678b2ae3736ea72d56eede3820bd3849823e7f305"},
{file = "mypy-1.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:4c8d8c6b80aa4a1689f2a179d31d86ae1367ea4a12855cc13aa3ba24bb36b2d8"}, {file = "mypy-1.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:44797d031a41516fcf5cbfa652265bb994e53e51994c1bd649ffcd0c3a7eccbf"},
{file = "mypy-1.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:70894c5345bea98321a2fe84df35f43ee7bb0feec117a71420c60459fc3e1eed"}, {file = "mypy-1.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ddae0f39ca146972ff6bb4399f3b2943884a774b8771ea0a8f50e971f5ea5ba8"},
{file = "mypy-1.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4a99fe1768925e4a139aace8f3fb66db3576ee1c30b9c0f70f744ead7e329c9f"}, {file = "mypy-1.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1c4c42c60a8103ead4c1c060ac3cdd3ff01e18fddce6f1016e08939647a0e703"},
{file = "mypy-1.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:023fe9e618182ca6317ae89833ba422c411469156b690fde6a315ad10695a521"}, {file = "mypy-1.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e86c2c6852f62f8f2b24cb7a613ebe8e0c7dc1402c61d36a609174f63e0ff017"},
{file = "mypy-1.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4d19f1a239d59f10fdc31263d48b7937c585810288376671eaf75380b074f238"}, {file = "mypy-1.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f9dca1e257d4cc129517779226753dbefb4f2266c4eaad610fc15c6a7e14283e"},
{file = "mypy-1.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:2de7babe398cb7a85ac7f1fd5c42f396c215ab3eff731b4d761d68d0f6a80f48"}, {file = "mypy-1.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:95d8d31a7713510685b05fbb18d6ac287a56c8f6554d88c19e73f724a445448a"},
{file = "mypy-1.2.0-py3-none-any.whl", hash = "sha256:d8e9187bfcd5ffedbe87403195e1fc340189a68463903c39e2b63307c9fa0394"}, {file = "mypy-1.3.0-py3-none-any.whl", hash = "sha256:a8763e72d5d9574d45ce5881962bc8e9046bf7b375b0abf031f3e6811732a897"},
{file = "mypy-1.2.0.tar.gz", hash = "sha256:f70a40410d774ae23fcb4afbbeca652905a04de7948eaf0b1789c8d1426b72d1"}, {file = "mypy-1.3.0.tar.gz", hash = "sha256:e1f4d16e296f5135624b34e8fb741eb0eadedca90862405b1f1fde2040b9bd11"},
] ]
[package.dependencies] [package.dependencies]
@ -2202,21 +2202,21 @@ docs = ["Sphinx (>=6.1.3,<7.0.0)"]
[[package]] [[package]]
name = "pylookyloomonitoring" name = "pylookyloomonitoring"
version = "1.0.2" version = "1.0.4"
description = "Python API to connect to lookyloo monitoring" description = "Python API to connect to lookyloo monitoring"
category = "main" category = "main"
optional = false optional = false
python-versions = ">=3.8,<4.0" python-versions = ">=3.8,<4.0"
files = [ files = [
{file = "pylookyloomonitoring-1.0.2-py3-none-any.whl", hash = "sha256:bb802e3cf8cd475e229f9b92c08e480dea1f33f636a64535c276ff0c0f110453"}, {file = "pylookyloomonitoring-1.0.4-py3-none-any.whl", hash = "sha256:53d77deed34d1bad0478914d0d734a8f0382893c0fb6ccdf379b6eda46f3ff82"},
{file = "pylookyloomonitoring-1.0.2.tar.gz", hash = "sha256:e86b66ba2b65fc1ac9370d7290f7ed053a1a6d4ff506cfc45b059f34ae95ab0c"}, {file = "pylookyloomonitoring-1.0.4.tar.gz", hash = "sha256:277bf20a2125e48c86d2af62614a1c132464fb65c21322706a5473abfb56916d"},
] ]
[package.dependencies] [package.dependencies]
requests = ">=2.28.2,<3.0.0" requests = ">=2.30.0,<3.0.0"
[package.extras] [package.extras]
docs = ["Sphinx (>=6.1.3,<7.0.0)"] docs = ["Sphinx (>=7.0.0,<8.0.0)"]
[[package]] [[package]]
name = "pymisp" name = "pymisp"
@ -2430,18 +2430,18 @@ tzdata = {version = "*", markers = "python_version >= \"3.6\""}
[[package]] [[package]]
name = "redis" name = "redis"
version = "4.5.4" version = "4.5.5"
description = "Python client for Redis database and key-value store" description = "Python client for Redis database and key-value store"
category = "main" category = "main"
optional = false optional = false
python-versions = ">=3.7" python-versions = ">=3.7"
files = [ files = [
{file = "redis-4.5.4-py3-none-any.whl", hash = "sha256:2c19e6767c474f2e85167909061d525ed65bea9301c0770bb151e041b7ac89a2"}, {file = "redis-4.5.5-py3-none-any.whl", hash = "sha256:77929bc7f5dab9adf3acba2d3bb7d7658f1e0c2f1cafe7eb36434e751c471119"},
{file = "redis-4.5.4.tar.gz", hash = "sha256:73ec35da4da267d6847e47f68730fdd5f62e2ca69e3ef5885c6a78a9374c3893"}, {file = "redis-4.5.5.tar.gz", hash = "sha256:dc87a0bdef6c8bfe1ef1e1c40be7034390c2ae02d92dcd0c7ca1729443899880"},
] ]
[package.dependencies] [package.dependencies]
async-timeout = {version = ">=4.0.2", markers = "python_version <= \"3.11.2\""} async-timeout = {version = ">=4.0.2", markers = "python_full_version <= \"3.11.2\""}
hiredis = {version = ">=1.0.0", optional = true, markers = "extra == \"hiredis\""} hiredis = {version = ">=1.0.0", optional = true, markers = "extra == \"hiredis\""}
[package.extras] [package.extras]
@ -2731,14 +2731,14 @@ test = ["argcomplete (>=2.0)", "pre-commit", "pytest", "pytest-mock"]
[[package]] [[package]]
name = "types-beautifulsoup4" name = "types-beautifulsoup4"
version = "4.12.0.4" version = "4.12.0.5"
description = "Typing stubs for beautifulsoup4" description = "Typing stubs for beautifulsoup4"
category = "dev" category = "dev"
optional = false optional = false
python-versions = "*" python-versions = "*"
files = [ files = [
{file = "types-beautifulsoup4-4.12.0.4.tar.gz", hash = "sha256:977e808b3e4fac0a25126d9f6c738886813862f840167a87c34df00b60154c8f"}, {file = "types-beautifulsoup4-4.12.0.5.tar.gz", hash = "sha256:d9be456416a62a5b9740559592e1063a69d4b0a1b83911d878558c8ae8e07074"},
{file = "types_beautifulsoup4-4.12.0.4-py3-none-any.whl", hash = "sha256:f5acd76487f220aa8b9906b29e60d6e833379788aca5ffb703d3ea4f7e1d31fb"}, {file = "types_beautifulsoup4-4.12.0.5-py3-none-any.whl", hash = "sha256:718364c8e6787884501c700b1d22b6c0a8711bf9d6c9bf96e1fba81495bc46a8"},
] ]
[package.dependencies] [package.dependencies]
@ -2758,26 +2758,26 @@ files = [
[[package]] [[package]]
name = "types-html5lib" name = "types-html5lib"
version = "1.1.11.13" version = "1.1.11.14"
description = "Typing stubs for html5lib" description = "Typing stubs for html5lib"
category = "dev" category = "dev"
optional = false optional = false
python-versions = "*" python-versions = "*"
files = [ files = [
{file = "types-html5lib-1.1.11.13.tar.gz", hash = "sha256:2a5ced934bcc7b47800b88ddb2541bbdf5bded7fb6d421693c978228e1fe631d"}, {file = "types-html5lib-1.1.11.14.tar.gz", hash = "sha256:091e9e74e0ee37c93fd789a164e99b2af80ecf5a314280450c6a763d027ea209"},
{file = "types_html5lib-1.1.11.13-py3-none-any.whl", hash = "sha256:d01603307f3bb39af9712e72eda5aa498420fe12179411533a54b2b03f6531eb"}, {file = "types_html5lib-1.1.11.14-py3-none-any.whl", hash = "sha256:758c1a27f3b63363a346f3646be9f8b1f25df4fc1f96f88af6d1d831f24ad675"},
] ]
[[package]] [[package]]
name = "types-pillow" name = "types-pillow"
version = "9.5.0.2" version = "9.5.0.4"
description = "Typing stubs for Pillow" description = "Typing stubs for Pillow"
category = "dev" category = "dev"
optional = false optional = false
python-versions = "*" python-versions = "*"
files = [ files = [
{file = "types-Pillow-9.5.0.2.tar.gz", hash = "sha256:b3f9f621f259566c19c1deca21901017c8b1e3e200ed2e49e0a2d83c0a5175db"}, {file = "types-Pillow-9.5.0.4.tar.gz", hash = "sha256:f1b6af47abd151847ee25911ffeba784899bc7dc7f9eba8ca6a5aac522b012ef"},
{file = "types_Pillow-9.5.0.2-py3-none-any.whl", hash = "sha256:58fdebd0ffa2353ecccdd622adde23bce89da5c0c8b96c34f2d1eca7b7e42d0e"}, {file = "types_Pillow-9.5.0.4-py3-none-any.whl", hash = "sha256:69427d9fa4320ff6e30f00fb9c0dd71185dc0a16de4757774220104759483466"},
] ]
[[package]] [[package]]
@ -2794,14 +2794,14 @@ files = [
[[package]] [[package]]
name = "types-pyopenssl" name = "types-pyopenssl"
version = "23.1.0.2" version = "23.1.0.3"
description = "Typing stubs for pyOpenSSL" description = "Typing stubs for pyOpenSSL"
category = "dev" category = "dev"
optional = false optional = false
python-versions = "*" python-versions = "*"
files = [ files = [
{file = "types-pyOpenSSL-23.1.0.2.tar.gz", hash = "sha256:20b80971b86240e8432a1832bd8124cea49c3088c7bfc77dfd23be27ffe4a517"}, {file = "types-pyOpenSSL-23.1.0.3.tar.gz", hash = "sha256:e7211088eff3e20d359888dedecb0994f7181d5cce0f26354dd47ca0484dc8a6"},
{file = "types_pyOpenSSL-23.1.0.2-py3-none-any.whl", hash = "sha256:b050641aeff6dfebf231ad719bdac12d53b8ee818d4afb67b886333484629957"}, {file = "types_pyOpenSSL-23.1.0.3-py3-none-any.whl", hash = "sha256:ad024b07a1f4bffbca44699543c71efd04733a6c22781fa9673a971e410a3086"},
] ]
[package.dependencies] [package.dependencies]
@ -2809,26 +2809,26 @@ cryptography = ">=35.0.0"
[[package]] [[package]]
name = "types-python-dateutil" name = "types-python-dateutil"
version = "2.8.19.12" version = "2.8.19.13"
description = "Typing stubs for python-dateutil" description = "Typing stubs for python-dateutil"
category = "dev" category = "dev"
optional = false optional = false
python-versions = "*" python-versions = "*"
files = [ files = [
{file = "types-python-dateutil-2.8.19.12.tar.gz", hash = "sha256:355b2cb82b31e556fd18e7b074de7c350c680ab80608f0cc55ba6770d986d67d"}, {file = "types-python-dateutil-2.8.19.13.tar.gz", hash = "sha256:09a0275f95ee31ce68196710ed2c3d1b9dc42e0b61cc43acc369a42cb939134f"},
{file = "types_python_dateutil-2.8.19.12-py3-none-any.whl", hash = "sha256:fe5b545e678ec13e3ddc83a0eee1545c1b5e2fba4cfc39b276ab6f4e7604a923"}, {file = "types_python_dateutil-2.8.19.13-py3-none-any.whl", hash = "sha256:0b0e7c68e7043b0354b26a1e0225cb1baea7abb1b324d02b50e2d08f1221043f"},
] ]
[[package]] [[package]]
name = "types-redis" name = "types-redis"
version = "4.5.4.2" version = "4.5.5.1"
description = "Typing stubs for redis" description = "Typing stubs for redis"
category = "dev" category = "dev"
optional = false optional = false
python-versions = "*" python-versions = "*"
files = [ files = [
{file = "types-redis-4.5.4.2.tar.gz", hash = "sha256:7979ce406cd7b4a0093b10a377e5060c5c890e8463cd1ef50423c1669efbc075"}, {file = "types-redis-4.5.5.1.tar.gz", hash = "sha256:0193d0522cccd6d46e9e17b811c30fc407e1800e0381da0e95f5c9239bdca58b"},
{file = "types_redis-4.5.4.2-py3-none-any.whl", hash = "sha256:b6f7e44aae1a79732f694cb6df6093e38361382d2be03780460684ef59745d62"}, {file = "types_redis-4.5.5.1-py3-none-any.whl", hash = "sha256:569b9795618c95d5f8fad2156dceb622614162761b4382892b99c519c378f776"},
] ]
[package.dependencies] [package.dependencies]
@ -2852,14 +2852,14 @@ types-urllib3 = "*"
[[package]] [[package]]
name = "types-urllib3" name = "types-urllib3"
version = "1.26.25.12" version = "1.26.25.13"
description = "Typing stubs for urllib3" description = "Typing stubs for urllib3"
category = "dev" category = "dev"
optional = false optional = false
python-versions = "*" python-versions = "*"
files = [ files = [
{file = "types-urllib3-1.26.25.12.tar.gz", hash = "sha256:a1557355ce8d350a555d142589f3001903757d2d36c18a66f588d9659bbc917d"}, {file = "types-urllib3-1.26.25.13.tar.gz", hash = "sha256:3300538c9dc11dad32eae4827ac313f5d986b8b21494801f1bf97a1ac6c03ae5"},
{file = "types_urllib3-1.26.25.12-py3-none-any.whl", hash = "sha256:3ba3d3a8ee46e0d5512c6bd0594da4f10b2584b47a470f8422044a2ab462f1df"}, {file = "types_urllib3-1.26.25.13-py3-none-any.whl", hash = "sha256:5dbd1d2bef14efee43f5318b5d36d805a489f6600252bb53626d4bfafd95e27c"},
] ]
[[package]] [[package]]
@ -2980,14 +2980,14 @@ files = [
[[package]] [[package]]
name = "werkzeug" name = "werkzeug"
version = "2.3.3" version = "2.3.4"
description = "The comprehensive WSGI web application library." description = "The comprehensive WSGI web application library."
category = "main" category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
{file = "Werkzeug-2.3.3-py3-none-any.whl", hash = "sha256:4866679a0722de00796a74086238bb3b98d90f423f05de039abb09315487254a"}, {file = "Werkzeug-2.3.4-py3-none-any.whl", hash = "sha256:48e5e61472fee0ddee27ebad085614ebedb7af41e88f687aaf881afb723a162f"},
{file = "Werkzeug-2.3.3.tar.gz", hash = "sha256:a987caf1092edc7523edb139edb20c70571c4a8d5eed02e0b547b4739174d091"}, {file = "Werkzeug-2.3.4.tar.gz", hash = "sha256:1d5a58e0377d1fe39d061a5de4469e414e78ccb1e1e59c0f5ad6fa1c36c52b76"},
] ]
[package.dependencies] [package.dependencies]
@ -3206,4 +3206,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more
[metadata] [metadata]
lock-version = "2.0" lock-version = "2.0"
python-versions = ">=3.8,<3.12" python-versions = ">=3.8,<3.12"
content-hash = "4e535f3fd77d8483046656b71a46cada24d0f741cf492d2076d38250e0da492b" content-hash = "6c9b9a4c8fc20da69e321da42eca10c856836102fef4867497c6dda180e3a1eb"

16
pyproject.toml
View File

@ -40,7 +40,7 @@ requests = "^2.30.0"
flask = "^2.3.2" flask = "^2.3.2"
gunicorn = "^20.1.0" gunicorn = "^20.1.0"
charset-normalizer = "^3.1.0" charset-normalizer = "^3.1.0"
redis = {version = "^4.5.4", extras = ["hiredis"]} redis = {version = "^4.5.5", extras = ["hiredis"]}
beautifulsoup4 = "^4.12.2" beautifulsoup4 = "^4.12.2"
bootstrap-flask = "^2.2.0" bootstrap-flask = "^2.2.0"
defang = "^0.5.3" defang = "^0.5.3"
@ -62,7 +62,7 @@ ua-parser = "^0.16.1"
Flask-Login = "^0.6.2" Flask-Login = "^0.6.2"
har2tree = "^1.19.3" har2tree = "^1.19.3"
passivetotal = "^2.5.9" passivetotal = "^2.5.9"
werkzeug = "^2.3.3" werkzeug = "^2.3.4"
filetype = "^1.2.0" filetype = "^1.2.0"
pypandora = "^1.4.0" pypandora = "^1.4.0"
lacuscore = "^1.4.12" lacuscore = "^1.4.12"
@ -72,21 +72,21 @@ publicsuffixlist = "^0.10.0.20230506"
pyfaup = "^1.2" pyfaup = "^1.2"
chardet = "^5.1.0" chardet = "^5.1.0"
pysecuritytxt = "^1.1.0" pysecuritytxt = "^1.1.0"
pylookyloomonitoring = "^1.0.2" pylookyloomonitoring = "^1.0.4"
[tool.poetry.group.dev.dependencies] [tool.poetry.group.dev.dependencies]
mypy = "^1.2.0" mypy = "^1.3.0"
ipython = [ ipython = [
{version = "<8.13.0", python = "<3.9"}, {version = "<8.13.0", python = "<3.9"},
{version = "^8.13.0", python = ">=3.9"} {version = "^8.13.0", python = ">=3.9"}
] ]
types-redis = {version = "^4.5.4.2"} types-redis = {version = "^4.5.5.1"}
types-requests = "^2.30.0.0" types-requests = "^2.30.0.0"
types-pkg-resources = "^0.1.3" types-pkg-resources = "^0.1.3"
types-Deprecated = "^1.2.9.2" types-Deprecated = "^1.2.9.2"
types-python-dateutil = "^2.8.19.12" types-python-dateutil = "^2.8.19.13"
types-beautifulsoup4 = "^4.12.0.4" types-beautifulsoup4 = "^4.12.0.5"
types-Pillow = "^9.5.0.2" types-Pillow = "^9.5.0.4"
[build-system] [build-system]
requires = ["poetry_core"] requires = ["poetry_core"]

3
website/web/genericapi.py
View File

@ -473,7 +473,7 @@ class CompareCaptures(Resource):
if not left_uuid or not right_uuid: if not left_uuid or not right_uuid:
return {'error': 'UUIDs of captures to compare missing', 'details': f'Left: {left_uuid} / Right: {right_uuid}'} return {'error': 'UUIDs of captures to compare missing', 'details': f'Left: {left_uuid} / Right: {right_uuid}'}
try: try:
result = comparator.compare_captures(left_uuid, right_uuid, settings=parameters.get('compare_settings')) different, result = comparator.compare_captures(left_uuid, right_uuid, settings=parameters.get('compare_settings'))
except MissingUUID as e: except MissingUUID as e:
# UUID non-existent, or capture still ongoing. # UUID non-existent, or capture still ongoing.
if left_uuid and right_uuid: if left_uuid and right_uuid:
@ -482,6 +482,7 @@ class CompareCaptures(Resource):
return {'error': str(e), 'details': {left_uuid: status_left, right_uuid: status_right}} return {'error': str(e), 'details': {left_uuid: status_left, right_uuid: status_right}}
else: else:
return {'error': str(e), 'details': 'Invalid request (left/right UUIDs missing.)'} return {'error': str(e), 'details': 'Invalid request (left/right UUIDs missing.)'}
result['different'] = different
return result return result