From c67f01c775c06cd5da8845b22e4fb8c0ef49d9a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Jan 2024 15:03:36 +0100 Subject: [PATCH] chg: Improve strict typing --- bin/archiver.py | 6 +- bin/async_capture.py | 4 +- lookyloo/capturecache.py | 2 +- lookyloo/helpers.py | 4 +- lookyloo/lookyloo.py | 14 ++-- lookyloo/modules/misp.py | 8 +-- lookyloo/modules/riskiq.py | 2 +- lookyloo/modules/vt.py | 6 +- poetry.lock | 136 ++++++++++++++++++------------------- pyproject.toml | 4 +- website/web/__init__.py | 12 ++-- website/web/genericapi.py | 6 +- website/web/helpers.py | 2 +- 13 files changed, 103 insertions(+), 103 deletions(-) diff --git a/bin/archiver.py b/bin/archiver.py index d6940ce..7cce3c7 100755 --- a/bin/archiver.py +++ b/bin/archiver.py @@ -14,7 +14,7 @@ from datetime import datetime, timedelta from pathlib import Path from redis import Redis -import s3fs # type: ignore +import s3fs # type: ignore[import-untyped] from lookyloo.default import AbstractManager, get_config, get_homedir, get_socket_path, try_make_file from lookyloo.helpers import get_captures_dir, is_locked, make_ts_from_dirname, make_dirs_list @@ -371,7 +371,7 @@ class Archiver(AbstractManager): self.logger.debug(f'Loading {index}') if recent_uuids := self.__load_index(index): self.logger.debug(f'{len(recent_uuids)} captures in directory {index.parent}.') - self.redis.hset('lookup_dirs', mapping=recent_uuids) # type: ignore + self.redis.hset('lookup_dirs', mapping=recent_uuids) # type: ignore[arg-type] else: index.unlink() total_recent_captures = self.redis.hlen('lookup_dirs') @@ -385,7 +385,7 @@ class Archiver(AbstractManager): self.logger.debug(f'Loading {index}') if archived_uuids := self.__load_index(index): self.logger.debug(f'{len(archived_uuids)} captures in directory {index.parent}.') - self.redis.hset('lookup_dirs_archived', mapping=archived_uuids) # type: ignore + self.redis.hset('lookup_dirs_archived', mapping=archived_uuids) # type: ignore[arg-type] else: index.unlink() total_archived_captures = self.redis.hlen('lookup_dirs_archived') diff --git a/bin/async_capture.py b/bin/async_capture.py index 6e7c0dc..4430669 100755 --- a/bin/async_capture.py +++ b/bin/async_capture.py @@ -79,10 +79,10 @@ class AsyncCapture(AbstractManager): if get_config('generic', 'default_public'): # By default, the captures are on the index, unless the user mark them as un-listed - listing = False if ('listing' in to_capture and to_capture['listing'].lower() in ['false', '0', '']) else True # type: ignore + listing = False if ('listing' in to_capture and isinstance(to_capture['listing'], str) and to_capture['listing'].lower() in ['false', '0', '']) else True else: # By default, the captures are not on the index, unless the user mark them as listed - listing = True if ('listing' in to_capture and to_capture['listing'].lower() in ['true', '1']) else False # type: ignore + listing = True if ('listing' in to_capture and isinstance(to_capture['listing'], str) and to_capture['listing'].lower() in ['true', '1']) else False self.lookyloo.store_capture( uuid, listing, diff --git a/lookyloo/capturecache.py b/lookyloo/capturecache.py index 7dd2bf9..f5869d5 100644 --- a/lookyloo/capturecache.py +++ b/lookyloo/capturecache.py @@ -448,7 +448,7 @@ class CapturesIndex(Mapping): # type: ignore[type-arg] p.hset('lookup_dirs_archived', uuid, capture_dir_str) p.delete(capture_dir_str) - p.hset(capture_dir_str, mapping=cache) # type: ignore + p.hset(capture_dir_str, mapping=cache) # type: ignore[arg-type] p.execute() return CaptureCache(cache) diff --git a/lookyloo/helpers.py b/lookyloo/helpers.py index 1e4194b..95da620 100644 --- a/lookyloo/helpers.py +++ b/lookyloo/helpers.py @@ -19,9 +19,9 @@ from urllib.parse import urlparse from har2tree import CrawledTree, HostNode, URLNode # type: ignore[attr-defined] from playwrightcapture import get_devices -from publicsuffixlist import PublicSuffixList # type: ignore +from publicsuffixlist import PublicSuffixList # type: ignore[import-untyped] from pytaxonomies import Taxonomies # type: ignore[attr-defined] -from ua_parser import user_agent_parser # type: ignore +from ua_parser import user_agent_parser # type: ignore[import-untyped] from werkzeug.user_agent import UserAgent from werkzeug.utils import cached_property diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index 1cbd250..ab42f0b 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -568,7 +568,7 @@ class Lookyloo(): def _prepare_lacus_query(self, query: CaptureSettings) -> CaptureSettings: # Remove the none, it makes redis unhappy - query = {k: v for k, v in query.items() if v is not None} # type: ignore + query = {k: v for k, v in query.items() if v is not None} # type: ignore[assignment] if 'url' in query and query['url'] is not None: # Make sure the URL does not have any space or newline @@ -626,9 +626,9 @@ class Lookyloo(): for key, value in query.items(): if isinstance(value, bool): - query[key] = 1 if value else 0 # type: ignore + query[key] = 1 if value else 0 # type: ignore[literal-required] elif isinstance(value, (list, dict)): - query[key] = json.dumps(value) if value else None # type: ignore + query[key] = json.dumps(value) if value else None # type: ignore[literal-required] query = self._prepare_lacus_query(query) @@ -679,7 +679,7 @@ class Lookyloo(): if value: mapping_capture[key] = json.dumps(value) elif value is not None: - mapping_capture[key] = value # type: ignore + mapping_capture[key] = value # type: ignore[assignment] p = self.redis.pipeline() p.zadd('to_capture', {perma_uuid: priority}) @@ -1157,7 +1157,7 @@ class Lookyloo(): event = self.misps.export(cache, self.is_public_instance) screenshot: MISPAttribute = event.add_attribute('attachment', 'screenshot_landing_page.png', data=self.get_screenshot(cache.uuid), - disable_correlation=True) # type: ignore + disable_correlation=True) # type: ignore[assignment] # If the last object attached to tht event is a file, it is the rendered page if event.objects and event.objects[-1].name == 'file': event.objects[-1].add_reference(screenshot, 'rendered-as', 'Screenshot of the page') @@ -1180,7 +1180,7 @@ class Lookyloo(): pt_entry = self.phishtank.get_url_lookup(urls[0].value) if not pt_entry or not pt_entry.get('phish_detail_url'): continue - pt_attribute: MISPAttribute = event.add_attribute('link', value=pt_entry['phish_detail_url'], comment='Phishtank permalink') # type: ignore + pt_attribute: MISPAttribute = event.add_attribute('link', value=pt_entry['phish_detail_url'], comment='Phishtank permalink') # type: ignore[assignment] e_obj.add_reference(pt_attribute, 'known-as', 'Permalink on Phishtank') if self.urlscan.available: @@ -1491,7 +1491,7 @@ class Lookyloo(): month_stats['uniq_urls'] = len(urls) month_stats['uniq_domains'] = len(uniq_domains(urls)) - year_stats['months'].append(month_stats) # type: ignore + year_stats['months'].append(month_stats) # type: ignore[union-attr] year_stats['yearly_submissions'] += month_stats['submissions'] statistics['years'].append(year_stats) diff --git a/lookyloo/modules/misp.py b/lookyloo/modules/misp.py index ec45ebe..78af518 100644 --- a/lookyloo/modules/misp.py +++ b/lookyloo/modules/misp.py @@ -100,7 +100,7 @@ class MISPs(Mapping, AbstractModule): # type: ignore[type-arg] self.__misp_add_ips_to_URLObject(initial_url, cache.tree.root_hartree.hostname_tree) initial_obj = event.add_object(initial_url) - lookyloo_link: MISPAttribute = event.add_attribute('link', f'https://{public_domain}/tree/{cache.uuid}') # type: ignore + lookyloo_link: MISPAttribute = event.add_attribute('link', f'https://{public_domain}/tree/{cache.uuid}') # type: ignore[assignment] if not is_public_instance: lookyloo_link.distribution = 0 initial_obj.add_reference(lookyloo_link, 'captured-by', 'Capture on lookyloo') @@ -165,7 +165,7 @@ class MISP(AbstractModule): self.enable_push = bool(self.config.get('enable_push', False)) self.allow_auto_trigger = bool(self.config.get('allow_auto_trigger', False)) - self.default_tags: list[str] = self.config.get('default_tags') # type: ignore + self.default_tags: list[str] = self.config.get('default_tags') # type: ignore[assignment] self.auto_publish = bool(self.config.get('auto_publish', False)) self.storage_dir_misp = get_homedir() / 'misp' self.storage_dir_misp.mkdir(parents=True, exist_ok=True) @@ -270,11 +270,11 @@ class MISP(AbstractModule): to_return: dict[str, set[str]] = defaultdict(set) # NOTE: We have MISPAttribute in that list for a in attributes: - to_return[a.event_id].add(a.value) # type: ignore + to_return[a.event_id].add(a.value) # type: ignore[union-attr,index] return to_return else: # The request returned an error - return attributes # type: ignore + return attributes # type: ignore[return-value] return {'info': 'No hits.'} else: return {'error': 'Module not available or lookup not enabled.'} diff --git a/lookyloo/modules/riskiq.py b/lookyloo/modules/riskiq.py index fc0250e..76fd40d 100644 --- a/lookyloo/modules/riskiq.py +++ b/lookyloo/modules/riskiq.py @@ -8,7 +8,7 @@ from datetime import date, datetime, timedelta from typing import Any, TYPE_CHECKING from urllib.parse import urlparse -from passivetotal import AccountClient, DnsRequest, WhoisRequest # type: ignore +from passivetotal import AccountClient, DnsRequest, WhoisRequest # type: ignore[import-untyped] from requests import Response from ..default import ConfigError, get_homedir diff --git a/lookyloo/modules/vt.py b/lookyloo/modules/vt.py index cffbc5d..975a098 100644 --- a/lookyloo/modules/vt.py +++ b/lookyloo/modules/vt.py @@ -7,9 +7,9 @@ import time from datetime import date from typing import Any, TYPE_CHECKING -import vt # type: ignore -from vt.error import APIError # type: ignore -from vt.object import WhistleBlowerDict # type: ignore +import vt # type: ignore[import-untyped] +from vt.error import APIError # type: ignore[import-untyped] +from vt.object import WhistleBlowerDict # type: ignore[import-untyped] from ..default import ConfigError, get_homedir from ..helpers import get_cache_directory diff --git a/poetry.lock b/poetry.lock index 8552454..0acb779 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2,24 +2,24 @@ [[package]] name = "aiobotocore" -version = "2.11.0" +version = "2.11.1" description = "Async client for aws services using botocore and aiohttp" optional = false python-versions = ">=3.8" files = [ - {file = "aiobotocore-2.11.0-py3-none-any.whl", hash = "sha256:6eaf48a6ccd3943ce7d26f75dc8fa0292b7a1a069bd5a9557d37fae5adf14d2d"}, - {file = "aiobotocore-2.11.0.tar.gz", hash = "sha256:4c5b1bf01e7aab74a29bd783159517ecf6d45b5a1647e53302e9554c064e420a"}, + {file = "aiobotocore-2.11.1-py3-none-any.whl", hash = "sha256:904a7ad7cc8671d662cfd596906dafe839118ea2a66332c37908e3dcfdee1e45"}, + {file = "aiobotocore-2.11.1.tar.gz", hash = "sha256:0b095af50da2d6f94e93ca959e2a4876f0f0d84d534b61b21d8e050832d04ab6"}, ] [package.dependencies] aiohttp = ">=3.7.4.post0,<4.0.0" aioitertools = ">=0.5.1,<1.0.0" -botocore = ">=1.33.2,<1.34.23" +botocore = ">=1.33.2,<1.34.28" wrapt = ">=1.10.10,<2.0.0" [package.extras] -awscli = ["awscli (>=1.31.2,<1.32.23)"] -boto3 = ["boto3 (>=1.33.2,<1.34.23)"] +awscli = ["awscli (>=1.31.2,<1.32.28)"] +boto3 = ["boto3 (>=1.33.2,<1.34.28)"] [[package]] name = "aiohttp" @@ -308,13 +308,13 @@ WTForms = "*" [[package]] name = "botocore" -version = "1.34.22" +version = "1.34.27" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">= 3.8" files = [ - {file = "botocore-1.34.22-py3-none-any.whl", hash = "sha256:e5f7775975b9213507fbcf846a96b7a2aec2a44fc12a44585197b014a4ab0889"}, - {file = "botocore-1.34.22.tar.gz", hash = "sha256:c47ba4286c576150d1b6ca6df69a87b5deff3d23bd84da8bcf8431ebac3c40ba"}, + {file = "botocore-1.34.27-py3-none-any.whl", hash = "sha256:1c10f247136ad17b6ef1588c1e043e294dbaebdebe9ce84dc56713029f515c53"}, + {file = "botocore-1.34.27.tar.gz", hash = "sha256:a0e68ba264275b358b8c1cca604161f4d9465cf7847d73e929543a9f30ff22d1"}, ] [package.dependencies] @@ -565,43 +565,43 @@ files = [ [[package]] name = "cryptography" -version = "42.0.0" +version = "42.0.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-42.0.0-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:c640b0ef54138fde761ec99a6c7dc4ce05e80420262c20fa239e694ca371d434"}, - {file = "cryptography-42.0.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:678cfa0d1e72ef41d48993a7be75a76b0725d29b820ff3cfd606a5b2b33fda01"}, - {file = "cryptography-42.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:146e971e92a6dd042214b537a726c9750496128453146ab0ee8971a0299dc9bd"}, - {file = "cryptography-42.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87086eae86a700307b544625e3ba11cc600c3c0ef8ab97b0fda0705d6db3d4e3"}, - {file = "cryptography-42.0.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0a68bfcf57a6887818307600c3c0ebc3f62fbb6ccad2240aa21887cda1f8df1b"}, - {file = "cryptography-42.0.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5a217bca51f3b91971400890905a9323ad805838ca3fa1e202a01844f485ee87"}, - {file = "cryptography-42.0.0-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:ca20550bb590db16223eb9ccc5852335b48b8f597e2f6f0878bbfd9e7314eb17"}, - {file = "cryptography-42.0.0-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:33588310b5c886dfb87dba5f013b8d27df7ffd31dc753775342a1e5ab139e59d"}, - {file = "cryptography-42.0.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9515ea7f596c8092fdc9902627e51b23a75daa2c7815ed5aa8cf4f07469212ec"}, - {file = "cryptography-42.0.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:35cf6ed4c38f054478a9df14f03c1169bb14bd98f0b1705751079b25e1cb58bc"}, - {file = "cryptography-42.0.0-cp37-abi3-win32.whl", hash = "sha256:8814722cffcfd1fbd91edd9f3451b88a8f26a5fd41b28c1c9193949d1c689dc4"}, - {file = "cryptography-42.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:a2a8d873667e4fd2f34aedab02ba500b824692c6542e017075a2efc38f60a4c0"}, - {file = "cryptography-42.0.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:8fedec73d590fd30c4e3f0d0f4bc961aeca8390c72f3eaa1a0874d180e868ddf"}, - {file = "cryptography-42.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be41b0c7366e5549265adf2145135dca107718fa44b6e418dc7499cfff6b4689"}, - {file = "cryptography-42.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ca482ea80626048975360c8e62be3ceb0f11803180b73163acd24bf014133a0"}, - {file = "cryptography-42.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:c58115384bdcfe9c7f644c72f10f6f42bed7cf59f7b52fe1bf7ae0a622b3a139"}, - {file = "cryptography-42.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:56ce0c106d5c3fec1038c3cca3d55ac320a5be1b44bf15116732d0bc716979a2"}, - {file = "cryptography-42.0.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:324721d93b998cb7367f1e6897370644751e5580ff9b370c0a50dc60a2003513"}, - {file = "cryptography-42.0.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:d97aae66b7de41cdf5b12087b5509e4e9805ed6f562406dfcf60e8481a9a28f8"}, - {file = "cryptography-42.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:85f759ed59ffd1d0baad296e72780aa62ff8a71f94dc1ab340386a1207d0ea81"}, - {file = "cryptography-42.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:206aaf42e031b93f86ad60f9f5d9da1b09164f25488238ac1dc488334eb5e221"}, - {file = "cryptography-42.0.0-cp39-abi3-win32.whl", hash = "sha256:74f18a4c8ca04134d2052a140322002fef535c99cdbc2a6afc18a8024d5c9d5b"}, - {file = "cryptography-42.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:14e4b909373bc5bf1095311fa0f7fcabf2d1a160ca13f1e9e467be1ac4cbdf94"}, - {file = "cryptography-42.0.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3005166a39b70c8b94455fdbe78d87a444da31ff70de3331cdec2c568cf25b7e"}, - {file = "cryptography-42.0.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:be14b31eb3a293fc6e6aa2807c8a3224c71426f7c4e3639ccf1a2f3ffd6df8c3"}, - {file = "cryptography-42.0.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:bd7cf7a8d9f34cc67220f1195884151426ce616fdc8285df9054bfa10135925f"}, - {file = "cryptography-42.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c310767268d88803b653fffe6d6f2f17bb9d49ffceb8d70aed50ad45ea49ab08"}, - {file = "cryptography-42.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:bdce70e562c69bb089523e75ef1d9625b7417c6297a76ac27b1b8b1eb51b7d0f"}, - {file = "cryptography-42.0.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:e9326ca78111e4c645f7e49cbce4ed2f3f85e17b61a563328c85a5208cf34440"}, - {file = "cryptography-42.0.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:69fd009a325cad6fbfd5b04c711a4da563c6c4854fc4c9544bff3088387c77c0"}, - {file = "cryptography-42.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:988b738f56c665366b1e4bfd9045c3efae89ee366ca3839cd5af53eaa1401bce"}, - {file = "cryptography-42.0.0.tar.gz", hash = "sha256:6cf9b76d6e93c62114bd19485e5cb003115c134cf9ce91f8ac924c44f8c8c3f4"}, + {file = "cryptography-42.0.1-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:265bdc693570b895eb641410b8fc9e8ddbce723a669236162b9d9cfb70bd8d77"}, + {file = "cryptography-42.0.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:160fa08dfa6dca9cb8ad9bd84e080c0db6414ba5ad9a7470bc60fb154f60111e"}, + {file = "cryptography-42.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:727387886c9c8de927c360a396c5edcb9340d9e960cda145fca75bdafdabd24c"}, + {file = "cryptography-42.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d84673c012aa698555d4710dcfe5f8a0ad76ea9dde8ef803128cc669640a2e0"}, + {file = "cryptography-42.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e6edc3a568667daf7d349d7e820783426ee4f1c0feab86c29bd1d6fe2755e009"}, + {file = "cryptography-42.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:d50718dd574a49d3ef3f7ef7ece66ef281b527951eb2267ce570425459f6a404"}, + {file = "cryptography-42.0.1-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:9544492e8024f29919eac2117edd8c950165e74eb551a22c53f6fdf6ba5f4cb8"}, + {file = "cryptography-42.0.1-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ab6b302d51fbb1dd339abc6f139a480de14d49d50f65fdc7dff782aa8631d035"}, + {file = "cryptography-42.0.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2fe16624637d6e3e765530bc55caa786ff2cbca67371d306e5d0a72e7c3d0407"}, + {file = "cryptography-42.0.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ed1b2130f5456a09a134cc505a17fc2830a1a48ed53efd37dcc904a23d7b82fa"}, + {file = "cryptography-42.0.1-cp37-abi3-win32.whl", hash = "sha256:e5edf189431b4d51f5c6fb4a95084a75cef6b4646c934eb6e32304fc720e1453"}, + {file = "cryptography-42.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:6bfd823b336fdcd8e06285ae8883d3d2624d3bdef312a0e2ef905f332f8e9302"}, + {file = "cryptography-42.0.1-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:351db02c1938c8e6b1fee8a78d6b15c5ccceca7a36b5ce48390479143da3b411"}, + {file = "cryptography-42.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:430100abed6d3652208ae1dd410c8396213baee2e01a003a4449357db7dc9e14"}, + {file = "cryptography-42.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dff7a32880a51321f5de7869ac9dde6b1fca00fc1fef89d60e93f215468e824"}, + {file = "cryptography-42.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:b512f33c6ab195852595187af5440d01bb5f8dd57cb7a91e1e009a17f1b7ebca"}, + {file = "cryptography-42.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:95d900d19a370ae36087cc728e6e7be9c964ffd8cbcb517fd1efb9c9284a6abc"}, + {file = "cryptography-42.0.1-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:6ac8924085ed8287545cba89dc472fc224c10cc634cdf2c3e2866fe868108e77"}, + {file = "cryptography-42.0.1-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cb2861a9364fa27d24832c718150fdbf9ce6781d7dc246a516435f57cfa31fe7"}, + {file = "cryptography-42.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25ec6e9e81de5d39f111a4114193dbd39167cc4bbd31c30471cebedc2a92c323"}, + {file = "cryptography-42.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9d61fcdf37647765086030d81872488e4cb3fafe1d2dda1d487875c3709c0a49"}, + {file = "cryptography-42.0.1-cp39-abi3-win32.whl", hash = "sha256:16b9260d04a0bfc8952b00335ff54f471309d3eb9d7e8dbfe9b0bd9e26e67881"}, + {file = "cryptography-42.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:7911586fc69d06cd0ab3f874a169433db1bc2f0e40988661408ac06c4527a986"}, + {file = "cryptography-42.0.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:d3594947d2507d4ef7a180a7f49a6db41f75fb874c2fd0e94f36b89bfd678bf2"}, + {file = "cryptography-42.0.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:8d7efb6bf427d2add2f40b6e1e8e476c17508fa8907234775214b153e69c2e11"}, + {file = "cryptography-42.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:126e0ba3cc754b200a2fb88f67d66de0d9b9e94070c5bc548318c8dab6383cb6"}, + {file = "cryptography-42.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:802d6f83233cf9696b59b09eb067e6b4d5ae40942feeb8e13b213c8fad47f1aa"}, + {file = "cryptography-42.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0b7cacc142260ada944de070ce810c3e2a438963ee3deb45aa26fd2cee94c9a4"}, + {file = "cryptography-42.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:32ea63ceeae870f1a62e87f9727359174089f7b4b01e4999750827bf10e15d60"}, + {file = "cryptography-42.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d3902c779a92151f134f68e555dd0b17c658e13429f270d8a847399b99235a3f"}, + {file = "cryptography-42.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:50aecd93676bcca78379604ed664c45da82bc1241ffb6f97f6b7392ed5bc6f04"}, + {file = "cryptography-42.0.1.tar.gz", hash = "sha256:fd33f53809bb363cf126bebe7a99d97735988d9b0131a2be59fbf83e1259a5b7"}, ] [package.dependencies] @@ -1462,18 +1462,18 @@ referencing = ">=0.31.0" [[package]] name = "lacuscore" -version = "1.7.10" +version = "1.7.11" description = "Core of Lacus, usable as a module" optional = false python-versions = ">=3.8,<4.0" files = [ - {file = "lacuscore-1.7.10-py3-none-any.whl", hash = "sha256:338ced22ab0045b25e4be9af7d2fa68a2e86dd25197dda91c212196ce440f56b"}, - {file = "lacuscore-1.7.10.tar.gz", hash = "sha256:89b7e56e70cf3471e927f9c79cbc654bc071021caec03ea898028351e60b2171"}, + {file = "lacuscore-1.7.11-py3-none-any.whl", hash = "sha256:b5bc6cf5b0ec2eed6e1072bb4234c06ae71e15ff4c7acc953f50d70f764c647d"}, + {file = "lacuscore-1.7.11.tar.gz", hash = "sha256:b99c92bac1e4730e3e9f150cfcf9910ad2645a3253aebb3f50684e7de817ef80"}, ] [package.dependencies] defang = ">=0.5.3,<0.6.0" -playwrightcapture = {version = ">=1.22.7,<2.0.0", extras = ["recaptcha"]} +playwrightcapture = {version = ">=1.22.8,<2.0.0", extras = ["recaptcha"]} redis = {version = ">=5.0.1,<6.0.0", extras = ["hiredis"]} requests = ">=2.31.0,<3.0.0" ua-parser = ">=0.18.0,<0.19.0" @@ -2131,18 +2131,18 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co [[package]] name = "playwright" -version = "1.41.0" +version = "1.41.1" description = "A high-level API to automate web browsers" optional = false python-versions = ">=3.8" files = [ - {file = "playwright-1.41.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:ad3ff4805fd21b91c9acbc5b27e62bf0ac6a235e93f91d9dd6d311d5e988b069"}, - {file = "playwright-1.41.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:9d34f472d174e55d8f12265e10d11ba21be99bf0661ccbf1f9b048312696d5e8"}, - {file = "playwright-1.41.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:f3f0841360652ddd77f6e059074d5079dd37b487168a2972efbf13cd23fbacb0"}, - {file = "playwright-1.41.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:b56c1d564df9a2b2ab393f8161b71bcc16df38ade60115e1162ad6d9f116c89b"}, - {file = "playwright-1.41.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ccf3214b0cdd932bcdc04794ecafe50bd386e8523ce4681969aac86d3326f87"}, - {file = "playwright-1.41.0-py3-none-win32.whl", hash = "sha256:46d4aaed0071db67361900a2027bb14dba74f9eb102cf2baf1876c0ff0c15da6"}, - {file = "playwright-1.41.0-py3-none-win_amd64.whl", hash = "sha256:925045824dc3d505c8f197c94a083e4619d7c268d7c3465d73a5a09040dc8b85"}, + {file = "playwright-1.41.1-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:b456f25db38e4d93afc3c671e1093f3995afb374f14cee284152a30f84cfff02"}, + {file = "playwright-1.41.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:53ff152506dbd8527aa815e92757be72f5df60810e8000e9419d29fd4445f53c"}, + {file = "playwright-1.41.1-py3-none-macosx_11_0_universal2.whl", hash = "sha256:70c432887b8b5e896fa804fb90ca2c8baf05b13a3590fb8bce8b3c3efba2842d"}, + {file = "playwright-1.41.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:f227a8d616fd3a02d45d68546ee69947dce4a058df134a9e7dc6167c543de3cd"}, + {file = "playwright-1.41.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:475130f879b4ba38b9db7232a043dd5bc3a8bd1a84567fbea7e21a02ee2fcb13"}, + {file = "playwright-1.41.1-py3-none-win32.whl", hash = "sha256:ef769414ea0ceb76085c67812ab6bc0cc6fac0adfc45aaa09d54ee161d7f637b"}, + {file = "playwright-1.41.1-py3-none-win_amd64.whl", hash = "sha256:316e1ba0854a712e9288b3fe49509438e648d43bade77bf724899de8c24848de"}, ] [package.dependencies] @@ -2169,19 +2169,19 @@ test = ["pytest"] [[package]] name = "playwrightcapture" -version = "1.22.7" +version = "1.22.8" description = "A simple library to capture websites using playwright" optional = false python-versions = ">=3.8,<4.0" files = [ - {file = "playwrightcapture-1.22.7-py3-none-any.whl", hash = "sha256:efe4ab5fd5ec81a0b56990cd2187e7ed4a34958f4f9aab6304b849db8a051526"}, - {file = "playwrightcapture-1.22.7.tar.gz", hash = "sha256:457825649eddd31eb201c63c5b520d48b9b22a071d8765fce18d3cb534654275"}, + {file = "playwrightcapture-1.22.8-py3-none-any.whl", hash = "sha256:0bd2b57dd2fd4278310b3af744eae7ade5e4e588c8cfcdb4a25f0d80a25c1a65"}, + {file = "playwrightcapture-1.22.8.tar.gz", hash = "sha256:bb9dfb6583b9fd4a42534607091af363b2a58bdecbd6c1b16ced425a91f8c122"}, ] [package.dependencies] beautifulsoup4 = {version = ">=4.12.3,<5.0.0", extras = ["charset-normalizer", "lxml"]} dateparser = ">=1.2.0,<2.0.0" -playwright = ">=1.41.0,<2.0.0" +playwright = ">=1.41.1,<2.0.0" playwright-stealth = ">=1.0.6,<2.0.0" pydub = {version = ">=0.25.1,<0.26.0", optional = true, markers = "extra == \"recaptcha\""} pytz = {version = ">=2023.3.post1,<2024.0", markers = "python_version < \"3.9\""} @@ -2221,13 +2221,13 @@ files = [ [[package]] name = "publicsuffixlist" -version = "0.10.0.20240108" +version = "0.10.0.20240125" description = "publicsuffixlist implement" optional = false python-versions = ">=2.6" files = [ - {file = "publicsuffixlist-0.10.0.20240108-py2.py3-none-any.whl", hash = "sha256:72ac774728036610501353789125a7adc57a793646cf6c6f1f7cc7458c913a8a"}, - {file = "publicsuffixlist-0.10.0.20240108.tar.gz", hash = "sha256:2d15301cbef4b5ecc9bfa47b38959af73350915748d44b2f91db2a8fc3b98d24"}, + {file = "publicsuffixlist-0.10.0.20240125-py2.py3-none-any.whl", hash = "sha256:cf9d83c5881b7348d5d9dc2d0e971832de5d5657df7fae048a544b8006859ab7"}, + {file = "publicsuffixlist-0.10.0.20240125.tar.gz", hash = "sha256:971c948000ac4542f340b3725364c5acb751cdd49b433bc409345a6103fc3b4e"}, ] [package.extras] @@ -3126,13 +3126,13 @@ files = [ [[package]] name = "types-pillow" -version = "10.2.0.20240111" +version = "10.2.0.20240125" description = "Typing stubs for Pillow" optional = false python-versions = ">=3.8" files = [ - {file = "types-Pillow-10.2.0.20240111.tar.gz", hash = "sha256:e8d359bfdc5a149a3c90a7e153cb2d0750ddf7fc3508a20dfadabd8a9435e354"}, - {file = "types_Pillow-10.2.0.20240111-py3-none-any.whl", hash = "sha256:1f4243b30c143b56b0646626f052e4269123e550f9096cdfb5fbd999daee7dbb"}, + {file = "types-Pillow-10.2.0.20240125.tar.gz", hash = "sha256:c449b2c43b9fdbe0494a7b950e6b39a4e50516091213fec24ef3f33c1d017717"}, + {file = "types_Pillow-10.2.0.20240125-py3-none-any.whl", hash = "sha256:322dbae32b4b7918da5e8a47c50ac0f24b0aa72a804a23857620f2722b03c858"}, ] [[package]] @@ -3213,13 +3213,13 @@ types-urllib3 = "*" [[package]] name = "types-requests" -version = "2.31.0.20240106" +version = "2.31.0.20240125" description = "Typing stubs for requests" optional = false python-versions = ">=3.8" files = [ - {file = "types-requests-2.31.0.20240106.tar.gz", hash = "sha256:0e1c731c17f33618ec58e022b614a1a2ecc25f7dc86800b36ef341380402c612"}, - {file = "types_requests-2.31.0.20240106-py3-none-any.whl", hash = "sha256:da997b3b6a72cc08d09f4dba9802fdbabc89104b35fe24ee588e674037689354"}, + {file = "types-requests-2.31.0.20240125.tar.gz", hash = "sha256:03a28ce1d7cd54199148e043b2079cdded22d6795d19a2c2a6791a4b2b5e2eb5"}, + {file = "types_requests-2.31.0.20240125-py3-none-any.whl", hash = "sha256:9592a9a4cb92d6d75d9b491a41477272b710e021011a2a3061157e2fb1f1a5d1"}, ] [package.dependencies] @@ -3607,4 +3607,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.12" -content-hash = "e08a28121c208e1ec579d1e3098caab4ff4dba1f7e82cebd64946918f55855f2" +content-hash = "7bb828635edbbbee36a4faf1a820314a58fb1dfcb98b24046f79159e9e7eaa75" diff --git a/pyproject.toml b/pyproject.toml index 7b94d3e..acf1680 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ passivetotal = "^2.5.9" werkzeug = "^3.0.1" filetype = "^1.2.0" pypandora = "^1.6.1" -lacuscore = "^1.7.10" +lacuscore = "^1.7.11" pylacus = "^1.7.3" pyipasnhistory = "^2.1.2" publicsuffixlist = "^0.10.0.20231214" @@ -97,7 +97,7 @@ types-pkg-resources = "^0.1.3" types-Deprecated = "^1.2.9.20240106" types-python-dateutil = "^2.8.19.20240106" types-beautifulsoup4 = "^4.12.0.20240106" -types-Pillow = "^10.2.0.20240111" +types-Pillow = "^10.2.0.20240125" types-pytz = "^2023.3.1.1" [build-system] diff --git a/website/web/__init__.py b/website/web/__init__.py index 4dcf540..94fb8ed 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -13,7 +13,7 @@ import os import sys import time -import filetype # type: ignore +import filetype # type: ignore[import-untyped] from datetime import date, datetime, timedelta, timezone from importlib.metadata import version @@ -23,12 +23,12 @@ from urllib.parse import quote_plus, unquote_plus, urlparse from uuid import uuid4 from zipfile import ZipFile -import flask_login # type: ignore +import flask_login # type: ignore[import-untyped] from flask import (Flask, Response, Request, flash, jsonify, redirect, render_template, request, send_file, url_for) -from flask_bootstrap import Bootstrap5 # type: ignore -from flask_cors import CORS # type: ignore -from flask_restx import Api # type: ignore +from flask_bootstrap import Bootstrap5 # type: ignore[import-untyped] +from flask_cors import CORS # type: ignore[import-untyped] +from flask_restx import Api # type: ignore[import-untyped] from lacuscore import CaptureStatus from pymisp import MISPEvent, MISPServerError # type: ignore[attr-defined] from werkzeug.security import check_password_hash @@ -54,7 +54,7 @@ from .proxied import ReverseProxied logging.config.dictConfig(get_config('logging')) app: Flask = Flask(__name__) -app.wsgi_app = ReverseProxied(app.wsgi_app) # type: ignore +app.wsgi_app = ReverseProxied(app.wsgi_app) # type: ignore[method-assign] app.config['SECRET_KEY'] = get_secret_key() diff --git a/website/web/genericapi.py b/website/web/genericapi.py index bc68157..bc29479 100644 --- a/website/web/genericapi.py +++ b/website/web/genericapi.py @@ -10,9 +10,9 @@ from io import BytesIO from typing import Any from zipfile import ZipFile -import flask_login # type: ignore +import flask_login # type: ignore[import-untyped] from flask import request, send_file, Response -from flask_restx import Namespace, Resource, abort, fields # type: ignore +from flask_restx import Namespace, Resource, abort, fields # type: ignore[import-untyped] from werkzeug.security import check_password_hash from lacuscore import CaptureStatus as CaptureStatusCore @@ -29,7 +29,7 @@ lookyloo: Lookyloo = get_lookyloo_instance() comparator: Comparator = Comparator() -def api_auth_check(method): # type: ignore +def api_auth_check(method): # type: ignore[no-untyped-def] if flask_login.current_user.is_authenticated or load_user_from_request(request): return method abort(403, 'Authentication required.') diff --git a/website/web/helpers.py b/website/web/helpers.py index be5a77d..4a107a8 100644 --- a/website/web/helpers.py +++ b/website/web/helpers.py @@ -8,7 +8,7 @@ import os from functools import lru_cache from pathlib import Path -import flask_login # type: ignore +import flask_login # type: ignore[import-untyped] from flask import Request from werkzeug.security import generate_password_hash