diff --git a/bin/async_capture.py b/bin/async_capture.py index 971a2193..8d00d2dd 100755 --- a/bin/async_capture.py +++ b/bin/async_capture.py @@ -4,13 +4,12 @@ import asyncio import logging import logging.config import signal -import time from pathlib import Path -from typing import Dict, Optional, Union +from typing import Dict, Optional, Set, Union from lacuscore import LacusCore, CaptureStatus as CaptureStatusCore, CaptureResponse as CaptureResponseCore -from pylacus import CaptureStatus as CaptureStatusPy, CaptureResponse as CaptureResponsePy +from pylacus import PyLacus, CaptureStatus as CaptureStatusPy, CaptureResponse as CaptureResponsePy from lookyloo.lookyloo import Lookyloo from lookyloo.default import AbstractManager, get_config @@ -30,7 +29,8 @@ class AsyncCapture(AbstractManager): self.capture_dir: Path = get_captures_dir() self.lookyloo = Lookyloo() - self.captures: Dict[asyncio.Task, float] = {} + if isinstance(self.lookyloo.lacus, LacusCore): + self.captures: Set[asyncio.Task] = set() self.fox = FOX(get_config('modules', 'FOX')) if not self.fox.available: @@ -40,104 +40,85 @@ class AsyncCapture(AbstractManager): if self.fox.available: self.fox.capture_default_trigger(url, auto_trigger=True) - async def process_capture_queue(self) -> None: - '''Process a query from the capture queue''' - self.set_running() - uuid: Optional[str] = None - entries: Union[CaptureResponseCore, CaptureResponsePy] - if isinstance(self.lookyloo.lacus, LacusCore): - if uuid := await self.lookyloo.lacus.consume_queue(): - entries = self.lookyloo.lacus.get_capture(uuid, decode=True) - if entries['status'] != CaptureStatusCore.DONE: - self.logger.warning(f'The capture {uuid} is reported as not done ({entries["status"]}) when it should.') - self.lookyloo.redis.zrem('to_capture', uuid) - self.lookyloo.redis.delete(uuid) - else: - # Find a capture that is done - try: - for uuid in self.lookyloo.redis.zrevrangebyscore('to_capture', 'Inf', '-Inf'): - if not uuid: - break - entries = self.lookyloo.lacus.get_capture(uuid) - if entries['status'] == CaptureStatusPy.DONE: - log = f'Got the capture for {uuid} from Lacus' - if runtime := entries.get('runtime'): - log = f'{log} - Runtime: {runtime}' - self.logger.info(log) - break - else: - # No captures are ready - uuid = None - except Exception as e: - self.logger.critical(f'Error when getting captures from lacus, will retry later: {e}') - uuid = None - await asyncio.sleep(10) - - if uuid is None: - self.unset_running() + async def _trigger_captures(self): + max_new_captures = get_config('generic', 'async_capture_processes') - len(self.captures) + self.logger.debug(f'{len(self.captures)} ongoing captures.') + if max_new_captures <= 0: + self.logger.info(f'Max amount of captures in parallel reached ({len(self.captures)})') return + for capture_task in self.lookyloo.lacus.consume_queue(max_new_captures): + self.captures.add(capture_task) + capture_task.add_done_callback(self.captures.discard) - self.lookyloo.redis.sadd('ongoing', uuid) - queue: Optional[str] = self.lookyloo.redis.getdel(f'{uuid}_mgmt') + def uuids_ready(self): + return [uuid for uuid in self.lookyloo.redis.zrevrangebyscore('to_capture', 'Inf', '-Inf') + if uuid and self.lookyloo.lacus.get_capture_status(uuid) in [CaptureStatusPy.DONE, CaptureStatusCore]] - to_capture: Dict[str, str] = self.lookyloo.redis.hgetall(uuid) + def process_capture_queue(self) -> None: + '''Process a query from the capture queue''' + entries: Union[CaptureResponseCore, CaptureResponsePy] + for uuid in self.uuids_ready(): + if isinstance(self.lookyloo.lacus, LacusCore): + entries = self.lookyloo.lacus.get_capture(uuid, decode=True) + elif isinstance(self.lookyloo.lacus, PyLacus): + entries = self.lookyloo.lacus.get_capture(uuid) + else: + raise Exception('Something is broken.') + log = f'Got the capture for {uuid} from Lacus' + if runtime := entries.get('runtime'): + log = f'{log} - Runtime: {runtime}' + self.logger.info(log) - if get_config('generic', 'default_public'): - # By default, the captures are on the index, unless the user mark them as un-listed - listing = False if ('listing' in to_capture and to_capture['listing'].lower() in ['false', '0', '']) else True - else: - # By default, the captures are not on the index, unless the user mark them as listed - listing = True if ('listing' in to_capture and to_capture['listing'].lower() in ['true', '1']) else False + self.lookyloo.redis.sadd('ongoing', uuid) + queue: Optional[str] = self.lookyloo.redis.getdel(f'{uuid}_mgmt') - self.lookyloo.store_capture( - uuid, listing, - os=to_capture.get('os'), browser=to_capture.get('browser'), - parent=to_capture.get('parent'), - downloaded_filename=entries.get('downloaded_filename'), - downloaded_file=entries.get('downloaded_file'), - error=entries.get('error'), har=entries.get('har'), - png=entries.get('png'), html=entries.get('html'), - last_redirected_url=entries.get('last_redirected_url'), - cookies=entries.get('cookies') # type: ignore - ) + to_capture: Dict[str, str] = self.lookyloo.redis.hgetall(uuid) - lazy_cleanup = self.lookyloo.redis.pipeline() - if queue and self.lookyloo.redis.zscore('queues', queue): - lazy_cleanup.zincrby('queues', -1, queue) - lazy_cleanup.zrem('to_capture', uuid) - lazy_cleanup.srem('ongoing', uuid) - lazy_cleanup.delete(uuid) - # make sure to expire the key if nothing was processed for a while (= queues empty) - lazy_cleanup.expire('queues', 600) - lazy_cleanup.execute() - self.unset_running() - self.logger.info(f'Done with {uuid}') + if get_config('generic', 'default_public'): + # By default, the captures are on the index, unless the user mark them as un-listed + listing = False if ('listing' in to_capture and to_capture['listing'].lower() in ['false', '0', '']) else True + else: + # By default, the captures are not on the index, unless the user mark them as listed + listing = True if ('listing' in to_capture and to_capture['listing'].lower() in ['true', '1']) else False - async def cancel_old_captures(self): - cancelled_tasks = [] - for task, timestamp in self.captures.items(): - if time.time() - timestamp >= get_config('generic', 'max_capture_time'): - task.cancel() - cancelled_tasks.append(task) - self.logger.warning('A capture has been going for too long, canceling it.') - if cancelled_tasks: - await asyncio.gather(*cancelled_tasks, return_exceptions=True) + self.lookyloo.store_capture( + uuid, listing, + os=to_capture.get('os'), browser=to_capture.get('browser'), + parent=to_capture.get('parent'), + downloaded_filename=entries.get('downloaded_filename'), + downloaded_file=entries.get('downloaded_file'), + error=entries.get('error'), har=entries.get('har'), + png=entries.get('png'), html=entries.get('html'), + last_redirected_url=entries.get('last_redirected_url'), + cookies=entries.get('cookies') # type: ignore + ) + + lazy_cleanup = self.lookyloo.redis.pipeline() + if queue and self.lookyloo.redis.zscore('queues', queue): + lazy_cleanup.zincrby('queues', -1, queue) + lazy_cleanup.zrem('to_capture', uuid) + lazy_cleanup.srem('ongoing', uuid) + lazy_cleanup.delete(uuid) + # make sure to expire the key if nothing was processed for a while (= queues empty) + lazy_cleanup.expire('queues', 600) + lazy_cleanup.execute() + self.unset_running() + self.logger.info(f'Done with {uuid}') async def _to_run_forever_async(self): - await self.cancel_old_captures() if self.force_stop: return - capture = asyncio.create_task(self.process_capture_queue()) - self.captures[capture] = time.time() - capture.add_done_callback(self.captures.pop) - while len(self.captures) >= get_config('generic', 'async_capture_processes'): - await self.cancel_old_captures() - await asyncio.sleep(1) + + if isinstance(self.lookyloo.lacus, LacusCore): + await self._trigger_captures() + + self.process_capture_queue() async def _wait_to_finish(self): - while self.captures: - self.logger.info(f'Waiting for {len(self.captures)} capture(s) to finish...') - await asyncio.sleep(5) + if isinstance(self.lookyloo.lacus, LacusCore): + while self.captures: + self.logger.info(f'Waiting for {len(self.captures)} capture(s) to finish...') + await asyncio.sleep(5) self.logger.info('No more captures') diff --git a/bin/start.py b/bin/start.py index f4bbf619..df48ac1a 100755 --- a/bin/start.py +++ b/bin/start.py @@ -2,7 +2,7 @@ from subprocess import Popen, run -from lookyloo.default import get_config, get_homedir +from lookyloo.default import get_homedir def main(): diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index 56bab1da..8d533e2f 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -158,8 +158,8 @@ class Lookyloo(): if not has_remote_lacus: # We need a redis connector that doesn't decode. redis: Redis = Redis(unix_socket_path=get_socket_path('cache')) - self._lacus = LacusCore(redis, get_config('generic', 'tor_proxy'), - get_config('generic', 'only_global_lookups'), + self._lacus = LacusCore(redis, tor_proxy=get_config('generic', 'tor_proxy'), + only_global_lookups=get_config('generic', 'only_global_lookups'), loglevel=get_config('generic', 'loglevel')) return self._lacus diff --git a/poetry.lock b/poetry.lock index c2970ad6..9ada4273 100644 --- a/poetry.lock +++ b/poetry.lock @@ -243,14 +243,14 @@ tzdata = ["tzdata"] [[package]] name = "beautifulsoup4" -version = "4.12.0" +version = "4.12.2" description = "Screen-scraping library" category = "main" optional = false python-versions = ">=3.6.0" files = [ - {file = "beautifulsoup4-4.12.0-py3-none-any.whl", hash = "sha256:2130a5ad7f513200fae61a17abb5e338ca980fa28c439c0571014bc0217e9591"}, - {file = "beautifulsoup4-4.12.0.tar.gz", hash = "sha256:c5fceeaec29d09c84970e47c65f2f0efe57872f7cff494c9691a26ec0ff13234"}, + {file = "beautifulsoup4-4.12.2-py3-none-any.whl", hash = "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"}, + {file = "beautifulsoup4-4.12.2.tar.gz", hash = "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da"}, ] [package.dependencies] @@ -644,18 +644,18 @@ tests = ["asttokens", "littleutils", "pytest", "rich"] [[package]] name = "filelock" -version = "3.10.7" +version = "3.11.0" description = "A platform independent file lock." category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "filelock-3.10.7-py3-none-any.whl", hash = "sha256:bde48477b15fde2c7e5a0713cbe72721cb5a5ad32ee0b8f419907960b9d75536"}, - {file = "filelock-3.10.7.tar.gz", hash = "sha256:892be14aa8efc01673b5ed6589dbccb95f9a8596f0507e232626155495c18105"}, + {file = "filelock-3.11.0-py3-none-any.whl", hash = "sha256:f08a52314748335c6460fc8fe40cd5638b85001225db78c2aa01c8c0db83b318"}, + {file = "filelock-3.11.0.tar.gz", hash = "sha256:3618c0da67adcc0506b015fd11ef7faf1b493f0b40d87728e19986b536890c37"}, ] [package.extras] -docs = ["furo (>=2022.12.7)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] +docs = ["furo (>=2023.3.27)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] testing = ["covdefaults (>=2.3)", "coverage (>=7.2.2)", "diff-cover (>=7.5)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] [[package]] @@ -1081,14 +1081,14 @@ files = [ [[package]] name = "importlib-metadata" -version = "6.1.0" +version = "6.2.0" description = "Read metadata from Python packages" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "importlib_metadata-6.1.0-py3-none-any.whl", hash = "sha256:ff80f3b5394912eb1b108fcfd444dc78b7f1f3e16b16188054bd01cb9cb86f09"}, - {file = "importlib_metadata-6.1.0.tar.gz", hash = "sha256:43ce9281e097583d758c2c708c4376371261a02c34682491a8e98352365aad20"}, + {file = "importlib_metadata-6.2.0-py3-none-any.whl", hash = "sha256:8388b74023a138c605fddd0d47cb81dd706232569f56c9aca7d9c7fdb54caeba"}, + {file = "importlib_metadata-6.2.0.tar.gz", hash = "sha256:9127aad2f49d7203e7112098c12b92e4fd1061ccd18548cdfdc49171a8c073cc"}, ] [package.dependencies] @@ -1232,14 +1232,14 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- [[package]] name = "lacuscore" -version = "1.3.2" +version = "1.4.0" description = "Core of Lacus, usable as a module" category = "main" optional = false python-versions = ">=3.8,<4.0" files = [ - {file = "lacuscore-1.3.2-py3-none-any.whl", hash = "sha256:1d01a200273a0e553ecac86d641e3bb86503a63a98fc7cadec2e362d74fa8d06"}, - {file = "lacuscore-1.3.2.tar.gz", hash = "sha256:2f422e17f2e4223d1d98c9e49d52212719aa2100208520afbd6403c302d5879b"}, + {file = "lacuscore-1.4.0-py3-none-any.whl", hash = "sha256:6c1f23d995b0b2082f5f8f6f91029916537336258a2dd4cdb64470de9c5eef6b"}, + {file = "lacuscore-1.4.0.tar.gz", hash = "sha256:f968d985069a02f18bc97b767286e6dc59f8d6207e9a2c858455ae93c19adace"}, ] [package.dependencies] @@ -1587,38 +1587,38 @@ files = [ [[package]] name = "mypy" -version = "1.1.1" +version = "1.2.0" description = "Optional static typing for Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "mypy-1.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39c7119335be05630611ee798cc982623b9e8f0cff04a0b48dfc26100e0b97af"}, - {file = "mypy-1.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:61bf08362e93b6b12fad3eab68c4ea903a077b87c90ac06c11e3d7a09b56b9c1"}, - {file = "mypy-1.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dbb19c9f662e41e474e0cff502b7064a7edc6764f5262b6cd91d698163196799"}, - {file = "mypy-1.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:315ac73cc1cce4771c27d426b7ea558fb4e2836f89cb0296cbe056894e3a1f78"}, - {file = "mypy-1.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:5cb14ff9919b7df3538590fc4d4c49a0f84392237cbf5f7a816b4161c061829e"}, - {file = "mypy-1.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:26cdd6a22b9b40b2fd71881a8a4f34b4d7914c679f154f43385ca878a8297389"}, - {file = "mypy-1.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5b5f81b40d94c785f288948c16e1f2da37203c6006546c5d947aab6f90aefef2"}, - {file = "mypy-1.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21b437be1c02712a605591e1ed1d858aba681757a1e55fe678a15c2244cd68a5"}, - {file = "mypy-1.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d809f88734f44a0d44959d795b1e6f64b2bbe0ea4d9cc4776aa588bb4229fc1c"}, - {file = "mypy-1.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:a380c041db500e1410bb5b16b3c1c35e61e773a5c3517926b81dfdab7582be54"}, - {file = "mypy-1.1.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b7c7b708fe9a871a96626d61912e3f4ddd365bf7f39128362bc50cbd74a634d5"}, - {file = "mypy-1.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1c10fa12df1232c936830839e2e935d090fc9ee315744ac33b8a32216b93707"}, - {file = "mypy-1.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0a28a76785bf57655a8ea5eb0540a15b0e781c807b5aa798bd463779988fa1d5"}, - {file = "mypy-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:ef6a01e563ec6a4940784c574d33f6ac1943864634517984471642908b30b6f7"}, - {file = "mypy-1.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d64c28e03ce40d5303450f547e07418c64c241669ab20610f273c9e6290b4b0b"}, - {file = "mypy-1.1.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:64cc3afb3e9e71a79d06e3ed24bb508a6d66f782aff7e56f628bf35ba2e0ba51"}, - {file = "mypy-1.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce61663faf7a8e5ec6f456857bfbcec2901fbdb3ad958b778403f63b9e606a1b"}, - {file = "mypy-1.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2b0c373d071593deefbcdd87ec8db91ea13bd8f1328d44947e88beae21e8d5e9"}, - {file = "mypy-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:2888ce4fe5aae5a673386fa232473014056967f3904f5abfcf6367b5af1f612a"}, - {file = "mypy-1.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:19ba15f9627a5723e522d007fe708007bae52b93faab00f95d72f03e1afa9598"}, - {file = "mypy-1.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:59bbd71e5c58eed2e992ce6523180e03c221dcd92b52f0e792f291d67b15a71c"}, - {file = "mypy-1.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9401e33814cec6aec8c03a9548e9385e0e228fc1b8b0a37b9ea21038e64cdd8a"}, - {file = "mypy-1.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4b398d8b1f4fba0e3c6463e02f8ad3346f71956b92287af22c9b12c3ec965a9f"}, - {file = "mypy-1.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:69b35d1dcb5707382810765ed34da9db47e7f95b3528334a3c999b0c90fe523f"}, - {file = "mypy-1.1.1-py3-none-any.whl", hash = "sha256:4e4e8b362cdf99ba00c2b218036002bdcdf1e0de085cdb296a49df03fb31dfc4"}, - {file = "mypy-1.1.1.tar.gz", hash = "sha256:ae9ceae0f5b9059f33dbc62dea087e942c0ccab4b7a003719cb70f9b8abfa32f"}, + {file = "mypy-1.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:701189408b460a2ff42b984e6bd45c3f41f0ac9f5f58b8873bbedc511900086d"}, + {file = "mypy-1.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fe91be1c51c90e2afe6827601ca14353bbf3953f343c2129fa1e247d55fd95ba"}, + {file = "mypy-1.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d26b513225ffd3eacece727f4387bdce6469192ef029ca9dd469940158bc89e"}, + {file = "mypy-1.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3a2d219775a120581a0ae8ca392b31f238d452729adbcb6892fa89688cb8306a"}, + {file = "mypy-1.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:2e93a8a553e0394b26c4ca683923b85a69f7ccdc0139e6acd1354cc884fe0128"}, + {file = "mypy-1.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3efde4af6f2d3ccf58ae825495dbb8d74abd6d176ee686ce2ab19bd025273f41"}, + {file = "mypy-1.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:695c45cea7e8abb6f088a34a6034b1d273122e5530aeebb9c09626cea6dca4cb"}, + {file = "mypy-1.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0e9464a0af6715852267bf29c9553e4555b61f5904a4fc538547a4d67617937"}, + {file = "mypy-1.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8293a216e902ac12779eb7a08f2bc39ec6c878d7c6025aa59464e0c4c16f7eb9"}, + {file = "mypy-1.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:f46af8d162f3d470d8ffc997aaf7a269996d205f9d746124a179d3abe05ac602"}, + {file = "mypy-1.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:031fc69c9a7e12bcc5660b74122ed84b3f1c505e762cc4296884096c6d8ee140"}, + {file = "mypy-1.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:390bc685ec209ada4e9d35068ac6988c60160b2b703072d2850457b62499e336"}, + {file = "mypy-1.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4b41412df69ec06ab141808d12e0bf2823717b1c363bd77b4c0820feaa37249e"}, + {file = "mypy-1.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:4e4a682b3f2489d218751981639cffc4e281d548f9d517addfd5a2917ac78119"}, + {file = "mypy-1.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a197ad3a774f8e74f21e428f0de7f60ad26a8d23437b69638aac2764d1e06a6a"}, + {file = "mypy-1.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c9a084bce1061e55cdc0493a2ad890375af359c766b8ac311ac8120d3a472950"}, + {file = "mypy-1.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eaeaa0888b7f3ccb7bcd40b50497ca30923dba14f385bde4af78fac713d6d6f6"}, + {file = "mypy-1.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bea55fc25b96c53affab852ad94bf111a3083bc1d8b0c76a61dd101d8a388cf5"}, + {file = "mypy-1.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:4c8d8c6b80aa4a1689f2a179d31d86ae1367ea4a12855cc13aa3ba24bb36b2d8"}, + {file = "mypy-1.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:70894c5345bea98321a2fe84df35f43ee7bb0feec117a71420c60459fc3e1eed"}, + {file = "mypy-1.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4a99fe1768925e4a139aace8f3fb66db3576ee1c30b9c0f70f744ead7e329c9f"}, + {file = "mypy-1.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:023fe9e618182ca6317ae89833ba422c411469156b690fde6a315ad10695a521"}, + {file = "mypy-1.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4d19f1a239d59f10fdc31263d48b7937c585810288376671eaf75380b074f238"}, + {file = "mypy-1.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:2de7babe398cb7a85ac7f1fd5c42f396c215ab3eff731b4d761d68d0f6a80f48"}, + {file = "mypy-1.2.0-py3-none-any.whl", hash = "sha256:d8e9187bfcd5ffedbe87403195e1fc340189a68463903c39e2b63307c9fa0394"}, + {file = "mypy-1.2.0.tar.gz", hash = "sha256:f70a40410d774ae23fcb4afbbeca652905a04de7948eaf0b1789c8d1426b72d1"}, ] [package.dependencies] @@ -2124,14 +2124,14 @@ docs = ["Sphinx (>=5.3.0,<6.0.0)"] [[package]] name = "pylacus" -version = "1.3.1" +version = "1.4.0" description = "Python CLI and module for lacus" category = "main" optional = false python-versions = ">=3.8,<4.0" files = [ - {file = "pylacus-1.3.1-py3-none-any.whl", hash = "sha256:0c42e135c8443293d690dd9d9a09861759b058da5f6c903a0de57e5a722ee51b"}, - {file = "pylacus-1.3.1.tar.gz", hash = "sha256:d205d7be51434f14da91dfce06f1edadf41c1e3af6fd311029676620450d6824"}, + {file = "pylacus-1.4.0-py3-none-any.whl", hash = "sha256:1a3dfb9d0c5891c54f6fd0287225194e190ab41e75617a74e9dd7b3c83daf9fe"}, + {file = "pylacus-1.4.0.tar.gz", hash = "sha256:a0521cf8ab781fe75c73a6942645dcf8b7c940f6d8db596da8e575a4451797a9"}, ] [package.dependencies] @@ -2661,14 +2661,14 @@ test = ["argcomplete (>=2.0)", "pre-commit", "pytest", "pytest-mock"] [[package]] name = "types-beautifulsoup4" -version = "4.12.0.1" +version = "4.12.0.2" description = "Typing stubs for beautifulsoup4" category = "dev" optional = false python-versions = "*" files = [ - {file = "types-beautifulsoup4-4.12.0.1.tar.gz", hash = "sha256:57bcd40e84c8d292c0a1de5267751a3a399ae49a3d826d9a999bc72fe94dcae3"}, - {file = "types_beautifulsoup4-4.12.0.1-py3-none-any.whl", hash = "sha256:8eaae29a605e5bb46a34160587f61a2a0af9ecfcefbaac983dd67b96ace1ed85"}, + {file = "types-beautifulsoup4-4.12.0.2.tar.gz", hash = "sha256:4d689bc581af6832ab7c47d67de9d4875eb8c23f6dea4d1c9efb2d28999c7ce9"}, + {file = "types_beautifulsoup4-4.12.0.2-py3-none-any.whl", hash = "sha256:b09bb3062ad3fbae02f38cc7de2d1acfa494b34818445be57b18b733b4cac370"}, ] [package.dependencies] @@ -3135,4 +3135,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "41e11a416c99f307a80e1a7f48cd24146314b56350fcafe77d06076242433aff" +content-hash = "6c10e9bb7722974a3e19d06e51e3231ead718aa933fdb322af1666af9ec7a58d" diff --git a/pyproject.toml b/pyproject.toml index 513a7744..a75d9063 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ flask = "^2.2.3" gunicorn = "^20.1.0" charset-normalizer = "^3.1.0" redis = {version = "^4.5.4", extras = ["hiredis"]} -beautifulsoup4 = "^4.12.0" +beautifulsoup4 = "^4.12.2" bootstrap-flask = "^2.2.0" defang = "^0.5.3" vt-py = "^0.17.5" @@ -65,8 +65,8 @@ passivetotal = "^2.5.9" werkzeug = "^2.2.3" filetype = "^1.2.0" pypandora = "^1.4.0" -lacuscore = "^1.3.2" -pylacus = "^1.3.1" +lacuscore = "^1.4.0" +pylacus = "^1.4.0" pyipasnhistory = "^2.1.2" publicsuffixlist = "^0.9.3" pyfaup = "^1.2" @@ -75,14 +75,14 @@ pysecuritytxt = "^1.1.0" pylookyloomonitoring = "^1.0.1" [tool.poetry.group.dev.dependencies] -mypy = "^1.1.1" +mypy = "^1.2.0" ipython = "^8.12.0" types-redis = {version = "^4.5.4.1"} types-requests = "^2.28.11.17" types-pkg-resources = "^0.1.3" types-Deprecated = "^1.2.9.2" types-python-dateutil = "^2.8.19.12" -types-beautifulsoup4 = "^4.12.0.1" +types-beautifulsoup4 = "^4.12.0.2" types-Pillow = "^9.4.0.19" [build-system]