mirror of https://github.com/CIRCL/lookyloo
chg: Fairly big refactoring/cleanup to support LacusCore 1.4.0
Raphaël Vinot
parent
b109f5e84b
commit
2ceda75eab
|
|
@ -4,13 +4,12 @@ import asyncio
|
|||
import logging
|
||||
import logging.config
|
||||
import signal
|
||||
import time
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional, Union
|
||||
from typing import Dict, Optional, Set, Union
|
||||
|
||||
from lacuscore import LacusCore, CaptureStatus as CaptureStatusCore, CaptureResponse as CaptureResponseCore
|
||||
from pylacus import CaptureStatus as CaptureStatusPy, CaptureResponse as CaptureResponsePy
|
||||
from pylacus import PyLacus, CaptureStatus as CaptureStatusPy, CaptureResponse as CaptureResponsePy
|
||||
|
||||
from lookyloo.lookyloo import Lookyloo
|
||||
from lookyloo.default import AbstractManager, get_config
|
||||
|
|
@ -30,7 +29,8 @@ class AsyncCapture(AbstractManager):
|
|||
self.capture_dir: Path = get_captures_dir()
|
||||
self.lookyloo = Lookyloo()
|
||||
|
||||
self.captures: Dict[asyncio.Task, float] = {}
|
||||
if isinstance(self.lookyloo.lacus, LacusCore):
|
||||
self.captures: Set[asyncio.Task] = set()
|
||||
|
||||
self.fox = FOX(get_config('modules', 'FOX'))
|
||||
if not self.fox.available:
|
||||
|
|
@ -40,104 +40,85 @@ class AsyncCapture(AbstractManager):
|
|||
if self.fox.available:
|
||||
self.fox.capture_default_trigger(url, auto_trigger=True)
|
||||
|
||||
async def process_capture_queue(self) -> None:
|
||||
'''Process a query from the capture queue'''
|
||||
self.set_running()
|
||||
uuid: Optional[str] = None
|
||||
entries: Union[CaptureResponseCore, CaptureResponsePy]
|
||||
if isinstance(self.lookyloo.lacus, LacusCore):
|
||||
if uuid := await self.lookyloo.lacus.consume_queue():
|
||||
entries = self.lookyloo.lacus.get_capture(uuid, decode=True)
|
||||
if entries['status'] != CaptureStatusCore.DONE:
|
||||
self.logger.warning(f'The capture {uuid} is reported as not done ({entries["status"]}) when it should.')
|
||||
self.lookyloo.redis.zrem('to_capture', uuid)
|
||||
self.lookyloo.redis.delete(uuid)
|
||||
else:
|
||||
# Find a capture that is done
|
||||
try:
|
||||
for uuid in self.lookyloo.redis.zrevrangebyscore('to_capture', 'Inf', '-Inf'):
|
||||
if not uuid:
|
||||
break
|
||||
entries = self.lookyloo.lacus.get_capture(uuid)
|
||||
if entries['status'] == CaptureStatusPy.DONE:
|
||||
log = f'Got the capture for {uuid} from Lacus'
|
||||
if runtime := entries.get('runtime'):
|
||||
log = f'{log} - Runtime: {runtime}'
|
||||
self.logger.info(log)
|
||||
break
|
||||
else:
|
||||
# No captures are ready
|
||||
uuid = None
|
||||
except Exception as e:
|
||||
self.logger.critical(f'Error when getting captures from lacus, will retry later: {e}')
|
||||
uuid = None
|
||||
await asyncio.sleep(10)
|
||||
|
||||
if uuid is None:
|
||||
self.unset_running()
|
||||
async def _trigger_captures(self):
|
||||
max_new_captures = get_config('generic', 'async_capture_processes') - len(self.captures)
|
||||
self.logger.debug(f'{len(self.captures)} ongoing captures.')
|
||||
if max_new_captures <= 0:
|
||||
self.logger.info(f'Max amount of captures in parallel reached ({len(self.captures)})')
|
||||
return
|
||||
for capture_task in self.lookyloo.lacus.consume_queue(max_new_captures):
|
||||
self.captures.add(capture_task)
|
||||
capture_task.add_done_callback(self.captures.discard)
|
||||
|
||||
self.lookyloo.redis.sadd('ongoing', uuid)
|
||||
queue: Optional[str] = self.lookyloo.redis.getdel(f'{uuid}_mgmt')
|
||||
def uuids_ready(self):
|
||||
return [uuid for uuid in self.lookyloo.redis.zrevrangebyscore('to_capture', 'Inf', '-Inf')
|
||||
if uuid and self.lookyloo.lacus.get_capture_status(uuid) in [CaptureStatusPy.DONE, CaptureStatusCore]]
|
||||
|
||||
to_capture: Dict[str, str] = self.lookyloo.redis.hgetall(uuid)
|
||||
def process_capture_queue(self) -> None:
|
||||
'''Process a query from the capture queue'''
|
||||
entries: Union[CaptureResponseCore, CaptureResponsePy]
|
||||
for uuid in self.uuids_ready():
|
||||
if isinstance(self.lookyloo.lacus, LacusCore):
|
||||
entries = self.lookyloo.lacus.get_capture(uuid, decode=True)
|
||||
elif isinstance(self.lookyloo.lacus, PyLacus):
|
||||
entries = self.lookyloo.lacus.get_capture(uuid)
|
||||
else:
|
||||
raise Exception('Something is broken.')
|
||||
log = f'Got the capture for {uuid} from Lacus'
|
||||
if runtime := entries.get('runtime'):
|
||||
log = f'{log} - Runtime: {runtime}'
|
||||
self.logger.info(log)
|
||||
|
||||
if get_config('generic', 'default_public'):
|
||||
# By default, the captures are on the index, unless the user mark them as un-listed
|
||||
listing = False if ('listing' in to_capture and to_capture['listing'].lower() in ['false', '0', '']) else True
|
||||
else:
|
||||
# By default, the captures are not on the index, unless the user mark them as listed
|
||||
listing = True if ('listing' in to_capture and to_capture['listing'].lower() in ['true', '1']) else False
|
||||
self.lookyloo.redis.sadd('ongoing', uuid)
|
||||
queue: Optional[str] = self.lookyloo.redis.getdel(f'{uuid}_mgmt')
|
||||
|
||||
self.lookyloo.store_capture(
|
||||
uuid, listing,
|
||||
os=to_capture.get('os'), browser=to_capture.get('browser'),
|
||||
parent=to_capture.get('parent'),
|
||||
downloaded_filename=entries.get('downloaded_filename'),
|
||||
downloaded_file=entries.get('downloaded_file'),
|
||||
error=entries.get('error'), har=entries.get('har'),
|
||||
png=entries.get('png'), html=entries.get('html'),
|
||||
last_redirected_url=entries.get('last_redirected_url'),
|
||||
cookies=entries.get('cookies') # type: ignore
|
||||
)
|
||||
to_capture: Dict[str, str] = self.lookyloo.redis.hgetall(uuid)
|
||||
|
||||
lazy_cleanup = self.lookyloo.redis.pipeline()
|
||||
if queue and self.lookyloo.redis.zscore('queues', queue):
|
||||
lazy_cleanup.zincrby('queues', -1, queue)
|
||||
lazy_cleanup.zrem('to_capture', uuid)
|
||||
lazy_cleanup.srem('ongoing', uuid)
|
||||
lazy_cleanup.delete(uuid)
|
||||
# make sure to expire the key if nothing was processed for a while (= queues empty)
|
||||
lazy_cleanup.expire('queues', 600)
|
||||
lazy_cleanup.execute()
|
||||
self.unset_running()
|
||||
self.logger.info(f'Done with {uuid}')
|
||||
if get_config('generic', 'default_public'):
|
||||
# By default, the captures are on the index, unless the user mark them as un-listed
|
||||
listing = False if ('listing' in to_capture and to_capture['listing'].lower() in ['false', '0', '']) else True
|
||||
else:
|
||||
# By default, the captures are not on the index, unless the user mark them as listed
|
||||
listing = True if ('listing' in to_capture and to_capture['listing'].lower() in ['true', '1']) else False
|
||||
|
||||
async def cancel_old_captures(self):
|
||||
cancelled_tasks = []
|
||||
for task, timestamp in self.captures.items():
|
||||
if time.time() - timestamp >= get_config('generic', 'max_capture_time'):
|
||||
task.cancel()
|
||||
cancelled_tasks.append(task)
|
||||
self.logger.warning('A capture has been going for too long, canceling it.')
|
||||
if cancelled_tasks:
|
||||
await asyncio.gather(*cancelled_tasks, return_exceptions=True)
|
||||
self.lookyloo.store_capture(
|
||||
uuid, listing,
|
||||
os=to_capture.get('os'), browser=to_capture.get('browser'),
|
||||
parent=to_capture.get('parent'),
|
||||
downloaded_filename=entries.get('downloaded_filename'),
|
||||
downloaded_file=entries.get('downloaded_file'),
|
||||
error=entries.get('error'), har=entries.get('har'),
|
||||
png=entries.get('png'), html=entries.get('html'),
|
||||
last_redirected_url=entries.get('last_redirected_url'),
|
||||
cookies=entries.get('cookies') # type: ignore
|
||||
)
|
||||
|
||||
lazy_cleanup = self.lookyloo.redis.pipeline()
|
||||
if queue and self.lookyloo.redis.zscore('queues', queue):
|
||||
lazy_cleanup.zincrby('queues', -1, queue)
|
||||
lazy_cleanup.zrem('to_capture', uuid)
|
||||
lazy_cleanup.srem('ongoing', uuid)
|
||||
lazy_cleanup.delete(uuid)
|
||||
# make sure to expire the key if nothing was processed for a while (= queues empty)
|
||||
lazy_cleanup.expire('queues', 600)
|
||||
lazy_cleanup.execute()
|
||||
self.unset_running()
|
||||
self.logger.info(f'Done with {uuid}')
|
||||
|
||||
async def _to_run_forever_async(self):
|
||||
await self.cancel_old_captures()
|
||||
if self.force_stop:
|
||||
return
|
||||
capture = asyncio.create_task(self.process_capture_queue())
|
||||
self.captures[capture] = time.time()
|
||||
capture.add_done_callback(self.captures.pop)
|
||||
while len(self.captures) >= get_config('generic', 'async_capture_processes'):
|
||||
await self.cancel_old_captures()
|
||||
await asyncio.sleep(1)
|
||||
|
||||
if isinstance(self.lookyloo.lacus, LacusCore):
|
||||
await self._trigger_captures()
|
||||
|
||||
self.process_capture_queue()
|
||||
|
||||
async def _wait_to_finish(self):
|
||||
while self.captures:
|
||||
self.logger.info(f'Waiting for {len(self.captures)} capture(s) to finish...')
|
||||
await asyncio.sleep(5)
|
||||
if isinstance(self.lookyloo.lacus, LacusCore):
|
||||
while self.captures:
|
||||
self.logger.info(f'Waiting for {len(self.captures)} capture(s) to finish...')
|
||||
await asyncio.sleep(5)
|
||||
self.logger.info('No more captures')
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
from subprocess import Popen, run
|
||||
|
||||
from lookyloo.default import get_config, get_homedir
|
||||
from lookyloo.default import get_homedir
|
||||
|
||||
|
||||
def main():
|
||||
|
|
|
|||
|
|
@ -158,8 +158,8 @@ class Lookyloo():
|
|||
if not has_remote_lacus:
|
||||
# We need a redis connector that doesn't decode.
|
||||
redis: Redis = Redis(unix_socket_path=get_socket_path('cache'))
|
||||
self._lacus = LacusCore(redis, get_config('generic', 'tor_proxy'),
|
||||
get_config('generic', 'only_global_lookups'),
|
||||
self._lacus = LacusCore(redis, tor_proxy=get_config('generic', 'tor_proxy'),
|
||||
only_global_lookups=get_config('generic', 'only_global_lookups'),
|
||||
loglevel=get_config('generic', 'loglevel'))
|
||||
return self._lacus
|
||||
|
||||
|
|
|
|||
|
|
@ -243,14 +243,14 @@ tzdata = ["tzdata"]
|
|||
|
||||
[[package]]
|
||||
name = "beautifulsoup4"
|
||||
version = "4.12.0"
|
||||
version = "4.12.2"
|
||||
description = "Screen-scraping library"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.6.0"
|
||||
files = [
|
||||
{file = "beautifulsoup4-4.12.0-py3-none-any.whl", hash = "sha256:2130a5ad7f513200fae61a17abb5e338ca980fa28c439c0571014bc0217e9591"},
|
||||
{file = "beautifulsoup4-4.12.0.tar.gz", hash = "sha256:c5fceeaec29d09c84970e47c65f2f0efe57872f7cff494c9691a26ec0ff13234"},
|
||||
{file = "beautifulsoup4-4.12.2-py3-none-any.whl", hash = "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"},
|
||||
{file = "beautifulsoup4-4.12.2.tar.gz", hash = "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
|
|
@ -644,18 +644,18 @@ tests = ["asttokens", "littleutils", "pytest", "rich"]
|
|||
|
||||
[[package]]
|
||||
name = "filelock"
|
||||
version = "3.10.7"
|
||||
version = "3.11.0"
|
||||
description = "A platform independent file lock."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
files = [
|
||||
{file = "filelock-3.10.7-py3-none-any.whl", hash = "sha256:bde48477b15fde2c7e5a0713cbe72721cb5a5ad32ee0b8f419907960b9d75536"},
|
||||
{file = "filelock-3.10.7.tar.gz", hash = "sha256:892be14aa8efc01673b5ed6589dbccb95f9a8596f0507e232626155495c18105"},
|
||||
{file = "filelock-3.11.0-py3-none-any.whl", hash = "sha256:f08a52314748335c6460fc8fe40cd5638b85001225db78c2aa01c8c0db83b318"},
|
||||
{file = "filelock-3.11.0.tar.gz", hash = "sha256:3618c0da67adcc0506b015fd11ef7faf1b493f0b40d87728e19986b536890c37"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
docs = ["furo (>=2022.12.7)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"]
|
||||
docs = ["furo (>=2023.3.27)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"]
|
||||
testing = ["covdefaults (>=2.3)", "coverage (>=7.2.2)", "diff-cover (>=7.5)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1081,14 +1081,14 @@ files = [
|
|||
|
||||
[[package]]
|
||||
name = "importlib-metadata"
|
||||
version = "6.1.0"
|
||||
version = "6.2.0"
|
||||
description = "Read metadata from Python packages"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
files = [
|
||||
{file = "importlib_metadata-6.1.0-py3-none-any.whl", hash = "sha256:ff80f3b5394912eb1b108fcfd444dc78b7f1f3e16b16188054bd01cb9cb86f09"},
|
||||
{file = "importlib_metadata-6.1.0.tar.gz", hash = "sha256:43ce9281e097583d758c2c708c4376371261a02c34682491a8e98352365aad20"},
|
||||
{file = "importlib_metadata-6.2.0-py3-none-any.whl", hash = "sha256:8388b74023a138c605fddd0d47cb81dd706232569f56c9aca7d9c7fdb54caeba"},
|
||||
{file = "importlib_metadata-6.2.0.tar.gz", hash = "sha256:9127aad2f49d7203e7112098c12b92e4fd1061ccd18548cdfdc49171a8c073cc"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
|
|
@ -1232,14 +1232,14 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-
|
|||
|
||||
[[package]]
|
||||
name = "lacuscore"
|
||||
version = "1.3.2"
|
||||
version = "1.4.0"
|
||||
description = "Core of Lacus, usable as a module"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.8,<4.0"
|
||||
files = [
|
||||
{file = "lacuscore-1.3.2-py3-none-any.whl", hash = "sha256:1d01a200273a0e553ecac86d641e3bb86503a63a98fc7cadec2e362d74fa8d06"},
|
||||
{file = "lacuscore-1.3.2.tar.gz", hash = "sha256:2f422e17f2e4223d1d98c9e49d52212719aa2100208520afbd6403c302d5879b"},
|
||||
{file = "lacuscore-1.4.0-py3-none-any.whl", hash = "sha256:6c1f23d995b0b2082f5f8f6f91029916537336258a2dd4cdb64470de9c5eef6b"},
|
||||
{file = "lacuscore-1.4.0.tar.gz", hash = "sha256:f968d985069a02f18bc97b767286e6dc59f8d6207e9a2c858455ae93c19adace"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
|
|
@ -1587,38 +1587,38 @@ files = [
|
|||
|
||||
[[package]]
|
||||
name = "mypy"
|
||||
version = "1.1.1"
|
||||
version = "1.2.0"
|
||||
description = "Optional static typing for Python"
|
||||
category = "dev"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
files = [
|
||||
{file = "mypy-1.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39c7119335be05630611ee798cc982623b9e8f0cff04a0b48dfc26100e0b97af"},
|
||||
{file = "mypy-1.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:61bf08362e93b6b12fad3eab68c4ea903a077b87c90ac06c11e3d7a09b56b9c1"},
|
||||
{file = "mypy-1.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dbb19c9f662e41e474e0cff502b7064a7edc6764f5262b6cd91d698163196799"},
|
||||
{file = "mypy-1.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:315ac73cc1cce4771c27d426b7ea558fb4e2836f89cb0296cbe056894e3a1f78"},
|
||||
{file = "mypy-1.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:5cb14ff9919b7df3538590fc4d4c49a0f84392237cbf5f7a816b4161c061829e"},
|
||||
{file = "mypy-1.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:26cdd6a22b9b40b2fd71881a8a4f34b4d7914c679f154f43385ca878a8297389"},
|
||||
{file = "mypy-1.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5b5f81b40d94c785f288948c16e1f2da37203c6006546c5d947aab6f90aefef2"},
|
||||
{file = "mypy-1.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21b437be1c02712a605591e1ed1d858aba681757a1e55fe678a15c2244cd68a5"},
|
||||
{file = "mypy-1.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d809f88734f44a0d44959d795b1e6f64b2bbe0ea4d9cc4776aa588bb4229fc1c"},
|
||||
{file = "mypy-1.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:a380c041db500e1410bb5b16b3c1c35e61e773a5c3517926b81dfdab7582be54"},
|
||||
{file = "mypy-1.1.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b7c7b708fe9a871a96626d61912e3f4ddd365bf7f39128362bc50cbd74a634d5"},
|
||||
{file = "mypy-1.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1c10fa12df1232c936830839e2e935d090fc9ee315744ac33b8a32216b93707"},
|
||||
{file = "mypy-1.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0a28a76785bf57655a8ea5eb0540a15b0e781c807b5aa798bd463779988fa1d5"},
|
||||
{file = "mypy-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:ef6a01e563ec6a4940784c574d33f6ac1943864634517984471642908b30b6f7"},
|
||||
{file = "mypy-1.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d64c28e03ce40d5303450f547e07418c64c241669ab20610f273c9e6290b4b0b"},
|
||||
{file = "mypy-1.1.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:64cc3afb3e9e71a79d06e3ed24bb508a6d66f782aff7e56f628bf35ba2e0ba51"},
|
||||
{file = "mypy-1.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce61663faf7a8e5ec6f456857bfbcec2901fbdb3ad958b778403f63b9e606a1b"},
|
||||
{file = "mypy-1.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2b0c373d071593deefbcdd87ec8db91ea13bd8f1328d44947e88beae21e8d5e9"},
|
||||
{file = "mypy-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:2888ce4fe5aae5a673386fa232473014056967f3904f5abfcf6367b5af1f612a"},
|
||||
{file = "mypy-1.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:19ba15f9627a5723e522d007fe708007bae52b93faab00f95d72f03e1afa9598"},
|
||||
{file = "mypy-1.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:59bbd71e5c58eed2e992ce6523180e03c221dcd92b52f0e792f291d67b15a71c"},
|
||||
{file = "mypy-1.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9401e33814cec6aec8c03a9548e9385e0e228fc1b8b0a37b9ea21038e64cdd8a"},
|
||||
{file = "mypy-1.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4b398d8b1f4fba0e3c6463e02f8ad3346f71956b92287af22c9b12c3ec965a9f"},
|
||||
{file = "mypy-1.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:69b35d1dcb5707382810765ed34da9db47e7f95b3528334a3c999b0c90fe523f"},
|
||||
{file = "mypy-1.1.1-py3-none-any.whl", hash = "sha256:4e4e8b362cdf99ba00c2b218036002bdcdf1e0de085cdb296a49df03fb31dfc4"},
|
||||
{file = "mypy-1.1.1.tar.gz", hash = "sha256:ae9ceae0f5b9059f33dbc62dea087e942c0ccab4b7a003719cb70f9b8abfa32f"},
|
||||
{file = "mypy-1.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:701189408b460a2ff42b984e6bd45c3f41f0ac9f5f58b8873bbedc511900086d"},
|
||||
{file = "mypy-1.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fe91be1c51c90e2afe6827601ca14353bbf3953f343c2129fa1e247d55fd95ba"},
|
||||
{file = "mypy-1.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d26b513225ffd3eacece727f4387bdce6469192ef029ca9dd469940158bc89e"},
|
||||
{file = "mypy-1.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3a2d219775a120581a0ae8ca392b31f238d452729adbcb6892fa89688cb8306a"},
|
||||
{file = "mypy-1.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:2e93a8a553e0394b26c4ca683923b85a69f7ccdc0139e6acd1354cc884fe0128"},
|
||||
{file = "mypy-1.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3efde4af6f2d3ccf58ae825495dbb8d74abd6d176ee686ce2ab19bd025273f41"},
|
||||
{file = "mypy-1.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:695c45cea7e8abb6f088a34a6034b1d273122e5530aeebb9c09626cea6dca4cb"},
|
||||
{file = "mypy-1.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0e9464a0af6715852267bf29c9553e4555b61f5904a4fc538547a4d67617937"},
|
||||
{file = "mypy-1.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8293a216e902ac12779eb7a08f2bc39ec6c878d7c6025aa59464e0c4c16f7eb9"},
|
||||
{file = "mypy-1.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:f46af8d162f3d470d8ffc997aaf7a269996d205f9d746124a179d3abe05ac602"},
|
||||
{file = "mypy-1.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:031fc69c9a7e12bcc5660b74122ed84b3f1c505e762cc4296884096c6d8ee140"},
|
||||
{file = "mypy-1.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:390bc685ec209ada4e9d35068ac6988c60160b2b703072d2850457b62499e336"},
|
||||
{file = "mypy-1.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4b41412df69ec06ab141808d12e0bf2823717b1c363bd77b4c0820feaa37249e"},
|
||||
{file = "mypy-1.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:4e4a682b3f2489d218751981639cffc4e281d548f9d517addfd5a2917ac78119"},
|
||||
{file = "mypy-1.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a197ad3a774f8e74f21e428f0de7f60ad26a8d23437b69638aac2764d1e06a6a"},
|
||||
{file = "mypy-1.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c9a084bce1061e55cdc0493a2ad890375af359c766b8ac311ac8120d3a472950"},
|
||||
{file = "mypy-1.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eaeaa0888b7f3ccb7bcd40b50497ca30923dba14f385bde4af78fac713d6d6f6"},
|
||||
{file = "mypy-1.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bea55fc25b96c53affab852ad94bf111a3083bc1d8b0c76a61dd101d8a388cf5"},
|
||||
{file = "mypy-1.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:4c8d8c6b80aa4a1689f2a179d31d86ae1367ea4a12855cc13aa3ba24bb36b2d8"},
|
||||
{file = "mypy-1.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:70894c5345bea98321a2fe84df35f43ee7bb0feec117a71420c60459fc3e1eed"},
|
||||
{file = "mypy-1.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4a99fe1768925e4a139aace8f3fb66db3576ee1c30b9c0f70f744ead7e329c9f"},
|
||||
{file = "mypy-1.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:023fe9e618182ca6317ae89833ba422c411469156b690fde6a315ad10695a521"},
|
||||
{file = "mypy-1.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4d19f1a239d59f10fdc31263d48b7937c585810288376671eaf75380b074f238"},
|
||||
{file = "mypy-1.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:2de7babe398cb7a85ac7f1fd5c42f396c215ab3eff731b4d761d68d0f6a80f48"},
|
||||
{file = "mypy-1.2.0-py3-none-any.whl", hash = "sha256:d8e9187bfcd5ffedbe87403195e1fc340189a68463903c39e2b63307c9fa0394"},
|
||||
{file = "mypy-1.2.0.tar.gz", hash = "sha256:f70a40410d774ae23fcb4afbbeca652905a04de7948eaf0b1789c8d1426b72d1"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
|
|
@ -2124,14 +2124,14 @@ docs = ["Sphinx (>=5.3.0,<6.0.0)"]
|
|||
|
||||
[[package]]
|
||||
name = "pylacus"
|
||||
version = "1.3.1"
|
||||
version = "1.4.0"
|
||||
description = "Python CLI and module for lacus"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.8,<4.0"
|
||||
files = [
|
||||
{file = "pylacus-1.3.1-py3-none-any.whl", hash = "sha256:0c42e135c8443293d690dd9d9a09861759b058da5f6c903a0de57e5a722ee51b"},
|
||||
{file = "pylacus-1.3.1.tar.gz", hash = "sha256:d205d7be51434f14da91dfce06f1edadf41c1e3af6fd311029676620450d6824"},
|
||||
{file = "pylacus-1.4.0-py3-none-any.whl", hash = "sha256:1a3dfb9d0c5891c54f6fd0287225194e190ab41e75617a74e9dd7b3c83daf9fe"},
|
||||
{file = "pylacus-1.4.0.tar.gz", hash = "sha256:a0521cf8ab781fe75c73a6942645dcf8b7c940f6d8db596da8e575a4451797a9"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
|
|
@ -2661,14 +2661,14 @@ test = ["argcomplete (>=2.0)", "pre-commit", "pytest", "pytest-mock"]
|
|||
|
||||
[[package]]
|
||||
name = "types-beautifulsoup4"
|
||||
version = "4.12.0.1"
|
||||
version = "4.12.0.2"
|
||||
description = "Typing stubs for beautifulsoup4"
|
||||
category = "dev"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
files = [
|
||||
{file = "types-beautifulsoup4-4.12.0.1.tar.gz", hash = "sha256:57bcd40e84c8d292c0a1de5267751a3a399ae49a3d826d9a999bc72fe94dcae3"},
|
||||
{file = "types_beautifulsoup4-4.12.0.1-py3-none-any.whl", hash = "sha256:8eaae29a605e5bb46a34160587f61a2a0af9ecfcefbaac983dd67b96ace1ed85"},
|
||||
{file = "types-beautifulsoup4-4.12.0.2.tar.gz", hash = "sha256:4d689bc581af6832ab7c47d67de9d4875eb8c23f6dea4d1c9efb2d28999c7ce9"},
|
||||
{file = "types_beautifulsoup4-4.12.0.2-py3-none-any.whl", hash = "sha256:b09bb3062ad3fbae02f38cc7de2d1acfa494b34818445be57b18b733b4cac370"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
|
|
@ -3135,4 +3135,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more
|
|||
[metadata]
|
||||
lock-version = "2.0"
|
||||
python-versions = ">=3.8,<3.12"
|
||||
content-hash = "41e11a416c99f307a80e1a7f48cd24146314b56350fcafe77d06076242433aff"
|
||||
content-hash = "6c10e9bb7722974a3e19d06e51e3231ead718aa933fdb322af1666af9ec7a58d"
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ flask = "^2.2.3"
|
|||
gunicorn = "^20.1.0"
|
||||
charset-normalizer = "^3.1.0"
|
||||
redis = {version = "^4.5.4", extras = ["hiredis"]}
|
||||
beautifulsoup4 = "^4.12.0"
|
||||
beautifulsoup4 = "^4.12.2"
|
||||
bootstrap-flask = "^2.2.0"
|
||||
defang = "^0.5.3"
|
||||
vt-py = "^0.17.5"
|
||||
|
|
@ -65,8 +65,8 @@ passivetotal = "^2.5.9"
|
|||
werkzeug = "^2.2.3"
|
||||
filetype = "^1.2.0"
|
||||
pypandora = "^1.4.0"
|
||||
lacuscore = "^1.3.2"
|
||||
pylacus = "^1.3.1"
|
||||
lacuscore = "^1.4.0"
|
||||
pylacus = "^1.4.0"
|
||||
pyipasnhistory = "^2.1.2"
|
||||
publicsuffixlist = "^0.9.3"
|
||||
pyfaup = "^1.2"
|
||||
|
|
@ -75,14 +75,14 @@ pysecuritytxt = "^1.1.0"
|
|||
pylookyloomonitoring = "^1.0.1"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
mypy = "^1.1.1"
|
||||
mypy = "^1.2.0"
|
||||
ipython = "^8.12.0"
|
||||
types-redis = {version = "^4.5.4.1"}
|
||||
types-requests = "^2.28.11.17"
|
||||
types-pkg-resources = "^0.1.3"
|
||||
types-Deprecated = "^1.2.9.2"
|
||||
types-python-dateutil = "^2.8.19.12"
|
||||
types-beautifulsoup4 = "^4.12.0.1"
|
||||
types-beautifulsoup4 = "^4.12.0.2"
|
||||
types-Pillow = "^9.4.0.19"
|
||||
|
||||
[build-system]
|
||||
|
|
|
|||
Loading…
Reference in New Issue