CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: Improve capture, ignore ssl issues.

pull/401/head
Raphaël Vinot 2022-04-26 13:49:24 +02:00
parent 77fbf47e73
commit 1679ccf90f
3 changed files with 12 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
bin/async_capture.py
View File

@ -131,22 +131,16 @@ class AsyncCapture(AbstractManager):
self.logger.info(f'Capturing {url}') self.logger.info(f'Capturing {url}')
try: try:
capture = Capture() async with Capture(proxy=proxy) as capture:
if proxy: capture.prepare_cookies(cookies)
await capture.prepare_capture(proxy=proxy) capture.user_agent = ua
else: if headers:
await capture.prepare_capture() capture.http_headers = headers
capture.prepare_cookies(cookies) await capture.prepare_context()
capture.user_agent = ua entries = await capture.capture_page(url, referer=referer)
if headers:
capture.http_headers = headers
await capture.prepare_context()
entries = await capture.capture_page(url, referer=referer)
except Exception as e: except Exception as e:
self.logger.exception(f'Something went terribly wrong when capturing {url} - {e}') self.logger.exception(f'Something went terribly wrong when capturing {url} - {e}')
return False, f'Something went terribly wrong when capturing {url}.' return False, f'Something went terribly wrong when capturing {url}.'
finally:
await capture.cleanup()
if not entries: if not entries:
# broken # broken

8
poetry.lock generated
View File

@ -662,7 +662,7 @@ websockets = "10.1"
[[package]] [[package]]
name = "playwrightcapture" name = "playwrightcapture"
version = "0.1.5" version = "0.1.7"
description = "A simple library to capture websites using playwright" description = "A simple library to capture websites using playwright"
category = "main" category = "main"
optional = false optional = false
@ -1241,7 +1241,7 @@ misp = ["python-magic", "pydeep"]
[metadata] [metadata]
lock-version = "1.1" lock-version = "1.1"
python-versions = ">=3.8,<3.11" python-versions = ">=3.8,<3.11"
content-hash = "75f32d9d7699fbdfb5582221ef9158b185f593ad78f815118911ec22aa066b9e" content-hash = "cd4f04ddb7118a7aca10b15364f28d6d271fdbad37482236232e1ea2050dfd61"
[metadata.files] [metadata.files]
aiohttp = [ aiohttp = [
@ -1965,8 +1965,8 @@ playwright = [
{file = "playwright-1.21.0-py3-none-win_amd64.whl", hash = "sha256:0cdd82d4d2ce176b596e960825a4be7b03b7637e9cb243e634e896d787160535"}, {file = "playwright-1.21.0-py3-none-win_amd64.whl", hash = "sha256:0cdd82d4d2ce176b596e960825a4be7b03b7637e9cb243e634e896d787160535"},
] ]
playwrightcapture = [ playwrightcapture = [
{file = "PlaywrightCapture-0.1.5-py3-none-any.whl", hash = "sha256:15bb8dcbbdd95f4772f5511b10888fe167a9e6a73d573aeebc7c5da70eb1d89e"}, {file = "PlaywrightCapture-0.1.7-py3-none-any.whl", hash = "sha256:b32b66a1b8d47590e4ff9b88d23f6359b0b19c40149678065cf77b3ccd5a242c"},
{file = "PlaywrightCapture-0.1.5.tar.gz", hash = "sha256:50bdb9792f5999a3688dbe22e6d9f50534b3d36afc4b3329811515e901ab1efe"}, {file = "PlaywrightCapture-0.1.7.tar.gz", hash = "sha256:b69b51321a9bd55ebbf3b029d6ba1eb98c57826cd6158112c7cdd8fd06d7850b"},
] ]
prompt-toolkit = [ prompt-toolkit = [
{file = "prompt_toolkit-3.0.29-py3-none-any.whl", hash = "sha256:62291dad495e665fca0bda814e342c69952086afb0f4094d0893d357e5c78752"}, {file = "prompt_toolkit-3.0.29-py3-none-any.whl", hash = "sha256:62291dad495e665fca0bda814e342c69952086afb0f4094d0893d357e5c78752"},

2
pyproject.toml
View File

@ -65,7 +65,7 @@ lief = "^0.12.1"
ua-parser = "^0.10.0" ua-parser = "^0.10.0"
Flask-Login = "^0.6.0" Flask-Login = "^0.6.0"
har2tree = "^1.11.1" har2tree = "^1.11.1"
playwrightcapture = "^0.1.5" playwrightcapture = "^0.1.7"
[tool.poetry.extras] [tool.poetry.extras]
misp = ['python-magic', 'pydeep'] misp = ['python-magic', 'pydeep']