CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: Bump har2tree, cleanup

pull/79/head
Raphaël Vinot 2020-04-01 14:33:35 +02:00
parent ce200717ec
commit e62d56e555
3 changed files with 13 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lookyloo/helpers.py
View File

@ -64,7 +64,7 @@ def load_configs(path_to_config_files: Optional[Union[str, Path]]=None) -> Dict[
def safe_create_dir(to_create: Path): def safe_create_dir(to_create: Path):
if to_create.exists() and not to_create.is_dir(): if to_create.exists() and not to_create.is_dir():
raise CreateDirectoryException(f'The path {to_create} already exists and is not a directory') raise CreateDirectoryException(f'The path {to_create} already exists and is not a directory')
os.makedirs(to_create, exist_ok=True) to_create.mkdir(parents=True, exist_ok=True)
def set_running(name: str) -> None: def set_running(name: str) -> None:

10
lookyloo/lookyloo.py
View File

@ -20,7 +20,7 @@ import base64
from uuid import uuid4 from uuid import uuid4
from pathlib import Path from pathlib import Path
from .helpers import get_homedir, get_socket_path, load_cookies, load_configs from .helpers import get_homedir, get_socket_path, load_cookies, load_configs, safe_create_dir
from .exceptions import NoValidHarFile from .exceptions import NoValidHarFile
from redis import Redis from redis import Redis
@ -48,8 +48,8 @@ class Lookyloo():
self.scrape_dir: Path = get_homedir() / 'scraped' self.scrape_dir: Path = get_homedir() / 'scraped'
self.splash_url: str = self.get_config('splash_url') self.splash_url: str = self.get_config('splash_url')
self.only_global_lookups: bool = self.get_config('only_global_lookups') self.only_global_lookups: bool = self.get_config('only_global_lookups')
if not self.scrape_dir.exists():
self.scrape_dir.mkdir(parents=True, exist_ok=True) safe_create_dir(self.scrape_dir)
# Initialize 3rd party components # Initialize 3rd party components
if 'modules' not in self.configs: if 'modules' not in self.configs:
@ -70,7 +70,7 @@ class Lookyloo():
else: else:
self.use_sane_js = True self.use_sane_js = True
def get_config(self, entry: str) -> Union[str, int, bool]: def get_config(self, entry: str) -> Any:
"""Get an entry from the generic config file. Automatic fallback to the sample file""" """Get an entry from the generic config file. Automatic fallback to the sample file"""
if 'generic' in self.configs: if 'generic' in self.configs:
if entry in self.configs['generic']: if entry in self.configs['generic']:
@ -260,7 +260,7 @@ class Lookyloo():
perma_uuid = str(uuid4()) perma_uuid = str(uuid4())
width = len(str(len(items))) width = len(str(len(items)))
dirpath = self.scrape_dir / datetime.now().isoformat() dirpath = self.scrape_dir / datetime.now().isoformat()
dirpath.mkdir() safe_create_dir(dirpath)
for i, item in enumerate(items): for i, item in enumerate(items):
if not listing: # Write no_index marker if not listing: # Write no_index marker
(dirpath / 'no_index').touch() (dirpath / 'no_index').touch()

14
poetry.lock generated
View File

@ -302,7 +302,7 @@ publicsuffix2 = "^2.20191221"
six = "^1.14.0" six = "^1.14.0"
[package.source] [package.source]
reference = "a8308b31d6259c270361059accf4d2eb8a39450c" reference = "c5d26f5bf3be2ffbe6991d326c81cc0f9a803d34"
type = "git" type = "git"
url = "https://github.com/viper-framework/har2tree.git" url = "https://github.com/viper-framework/har2tree.git"
[[package]] [[package]]
@ -1001,7 +1001,7 @@ description = "The official Python client library for VirusTotal"
name = "vt-py" name = "vt-py"
optional = false optional = false
python-versions = ">=3.6.0" python-versions = ">=3.6.0"
version = "0.5.2" version = "0.5.3"
[package.dependencies] [package.dependencies]
aiohttp = "*" aiohttp = "*"
@ -1034,10 +1034,10 @@ description = "The comprehensive WSGI web application library."
name = "werkzeug" name = "werkzeug"
optional = false optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "1.0.0" version = "1.0.1"
[package.extras] [package.extras]
dev = ["pytest", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinx-issues"] dev = ["pytest", "pytest-timeout", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinx-issues"]
watchdog = ["watchdog"] watchdog = ["watchdog"]
[[package]] [[package]]
@ -1620,7 +1620,7 @@ urllib3 = [
{file = "urllib3-1.25.8.tar.gz", hash = "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc"}, {file = "urllib3-1.25.8.tar.gz", hash = "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc"},
] ]
vt-py = [ vt-py = [
{file = "vt-py-0.5.2.tar.gz", hash = "sha256:e4ff54466b12d3e66ac77d2acbe4eb93c916ecf2a83b9f1e39057b7fc9f29067"}, {file = "vt-py-0.5.3.tar.gz", hash = "sha256:0a52d58976ec3baf24ade11d0473773d6c7a8ccf862c86f34bc74216ffbe920f"},
] ]
w3lib = [ w3lib = [
{file = "w3lib-1.21.0-py2.py3-none-any.whl", hash = "sha256:847704b837b2b973cddef6938325d466628e6078266bc2e1f7ac49ba85c34823"}, {file = "w3lib-1.21.0-py2.py3-none-any.whl", hash = "sha256:847704b837b2b973cddef6938325d466628e6078266bc2e1f7ac49ba85c34823"},
@ -1631,8 +1631,8 @@ wcwidth = [
{file = "wcwidth-0.1.9.tar.gz", hash = "sha256:ee73862862a156bf77ff92b09034fc4825dd3af9cf81bc5b360668d425f3c5f1"}, {file = "wcwidth-0.1.9.tar.gz", hash = "sha256:ee73862862a156bf77ff92b09034fc4825dd3af9cf81bc5b360668d425f3c5f1"},
] ]
werkzeug = [ werkzeug = [
{file = "Werkzeug-1.0.0-py2.py3-none-any.whl", hash = "sha256:6dc65cf9091cf750012f56f2cad759fa9e879f511b5ff8685e456b4e3bf90d16"}, {file = "Werkzeug-1.0.1-py2.py3-none-any.whl", hash = "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43"},
{file = "Werkzeug-1.0.0.tar.gz", hash = "sha256:169ba8a33788476292d04186ab33b01d6add475033dfc07215e6d219cc077096"}, {file = "Werkzeug-1.0.1.tar.gz", hash = "sha256:6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c"},
] ]
yarl = [ yarl = [
{file = "yarl-1.4.2-cp35-cp35m-macosx_10_13_x86_64.whl", hash = "sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b"}, {file = "yarl-1.4.2-cp35-cp35m-macosx_10_13_x86_64.whl", hash = "sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b"},