From de7f50c3b0ab65bf4e26c1e7a1d1b29325c1f80d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 3 Apr 2020 17:51:58 +0200 Subject: [PATCH] chg: Improve logging --- config/generic.json.sample | 5 ++- lookyloo/__init__.py | 2 ++ lookyloo/lookyloo.py | 9 ++++-- poetry.lock | 65 ++++++++++++++++++-------------------- website/web/__init__.py | 4 +++ 5 files changed, 46 insertions(+), 39 deletions(-) diff --git a/config/generic.json.sample b/config/generic.json.sample index b2ddb478..86531e3a 100644 --- a/config/generic.json.sample +++ b/config/generic.json.sample @@ -1,11 +1,14 @@ { "loglevel": "INFO", + "splash_loglevel": "WARNING", "only_global_lookups": true, "splash_url": "http://127.0.0.1:8050", "cache_clean_user": {}, "_notes": { + "loglevel": "(lookyloo) Can be one of the value listed here: https://docs.python.org/3/library/logging.html#levels", + "splash_loglevel": "(Splash) INFO is *very* verbose.", "only_global_lookups": "Set it to True if your instance is publicly available so users aren't able to scan your internal network", - "loglevel": "Can be one of the value listed here: https://docs.python.org/3/library/logging.html#levels", + "splash_url": "URL to connect to splash", "cache_clean_user": "Format: {username: password}" } } diff --git a/lookyloo/__init__.py b/lookyloo/__init__.py index e69de29b..26346d78 100644 --- a/lookyloo/__init__.py +++ b/lookyloo/__init__.py @@ -0,0 +1,2 @@ +import logging +logging.getLogger(__name__).addHandler(logging.NullHandler()) diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index 4736c954..76d8846f 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -89,7 +89,7 @@ class Lookyloo(): if entry in self.configs['generic']: return self.configs['generic'][entry] else: - self.logger.warning(f'Unable to fing {entry} in config file.') + self.logger.warning(f'Unable to find {entry} in config file.') else: self.logger.warning('No generic config file available.') self.logger.warning('Falling back on sample config, please initialize the generic config file.') @@ -223,7 +223,9 @@ class Lookyloo(): meta = json.load(f) ct = self._load_pickle(pickle_file) if not ct: - ct = CrawledTree(har_files) + with open((capture_dir / 'uuid'), 'r') as f: + uuid = f.read() + ct = CrawledTree(har_files, uuid) with pickle_file.open('wb') as _p: pickle.dump(ct, _p) return str(pickle_file), ct.to_json(), ct.start_time.isoformat(), ct.user_agent, ct.root_url, meta @@ -265,7 +267,8 @@ class Lookyloo(): return False cookies = load_cookies(cookies_pseudofile) - items = crawl(self.splash_url, url, cookies=cookies, depth=depth, user_agent=user_agent, log_enabled=True, log_level='INFO') + items = crawl(self.splash_url, url, cookies=cookies, depth=depth, user_agent=user_agent, + log_enabled=True, log_level=self.get_config('splash_loglevel')) if not items: # broken return False diff --git a/poetry.lock b/poetry.lock index b660e31f..3996e292 100644 --- a/poetry.lock +++ b/poetry.lock @@ -175,7 +175,7 @@ description = "A Python module to bypass Cloudflare's anti-bot page." name = "cloudscraper" optional = false python-versions = "*" -version = "1.2.30" +version = "1.2.33" [package.dependencies] requests = ">=2.9.2" @@ -204,8 +204,8 @@ category = "main" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." name = "cryptography" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" -version = "2.8" +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" +version = "2.9" [package.dependencies] cffi = ">=1.8,<1.11.3 || >1.11.3" @@ -313,10 +313,9 @@ publicsuffix2 = "^2.20191221" six = "^1.14.0" [package.source] -reference = "9ec997b0f1c09d3034fd861fd17d972da479f505" +reference = "ebd7dac3f1fd9e2b4b858200a0b1f82a7cfd1c9c" type = "git" url = "https://github.com/viper-framework/har2tree.git" - [[package]] category = "main" description = "A featureful, immutable, and correct URL for Python." @@ -756,7 +755,6 @@ requests = "^2.22.0" reference = "3ea143f44d37ab701c70ffb38408528ddb4e2b6e" type = "git" url = "https://github.com/CIRCL/PySaneJS.git" - [[package]] category = "main" description = "pytest: simple powerful testing with Python" @@ -894,10 +892,9 @@ scrapy = "^1.8.0" scrapy-splash = "^0.7.2" [package.source] -reference = "300ee49cb21784514dd2a35b374ce06c4b7f04a6" +reference = "3a58d1d751efc0dee54c6d2f485f9d226a5c2e79" type = "git" url = "https://github.com/viper-framework/ScrapySplashWrapper.git" - [[package]] category = "main" description = "Service identity verification for pyOpenSSL & cryptography." @@ -994,7 +991,7 @@ description = "Backported and Experimental Type Hints for Python 3.5+" name = "typing-extensions" optional = false python-versions = "*" -version = "3.7.4.1" +version = "3.7.4.2" [[package]] category = "main" @@ -1227,8 +1224,8 @@ click = [ {file = "click-7.1.1.tar.gz", hash = "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc"}, ] cloudscraper = [ - {file = "cloudscraper-1.2.30-py2.py3-none-any.whl", hash = "sha256:75f978fd2dc10582768c99bcc2dfaab49e099a69ba681b74baaf84e00d198e05"}, - {file = "cloudscraper-1.2.30.tar.gz", hash = "sha256:16715a32c140629e232c309d295e164a2ca3f119bcda7686bc5fe12379f947d0"}, + {file = "cloudscraper-1.2.33-py2.py3-none-any.whl", hash = "sha256:74618cfd55841dd0df959a164ee30b9456ecaddcbc7215cc13f89e7a0e3cadf0"}, + {file = "cloudscraper-1.2.33.tar.gz", hash = "sha256:2bcd017740c7dc0b8d64b55742638a50960ba82b9fd75fc38b9e94c5cd641aa5"}, ] colorama = [ {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"}, @@ -1239,27 +1236,25 @@ constantly = [ {file = "constantly-15.1.0.tar.gz", hash = "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35"}, ] cryptography = [ - {file = "cryptography-2.8-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8"}, - {file = "cryptography-2.8-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2"}, - {file = "cryptography-2.8-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad"}, - {file = "cryptography-2.8-cp27-cp27m-win32.whl", hash = "sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2"}, - {file = "cryptography-2.8-cp27-cp27m-win_amd64.whl", hash = "sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912"}, - {file = "cryptography-2.8-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d"}, - {file = "cryptography-2.8-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42"}, - {file = "cryptography-2.8-cp34-abi3-macosx_10_6_intel.whl", hash = "sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879"}, - {file = "cryptography-2.8-cp34-abi3-manylinux1_x86_64.whl", hash = "sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d"}, - {file = "cryptography-2.8-cp34-abi3-manylinux2010_x86_64.whl", hash = "sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9"}, - {file = "cryptography-2.8-cp34-cp34m-win32.whl", hash = "sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c"}, - {file = "cryptography-2.8-cp34-cp34m-win_amd64.whl", hash = "sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0"}, - {file = "cryptography-2.8-cp35-cp35m-win32.whl", hash = "sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf"}, - {file = "cryptography-2.8-cp35-cp35m-win_amd64.whl", hash = "sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793"}, - {file = "cryptography-2.8-cp36-cp36m-win32.whl", hash = "sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595"}, - {file = "cryptography-2.8-cp36-cp36m-win_amd64.whl", hash = "sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7"}, - {file = "cryptography-2.8-cp37-cp37m-win32.whl", hash = "sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff"}, - {file = "cryptography-2.8-cp37-cp37m-win_amd64.whl", hash = "sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f"}, - {file = "cryptography-2.8-cp38-cp38-win32.whl", hash = "sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e"}, - {file = "cryptography-2.8-cp38-cp38-win_amd64.whl", hash = "sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13"}, - {file = "cryptography-2.8.tar.gz", hash = "sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651"}, + {file = "cryptography-2.9-cp27-cp27m-macosx_10_9_intel.whl", hash = "sha256:ef9a55013676907df6c9d7dd943eb1770d014f68beaa7e73250fb43c759f4585"}, + {file = "cryptography-2.9-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:2a2ad24d43398d89f92209289f15265107928f22a8d10385f70def7a698d6a02"}, + {file = "cryptography-2.9-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:95e1296e0157361fe2f5f0ed307fd31f94b0ca13372e3673fa95095a627636a1"}, + {file = "cryptography-2.9-cp27-cp27m-win32.whl", hash = "sha256:192ca04a36852a994ef21df13cca4d822adbbdc9d5009c0f96f1d2929e375d4f"}, + {file = "cryptography-2.9-cp27-cp27m-win_amd64.whl", hash = "sha256:ed1d0760c7e46436ec90834d6f10477ff09475c692ed1695329d324b2c5cd547"}, + {file = "cryptography-2.9-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:19ae795137682a9778892fb4390c07811828b173741bce91e30f899424b3934d"}, + {file = "cryptography-2.9-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d1bf5a1a0d60c7f9a78e448adcb99aa101f3f9588b16708044638881be15d6bc"}, + {file = "cryptography-2.9-cp35-abi3-macosx_10_9_intel.whl", hash = "sha256:1b9b535d6b55936a79dbe4990b64bb16048f48747c76c29713fea8c50eca2acf"}, + {file = "cryptography-2.9-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:7a279f33a081d436e90e91d1a7c338553c04e464de1c9302311a5e7e4b746088"}, + {file = "cryptography-2.9-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:49870684da168b90110bbaf86140d4681032c5e6a2461adc7afdd93be5634216"}, + {file = "cryptography-2.9-cp35-cp35m-win32.whl", hash = "sha256:6b91cab3841b4c7cb70e4db1697c69f036c8bc0a253edc0baa6783154f1301e4"}, + {file = "cryptography-2.9-cp35-cp35m-win_amd64.whl", hash = "sha256:587f98ce27ac4547177a0c6fe0986b8736058daffe9160dcf5f1bd411b7fbaa1"}, + {file = "cryptography-2.9-cp36-cp36m-win32.whl", hash = "sha256:cc20316e3f5a6b582fc3b029d8dc03aabeb645acfcb7fc1d9848841a33265748"}, + {file = "cryptography-2.9-cp36-cp36m-win_amd64.whl", hash = "sha256:3be7a5722d5bfe69894d3f7bbed15547b17619f3a88a318aab2e37f457524164"}, + {file = "cryptography-2.9-cp37-cp37m-win32.whl", hash = "sha256:7598974f6879a338c785c513e7c5a4329fbc58b9f6b9a6305035fca5b1076552"}, + {file = "cryptography-2.9-cp37-cp37m-win_amd64.whl", hash = "sha256:5aca6f00b2f42546b9bdf11a69f248d1881212ce5b9e2618b04935b87f6f82a1"}, + {file = "cryptography-2.9-cp38-cp38-win32.whl", hash = "sha256:9fc9da390e98cb6975eadf251b6e5fa088820141061bf041cd5c72deba1dc526"}, + {file = "cryptography-2.9-cp38-cp38-win_amd64.whl", hash = "sha256:6b744039b55988519cc183149cceb573189b3e46e16ccf6f8c46798bb767c9dc"}, + {file = "cryptography-2.9.tar.gz", hash = "sha256:0cacd3ef5c604b8e5f59bf2582c076c98a37fe206b31430d0cd08138aff0986e"}, ] cssselect = [ {file = "cssselect-1.1.0-py2.py3-none-any.whl", hash = "sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf"}, @@ -1629,9 +1624,9 @@ typed-ast = [ {file = "typed_ast-1.4.1.tar.gz", hash = "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b"}, ] typing-extensions = [ - {file = "typing_extensions-3.7.4.1-py2-none-any.whl", hash = "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d"}, - {file = "typing_extensions-3.7.4.1-py3-none-any.whl", hash = "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575"}, - {file = "typing_extensions-3.7.4.1.tar.gz", hash = "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2"}, + {file = "typing_extensions-3.7.4.2-py2-none-any.whl", hash = "sha256:f8d2bd89d25bc39dabe7d23df520442fa1d8969b82544370e03d88b5a591c392"}, + {file = "typing_extensions-3.7.4.2-py3-none-any.whl", hash = "sha256:6e95524d8a547a91e08f404ae485bbb71962de46967e1b71a0cb89af24e761c5"}, + {file = "typing_extensions-3.7.4.2.tar.gz", hash = "sha256:79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae"}, ] urllib3 = [ {file = "urllib3-1.25.8-py2.py3-none-any.whl", hash = "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc"}, diff --git a/website/web/__init__.py b/website/web/__init__.py index 9f91019b..abdc3361 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -18,6 +18,8 @@ from lookyloo.exceptions import NoValidHarFile from typing import Tuple +import logging + app: Flask = Flask(__name__) secret_file_path: Path = get_homedir() / 'secret_key' @@ -39,6 +41,8 @@ lookyloo: Lookyloo = Lookyloo() user = lookyloo.get_config('cache_clean_user') +logging.basicConfig(level=lookyloo.get_config('loglevel')) + @auth.get_password def get_pw(username):