mirror of https://github.com/CIRCL/lookyloo
chg: Improve logging
Raphaël Vinot
parent
2092038a69
commit
de7f50c3b0
|
|
@ -1,11 +1,14 @@
|
|||
{
|
||||
"loglevel": "INFO",
|
||||
"splash_loglevel": "WARNING",
|
||||
"only_global_lookups": true,
|
||||
"splash_url": "http://127.0.0.1:8050",
|
||||
"cache_clean_user": {},
|
||||
"_notes": {
|
||||
"loglevel": "(lookyloo) Can be one of the value listed here: https://docs.python.org/3/library/logging.html#levels",
|
||||
"splash_loglevel": "(Splash) INFO is *very* verbose.",
|
||||
"only_global_lookups": "Set it to True if your instance is publicly available so users aren't able to scan your internal network",
|
||||
"loglevel": "Can be one of the value listed here: https://docs.python.org/3/library/logging.html#levels",
|
||||
"splash_url": "URL to connect to splash",
|
||||
"cache_clean_user": "Format: {username: password}"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
import logging
|
||||
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
||||
|
|
@ -89,7 +89,7 @@ class Lookyloo():
|
|||
if entry in self.configs['generic']:
|
||||
return self.configs['generic'][entry]
|
||||
else:
|
||||
self.logger.warning(f'Unable to fing {entry} in config file.')
|
||||
self.logger.warning(f'Unable to find {entry} in config file.')
|
||||
else:
|
||||
self.logger.warning('No generic config file available.')
|
||||
self.logger.warning('Falling back on sample config, please initialize the generic config file.')
|
||||
|
|
@ -223,7 +223,9 @@ class Lookyloo():
|
|||
meta = json.load(f)
|
||||
ct = self._load_pickle(pickle_file)
|
||||
if not ct:
|
||||
ct = CrawledTree(har_files)
|
||||
with open((capture_dir / 'uuid'), 'r') as f:
|
||||
uuid = f.read()
|
||||
ct = CrawledTree(har_files, uuid)
|
||||
with pickle_file.open('wb') as _p:
|
||||
pickle.dump(ct, _p)
|
||||
return str(pickle_file), ct.to_json(), ct.start_time.isoformat(), ct.user_agent, ct.root_url, meta
|
||||
|
|
@ -265,7 +267,8 @@ class Lookyloo():
|
|||
return False
|
||||
|
||||
cookies = load_cookies(cookies_pseudofile)
|
||||
items = crawl(self.splash_url, url, cookies=cookies, depth=depth, user_agent=user_agent, log_enabled=True, log_level='INFO')
|
||||
items = crawl(self.splash_url, url, cookies=cookies, depth=depth, user_agent=user_agent,
|
||||
log_enabled=True, log_level=self.get_config('splash_loglevel'))
|
||||
if not items:
|
||||
# broken
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -175,7 +175,7 @@ description = "A Python module to bypass Cloudflare's anti-bot page."
|
|||
name = "cloudscraper"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
version = "1.2.30"
|
||||
version = "1.2.33"
|
||||
|
||||
[package.dependencies]
|
||||
requests = ">=2.9.2"
|
||||
|
|
@ -204,8 +204,8 @@ category = "main"
|
|||
description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers."
|
||||
name = "cryptography"
|
||||
optional = false
|
||||
python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*"
|
||||
version = "2.8"
|
||||
python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*"
|
||||
version = "2.9"
|
||||
|
||||
[package.dependencies]
|
||||
cffi = ">=1.8,<1.11.3 || >1.11.3"
|
||||
|
|
@ -313,10 +313,9 @@ publicsuffix2 = "^2.20191221"
|
|||
six = "^1.14.0"
|
||||
|
||||
[package.source]
|
||||
reference = "9ec997b0f1c09d3034fd861fd17d972da479f505"
|
||||
reference = "ebd7dac3f1fd9e2b4b858200a0b1f82a7cfd1c9c"
|
||||
type = "git"
|
||||
url = "https://github.com/viper-framework/har2tree.git"
|
||||
|
||||
[[package]]
|
||||
category = "main"
|
||||
description = "A featureful, immutable, and correct URL for Python."
|
||||
|
|
@ -756,7 +755,6 @@ requests = "^2.22.0"
|
|||
reference = "3ea143f44d37ab701c70ffb38408528ddb4e2b6e"
|
||||
type = "git"
|
||||
url = "https://github.com/CIRCL/PySaneJS.git"
|
||||
|
||||
[[package]]
|
||||
category = "main"
|
||||
description = "pytest: simple powerful testing with Python"
|
||||
|
|
@ -894,10 +892,9 @@ scrapy = "^1.8.0"
|
|||
scrapy-splash = "^0.7.2"
|
||||
|
||||
[package.source]
|
||||
reference = "300ee49cb21784514dd2a35b374ce06c4b7f04a6"
|
||||
reference = "3a58d1d751efc0dee54c6d2f485f9d226a5c2e79"
|
||||
type = "git"
|
||||
url = "https://github.com/viper-framework/ScrapySplashWrapper.git"
|
||||
|
||||
[[package]]
|
||||
category = "main"
|
||||
description = "Service identity verification for pyOpenSSL & cryptography."
|
||||
|
|
@ -994,7 +991,7 @@ description = "Backported and Experimental Type Hints for Python 3.5+"
|
|||
name = "typing-extensions"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
version = "3.7.4.1"
|
||||
version = "3.7.4.2"
|
||||
|
||||
[[package]]
|
||||
category = "main"
|
||||
|
|
@ -1227,8 +1224,8 @@ click = [
|
|||
{file = "click-7.1.1.tar.gz", hash = "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc"},
|
||||
]
|
||||
cloudscraper = [
|
||||
{file = "cloudscraper-1.2.30-py2.py3-none-any.whl", hash = "sha256:75f978fd2dc10582768c99bcc2dfaab49e099a69ba681b74baaf84e00d198e05"},
|
||||
{file = "cloudscraper-1.2.30.tar.gz", hash = "sha256:16715a32c140629e232c309d295e164a2ca3f119bcda7686bc5fe12379f947d0"},
|
||||
{file = "cloudscraper-1.2.33-py2.py3-none-any.whl", hash = "sha256:74618cfd55841dd0df959a164ee30b9456ecaddcbc7215cc13f89e7a0e3cadf0"},
|
||||
{file = "cloudscraper-1.2.33.tar.gz", hash = "sha256:2bcd017740c7dc0b8d64b55742638a50960ba82b9fd75fc38b9e94c5cd641aa5"},
|
||||
]
|
||||
colorama = [
|
||||
{file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"},
|
||||
|
|
@ -1239,27 +1236,25 @@ constantly = [
|
|||
{file = "constantly-15.1.0.tar.gz", hash = "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35"},
|
||||
]
|
||||
cryptography = [
|
||||
{file = "cryptography-2.8-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8"},
|
||||
{file = "cryptography-2.8-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2"},
|
||||
{file = "cryptography-2.8-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad"},
|
||||
{file = "cryptography-2.8-cp27-cp27m-win32.whl", hash = "sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2"},
|
||||
{file = "cryptography-2.8-cp27-cp27m-win_amd64.whl", hash = "sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912"},
|
||||
{file = "cryptography-2.8-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d"},
|
||||
{file = "cryptography-2.8-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42"},
|
||||
{file = "cryptography-2.8-cp34-abi3-macosx_10_6_intel.whl", hash = "sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879"},
|
||||
{file = "cryptography-2.8-cp34-abi3-manylinux1_x86_64.whl", hash = "sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d"},
|
||||
{file = "cryptography-2.8-cp34-abi3-manylinux2010_x86_64.whl", hash = "sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9"},
|
||||
{file = "cryptography-2.8-cp34-cp34m-win32.whl", hash = "sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c"},
|
||||
{file = "cryptography-2.8-cp34-cp34m-win_amd64.whl", hash = "sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0"},
|
||||
{file = "cryptography-2.8-cp35-cp35m-win32.whl", hash = "sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf"},
|
||||
{file = "cryptography-2.8-cp35-cp35m-win_amd64.whl", hash = "sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793"},
|
||||
{file = "cryptography-2.8-cp36-cp36m-win32.whl", hash = "sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595"},
|
||||
{file = "cryptography-2.8-cp36-cp36m-win_amd64.whl", hash = "sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7"},
|
||||
{file = "cryptography-2.8-cp37-cp37m-win32.whl", hash = "sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff"},
|
||||
{file = "cryptography-2.8-cp37-cp37m-win_amd64.whl", hash = "sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f"},
|
||||
{file = "cryptography-2.8-cp38-cp38-win32.whl", hash = "sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e"},
|
||||
{file = "cryptography-2.8-cp38-cp38-win_amd64.whl", hash = "sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13"},
|
||||
{file = "cryptography-2.8.tar.gz", hash = "sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651"},
|
||||
{file = "cryptography-2.9-cp27-cp27m-macosx_10_9_intel.whl", hash = "sha256:ef9a55013676907df6c9d7dd943eb1770d014f68beaa7e73250fb43c759f4585"},
|
||||
{file = "cryptography-2.9-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:2a2ad24d43398d89f92209289f15265107928f22a8d10385f70def7a698d6a02"},
|
||||
{file = "cryptography-2.9-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:95e1296e0157361fe2f5f0ed307fd31f94b0ca13372e3673fa95095a627636a1"},
|
||||
{file = "cryptography-2.9-cp27-cp27m-win32.whl", hash = "sha256:192ca04a36852a994ef21df13cca4d822adbbdc9d5009c0f96f1d2929e375d4f"},
|
||||
{file = "cryptography-2.9-cp27-cp27m-win_amd64.whl", hash = "sha256:ed1d0760c7e46436ec90834d6f10477ff09475c692ed1695329d324b2c5cd547"},
|
||||
{file = "cryptography-2.9-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:19ae795137682a9778892fb4390c07811828b173741bce91e30f899424b3934d"},
|
||||
{file = "cryptography-2.9-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d1bf5a1a0d60c7f9a78e448adcb99aa101f3f9588b16708044638881be15d6bc"},
|
||||
{file = "cryptography-2.9-cp35-abi3-macosx_10_9_intel.whl", hash = "sha256:1b9b535d6b55936a79dbe4990b64bb16048f48747c76c29713fea8c50eca2acf"},
|
||||
{file = "cryptography-2.9-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:7a279f33a081d436e90e91d1a7c338553c04e464de1c9302311a5e7e4b746088"},
|
||||
{file = "cryptography-2.9-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:49870684da168b90110bbaf86140d4681032c5e6a2461adc7afdd93be5634216"},
|
||||
{file = "cryptography-2.9-cp35-cp35m-win32.whl", hash = "sha256:6b91cab3841b4c7cb70e4db1697c69f036c8bc0a253edc0baa6783154f1301e4"},
|
||||
{file = "cryptography-2.9-cp35-cp35m-win_amd64.whl", hash = "sha256:587f98ce27ac4547177a0c6fe0986b8736058daffe9160dcf5f1bd411b7fbaa1"},
|
||||
{file = "cryptography-2.9-cp36-cp36m-win32.whl", hash = "sha256:cc20316e3f5a6b582fc3b029d8dc03aabeb645acfcb7fc1d9848841a33265748"},
|
||||
{file = "cryptography-2.9-cp36-cp36m-win_amd64.whl", hash = "sha256:3be7a5722d5bfe69894d3f7bbed15547b17619f3a88a318aab2e37f457524164"},
|
||||
{file = "cryptography-2.9-cp37-cp37m-win32.whl", hash = "sha256:7598974f6879a338c785c513e7c5a4329fbc58b9f6b9a6305035fca5b1076552"},
|
||||
{file = "cryptography-2.9-cp37-cp37m-win_amd64.whl", hash = "sha256:5aca6f00b2f42546b9bdf11a69f248d1881212ce5b9e2618b04935b87f6f82a1"},
|
||||
{file = "cryptography-2.9-cp38-cp38-win32.whl", hash = "sha256:9fc9da390e98cb6975eadf251b6e5fa088820141061bf041cd5c72deba1dc526"},
|
||||
{file = "cryptography-2.9-cp38-cp38-win_amd64.whl", hash = "sha256:6b744039b55988519cc183149cceb573189b3e46e16ccf6f8c46798bb767c9dc"},
|
||||
{file = "cryptography-2.9.tar.gz", hash = "sha256:0cacd3ef5c604b8e5f59bf2582c076c98a37fe206b31430d0cd08138aff0986e"},
|
||||
]
|
||||
cssselect = [
|
||||
{file = "cssselect-1.1.0-py2.py3-none-any.whl", hash = "sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf"},
|
||||
|
|
@ -1629,9 +1624,9 @@ typed-ast = [
|
|||
{file = "typed_ast-1.4.1.tar.gz", hash = "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b"},
|
||||
]
|
||||
typing-extensions = [
|
||||
{file = "typing_extensions-3.7.4.1-py2-none-any.whl", hash = "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d"},
|
||||
{file = "typing_extensions-3.7.4.1-py3-none-any.whl", hash = "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575"},
|
||||
{file = "typing_extensions-3.7.4.1.tar.gz", hash = "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2"},
|
||||
{file = "typing_extensions-3.7.4.2-py2-none-any.whl", hash = "sha256:f8d2bd89d25bc39dabe7d23df520442fa1d8969b82544370e03d88b5a591c392"},
|
||||
{file = "typing_extensions-3.7.4.2-py3-none-any.whl", hash = "sha256:6e95524d8a547a91e08f404ae485bbb71962de46967e1b71a0cb89af24e761c5"},
|
||||
{file = "typing_extensions-3.7.4.2.tar.gz", hash = "sha256:79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae"},
|
||||
]
|
||||
urllib3 = [
|
||||
{file = "urllib3-1.25.8-py2.py3-none-any.whl", hash = "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc"},
|
||||
|
|
|
|||
|
|
@ -18,6 +18,8 @@ from lookyloo.exceptions import NoValidHarFile
|
|||
|
||||
from typing import Tuple
|
||||
|
||||
import logging
|
||||
|
||||
app: Flask = Flask(__name__)
|
||||
|
||||
secret_file_path: Path = get_homedir() / 'secret_key'
|
||||
|
|
@ -39,6 +41,8 @@ lookyloo: Lookyloo = Lookyloo()
|
|||
|
||||
user = lookyloo.get_config('cache_clean_user')
|
||||
|
||||
logging.basicConfig(level=lookyloo.get_config('loglevel'))
|
||||
|
||||
|
||||
@auth.get_password
|
||||
def get_pw(username):
|
||||
|
|
|
|||
Loading…
Reference in New Issue