CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: Improve logging

pull/79/head
Raphaël Vinot 2020-04-03 17:51:58 +02:00
parent 2092038a69
commit de7f50c3b0
5 changed files with 46 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
config/generic.json.sample
View File

@ -1,11 +1,14 @@
{ {
"loglevel": "INFO", "loglevel": "INFO",
"splash_loglevel": "WARNING",
"only_global_lookups": true, "only_global_lookups": true,
"splash_url": "http://127.0.0.1:8050", "splash_url": "http://127.0.0.1:8050",
"cache_clean_user": {}, "cache_clean_user": {},
"_notes": { "_notes": {
"loglevel": "(lookyloo) Can be one of the value listed here: https://docs.python.org/3/library/logging.html#levels",
"splash_loglevel": "(Splash) INFO is *very* verbose.",
"only_global_lookups": "Set it to True if your instance is publicly available so users aren't able to scan your internal network", "only_global_lookups": "Set it to True if your instance is publicly available so users aren't able to scan your internal network",
"loglevel": "Can be one of the value listed here: https://docs.python.org/3/library/logging.html#levels", "splash_url": "URL to connect to splash",
"cache_clean_user": "Format: {username: password}" "cache_clean_user": "Format: {username: password}"
} }
} }

2
lookyloo/__init__.py
View File

@ -0,0 +1,2 @@
import logging
logging.getLogger(__name__).addHandler(logging.NullHandler())

9
lookyloo/lookyloo.py
View File

@ -89,7 +89,7 @@ class Lookyloo():
if entry in self.configs['generic']: if entry in self.configs['generic']:
return self.configs['generic'][entry] return self.configs['generic'][entry]
else: else:
self.logger.warning(f'Unable to fing {entry} in config file.') self.logger.warning(f'Unable to find {entry} in config file.')
else: else:
self.logger.warning('No generic config file available.') self.logger.warning('No generic config file available.')
self.logger.warning('Falling back on sample config, please initialize the generic config file.') self.logger.warning('Falling back on sample config, please initialize the generic config file.')
@ -223,7 +223,9 @@ class Lookyloo():
meta = json.load(f) meta = json.load(f)
ct = self._load_pickle(pickle_file) ct = self._load_pickle(pickle_file)
if not ct: if not ct:
ct = CrawledTree(har_files) with open((capture_dir / 'uuid'), 'r') as f:
uuid = f.read()
ct = CrawledTree(har_files, uuid)
with pickle_file.open('wb') as _p: with pickle_file.open('wb') as _p:
pickle.dump(ct, _p) pickle.dump(ct, _p)
return str(pickle_file), ct.to_json(), ct.start_time.isoformat(), ct.user_agent, ct.root_url, meta return str(pickle_file), ct.to_json(), ct.start_time.isoformat(), ct.user_agent, ct.root_url, meta
@ -265,7 +267,8 @@ class Lookyloo():
return False return False
cookies = load_cookies(cookies_pseudofile) cookies = load_cookies(cookies_pseudofile)
items = crawl(self.splash_url, url, cookies=cookies, depth=depth, user_agent=user_agent, log_enabled=True, log_level='INFO') items = crawl(self.splash_url, url, cookies=cookies, depth=depth, user_agent=user_agent,
log_enabled=True, log_level=self.get_config('splash_loglevel'))
if not items: if not items:
# broken # broken
return False return False

65
poetry.lock generated
View File

@ -175,7 +175,7 @@ description = "A Python module to bypass Cloudflare's anti-bot page."
name = "cloudscraper" name = "cloudscraper"
optional = false optional = false
python-versions = "*" python-versions = "*"
version = "1.2.30" version = "1.2.33"
[package.dependencies] [package.dependencies]
requests = ">=2.9.2" requests = ">=2.9.2"
@ -204,8 +204,8 @@ category = "main"
description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers."
name = "cryptography" name = "cryptography"
optional = false optional = false
python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*"
version = "2.8" version = "2.9"
[package.dependencies] [package.dependencies]
cffi = ">=1.8,<1.11.3 || >1.11.3" cffi = ">=1.8,<1.11.3 || >1.11.3"
@ -313,10 +313,9 @@ publicsuffix2 = "^2.20191221"
six = "^1.14.0" six = "^1.14.0"
[package.source] [package.source]
reference = "9ec997b0f1c09d3034fd861fd17d972da479f505" reference = "ebd7dac3f1fd9e2b4b858200a0b1f82a7cfd1c9c"
type = "git" type = "git"
url = "https://github.com/viper-framework/har2tree.git" url = "https://github.com/viper-framework/har2tree.git"
[[package]] [[package]]
category = "main" category = "main"
description = "A featureful, immutable, and correct URL for Python." description = "A featureful, immutable, and correct URL for Python."
@ -756,7 +755,6 @@ requests = "^2.22.0"
reference = "3ea143f44d37ab701c70ffb38408528ddb4e2b6e" reference = "3ea143f44d37ab701c70ffb38408528ddb4e2b6e"
type = "git" type = "git"
url = "https://github.com/CIRCL/PySaneJS.git" url = "https://github.com/CIRCL/PySaneJS.git"
[[package]] [[package]]
category = "main" category = "main"
description = "pytest: simple powerful testing with Python" description = "pytest: simple powerful testing with Python"
@ -894,10 +892,9 @@ scrapy = "^1.8.0"
scrapy-splash = "^0.7.2" scrapy-splash = "^0.7.2"
[package.source] [package.source]
reference = "300ee49cb21784514dd2a35b374ce06c4b7f04a6" reference = "3a58d1d751efc0dee54c6d2f485f9d226a5c2e79"
type = "git" type = "git"
url = "https://github.com/viper-framework/ScrapySplashWrapper.git" url = "https://github.com/viper-framework/ScrapySplashWrapper.git"
[[package]] [[package]]
category = "main" category = "main"
description = "Service identity verification for pyOpenSSL & cryptography." description = "Service identity verification for pyOpenSSL & cryptography."
@ -994,7 +991,7 @@ description = "Backported and Experimental Type Hints for Python 3.5+"
name = "typing-extensions" name = "typing-extensions"
optional = false optional = false
python-versions = "*" python-versions = "*"
version = "3.7.4.1" version = "3.7.4.2"
[[package]] [[package]]
category = "main" category = "main"
@ -1227,8 +1224,8 @@ click = [
{file = "click-7.1.1.tar.gz", hash = "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc"}, {file = "click-7.1.1.tar.gz", hash = "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc"},
] ]
cloudscraper = [ cloudscraper = [
{file = "cloudscraper-1.2.30-py2.py3-none-any.whl", hash = "sha256:75f978fd2dc10582768c99bcc2dfaab49e099a69ba681b74baaf84e00d198e05"}, {file = "cloudscraper-1.2.33-py2.py3-none-any.whl", hash = "sha256:74618cfd55841dd0df959a164ee30b9456ecaddcbc7215cc13f89e7a0e3cadf0"},
{file = "cloudscraper-1.2.30.tar.gz", hash = "sha256:16715a32c140629e232c309d295e164a2ca3f119bcda7686bc5fe12379f947d0"}, {file = "cloudscraper-1.2.33.tar.gz", hash = "sha256:2bcd017740c7dc0b8d64b55742638a50960ba82b9fd75fc38b9e94c5cd641aa5"},
] ]
colorama = [ colorama = [
{file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"}, {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"},
@ -1239,27 +1236,25 @@ constantly = [
{file = "constantly-15.1.0.tar.gz", hash = "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35"}, {file = "constantly-15.1.0.tar.gz", hash = "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35"},
] ]
cryptography = [ cryptography = [
{file = "cryptography-2.8-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8"}, {file = "cryptography-2.9-cp27-cp27m-macosx_10_9_intel.whl", hash = "sha256:ef9a55013676907df6c9d7dd943eb1770d014f68beaa7e73250fb43c759f4585"},
{file = "cryptography-2.8-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2"}, {file = "cryptography-2.9-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:2a2ad24d43398d89f92209289f15265107928f22a8d10385f70def7a698d6a02"},
{file = "cryptography-2.8-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad"}, {file = "cryptography-2.9-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:95e1296e0157361fe2f5f0ed307fd31f94b0ca13372e3673fa95095a627636a1"},
{file = "cryptography-2.8-cp27-cp27m-win32.whl", hash = "sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2"}, {file = "cryptography-2.9-cp27-cp27m-win32.whl", hash = "sha256:192ca04a36852a994ef21df13cca4d822adbbdc9d5009c0f96f1d2929e375d4f"},
{file = "cryptography-2.8-cp27-cp27m-win_amd64.whl", hash = "sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912"}, {file = "cryptography-2.9-cp27-cp27m-win_amd64.whl", hash = "sha256:ed1d0760c7e46436ec90834d6f10477ff09475c692ed1695329d324b2c5cd547"},
{file = "cryptography-2.8-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d"}, {file = "cryptography-2.9-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:19ae795137682a9778892fb4390c07811828b173741bce91e30f899424b3934d"},
{file = "cryptography-2.8-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42"}, {file = "cryptography-2.9-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d1bf5a1a0d60c7f9a78e448adcb99aa101f3f9588b16708044638881be15d6bc"},
{file = "cryptography-2.8-cp34-abi3-macosx_10_6_intel.whl", hash = "sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879"}, {file = "cryptography-2.9-cp35-abi3-macosx_10_9_intel.whl", hash = "sha256:1b9b535d6b55936a79dbe4990b64bb16048f48747c76c29713fea8c50eca2acf"},
{file = "cryptography-2.8-cp34-abi3-manylinux1_x86_64.whl", hash = "sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d"}, {file = "cryptography-2.9-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:7a279f33a081d436e90e91d1a7c338553c04e464de1c9302311a5e7e4b746088"},
{file = "cryptography-2.8-cp34-abi3-manylinux2010_x86_64.whl", hash = "sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9"}, {file = "cryptography-2.9-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:49870684da168b90110bbaf86140d4681032c5e6a2461adc7afdd93be5634216"},
{file = "cryptography-2.8-cp34-cp34m-win32.whl", hash = "sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c"}, {file = "cryptography-2.9-cp35-cp35m-win32.whl", hash = "sha256:6b91cab3841b4c7cb70e4db1697c69f036c8bc0a253edc0baa6783154f1301e4"},
{file = "cryptography-2.8-cp34-cp34m-win_amd64.whl", hash = "sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0"}, {file = "cryptography-2.9-cp35-cp35m-win_amd64.whl", hash = "sha256:587f98ce27ac4547177a0c6fe0986b8736058daffe9160dcf5f1bd411b7fbaa1"},
{file = "cryptography-2.8-cp35-cp35m-win32.whl", hash = "sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf"}, {file = "cryptography-2.9-cp36-cp36m-win32.whl", hash = "sha256:cc20316e3f5a6b582fc3b029d8dc03aabeb645acfcb7fc1d9848841a33265748"},
{file = "cryptography-2.8-cp35-cp35m-win_amd64.whl", hash = "sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793"}, {file = "cryptography-2.9-cp36-cp36m-win_amd64.whl", hash = "sha256:3be7a5722d5bfe69894d3f7bbed15547b17619f3a88a318aab2e37f457524164"},
{file = "cryptography-2.8-cp36-cp36m-win32.whl", hash = "sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595"}, {file = "cryptography-2.9-cp37-cp37m-win32.whl", hash = "sha256:7598974f6879a338c785c513e7c5a4329fbc58b9f6b9a6305035fca5b1076552"},
{file = "cryptography-2.8-cp36-cp36m-win_amd64.whl", hash = "sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7"}, {file = "cryptography-2.9-cp37-cp37m-win_amd64.whl", hash = "sha256:5aca6f00b2f42546b9bdf11a69f248d1881212ce5b9e2618b04935b87f6f82a1"},
{file = "cryptography-2.8-cp37-cp37m-win32.whl", hash = "sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff"}, {file = "cryptography-2.9-cp38-cp38-win32.whl", hash = "sha256:9fc9da390e98cb6975eadf251b6e5fa088820141061bf041cd5c72deba1dc526"},
{file = "cryptography-2.8-cp37-cp37m-win_amd64.whl", hash = "sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f"}, {file = "cryptography-2.9-cp38-cp38-win_amd64.whl", hash = "sha256:6b744039b55988519cc183149cceb573189b3e46e16ccf6f8c46798bb767c9dc"},
{file = "cryptography-2.8-cp38-cp38-win32.whl", hash = "sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e"}, {file = "cryptography-2.9.tar.gz", hash = "sha256:0cacd3ef5c604b8e5f59bf2582c076c98a37fe206b31430d0cd08138aff0986e"},
{file = "cryptography-2.8-cp38-cp38-win_amd64.whl", hash = "sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13"},
{file = "cryptography-2.8.tar.gz", hash = "sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651"},
] ]
cssselect = [ cssselect = [
{file = "cssselect-1.1.0-py2.py3-none-any.whl", hash = "sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf"}, {file = "cssselect-1.1.0-py2.py3-none-any.whl", hash = "sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf"},
@ -1629,9 +1624,9 @@ typed-ast = [
{file = "typed_ast-1.4.1.tar.gz", hash = "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b"}, {file = "typed_ast-1.4.1.tar.gz", hash = "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b"},
] ]
typing-extensions = [ typing-extensions = [
{file = "typing_extensions-3.7.4.1-py2-none-any.whl", hash = "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d"}, {file = "typing_extensions-3.7.4.2-py2-none-any.whl", hash = "sha256:f8d2bd89d25bc39dabe7d23df520442fa1d8969b82544370e03d88b5a591c392"},
{file = "typing_extensions-3.7.4.1-py3-none-any.whl", hash = "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575"}, {file = "typing_extensions-3.7.4.2-py3-none-any.whl", hash = "sha256:6e95524d8a547a91e08f404ae485bbb71962de46967e1b71a0cb89af24e761c5"},
{file = "typing_extensions-3.7.4.1.tar.gz", hash = "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2"}, {file = "typing_extensions-3.7.4.2.tar.gz", hash = "sha256:79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae"},
] ]
urllib3 = [ urllib3 = [
{file = "urllib3-1.25.8-py2.py3-none-any.whl", hash = "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc"}, {file = "urllib3-1.25.8-py2.py3-none-any.whl", hash = "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc"},

4
website/web/__init__.py
View File

@ -18,6 +18,8 @@ from lookyloo.exceptions import NoValidHarFile
from typing import Tuple from typing import Tuple
import logging
app: Flask = Flask(__name__) app: Flask = Flask(__name__)
secret_file_path: Path = get_homedir() / 'secret_key' secret_file_path: Path = get_homedir() / 'secret_key'
@ -39,6 +41,8 @@ lookyloo: Lookyloo = Lookyloo()
user = lookyloo.get_config('cache_clean_user') user = lookyloo.get_config('cache_clean_user')
logging.basicConfig(level=lookyloo.get_config('loglevel'))
@auth.get_password @auth.get_password
def get_pw(username): def get_pw(username):