CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

new: Add error page

pull/42/head
Raphaël Vinot 2019-02-18 13:52:48 +01:00
parent 852fb708f6
commit 9f2b7f7b93
6 changed files with 96 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
Pipfile.lock generated
View File

@ -83,40 +83,36 @@
}, },
"cffi": { "cffi": {
"hashes": [ "hashes": [
"sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743", "sha256:0b5f895714a7a9905148fc51978c62e8a6cbcace30904d39dcd0d9e2265bb2f6",
"sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef", "sha256:27cdc7ba35ee6aa443271d11583b50815c4bb52be89a909d0028e86c21961709",
"sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50", "sha256:2d4a38049ea93d5ce3c7659210393524c1efc3efafa151bd85d196fa98fce50a",
"sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f", "sha256:3262573d0d60fc6b9d0e0e6e666db0e5045cbe8a531779aa0deb3b425ec5a282",
"sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30", "sha256:358e96cfffc185ab8f6e7e425c7bb028931ed08d65402fbcf3f4e1bff6e66556",
"sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93", "sha256:37c7db824b5687fbd7ea5519acfd054c905951acc53503547c86be3db0580134",
"sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257", "sha256:39b9554dfe60f878e0c6ff8a460708db6e1b1c9cc6da2c74df2955adf83e355d",
"sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b", "sha256:42b96a77acf8b2d06821600fa87c208046decc13bd22a4a0e65c5c973443e0da",
"sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3", "sha256:5b37dde5035d3c219324cac0e69d96495970977f310b306fa2df5910e1f329a1",
"sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e", "sha256:5d35819f5566d0dd254f273d60cf4a2dcdd3ae3003dfd412d40b3fe8ffd87509",
"sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc", "sha256:5df73aa465e53549bd03c819c1bc69fb85529a5e1a693b7b6cb64408dd3970d1",
"sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04", "sha256:7075b361f7a4d0d4165439992d0b8a3cdfad1f302bf246ed9308a2e33b046bd3",
"sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6", "sha256:7678b5a667b0381c173abe530d7bdb0e6e3b98e062490618f04b80ca62686d96",
"sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359", "sha256:7dfd996192ff8a535458c17f22ff5eb78b83504c34d10eefac0c77b1322609e2",
"sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596", "sha256:8a3be5d31d02c60f84c4fd4c98c5e3a97b49f32e16861367f67c49425f955b28",
"sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b", "sha256:9812e53369c469506b123aee9dcb56d50c82fad60c5df87feb5ff59af5b5f55c",
"sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd", "sha256:9b6f7ba4e78c52c1a291d0c0c0bd745d19adde1a9e1c03cb899f0c6efd6f8033",
"sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95", "sha256:a85bc1d7c3bba89b3d8c892bc0458de504f8b3bcca18892e6ed15b5f7a52ad9d",
"sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5", "sha256:aa6b9c843ad645ebb12616de848cc4e25a40f633ccc293c3c9fe34107c02c2ea",
"sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e", "sha256:bae1aa56ee00746798beafe486daa7cfb586cd395c6ce822ba3068e48d761bc0",
"sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6", "sha256:bae96e26510e4825d5910a196bf6b5a11a18b87d9278db6d08413be8ea799469",
"sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca", "sha256:bd78df3b594013b227bf31d0301566dc50ba6f40df38a70ded731d5a8f2cb071",
"sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31", "sha256:c2711197154f46d06f73542c539a0ff5411f1951fab391e0a4ac8359badef719",
"sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1", "sha256:d998c20e3deed234fca993fd6c8314cb7cbfda05fd170f1bd75bb5d7421c3c5a",
"sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2", "sha256:df4f840d77d9e37136f8e6b432fecc9d6b8730f18f896e90628712c793466ce6",
"sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085", "sha256:f5653c2581acb038319e6705d4e3593677676df14b112f13e0b5b44b6a18df1a",
"sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801", "sha256:f7c7aa485a2e2250d455148470ffd0195eecc3d845122635202d7467d6f7b4cf",
"sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4", "sha256:f9e2c66a6493147de835f207f198540a56b26745ce4f272fbc7c2f2cfebeb729"
"sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184",
"sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917",
"sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f",
"sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb"
], ],
"version": "==1.11.5" "version": "==1.12.1"
}, },
"chardet": { "chardet": {
"hashes": [ "hashes": [
@ -250,7 +246,7 @@
"har2tree": { "har2tree": {
"editable": true, "editable": true,
"git": "https://github.com/viper-framework/har2tree.git", "git": "https://github.com/viper-framework/har2tree.git",
"ref": "98305ff26ddfb7ebc7f5fe28239351db00900175" "ref": "a90418ace183e5ebf8891bebe0a7efc7a8d910f7"
}, },
"hyperlink": { "hyperlink": {
"hashes": [ "hashes": [
@ -293,34 +289,34 @@
}, },
"lxml": { "lxml": {
"hashes": [ "hashes": [
"sha256:0dd6589fa75d369ba06d2b5f38dae107f76ea127f212f6a7bee134f6df2d1d21", "sha256:0537eee4902e8bf4f41bfee8133f7edf96533dd175930a12086d6a40d62376b2",
"sha256:1afbac344aa68c29e81ab56c1a9411c3663157b5aee5065b7fa030b398d4f7e0", "sha256:0562ec748abd230ab87d73384e08fa784f9b9cee89e28696087d2d22c052cc27",
"sha256:1baad9d073692421ad5dbbd81430aba6c7f5fdc347f03537ae046ddf2c9b2297", "sha256:09e91831e749fbf0f24608694e4573be0ef51430229450c39c83176cc2e2d353",
"sha256:1d8736421a2358becd3edf20260e41a06a0bf08a560480d3a5734a6bcbacf591", "sha256:1ae4c0722fc70c0d4fba43ae33c2885f705e96dce1db41f75ae14a2d2749b428",
"sha256:1e1d9bddc5afaddf0de76246d3f2152f961697ad7439c559f179002682c45801", "sha256:1c630c083d782cbaf1f7f37f6cac87bda9cff643cf2803a5f180f30d97955cef",
"sha256:1f179dc8b2643715f020f4d119d5529b02cd794c1c8f305868b73b8674d2a03f", "sha256:2fe74e3836bd8c0fa7467ffae05545233c7f37de1eb765cacfda15ad20c6574a",
"sha256:241fb7bdf97cb1df1edfa8f0bcdfd80525d4023dac4523a241907c8b2f44e541", "sha256:37af783c2667ead34a811037bda56a0b142ac8438f7ed29ae93f82ddb812fbd6",
"sha256:2f9765ee5acd3dbdcdc0d0c79309e01f7c16bc8d39b49250bf88de7b46daaf58", "sha256:3f2d9eafbb0b24a33f56acd16f39fc935756524dcb3172892721c54713964c70",
"sha256:312e1e1b1c3ce0c67e0b8105317323e12807955e8186872affb667dbd67971f6", "sha256:47d8365a8ef14097aa4c65730689be51851b4ade677285a3b2daa03b37893e26",
"sha256:3273db1a8055ca70257fd3691c6d2c216544e1a70b673543e15cc077d8e9c730", "sha256:510e904079bc56ea784677348e151e1156040dbfb736f1d8ea4b9e6d0ab2d9f4",
"sha256:34dfaa8c02891f9a246b17a732ca3e99c5e42802416628e740a5d1cb2f50ff49", "sha256:58d0851da422bba31c7f652a7e9335313cf94a641aa6d73b8f3c67602f75b593",
"sha256:3aa3f5288af349a0f3a96448ebf2e57e17332d99f4f30b02093b7948bd9f94cc", "sha256:7940d5c2185ffb989203dacbb28e6ae88b4f1bb25d04e17f94b0edd82232bcbd",
"sha256:51102e160b9d83c1cc435162d90b8e3c8c93b28d18d87b60c56522d332d26879", "sha256:7cf39bb3a905579836f7a8f3a45320d9eb22f16ab0c1e112efb940ced4d057a5",
"sha256:56115fc2e2a4140e8994eb9585119a1ae9223b506826089a3ba753a62bd194a6", "sha256:9563a23c1456c0ab550c087833bc13fcc61013a66c6420921d5b70550ea312bf",
"sha256:69d83de14dbe8fe51dccfd36f88bf0b40f5debeac763edf9f8325180190eba6e", "sha256:95b392952935947e0786a90b75cc33388549dcb19af716b525dae65b186138fc",
"sha256:99fdce94aeaa3ccbdfcb1e23b34273605c5853aa92ec23d84c84765178662c6c", "sha256:983129f3fd3cef5c3cf067adcca56e30a169656c00fcc6c648629dbb850b27fa",
"sha256:a7c0cd5b8a20f3093ee4a67374ccb3b8a126743b15a4d759e2a1bf098faac2b2", "sha256:a0b75b1f1854771844c647c464533def3e0a899dd094a85d1d4ed72ecaaee93d",
"sha256:abe12886554634ed95416a46701a917784cb2b4c77bfacac6916681d49bbf83d", "sha256:b5db89cc0ef624f3a81214b7961a99f443b8c91e88188376b6b322fd10d5b118",
"sha256:b4f67b5183bd5f9bafaeb76ad119e977ba570d2b0e61202f534ac9b5c33b4485", "sha256:c0a7751ba1a4bfbe7831920d98cee3ce748007eab8dfda74593d44079568219a",
"sha256:bdd7c1658475cc1b867b36d5c4ed4bc316be8d3368abe03d348ba906a1f83b0e", "sha256:c0c5a7d4aafcc30c9b6d8613a362567e32e5f5b708dc41bc3a81dac56f8af8bb",
"sha256:c6f24149a19f611a415a51b9bc5f17b6c2f698e0d6b41ffb3fa9f24d35d05d73", "sha256:d4d63d85eacc6cb37b459b16061e1f100d154bee89dc8d8f9a6128a5a538e92e",
"sha256:d1e111b3ab98613115a208c1017f266478b0ab224a67bc8eac670fa0bad7d488", "sha256:da5e7e941d6e71c9c9a717c93725cda0708c2474f532e3680ac5e39ec57d224d",
"sha256:d6520aa965773bbab6cb7a791d5895b00d02cf9adc93ac2bf4edb9ac1a6addc5", "sha256:dccad2b3c583f036f43f80ac99ee212c2fa9a45151358d55f13004d095e683b2",
"sha256:dd185cde2ccad7b649593b0cda72021bc8a91667417001dbaf24cd746ecb7c11", "sha256:df46307d39f2aeaafa1d25309b8a8d11738b73e9861f72d4d0a092528f498baa",
"sha256:de2e5b0828a9d285f909b5d2e9d43f1cf6cf21fe65bc7660bdaa1780c7b58298", "sha256:e70b5e1cb48828ddd2818f99b1662cb9226dc6f57d07fc75485405c77da17436",
"sha256:f726444b8e909c4f41b4fde416e1071cf28fa84634bfb4befdf400933b6463af" "sha256:ea825562b8cd057cbc9810d496b8b5dec37a1e2fc7b27bc7c1e72ce94462a09a"
], ],
"version": "==4.3.0" "version": "==4.3.1"
}, },
"markupsafe": { "markupsafe": {
"hashes": [ "hashes": [
@ -428,11 +424,11 @@
}, },
"redis": { "redis": {
"hashes": [ "hashes": [
"sha256:74c892041cba46078ae1ef845241548baa3bd3634f9a6f0f952f006eb1619c71", "sha256:724932360d48e5407e8f82e405ab3650a36ed02c7e460d1e6fddf0f038422b54",
"sha256:7ba8612bbfd966dea8c62322543fed0095da2834dbd5a7c124afbc617a156aa7" "sha256:9b19425a38fd074eb5795ff2b0d9a55b46a44f91f5347995f27e3ad257a7d775"
], ],
"index": "pypi", "index": "pypi",
"version": "==3.1.0" "version": "==3.2.0"
}, },
"requests": { "requests": {
"hashes": [ "hashes": [
@ -477,10 +473,10 @@
}, },
"soupsieve": { "soupsieve": {
"hashes": [ "hashes": [
"sha256:466910df7561796a60748826781ebe9a888f7a1668a636ae86783f44d10aae73", "sha256:afa56bf14907bb09403e5d15fbed6275caa4174d36b975226e3b67a3bb6e2c4b",
"sha256:87db12ae79194f0ff9808d2b1641c4f031ae39ffa3cab6b907ea7c1e5e5ed445" "sha256:eaed742b48b1f3e2d45ba6f79401b2ed5dc33b2123dfe216adb90d4bfa0ade26"
], ],
"version": "==1.7.3" "version": "==1.8"
}, },
"twisted": { "twisted": {
"hashes": [ "hashes": [

6
lookyloo/exceptions.py
View File

@ -3,8 +3,12 @@
class LookylooException(Exception): class LookylooException(Exception):
pass pass
class MissingEnv(LookylooException): class MissingEnv(LookylooException):
pass pass
class NoValidHarFile(LookylooException):
pass

1
lookyloo/helpers.py
View File

@ -43,7 +43,6 @@ def get_socket_path(name: str) -> str:
def check_running(name: str) -> bool: def check_running(name: str) -> bool:
socket_path = get_socket_path(name) socket_path = get_socket_path(name)
print(socket_path)
try: try:
r = Redis(unix_socket_path=socket_path) r = Redis(unix_socket_path=socket_path)
if r.ping(): if r.ping():

20
lookyloo/lookyloo.py
View File

@ -4,7 +4,7 @@
import json import json
from scrapysplashwrapper import crawl from scrapysplashwrapper import crawl
from har2tree import CrawledTree from har2tree import CrawledTree, Har2TreeError
import pickle import pickle
from datetime import datetime from datetime import datetime
@ -21,6 +21,7 @@ from pysanejs import SaneJS
from pathlib import Path from pathlib import Path
from .helpers import get_homedir, get_socket_path from .helpers import get_homedir, get_socket_path
from .exceptions import NoValidHarFile
from redis import Redis from redis import Redis
import logging import logging
@ -109,13 +110,16 @@ class Lookyloo():
def load_tree(self, report_dir: Path): def load_tree(self, report_dir: Path):
har_files = sorted(report_dir.glob('*.har')) har_files = sorted(report_dir.glob('*.har'))
ct = CrawledTree(har_files) try:
ct.find_parents() ct = CrawledTree(har_files)
ct.join_trees() ct.find_parents()
temp = tempfile.NamedTemporaryFile(prefix='lookyloo', delete=False) ct.join_trees()
pickle.dump(ct, temp) temp = tempfile.NamedTemporaryFile(prefix='lookyloo', delete=False)
temp.close() pickle.dump(ct, temp)
return temp.name, ct.to_json(), ct.start_time.isoformat(), ct.user_agent, ct.root_url temp.close()
return temp.name, ct.to_json(), ct.start_time.isoformat(), ct.user_agent, ct.root_url
except Har2TreeError as e:
raise NoValidHarFile(e.message)
def cleanup_old_tmpfiles(self): def cleanup_old_tmpfiles(self):
for tmpfile in pathlib.Path(tempfile.gettempdir()).glob('lookyloo*'): for tmpfile in pathlib.Path(tempfile.gettempdir()).glob('lookyloo*'):

10
website/web/__init__.py
View File

@ -12,6 +12,7 @@ from flask_bootstrap import Bootstrap
from lookyloo.helpers import get_homedir from lookyloo.helpers import get_homedir
from lookyloo.lookyloo import Lookyloo from lookyloo.lookyloo import Lookyloo
from lookyloo.exceptions import NoValidHarFile
app = Flask(__name__) app = Flask(__name__)
@ -127,9 +128,12 @@ def tree(tree_uuid):
if not report_dir: if not report_dir:
return redirect(url_for('index')) return redirect(url_for('index'))
tree_json, start_time, user_agent, root_url = load_tree(report_dir) try:
return render_template('tree.html', tree_json=tree_json, start_time=start_time, tree_json, start_time, user_agent, root_url = load_tree(report_dir)
user_agent=user_agent, root_url=root_url, tree_uuid=tree_uuid) return render_template('tree.html', tree_json=tree_json, start_time=start_time,
user_agent=user_agent, root_url=root_url, tree_uuid=tree_uuid)
except NoValidHarFile as e:
return render_template('error.html', error_message=e.message)
@app.route('/', methods=['GET']) @app.route('/', methods=['GET'])

9
website/web/templates/error.html Normal file
View File

@ -0,0 +1,9 @@
{% extends "main.html" %}
{% block title %}Error{% endblock %}
{% block content %}
<div class="container">
<h1>Something went wrong</h1>
<b>{{ error_message }}</b>
</div>
{% endblock %}