new: Allow admin to rebuild cache

pull/79/head
Raphaël Vinot 2020-04-01 17:44:06 +02:00
parent c960a94702
commit 1c2cdad38b
5 changed files with 71 additions and 2 deletions

View File

@ -2,8 +2,10 @@
"loglevel": "INFO",
"only_global_lookups": true,
"splash_url": "http://127.0.0.1:8050",
"cache_clean_user": {},
"_notes": {
"only_global_lookups": "Set it to True if your instance is publicly available so users aren't able to scan your internal network",
"loglevel": "Can be one of the value listed here: https://docs.python.org/3/library/logging.html#levels"
"loglevel": "Can be one of the value listed here: https://docs.python.org/3/library/logging.html#levels",
"cache_clean_user": "Format: {username: password}"
}
}

View File

@ -70,6 +70,19 @@ class Lookyloo():
else:
self.use_sane_js = True
def rebuild_cache(self):
self.redis.flushdb()
self._init_existing_dumps()
def remove_pickle(self, capture_dir: Path):
if (capture_dir / 'tree.pickle').exists():
(capture_dir / 'tree.pickle').unlink()
def rebuild_all(self):
for capture_dir in self.capture_dirs:
self.remove_pickle(capture_dir)
self.rebuild_cache()
def get_config(self, entry: str) -> Any:
"""Get an entry from the generic config file. Automatic fallback to the sample file"""
if 'generic' in self.configs:

20
poetry.lock generated
View File

@ -269,6 +269,17 @@ dev = ["pytest", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinxco
docs = ["sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"]
dotenv = ["python-dotenv"]
[[package]]
category = "main"
description = "Basic and Digest HTTP authentication for Flask routes"
name = "flask-httpauth"
optional = false
python-versions = "*"
version = "3.3.0"
[package.dependencies]
Flask = "*"
[[package]]
category = "main"
description = "WSGI HTTP Server for UNIX"
@ -305,6 +316,7 @@ six = "^1.14.0"
reference = "9ec997b0f1c09d3034fd861fd17d972da479f505"
type = "git"
url = "https://github.com/viper-framework/har2tree.git"
[[package]]
category = "main"
description = "A featureful, immutable, and correct URL for Python."
@ -744,6 +756,7 @@ requests = "^2.22.0"
reference = "3ea143f44d37ab701c70ffb38408528ddb4e2b6e"
type = "git"
url = "https://github.com/CIRCL/PySaneJS.git"
[[package]]
category = "main"
description = "pytest: simple powerful testing with Python"
@ -884,6 +897,7 @@ scrapy-splash = "^0.7.2"
reference = "300ee49cb21784514dd2a35b374ce06c4b7f04a6"
type = "git"
url = "https://github.com/viper-framework/ScrapySplashWrapper.git"
[[package]]
category = "main"
description = "Service identity verification for pyOpenSSL & cryptography."
@ -1082,7 +1096,7 @@ test = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
[metadata]
content-hash = "4294a7aa675b6d425096a74ce78486ed9f3c4a50e2549837dfc86c389290fa68"
content-hash = "23715c6f42d23f07b8be767480974259f8ac2e99a209d04cb076b5d2b6d0e6b4"
python-versions = "^3.6"
[metadata.files]
@ -1265,6 +1279,10 @@ flask = [
{file = "Flask-1.1.1-py2.py3-none-any.whl", hash = "sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6"},
{file = "Flask-1.1.1.tar.gz", hash = "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52"},
]
flask-httpauth = [
{file = "Flask-HTTPAuth-3.3.0.tar.gz", hash = "sha256:6ef8b761332e780f9ff74d5f9056c2616f52babc1998b01d9f361a1e439e61b9"},
{file = "Flask_HTTPAuth-3.3.0-py2.py3-none-any.whl", hash = "sha256:0149953720489407e51ec24bc2f86273597b7973d71cd51f9443bd0e2a89bd72"},
]
gunicorn = [
{file = "gunicorn-20.0.4-py2.py3-none-any.whl", hash = "sha256:cd4a810dd51bf497552cf3f863b575dabd73d6ad6a91075b65936b151cbf4f9c"},
{file = "gunicorn-20.0.4.tar.gz", hash = "sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626"},

View File

@ -44,6 +44,7 @@ bootstrap-flask = "^1.2.0"
cloudscraper = "^1.2.20"
defang = "^0.5.3"
vt-py = "^0.5.2"
Flask-HTTPAuth = "^3.3.0"
[tool.poetry.dev-dependencies]
mypy = "^0.761"

View File

@ -10,6 +10,7 @@ from pathlib import Path
from flask import Flask, render_template, request, session, send_file, redirect, url_for, Response, flash
from flask_bootstrap import Bootstrap # type: ignore
from flask_httpauth import HTTPDigestAuth # type: ignore
from lookyloo.helpers import get_homedir, update_user_agents, get_user_agents
from lookyloo.lookyloo import Lookyloo
@ -32,9 +33,43 @@ Bootstrap(app)
app.config['BOOTSTRAP_SERVE_LOCAL'] = True
app.config['SESSION_COOKIE_NAME'] = 'lookyloo'
app.debug = False
auth = HTTPDigestAuth()
lookyloo: Lookyloo = Lookyloo()
user = lookyloo.get_config('cache_clean_user')
@auth.get_password
def get_pw(username):
if username in user:
return user.get(username)
return None
@app.route('/rebuild_all')
@auth.login_required
def rebuild_all():
lookyloo.rebuild_all()
return redirect(url_for('index'))
@app.route('/rebuild_cache')
@auth.login_required
def rebuild_cache():
lookyloo.rebuild_cache()
return redirect(url_for('index'))
@app.route('/tree/<tree_uuid>/rebuild')
@auth.login_required
def rebuild_tree(tree_uuid):
capture_dir = lookyloo.lookup_capture_dir(tree_uuid)
if capture_dir:
lookyloo.remove_pickle(capture_dir)
return redirect(url_for('tree', tree_uuid=tree_uuid))
return redirect(url_for('index'))
# keep
def load_tree(capture_dir: Path) -> Tuple[dict, str, str, str, dict]: