mirror of https://github.com/CIRCL/lookyloo
new: Use flask-restx for the API
Raphaël Vinot
parent
88eb85df46
commit
7bf0b78754
|
|
@ -17,6 +17,17 @@ yarl = ">=1.0,<2.0"
|
|||
[package.extras]
|
||||
speedups = ["aiodns", "brotlipy", "cchardet"]
|
||||
|
||||
[[package]]
|
||||
name = "aniso8601"
|
||||
version = "9.0.1"
|
||||
description = "A library for parsing ISO 8601 strings."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
|
||||
[package.extras]
|
||||
dev = ["black", "coverage", "isort", "pre-commit", "pyenchant", "pylint"]
|
||||
|
||||
[[package]]
|
||||
name = "appnope"
|
||||
version = "0.1.2"
|
||||
|
|
@ -137,20 +148,17 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
|
|||
|
||||
[[package]]
|
||||
name = "click"
|
||||
version = "8.0.1"
|
||||
version = "7.1.2"
|
||||
description = "Composable command line interface toolkit"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
|
||||
[package.dependencies]
|
||||
colorama = {version = "*", markers = "platform_system == \"Windows\""}
|
||||
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
|
||||
|
||||
[[package]]
|
||||
name = "colorama"
|
||||
version = "0.4.4"
|
||||
description = "Cross-platform colored terminal text."
|
||||
category = "main"
|
||||
category = "dev"
|
||||
optional = false
|
||||
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
|
||||
|
||||
|
|
@ -252,20 +260,21 @@ python-versions = "*"
|
|||
|
||||
[[package]]
|
||||
name = "flask"
|
||||
version = "2.0.1"
|
||||
version = "1.1.4"
|
||||
description = "A simple framework for building complex web applications."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
|
||||
|
||||
[package.dependencies]
|
||||
click = ">=7.1.2"
|
||||
itsdangerous = ">=2.0"
|
||||
Jinja2 = ">=3.0"
|
||||
Werkzeug = ">=2.0"
|
||||
click = ">=5.1,<8.0"
|
||||
itsdangerous = ">=0.24,<2.0"
|
||||
Jinja2 = ">=2.10.1,<3.0"
|
||||
Werkzeug = ">=0.15,<2.0"
|
||||
|
||||
[package.extras]
|
||||
async = ["asgiref (>=3.2)"]
|
||||
dev = ["pytest", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"]
|
||||
docs = ["sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"]
|
||||
dotenv = ["python-dotenv"]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -279,6 +288,27 @@ python-versions = "*"
|
|||
[package.dependencies]
|
||||
Flask = "*"
|
||||
|
||||
[[package]]
|
||||
name = "flask-restx"
|
||||
version = "0.4.0"
|
||||
description = "Fully featured framework for fast, easy and documented API development with Flask"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
|
||||
[package.dependencies]
|
||||
aniso8601 = {version = ">=0.82", markers = "python_version >= \"3.5\""}
|
||||
Flask = ">=0.8,<2.0.0"
|
||||
jsonschema = "*"
|
||||
pytz = "*"
|
||||
six = ">=1.3.0"
|
||||
werkzeug = "<2.0.0"
|
||||
|
||||
[package.extras]
|
||||
dev = ["blinker", "Faker (==2.0.0)", "mock (==3.0.5)", "pytest-benchmark (==3.2.2)", "pytest-cov (==2.7.1)", "pytest-flask (==0.15.1)", "pytest-mock (==1.10.4)", "pytest-profiling (==1.7.0)", "tzlocal", "invoke (==1.3.0)", "readme-renderer (==24.0)", "twine (==1.15.0)", "tox", "pytest (==4.6.5)", "pytest (==5.4.1)", "ossaudit", "black"]
|
||||
doc = ["alabaster (==0.7.12)", "Sphinx (==2.1.2)", "sphinx-issues (==1.2.0)"]
|
||||
test = ["blinker", "Faker (==2.0.0)", "mock (==3.0.5)", "pytest-benchmark (==3.2.2)", "pytest-cov (==2.7.1)", "pytest-flask (==0.15.1)", "pytest-mock (==1.10.4)", "pytest-profiling (==1.7.0)", "tzlocal", "invoke (==1.3.0)", "readme-renderer (==24.0)", "twine (==1.15.0)", "pytest (==4.6.5)", "pytest (==5.4.1)", "ossaudit"]
|
||||
|
||||
[[package]]
|
||||
name = "gunicorn"
|
||||
version = "20.1.0"
|
||||
|
|
@ -437,11 +467,11 @@ w3lib = ">=1.17.0"
|
|||
|
||||
[[package]]
|
||||
name = "itsdangerous"
|
||||
version = "2.0.1"
|
||||
description = "Safely pass data to untrusted environments and back."
|
||||
version = "1.1.0"
|
||||
description = "Various helpers to pass data to untrusted environments and back."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
|
||||
|
||||
[[package]]
|
||||
name = "jedi"
|
||||
|
|
@ -460,17 +490,17 @@ testing = ["Django (<3.1)", "colorama", "docopt", "pytest (<6.0.0)"]
|
|||
|
||||
[[package]]
|
||||
name = "jinja2"
|
||||
version = "3.0.1"
|
||||
version = "2.11.3"
|
||||
description = "A very fast and expressive template engine."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
|
||||
|
||||
[package.dependencies]
|
||||
MarkupSafe = ">=2.0"
|
||||
MarkupSafe = ">=0.23"
|
||||
|
||||
[package.extras]
|
||||
i18n = ["Babel (>=2.7)"]
|
||||
i18n = ["Babel (>=0.8)"]
|
||||
|
||||
[[package]]
|
||||
name = "jmespath"
|
||||
|
|
@ -860,6 +890,14 @@ category = "main"
|
|||
optional = true
|
||||
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
|
||||
|
||||
[[package]]
|
||||
name = "pytz"
|
||||
version = "2021.1"
|
||||
description = "World timezone definitions, modern and historical"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
|
||||
[[package]]
|
||||
name = "queuelib"
|
||||
version = "1.6.1"
|
||||
|
|
@ -1102,13 +1140,14 @@ python-versions = "*"
|
|||
|
||||
[[package]]
|
||||
name = "werkzeug"
|
||||
version = "2.0.1"
|
||||
version = "1.0.1"
|
||||
description = "The comprehensive WSGI web application library."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
|
||||
|
||||
[package.extras]
|
||||
dev = ["pytest", "pytest-timeout", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinx-issues"]
|
||||
watchdog = ["watchdog"]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1150,7 +1189,7 @@ misp = ["python-magic", "pydeep"]
|
|||
[metadata]
|
||||
lock-version = "1.1"
|
||||
python-versions = "^3.8"
|
||||
content-hash = "e8e3df6d57b274c350ed4df996c608cdf1ecb06ffd3f985c3ed0912a42da310b"
|
||||
content-hash = "27c09f95d480f1517a425ed98847d3096d0ec22b7cccf0727189a31e4206a9ed"
|
||||
|
||||
[metadata.files]
|
||||
aiohttp = [
|
||||
|
|
@ -1192,6 +1231,10 @@ aiohttp = [
|
|||
{file = "aiohttp-3.7.4.post0-cp39-cp39-win_amd64.whl", hash = "sha256:02f46fc0e3c5ac58b80d4d56eb0a7c7d97fcef69ace9326289fb9f1955e65cfe"},
|
||||
{file = "aiohttp-3.7.4.post0.tar.gz", hash = "sha256:493d3299ebe5f5a7c66b9819eacdcfbbaaf1a8e84911ddffcdc48888497afecf"},
|
||||
]
|
||||
aniso8601 = [
|
||||
{file = "aniso8601-9.0.1-py2.py3-none-any.whl", hash = "sha256:1d2b7ef82963909e93c4f24ce48d4de9e66009a21bf1c1e1c85bdd0812fe412f"},
|
||||
{file = "aniso8601-9.0.1.tar.gz", hash = "sha256:72e3117667eedf66951bb2d93f4296a56b94b078a8a95905a052611fb3f1b973"},
|
||||
]
|
||||
appnope = [
|
||||
{file = "appnope-0.1.2-py2.py3-none-any.whl", hash = "sha256:93aa393e9d6c54c5cd570ccadd8edad61ea0c4b9ea7a01409020c9aa019eb442"},
|
||||
{file = "appnope-0.1.2.tar.gz", hash = "sha256:dd83cd4b5b460958838f6eb3000c660b1f9caf2a5b1de4264e941512f603258a"},
|
||||
|
|
@ -1300,8 +1343,8 @@ chardet = [
|
|||
{file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"},
|
||||
]
|
||||
click = [
|
||||
{file = "click-8.0.1-py3-none-any.whl", hash = "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6"},
|
||||
{file = "click-8.0.1.tar.gz", hash = "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a"},
|
||||
{file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"},
|
||||
{file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"},
|
||||
]
|
||||
colorama = [
|
||||
{file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"},
|
||||
|
|
@ -1351,13 +1394,17 @@ filetype = [
|
|||
{file = "filetype-1.0.7.tar.gz", hash = "sha256:da393ece8d98b47edf2dd5a85a2c8733e44b769e32c71af4cd96ed8d38d96aa7"},
|
||||
]
|
||||
flask = [
|
||||
{file = "Flask-2.0.1-py3-none-any.whl", hash = "sha256:a6209ca15eb63fc9385f38e452704113d679511d9574d09b2cf9183ae7d20dc9"},
|
||||
{file = "Flask-2.0.1.tar.gz", hash = "sha256:1c4c257b1892aec1398784c63791cbaa43062f1f7aeb555c4da961b20ee68f55"},
|
||||
{file = "Flask-1.1.4-py2.py3-none-any.whl", hash = "sha256:c34f04500f2cbbea882b1acb02002ad6fe6b7ffa64a6164577995657f50aed22"},
|
||||
{file = "Flask-1.1.4.tar.gz", hash = "sha256:0fbeb6180d383a9186d0d6ed954e0042ad9f18e0e8de088b2b419d526927d196"},
|
||||
]
|
||||
flask-login = [
|
||||
{file = "Flask-Login-0.5.0.tar.gz", hash = "sha256:6d33aef15b5bcead780acc339464aae8a6e28f13c90d8b1cf9de8b549d1c0b4b"},
|
||||
{file = "Flask_Login-0.5.0-py2.py3-none-any.whl", hash = "sha256:7451b5001e17837ba58945aead261ba425fdf7b4f0448777e597ddab39f4fba0"},
|
||||
]
|
||||
flask-restx = [
|
||||
{file = "flask-restx-0.4.0.tar.gz", hash = "sha256:8bdbfd8ad2949383c490b3b180481c0e9f70163230a5b567a56592c82f7c5cb0"},
|
||||
{file = "flask_restx-0.4.0-py2.py3-none-any.whl", hash = "sha256:780ac57ebda25e9f17ffccf3ff1b47a2c4efe5787cebddbbba1a91f53560d79f"},
|
||||
]
|
||||
gunicorn = [
|
||||
{file = "gunicorn-20.1.0.tar.gz", hash = "sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8"},
|
||||
]
|
||||
|
|
@ -1406,16 +1453,16 @@ itemloaders = [
|
|||
{file = "itemloaders-1.0.4.tar.gz", hash = "sha256:1277cd8ca3e4c02dcdfbc1bcae9134ad89acfa6041bd15b4561c6290203a0c96"},
|
||||
]
|
||||
itsdangerous = [
|
||||
{file = "itsdangerous-2.0.1-py3-none-any.whl", hash = "sha256:5174094b9637652bdb841a3029700391451bd092ba3db90600dea710ba28e97c"},
|
||||
{file = "itsdangerous-2.0.1.tar.gz", hash = "sha256:9e724d68fc22902a1435351f84c3fb8623f303fffcc566a4cb952df8c572cff0"},
|
||||
{file = "itsdangerous-1.1.0-py2.py3-none-any.whl", hash = "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749"},
|
||||
{file = "itsdangerous-1.1.0.tar.gz", hash = "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19"},
|
||||
]
|
||||
jedi = [
|
||||
{file = "jedi-0.18.0-py2.py3-none-any.whl", hash = "sha256:18456d83f65f400ab0c2d3319e48520420ef43b23a086fdc05dff34132f0fb93"},
|
||||
{file = "jedi-0.18.0.tar.gz", hash = "sha256:92550a404bad8afed881a137ec9a461fed49eca661414be45059329614ed0707"},
|
||||
]
|
||||
jinja2 = [
|
||||
{file = "Jinja2-3.0.1-py3-none-any.whl", hash = "sha256:1f06f2da51e7b56b8f238affdd6b4e2c61e39598a378cc49345bc1bd42a978a4"},
|
||||
{file = "Jinja2-3.0.1.tar.gz", hash = "sha256:703f484b47a6af502e743c9122595cc812b0271f661722403114f71a79d0f5a4"},
|
||||
{file = "Jinja2-2.11.3-py2.py3-none-any.whl", hash = "sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419"},
|
||||
{file = "Jinja2-2.11.3.tar.gz", hash = "sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6"},
|
||||
]
|
||||
jmespath = [
|
||||
{file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"},
|
||||
|
|
@ -1778,6 +1825,10 @@ python-magic = [
|
|||
{file = "python-magic-0.4.24.tar.gz", hash = "sha256:de800df9fb50f8ec5974761054a708af6e4246b03b4bdaee993f948947b0ebcf"},
|
||||
{file = "python_magic-0.4.24-py2.py3-none-any.whl", hash = "sha256:4fec8ee805fea30c07afccd1592c0f17977089895bdfaae5fec870a84e997626"},
|
||||
]
|
||||
pytz = [
|
||||
{file = "pytz-2021.1-py2.py3-none-any.whl", hash = "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"},
|
||||
{file = "pytz-2021.1.tar.gz", hash = "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da"},
|
||||
]
|
||||
queuelib = [
|
||||
{file = "queuelib-1.6.1-py2.py3-none-any.whl", hash = "sha256:90ee30ebb0b57112606358b63c09a681bbb9a7dd1120af09c836b475504cea85"},
|
||||
{file = "queuelib-1.6.1.tar.gz", hash = "sha256:631d067c9be57e395c382d680d3653ca1452cd29e8da25c5e8d94b5c0c528c31"},
|
||||
|
|
@ -1893,8 +1944,8 @@ wcwidth = [
|
|||
{file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"},
|
||||
]
|
||||
werkzeug = [
|
||||
{file = "Werkzeug-2.0.1-py3-none-any.whl", hash = "sha256:6c1ec500dcdba0baa27600f6a22f6333d8b662d22027ff9f6202e3367413caa8"},
|
||||
{file = "Werkzeug-2.0.1.tar.gz", hash = "sha256:1de1db30d010ff1af14a009224ec49ab2329ad2cde454c8a708130642d579c42"},
|
||||
{file = "Werkzeug-1.0.1-py2.py3-none-any.whl", hash = "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43"},
|
||||
{file = "Werkzeug-1.0.1.tar.gz", hash = "sha256:6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c"},
|
||||
]
|
||||
wrapt = [
|
||||
{file = "wrapt-1.12.1.tar.gz", hash = "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7"},
|
||||
|
|
|
|||
|
|
@ -36,7 +36,6 @@ background_indexer = "bin.background_indexer:main"
|
|||
[tool.poetry.dependencies]
|
||||
python = "^3.8"
|
||||
requests = "^2.25.1"
|
||||
flask = "^2.0.0"
|
||||
gunicorn = "^20.1.0"
|
||||
cchardet = "^2.1.7"
|
||||
redis = "^3.5.3"
|
||||
|
|
@ -58,6 +57,7 @@ pydeep = {version = "^0.4", optional = true}
|
|||
Pillow = "^8.2.0"
|
||||
lief = "^0.11.4"
|
||||
Flask-Login = "^0.5.0"
|
||||
flask-restx = "^0.4.0"
|
||||
|
||||
[tool.poetry.extras]
|
||||
misp = ['python-magic', 'pydeep']
|
||||
|
|
|
|||
|
|
@ -14,10 +14,12 @@ import logging
|
|||
import hashlib
|
||||
from urllib.parse import quote_plus, unquote_plus
|
||||
import time
|
||||
import pkg_resources
|
||||
|
||||
from flask import Flask, render_template, request, send_file, redirect, url_for, Response, flash, jsonify
|
||||
from flask_bootstrap import Bootstrap # type: ignore
|
||||
import flask_login # type: ignore
|
||||
from flask_restx import Resource, Api, fields, abort # type: ignore
|
||||
from werkzeug.security import generate_password_hash, check_password_hash
|
||||
|
||||
from pymisp import MISPEvent, MISPServerError
|
||||
|
|
@ -577,7 +579,7 @@ def tree(tree_uuid: str, node_uuid: Optional[str]=None):
|
|||
@flask_login.login_required
|
||||
def mark_as_legitimate(tree_uuid: str):
|
||||
if request.data:
|
||||
legitimate_entries: Dict = request.get_json(force=True) # type: ignore
|
||||
legitimate_entries: Dict = request.get_json(force=True)
|
||||
lookyloo.add_to_legitimate(tree_uuid, **legitimate_entries)
|
||||
else:
|
||||
lookyloo.add_to_legitimate(tree_uuid)
|
||||
|
|
@ -696,17 +698,6 @@ def rebuild_cache():
|
|||
return redirect(url_for('index'))
|
||||
|
||||
|
||||
@app.route('/submit', methods=['POST'])
|
||||
def submit():
|
||||
if flask_login.current_user.is_authenticated:
|
||||
user = flask_login.current_user.get_id()
|
||||
else:
|
||||
user = src_request_ip(request)
|
||||
to_query: Dict = request.get_json(force=True) # type: ignore
|
||||
perma_uuid = lookyloo.enqueue_capture(to_query, source='api', user=user, authenticated=flask_login.current_user.is_authenticated)
|
||||
return Response(perma_uuid, mimetype='text/text')
|
||||
|
||||
|
||||
@app.route('/search', methods=['GET', 'POST'])
|
||||
def search():
|
||||
if request.form.get('url'):
|
||||
|
|
@ -1021,122 +1012,255 @@ def web_misp_push_view(tree_uuid: str):
|
|||
default_tags=lookyloo.misp.default_tags)
|
||||
|
||||
|
||||
# Query API
|
||||
|
||||
@app.route('/json/get_token', methods=['POST'])
|
||||
def json_get_token():
|
||||
auth: Dict = request.get_json(force=True) # type: ignore
|
||||
if 'username' in auth and 'password' in auth: # Expected keys in json
|
||||
if (auth['username'] in users_table
|
||||
and check_password_hash(users_table[auth['username']]['password'], auth['password'])):
|
||||
return jsonify({'authkey': users_table[auth['username']]['authkey']})
|
||||
return jsonify({'error': 'User/Password invalid.'})
|
||||
|
||||
|
||||
@app.route('/json/<string:tree_uuid>/status', methods=['GET'])
|
||||
def get_capture_status(tree_uuid: str):
|
||||
return jsonify({'status_code': lookyloo.get_capture_status(tree_uuid)})
|
||||
|
||||
|
||||
@app.route('/json/<string:tree_uuid>/redirects', methods=['GET'])
|
||||
def json_redirects(tree_uuid: str):
|
||||
cache = lookyloo.capture_cache(tree_uuid)
|
||||
if not cache:
|
||||
return {'error': 'UUID missing in cache, try again later.'}
|
||||
|
||||
to_return: Dict[str, Any] = {'response': {'url': cache.url, 'redirects': []}}
|
||||
if not cache.redirects:
|
||||
to_return['response']['info'] = 'No redirects'
|
||||
return to_return
|
||||
if cache.incomplete_redirects:
|
||||
# Trigger tree build, get all redirects
|
||||
lookyloo.get_crawled_tree(tree_uuid)
|
||||
cache = lookyloo.capture_cache(tree_uuid)
|
||||
if cache:
|
||||
to_return['response']['redirects'] = cache.redirects
|
||||
else:
|
||||
to_return['response']['redirects'] = cache.redirects
|
||||
|
||||
return jsonify(to_return)
|
||||
|
||||
|
||||
@app.route('/json/<string:tree_uuid>/misp_export', methods=['GET'])
|
||||
def misp_export(tree_uuid: str):
|
||||
with_parents = request.args.get('with_parents')
|
||||
event = lookyloo.misp_export(tree_uuid, True if with_parents else False)
|
||||
if isinstance(event, dict):
|
||||
return jsonify(event)
|
||||
|
||||
to_return = []
|
||||
for e in event:
|
||||
to_return.append(e.to_json(indent=2))
|
||||
return jsonify(to_return)
|
||||
|
||||
|
||||
@app.route('/json/<string:tree_uuid>/misp_push', methods=['GET', 'POST'])
|
||||
@flask_login.login_required
|
||||
def misp_push(tree_uuid: str):
|
||||
if request.method == 'POST':
|
||||
parameters: Dict = request.get_json(force=True) # type: ignore
|
||||
with_parents = True if 'with_parents' in parameters else False
|
||||
allow_duplicates = True if 'allow_duplicates' in parameters else False
|
||||
else:
|
||||
with_parents = False
|
||||
allow_duplicates = False
|
||||
to_return: Dict = {}
|
||||
if not lookyloo.misp.available:
|
||||
to_return['error'] = 'MISP module not available.'
|
||||
elif not lookyloo.misp.enable_push:
|
||||
to_return['error'] = 'Push not enabled in MISP module.'
|
||||
else:
|
||||
event = lookyloo.misp_export(tree_uuid, with_parents)
|
||||
if isinstance(event, dict):
|
||||
to_return['error'] = event
|
||||
else:
|
||||
new_events = lookyloo.misp.push(event, allow_duplicates)
|
||||
if isinstance(new_events, dict):
|
||||
to_return['error'] = new_events
|
||||
else:
|
||||
events_to_return = []
|
||||
for e in new_events:
|
||||
events_to_return.append(e.to_json(indent=2))
|
||||
jsonify(events_to_return)
|
||||
|
||||
return jsonify(to_return)
|
||||
|
||||
|
||||
@app.route('/json/hash_info/<h>', methods=['GET'])
|
||||
def json_hash_info(h: str):
|
||||
details, body = lookyloo.get_body_hash_full(h)
|
||||
if not details:
|
||||
return {'error': 'Unknown Hash.'}
|
||||
to_return: Dict[str, Any] = {'response': {'hash': h, 'details': details,
|
||||
'body': base64.b64encode(body.getvalue()).decode()}}
|
||||
return jsonify(to_return)
|
||||
|
||||
|
||||
@app.route('/json/url_info', methods=['POST'])
|
||||
def json_url_info():
|
||||
to_query: Dict = request.get_json(force=True) # type: ignore
|
||||
occurrences = lookyloo.get_url_occurrences(to_query.pop('url'), **to_query)
|
||||
return jsonify(occurrences)
|
||||
|
||||
|
||||
@app.route('/json/hostname_info', methods=['POST'])
|
||||
def json_hostname_info():
|
||||
to_query: Dict = request.get_json(force=True) # type: ignore
|
||||
occurrences = lookyloo.get_hostname_occurrences(to_query.pop('hostname'), **to_query)
|
||||
return jsonify(occurrences)
|
||||
|
||||
|
||||
@app.route('/json/stats', methods=['GET'])
|
||||
def json_stats():
|
||||
to_return = lookyloo.get_stats()
|
||||
return Response(json.dumps(to_return), mimetype='application/json')
|
||||
|
||||
|
||||
@app.route('/whois/<string:query>', methods=['GET'])
|
||||
def whois(query: str):
|
||||
to_return = lookyloo.uwhois.whois(query)
|
||||
return send_file(BytesIO(to_return.encode()),
|
||||
mimetype='test/plain', as_attachment=True, attachment_filename=f'whois.{query}.txt')
|
||||
|
||||
|
||||
# Query API
|
||||
|
||||
authorizations = {
|
||||
'apikey': {
|
||||
'type': 'apiKey',
|
||||
'in': 'header',
|
||||
'name': 'Authorization'
|
||||
}
|
||||
}
|
||||
|
||||
api = Api(app, title='Lookyloo API',
|
||||
description='API to submit captures and query a lookyloo instance.',
|
||||
doc='/doc/',
|
||||
authorizations=authorizations,
|
||||
version=pkg_resources.get_distribution('lookyloo').version)
|
||||
|
||||
|
||||
def api_auth_check(method):
|
||||
if flask_login.current_user.is_authenticated or load_user_from_request(request):
|
||||
return method
|
||||
abort(403, 'Authentication required.')
|
||||
|
||||
|
||||
token_request_fields = api.model('AuthTokenFields', {
|
||||
'username': fields.String(description="Your username", required=True),
|
||||
'password': fields.String(description="Your password", required=True),
|
||||
})
|
||||
|
||||
|
||||
@api.route('/json/get_token')
|
||||
@api.doc(description='Get the API token required for authenticated calls')
|
||||
class AuthToken(Resource):
|
||||
|
||||
@api.param('username', 'Your username')
|
||||
@api.param('password', 'Your password')
|
||||
def get(self):
|
||||
username = request.args['username'] if request.args.get('username') else False
|
||||
password = request.args['password'] if request.args.get('password') else False
|
||||
if username in users_table and check_password_hash(users_table[username]['password'], password):
|
||||
return {'authkey': users_table[username]['authkey']}
|
||||
return {'error': 'User/Password invalid.'}
|
||||
|
||||
@api.doc(body=token_request_fields)
|
||||
def post(self):
|
||||
auth: Dict = request.get_json(force=True)
|
||||
if 'username' in auth and 'password' in auth: # Expected keys in json
|
||||
if (auth['username'] in users_table
|
||||
and check_password_hash(users_table[auth['username']]['password'], auth['password'])):
|
||||
return {'authkey': users_table[auth['username']]['authkey']}
|
||||
return {'error': 'User/Password invalid.'}
|
||||
|
||||
|
||||
@api.route('/json/<string:tree_uuid>/status')
|
||||
@api.doc(description='Get the status of a capture',
|
||||
params={'tree_uuid': 'The UUID of the capture'})
|
||||
class CaptureStatusQuery(Resource):
|
||||
def get(self, tree_uuid: str):
|
||||
return {'status_code': lookyloo.get_capture_status(tree_uuid)}
|
||||
|
||||
|
||||
@api.route('/json/<string:tree_uuid>/redirects')
|
||||
@api.doc(description='Get all the redirects of a capture',
|
||||
params={'tree_uuid': 'The UUID of the capture'})
|
||||
class CaptureRedirects(Resource):
|
||||
def get(self, tree_uuid: str):
|
||||
cache = lookyloo.capture_cache(tree_uuid)
|
||||
if not cache:
|
||||
return {'error': 'UUID missing in cache, try again later.'}
|
||||
|
||||
to_return: Dict[str, Any] = {'response': {'url': cache.url, 'redirects': []}}
|
||||
if not cache.redirects:
|
||||
to_return['response']['info'] = 'No redirects'
|
||||
return to_return
|
||||
if cache.incomplete_redirects:
|
||||
# Trigger tree build, get all redirects
|
||||
lookyloo.get_crawled_tree(tree_uuid)
|
||||
cache = lookyloo.capture_cache(tree_uuid)
|
||||
if cache:
|
||||
to_return['response']['redirects'] = cache.redirects
|
||||
else:
|
||||
to_return['response']['redirects'] = cache.redirects
|
||||
|
||||
return to_return
|
||||
|
||||
|
||||
@api.route('/json/<string:tree_uuid>/misp_export')
|
||||
@api.doc(description='Get an export of the capture in MISP format',
|
||||
params={'tree_uuid': 'The UUID of the capture'})
|
||||
class MISPExport(Resource):
|
||||
def get(self, tree_uuid: str):
|
||||
with_parents = request.args.get('with_parents')
|
||||
event = lookyloo.misp_export(tree_uuid, True if with_parents else False)
|
||||
if isinstance(event, dict):
|
||||
return event
|
||||
|
||||
to_return = []
|
||||
for e in event:
|
||||
to_return.append(e.to_json(indent=2))
|
||||
return to_return
|
||||
|
||||
|
||||
misp_push_fields = api.model('MISPPushFields', {
|
||||
'allow_duplicates': fields.Integer(description="Push the event even if it is already present on the MISP instance",
|
||||
example=0, min=0, max=1),
|
||||
'with_parents': fields.Integer(description="Also push the parents of the capture (if any)",
|
||||
example=0, min=0, max=1),
|
||||
})
|
||||
|
||||
|
||||
@api.route('/json/<string:tree_uuid>/misp_push')
|
||||
@api.doc(description='Push an event to a pre-configured MISP instance',
|
||||
params={'tree_uuid': 'The UUID of the capture'},
|
||||
security='apikey')
|
||||
class MISPPush(Resource):
|
||||
method_decorators = [api_auth_check]
|
||||
|
||||
@api.param('with_parents', 'Also push the parents of the capture (if any)')
|
||||
@api.param('allow_duplicates', 'Push the event even if it is already present on the MISP instance')
|
||||
def get(self, tree_uuid: str):
|
||||
with_parents = True if request.args.get('with_parents') else False
|
||||
allow_duplicates = True if request.args.get('allow_duplicates') else False
|
||||
to_return: Dict = {}
|
||||
if not lookyloo.misp.available:
|
||||
to_return['error'] = 'MISP module not available.'
|
||||
elif not lookyloo.misp.enable_push:
|
||||
to_return['error'] = 'Push not enabled in MISP module.'
|
||||
else:
|
||||
event = lookyloo.misp_export(tree_uuid, with_parents)
|
||||
if isinstance(event, dict):
|
||||
to_return['error'] = event
|
||||
else:
|
||||
new_events = lookyloo.misp.push(event, allow_duplicates)
|
||||
if isinstance(new_events, dict):
|
||||
to_return['error'] = new_events
|
||||
else:
|
||||
events_to_return = []
|
||||
for e in new_events:
|
||||
events_to_return.append(e.to_json(indent=2))
|
||||
return events_to_return
|
||||
|
||||
return to_return
|
||||
|
||||
@api.doc(body=misp_push_fields)
|
||||
def post(self, tree_uuid: str):
|
||||
parameters: Dict = request.get_json(force=True)
|
||||
with_parents = True if parameters.get('with_parents') else False
|
||||
allow_duplicates = True if parameters.get('allow_duplicates') else False
|
||||
|
||||
to_return: Dict = {}
|
||||
if not lookyloo.misp.available:
|
||||
to_return['error'] = 'MISP module not available.'
|
||||
elif not lookyloo.misp.enable_push:
|
||||
to_return['error'] = 'Push not enabled in MISP module.'
|
||||
else:
|
||||
event = lookyloo.misp_export(tree_uuid, with_parents)
|
||||
if isinstance(event, dict):
|
||||
to_return['error'] = event
|
||||
else:
|
||||
new_events = lookyloo.misp.push(event, allow_duplicates)
|
||||
if isinstance(new_events, dict):
|
||||
to_return['error'] = new_events
|
||||
else:
|
||||
events_to_return = []
|
||||
for e in new_events:
|
||||
events_to_return.append(e.to_json(indent=2))
|
||||
return events_to_return
|
||||
|
||||
return to_return
|
||||
|
||||
|
||||
@api.route('/json/hash_info/<h>')
|
||||
@api.doc(description='Search for a ressource with a specific hash (sha512)',
|
||||
params={'h': 'The hash (sha512)'})
|
||||
class HashInfo(Resource):
|
||||
def get(self, h: str):
|
||||
details, body = lookyloo.get_body_hash_full(h)
|
||||
if not details:
|
||||
return {'error': 'Unknown Hash.'}
|
||||
to_return: Dict[str, Any] = {'response': {'hash': h, 'details': details,
|
||||
'body': base64.b64encode(body.getvalue()).decode()}}
|
||||
return to_return
|
||||
|
||||
|
||||
url_info_fields = api.model('URLInfoFields', {
|
||||
'url': fields.String(description="The URL to search", required=True),
|
||||
'limit': fields.Integer(description="The maximal amount of captures to return", example=20),
|
||||
})
|
||||
|
||||
|
||||
@api.route('/json/url_info')
|
||||
@api.doc(description='Search for a URL')
|
||||
class URLInfo(Resource):
|
||||
|
||||
@api.doc(body=url_info_fields)
|
||||
def post(self):
|
||||
to_query: Dict = request.get_json(force=True)
|
||||
occurrences = lookyloo.get_url_occurrences(to_query.pop('url'), **to_query)
|
||||
return occurrences
|
||||
|
||||
|
||||
hostname_info_fields = api.model('HostnameInfoFields', {
|
||||
'hostname': fields.String(description="The hostname to search", required=True),
|
||||
'limit': fields.Integer(description="The maximal amount of captures to return", example=20),
|
||||
})
|
||||
|
||||
|
||||
@api.route('/json/hostname_info')
|
||||
@api.doc(description='Search for a hostname')
|
||||
class HostnameInfo(Resource):
|
||||
|
||||
@api.doc(body=hostname_info_fields)
|
||||
def post(self):
|
||||
to_query: Dict = request.get_json(force=True)
|
||||
occurrences = lookyloo.get_hostname_occurrences(to_query.pop('hostname'), **to_query)
|
||||
return occurrences
|
||||
|
||||
|
||||
@api.route('/json/stats')
|
||||
@api.doc(description='Get the statistics of the lookyloo instance.')
|
||||
class InstanceStats(Resource):
|
||||
def get(self):
|
||||
return lookyloo.get_stats()
|
||||
|
||||
|
||||
submit_fields = api.model('SubmitFields', {
|
||||
'url': fields.String(description="The URL to capture", required=True),
|
||||
'listing': fields.Integer(description="Display the capture on the index", min=0, max=1, example=1),
|
||||
'user_agent': fields.String(description="User agent to use for the capture", example=''),
|
||||
'referer': fields.String(description="Referer to pass to the capture", example=''),
|
||||
'cookies': fields.String(description="JSON export of a list of cookies as exported from an other capture", example='')
|
||||
})
|
||||
|
||||
|
||||
@api.route('/submit')
|
||||
class SubmitCapture(Resource):
|
||||
|
||||
@api.doc(body=submit_fields)
|
||||
def post(self):
|
||||
if flask_login.current_user.is_authenticated:
|
||||
user = flask_login.current_user.get_id()
|
||||
else:
|
||||
user = src_request_ip(request)
|
||||
to_query: Dict = request.get_json(force=True)
|
||||
perma_uuid = lookyloo.enqueue_capture(to_query, source='api', user=user, authenticated=flask_login.current_user.is_authenticated)
|
||||
return Response(perma_uuid, mimetype='text/text')
|
||||
|
|
|
|||
Loading…
Reference in New Issue