From 7bf0b78754ee254245d1e66d2308bf52fa3d33a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 4 Jun 2021 17:30:14 -0700 Subject: [PATCH] new: Use flask-restx for the API --- poetry.lock | 117 +++++++++---- pyproject.toml | 2 +- website/web/__init__.py | 376 ++++++++++++++++++++++++++-------------- 3 files changed, 335 insertions(+), 160 deletions(-) diff --git a/poetry.lock b/poetry.lock index 852e9c48..773ff23f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -17,6 +17,17 @@ yarl = ">=1.0,<2.0" [package.extras] speedups = ["aiodns", "brotlipy", "cchardet"] +[[package]] +name = "aniso8601" +version = "9.0.1" +description = "A library for parsing ISO 8601 strings." +category = "main" +optional = false +python-versions = "*" + +[package.extras] +dev = ["black", "coverage", "isort", "pre-commit", "pyenchant", "pylint"] + [[package]] name = "appnope" version = "0.1.2" @@ -137,20 +148,17 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "click" -version = "8.0.1" +version = "7.1.2" description = "Composable command line interface toolkit" category = "main" optional = false -python-versions = ">=3.6" - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "colorama" version = "0.4.4" description = "Cross-platform colored terminal text." -category = "main" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -252,20 +260,21 @@ python-versions = "*" [[package]] name = "flask" -version = "2.0.1" +version = "1.1.4" description = "A simple framework for building complex web applications." category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.dependencies] -click = ">=7.1.2" -itsdangerous = ">=2.0" -Jinja2 = ">=3.0" -Werkzeug = ">=2.0" +click = ">=5.1,<8.0" +itsdangerous = ">=0.24,<2.0" +Jinja2 = ">=2.10.1,<3.0" +Werkzeug = ">=0.15,<2.0" [package.extras] -async = ["asgiref (>=3.2)"] +dev = ["pytest", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"] +docs = ["sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"] dotenv = ["python-dotenv"] [[package]] @@ -279,6 +288,27 @@ python-versions = "*" [package.dependencies] Flask = "*" +[[package]] +name = "flask-restx" +version = "0.4.0" +description = "Fully featured framework for fast, easy and documented API development with Flask" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +aniso8601 = {version = ">=0.82", markers = "python_version >= \"3.5\""} +Flask = ">=0.8,<2.0.0" +jsonschema = "*" +pytz = "*" +six = ">=1.3.0" +werkzeug = "<2.0.0" + +[package.extras] +dev = ["blinker", "Faker (==2.0.0)", "mock (==3.0.5)", "pytest-benchmark (==3.2.2)", "pytest-cov (==2.7.1)", "pytest-flask (==0.15.1)", "pytest-mock (==1.10.4)", "pytest-profiling (==1.7.0)", "tzlocal", "invoke (==1.3.0)", "readme-renderer (==24.0)", "twine (==1.15.0)", "tox", "pytest (==4.6.5)", "pytest (==5.4.1)", "ossaudit", "black"] +doc = ["alabaster (==0.7.12)", "Sphinx (==2.1.2)", "sphinx-issues (==1.2.0)"] +test = ["blinker", "Faker (==2.0.0)", "mock (==3.0.5)", "pytest-benchmark (==3.2.2)", "pytest-cov (==2.7.1)", "pytest-flask (==0.15.1)", "pytest-mock (==1.10.4)", "pytest-profiling (==1.7.0)", "tzlocal", "invoke (==1.3.0)", "readme-renderer (==24.0)", "twine (==1.15.0)", "pytest (==4.6.5)", "pytest (==5.4.1)", "ossaudit"] + [[package]] name = "gunicorn" version = "20.1.0" @@ -437,11 +467,11 @@ w3lib = ">=1.17.0" [[package]] name = "itsdangerous" -version = "2.0.1" -description = "Safely pass data to untrusted environments and back." +version = "1.1.0" +description = "Various helpers to pass data to untrusted environments and back." category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "jedi" @@ -460,17 +490,17 @@ testing = ["Django (<3.1)", "colorama", "docopt", "pytest (<6.0.0)"] [[package]] name = "jinja2" -version = "3.0.1" +version = "2.11.3" description = "A very fast and expressive template engine." category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.dependencies] -MarkupSafe = ">=2.0" +MarkupSafe = ">=0.23" [package.extras] -i18n = ["Babel (>=2.7)"] +i18n = ["Babel (>=0.8)"] [[package]] name = "jmespath" @@ -860,6 +890,14 @@ category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "pytz" +version = "2021.1" +description = "World timezone definitions, modern and historical" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "queuelib" version = "1.6.1" @@ -1102,13 +1140,14 @@ python-versions = "*" [[package]] name = "werkzeug" -version = "2.0.1" +version = "1.0.1" description = "The comprehensive WSGI web application library." category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.extras] +dev = ["pytest", "pytest-timeout", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinx-issues"] watchdog = ["watchdog"] [[package]] @@ -1150,7 +1189,7 @@ misp = ["python-magic", "pydeep"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "e8e3df6d57b274c350ed4df996c608cdf1ecb06ffd3f985c3ed0912a42da310b" +content-hash = "27c09f95d480f1517a425ed98847d3096d0ec22b7cccf0727189a31e4206a9ed" [metadata.files] aiohttp = [ @@ -1192,6 +1231,10 @@ aiohttp = [ {file = "aiohttp-3.7.4.post0-cp39-cp39-win_amd64.whl", hash = "sha256:02f46fc0e3c5ac58b80d4d56eb0a7c7d97fcef69ace9326289fb9f1955e65cfe"}, {file = "aiohttp-3.7.4.post0.tar.gz", hash = "sha256:493d3299ebe5f5a7c66b9819eacdcfbbaaf1a8e84911ddffcdc48888497afecf"}, ] +aniso8601 = [ + {file = "aniso8601-9.0.1-py2.py3-none-any.whl", hash = "sha256:1d2b7ef82963909e93c4f24ce48d4de9e66009a21bf1c1e1c85bdd0812fe412f"}, + {file = "aniso8601-9.0.1.tar.gz", hash = "sha256:72e3117667eedf66951bb2d93f4296a56b94b078a8a95905a052611fb3f1b973"}, +] appnope = [ {file = "appnope-0.1.2-py2.py3-none-any.whl", hash = "sha256:93aa393e9d6c54c5cd570ccadd8edad61ea0c4b9ea7a01409020c9aa019eb442"}, {file = "appnope-0.1.2.tar.gz", hash = "sha256:dd83cd4b5b460958838f6eb3000c660b1f9caf2a5b1de4264e941512f603258a"}, @@ -1300,8 +1343,8 @@ chardet = [ {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"}, ] click = [ - {file = "click-8.0.1-py3-none-any.whl", hash = "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6"}, - {file = "click-8.0.1.tar.gz", hash = "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a"}, + {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, + {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, ] colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, @@ -1351,13 +1394,17 @@ filetype = [ {file = "filetype-1.0.7.tar.gz", hash = "sha256:da393ece8d98b47edf2dd5a85a2c8733e44b769e32c71af4cd96ed8d38d96aa7"}, ] flask = [ - {file = "Flask-2.0.1-py3-none-any.whl", hash = "sha256:a6209ca15eb63fc9385f38e452704113d679511d9574d09b2cf9183ae7d20dc9"}, - {file = "Flask-2.0.1.tar.gz", hash = "sha256:1c4c257b1892aec1398784c63791cbaa43062f1f7aeb555c4da961b20ee68f55"}, + {file = "Flask-1.1.4-py2.py3-none-any.whl", hash = "sha256:c34f04500f2cbbea882b1acb02002ad6fe6b7ffa64a6164577995657f50aed22"}, + {file = "Flask-1.1.4.tar.gz", hash = "sha256:0fbeb6180d383a9186d0d6ed954e0042ad9f18e0e8de088b2b419d526927d196"}, ] flask-login = [ {file = "Flask-Login-0.5.0.tar.gz", hash = "sha256:6d33aef15b5bcead780acc339464aae8a6e28f13c90d8b1cf9de8b549d1c0b4b"}, {file = "Flask_Login-0.5.0-py2.py3-none-any.whl", hash = "sha256:7451b5001e17837ba58945aead261ba425fdf7b4f0448777e597ddab39f4fba0"}, ] +flask-restx = [ + {file = "flask-restx-0.4.0.tar.gz", hash = "sha256:8bdbfd8ad2949383c490b3b180481c0e9f70163230a5b567a56592c82f7c5cb0"}, + {file = "flask_restx-0.4.0-py2.py3-none-any.whl", hash = "sha256:780ac57ebda25e9f17ffccf3ff1b47a2c4efe5787cebddbbba1a91f53560d79f"}, +] gunicorn = [ {file = "gunicorn-20.1.0.tar.gz", hash = "sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8"}, ] @@ -1406,16 +1453,16 @@ itemloaders = [ {file = "itemloaders-1.0.4.tar.gz", hash = "sha256:1277cd8ca3e4c02dcdfbc1bcae9134ad89acfa6041bd15b4561c6290203a0c96"}, ] itsdangerous = [ - {file = "itsdangerous-2.0.1-py3-none-any.whl", hash = "sha256:5174094b9637652bdb841a3029700391451bd092ba3db90600dea710ba28e97c"}, - {file = "itsdangerous-2.0.1.tar.gz", hash = "sha256:9e724d68fc22902a1435351f84c3fb8623f303fffcc566a4cb952df8c572cff0"}, + {file = "itsdangerous-1.1.0-py2.py3-none-any.whl", hash = "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749"}, + {file = "itsdangerous-1.1.0.tar.gz", hash = "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19"}, ] jedi = [ {file = "jedi-0.18.0-py2.py3-none-any.whl", hash = "sha256:18456d83f65f400ab0c2d3319e48520420ef43b23a086fdc05dff34132f0fb93"}, {file = "jedi-0.18.0.tar.gz", hash = "sha256:92550a404bad8afed881a137ec9a461fed49eca661414be45059329614ed0707"}, ] jinja2 = [ - {file = "Jinja2-3.0.1-py3-none-any.whl", hash = "sha256:1f06f2da51e7b56b8f238affdd6b4e2c61e39598a378cc49345bc1bd42a978a4"}, - {file = "Jinja2-3.0.1.tar.gz", hash = "sha256:703f484b47a6af502e743c9122595cc812b0271f661722403114f71a79d0f5a4"}, + {file = "Jinja2-2.11.3-py2.py3-none-any.whl", hash = "sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419"}, + {file = "Jinja2-2.11.3.tar.gz", hash = "sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6"}, ] jmespath = [ {file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"}, @@ -1778,6 +1825,10 @@ python-magic = [ {file = "python-magic-0.4.24.tar.gz", hash = "sha256:de800df9fb50f8ec5974761054a708af6e4246b03b4bdaee993f948947b0ebcf"}, {file = "python_magic-0.4.24-py2.py3-none-any.whl", hash = "sha256:4fec8ee805fea30c07afccd1592c0f17977089895bdfaae5fec870a84e997626"}, ] +pytz = [ + {file = "pytz-2021.1-py2.py3-none-any.whl", hash = "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"}, + {file = "pytz-2021.1.tar.gz", hash = "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da"}, +] queuelib = [ {file = "queuelib-1.6.1-py2.py3-none-any.whl", hash = "sha256:90ee30ebb0b57112606358b63c09a681bbb9a7dd1120af09c836b475504cea85"}, {file = "queuelib-1.6.1.tar.gz", hash = "sha256:631d067c9be57e395c382d680d3653ca1452cd29e8da25c5e8d94b5c0c528c31"}, @@ -1893,8 +1944,8 @@ wcwidth = [ {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, ] werkzeug = [ - {file = "Werkzeug-2.0.1-py3-none-any.whl", hash = "sha256:6c1ec500dcdba0baa27600f6a22f6333d8b662d22027ff9f6202e3367413caa8"}, - {file = "Werkzeug-2.0.1.tar.gz", hash = "sha256:1de1db30d010ff1af14a009224ec49ab2329ad2cde454c8a708130642d579c42"}, + {file = "Werkzeug-1.0.1-py2.py3-none-any.whl", hash = "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43"}, + {file = "Werkzeug-1.0.1.tar.gz", hash = "sha256:6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c"}, ] wrapt = [ {file = "wrapt-1.12.1.tar.gz", hash = "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7"}, diff --git a/pyproject.toml b/pyproject.toml index d85ea309..8cf6acbc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,6 @@ background_indexer = "bin.background_indexer:main" [tool.poetry.dependencies] python = "^3.8" requests = "^2.25.1" -flask = "^2.0.0" gunicorn = "^20.1.0" cchardet = "^2.1.7" redis = "^3.5.3" @@ -58,6 +57,7 @@ pydeep = {version = "^0.4", optional = true} Pillow = "^8.2.0" lief = "^0.11.4" Flask-Login = "^0.5.0" +flask-restx = "^0.4.0" [tool.poetry.extras] misp = ['python-magic', 'pydeep'] diff --git a/website/web/__init__.py b/website/web/__init__.py index d6258952..0a7b81c1 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -14,10 +14,12 @@ import logging import hashlib from urllib.parse import quote_plus, unquote_plus import time +import pkg_resources from flask import Flask, render_template, request, send_file, redirect, url_for, Response, flash, jsonify from flask_bootstrap import Bootstrap # type: ignore import flask_login # type: ignore +from flask_restx import Resource, Api, fields, abort # type: ignore from werkzeug.security import generate_password_hash, check_password_hash from pymisp import MISPEvent, MISPServerError @@ -577,7 +579,7 @@ def tree(tree_uuid: str, node_uuid: Optional[str]=None): @flask_login.login_required def mark_as_legitimate(tree_uuid: str): if request.data: - legitimate_entries: Dict = request.get_json(force=True) # type: ignore + legitimate_entries: Dict = request.get_json(force=True) lookyloo.add_to_legitimate(tree_uuid, **legitimate_entries) else: lookyloo.add_to_legitimate(tree_uuid) @@ -696,17 +698,6 @@ def rebuild_cache(): return redirect(url_for('index')) -@app.route('/submit', methods=['POST']) -def submit(): - if flask_login.current_user.is_authenticated: - user = flask_login.current_user.get_id() - else: - user = src_request_ip(request) - to_query: Dict = request.get_json(force=True) # type: ignore - perma_uuid = lookyloo.enqueue_capture(to_query, source='api', user=user, authenticated=flask_login.current_user.is_authenticated) - return Response(perma_uuid, mimetype='text/text') - - @app.route('/search', methods=['GET', 'POST']) def search(): if request.form.get('url'): @@ -1021,122 +1012,255 @@ def web_misp_push_view(tree_uuid: str): default_tags=lookyloo.misp.default_tags) -# Query API - -@app.route('/json/get_token', methods=['POST']) -def json_get_token(): - auth: Dict = request.get_json(force=True) # type: ignore - if 'username' in auth and 'password' in auth: # Expected keys in json - if (auth['username'] in users_table - and check_password_hash(users_table[auth['username']]['password'], auth['password'])): - return jsonify({'authkey': users_table[auth['username']]['authkey']}) - return jsonify({'error': 'User/Password invalid.'}) - - -@app.route('/json//status', methods=['GET']) -def get_capture_status(tree_uuid: str): - return jsonify({'status_code': lookyloo.get_capture_status(tree_uuid)}) - - -@app.route('/json//redirects', methods=['GET']) -def json_redirects(tree_uuid: str): - cache = lookyloo.capture_cache(tree_uuid) - if not cache: - return {'error': 'UUID missing in cache, try again later.'} - - to_return: Dict[str, Any] = {'response': {'url': cache.url, 'redirects': []}} - if not cache.redirects: - to_return['response']['info'] = 'No redirects' - return to_return - if cache.incomplete_redirects: - # Trigger tree build, get all redirects - lookyloo.get_crawled_tree(tree_uuid) - cache = lookyloo.capture_cache(tree_uuid) - if cache: - to_return['response']['redirects'] = cache.redirects - else: - to_return['response']['redirects'] = cache.redirects - - return jsonify(to_return) - - -@app.route('/json//misp_export', methods=['GET']) -def misp_export(tree_uuid: str): - with_parents = request.args.get('with_parents') - event = lookyloo.misp_export(tree_uuid, True if with_parents else False) - if isinstance(event, dict): - return jsonify(event) - - to_return = [] - for e in event: - to_return.append(e.to_json(indent=2)) - return jsonify(to_return) - - -@app.route('/json//misp_push', methods=['GET', 'POST']) -@flask_login.login_required -def misp_push(tree_uuid: str): - if request.method == 'POST': - parameters: Dict = request.get_json(force=True) # type: ignore - with_parents = True if 'with_parents' in parameters else False - allow_duplicates = True if 'allow_duplicates' in parameters else False - else: - with_parents = False - allow_duplicates = False - to_return: Dict = {} - if not lookyloo.misp.available: - to_return['error'] = 'MISP module not available.' - elif not lookyloo.misp.enable_push: - to_return['error'] = 'Push not enabled in MISP module.' - else: - event = lookyloo.misp_export(tree_uuid, with_parents) - if isinstance(event, dict): - to_return['error'] = event - else: - new_events = lookyloo.misp.push(event, allow_duplicates) - if isinstance(new_events, dict): - to_return['error'] = new_events - else: - events_to_return = [] - for e in new_events: - events_to_return.append(e.to_json(indent=2)) - jsonify(events_to_return) - - return jsonify(to_return) - - -@app.route('/json/hash_info/', methods=['GET']) -def json_hash_info(h: str): - details, body = lookyloo.get_body_hash_full(h) - if not details: - return {'error': 'Unknown Hash.'} - to_return: Dict[str, Any] = {'response': {'hash': h, 'details': details, - 'body': base64.b64encode(body.getvalue()).decode()}} - return jsonify(to_return) - - -@app.route('/json/url_info', methods=['POST']) -def json_url_info(): - to_query: Dict = request.get_json(force=True) # type: ignore - occurrences = lookyloo.get_url_occurrences(to_query.pop('url'), **to_query) - return jsonify(occurrences) - - -@app.route('/json/hostname_info', methods=['POST']) -def json_hostname_info(): - to_query: Dict = request.get_json(force=True) # type: ignore - occurrences = lookyloo.get_hostname_occurrences(to_query.pop('hostname'), **to_query) - return jsonify(occurrences) - - -@app.route('/json/stats', methods=['GET']) -def json_stats(): - to_return = lookyloo.get_stats() - return Response(json.dumps(to_return), mimetype='application/json') - - @app.route('/whois/', methods=['GET']) def whois(query: str): to_return = lookyloo.uwhois.whois(query) return send_file(BytesIO(to_return.encode()), mimetype='test/plain', as_attachment=True, attachment_filename=f'whois.{query}.txt') + + +# Query API + +authorizations = { + 'apikey': { + 'type': 'apiKey', + 'in': 'header', + 'name': 'Authorization' + } +} + +api = Api(app, title='Lookyloo API', + description='API to submit captures and query a lookyloo instance.', + doc='/doc/', + authorizations=authorizations, + version=pkg_resources.get_distribution('lookyloo').version) + + +def api_auth_check(method): + if flask_login.current_user.is_authenticated or load_user_from_request(request): + return method + abort(403, 'Authentication required.') + + +token_request_fields = api.model('AuthTokenFields', { + 'username': fields.String(description="Your username", required=True), + 'password': fields.String(description="Your password", required=True), +}) + + +@api.route('/json/get_token') +@api.doc(description='Get the API token required for authenticated calls') +class AuthToken(Resource): + + @api.param('username', 'Your username') + @api.param('password', 'Your password') + def get(self): + username = request.args['username'] if request.args.get('username') else False + password = request.args['password'] if request.args.get('password') else False + if username in users_table and check_password_hash(users_table[username]['password'], password): + return {'authkey': users_table[username]['authkey']} + return {'error': 'User/Password invalid.'} + + @api.doc(body=token_request_fields) + def post(self): + auth: Dict = request.get_json(force=True) + if 'username' in auth and 'password' in auth: # Expected keys in json + if (auth['username'] in users_table + and check_password_hash(users_table[auth['username']]['password'], auth['password'])): + return {'authkey': users_table[auth['username']]['authkey']} + return {'error': 'User/Password invalid.'} + + +@api.route('/json//status') +@api.doc(description='Get the status of a capture', + params={'tree_uuid': 'The UUID of the capture'}) +class CaptureStatusQuery(Resource): + def get(self, tree_uuid: str): + return {'status_code': lookyloo.get_capture_status(tree_uuid)} + + +@api.route('/json//redirects') +@api.doc(description='Get all the redirects of a capture', + params={'tree_uuid': 'The UUID of the capture'}) +class CaptureRedirects(Resource): + def get(self, tree_uuid: str): + cache = lookyloo.capture_cache(tree_uuid) + if not cache: + return {'error': 'UUID missing in cache, try again later.'} + + to_return: Dict[str, Any] = {'response': {'url': cache.url, 'redirects': []}} + if not cache.redirects: + to_return['response']['info'] = 'No redirects' + return to_return + if cache.incomplete_redirects: + # Trigger tree build, get all redirects + lookyloo.get_crawled_tree(tree_uuid) + cache = lookyloo.capture_cache(tree_uuid) + if cache: + to_return['response']['redirects'] = cache.redirects + else: + to_return['response']['redirects'] = cache.redirects + + return to_return + + +@api.route('/json//misp_export') +@api.doc(description='Get an export of the capture in MISP format', + params={'tree_uuid': 'The UUID of the capture'}) +class MISPExport(Resource): + def get(self, tree_uuid: str): + with_parents = request.args.get('with_parents') + event = lookyloo.misp_export(tree_uuid, True if with_parents else False) + if isinstance(event, dict): + return event + + to_return = [] + for e in event: + to_return.append(e.to_json(indent=2)) + return to_return + + +misp_push_fields = api.model('MISPPushFields', { + 'allow_duplicates': fields.Integer(description="Push the event even if it is already present on the MISP instance", + example=0, min=0, max=1), + 'with_parents': fields.Integer(description="Also push the parents of the capture (if any)", + example=0, min=0, max=1), +}) + + +@api.route('/json//misp_push') +@api.doc(description='Push an event to a pre-configured MISP instance', + params={'tree_uuid': 'The UUID of the capture'}, + security='apikey') +class MISPPush(Resource): + method_decorators = [api_auth_check] + + @api.param('with_parents', 'Also push the parents of the capture (if any)') + @api.param('allow_duplicates', 'Push the event even if it is already present on the MISP instance') + def get(self, tree_uuid: str): + with_parents = True if request.args.get('with_parents') else False + allow_duplicates = True if request.args.get('allow_duplicates') else False + to_return: Dict = {} + if not lookyloo.misp.available: + to_return['error'] = 'MISP module not available.' + elif not lookyloo.misp.enable_push: + to_return['error'] = 'Push not enabled in MISP module.' + else: + event = lookyloo.misp_export(tree_uuid, with_parents) + if isinstance(event, dict): + to_return['error'] = event + else: + new_events = lookyloo.misp.push(event, allow_duplicates) + if isinstance(new_events, dict): + to_return['error'] = new_events + else: + events_to_return = [] + for e in new_events: + events_to_return.append(e.to_json(indent=2)) + return events_to_return + + return to_return + + @api.doc(body=misp_push_fields) + def post(self, tree_uuid: str): + parameters: Dict = request.get_json(force=True) + with_parents = True if parameters.get('with_parents') else False + allow_duplicates = True if parameters.get('allow_duplicates') else False + + to_return: Dict = {} + if not lookyloo.misp.available: + to_return['error'] = 'MISP module not available.' + elif not lookyloo.misp.enable_push: + to_return['error'] = 'Push not enabled in MISP module.' + else: + event = lookyloo.misp_export(tree_uuid, with_parents) + if isinstance(event, dict): + to_return['error'] = event + else: + new_events = lookyloo.misp.push(event, allow_duplicates) + if isinstance(new_events, dict): + to_return['error'] = new_events + else: + events_to_return = [] + for e in new_events: + events_to_return.append(e.to_json(indent=2)) + return events_to_return + + return to_return + + +@api.route('/json/hash_info/') +@api.doc(description='Search for a ressource with a specific hash (sha512)', + params={'h': 'The hash (sha512)'}) +class HashInfo(Resource): + def get(self, h: str): + details, body = lookyloo.get_body_hash_full(h) + if not details: + return {'error': 'Unknown Hash.'} + to_return: Dict[str, Any] = {'response': {'hash': h, 'details': details, + 'body': base64.b64encode(body.getvalue()).decode()}} + return to_return + + +url_info_fields = api.model('URLInfoFields', { + 'url': fields.String(description="The URL to search", required=True), + 'limit': fields.Integer(description="The maximal amount of captures to return", example=20), +}) + + +@api.route('/json/url_info') +@api.doc(description='Search for a URL') +class URLInfo(Resource): + + @api.doc(body=url_info_fields) + def post(self): + to_query: Dict = request.get_json(force=True) + occurrences = lookyloo.get_url_occurrences(to_query.pop('url'), **to_query) + return occurrences + + +hostname_info_fields = api.model('HostnameInfoFields', { + 'hostname': fields.String(description="The hostname to search", required=True), + 'limit': fields.Integer(description="The maximal amount of captures to return", example=20), +}) + + +@api.route('/json/hostname_info') +@api.doc(description='Search for a hostname') +class HostnameInfo(Resource): + + @api.doc(body=hostname_info_fields) + def post(self): + to_query: Dict = request.get_json(force=True) + occurrences = lookyloo.get_hostname_occurrences(to_query.pop('hostname'), **to_query) + return occurrences + + +@api.route('/json/stats') +@api.doc(description='Get the statistics of the lookyloo instance.') +class InstanceStats(Resource): + def get(self): + return lookyloo.get_stats() + + +submit_fields = api.model('SubmitFields', { + 'url': fields.String(description="The URL to capture", required=True), + 'listing': fields.Integer(description="Display the capture on the index", min=0, max=1, example=1), + 'user_agent': fields.String(description="User agent to use for the capture", example=''), + 'referer': fields.String(description="Referer to pass to the capture", example=''), + 'cookies': fields.String(description="JSON export of a list of cookies as exported from an other capture", example='') +}) + + +@api.route('/submit') +class SubmitCapture(Resource): + + @api.doc(body=submit_fields) + def post(self): + if flask_login.current_user.is_authenticated: + user = flask_login.current_user.get_id() + else: + user = src_request_ip(request) + to_query: Dict = request.get_json(force=True) + perma_uuid = lookyloo.enqueue_capture(to_query, source='api', user=user, authenticated=flask_login.current_user.is_authenticated) + return Response(perma_uuid, mimetype='text/text')