From 7bf0b78754ee254245d1e66d2308bf52fa3d33a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 4 Jun 2021 17:30:14 -0700 Subject: [PATCH 01/18] new: Use flask-restx for the API --- poetry.lock | 117 +++++++++---- pyproject.toml | 2 +- website/web/__init__.py | 376 ++++++++++++++++++++++++++-------------- 3 files changed, 335 insertions(+), 160 deletions(-) diff --git a/poetry.lock b/poetry.lock index 852e9c48..773ff23f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -17,6 +17,17 @@ yarl = ">=1.0,<2.0" [package.extras] speedups = ["aiodns", "brotlipy", "cchardet"] +[[package]] +name = "aniso8601" +version = "9.0.1" +description = "A library for parsing ISO 8601 strings." +category = "main" +optional = false +python-versions = "*" + +[package.extras] +dev = ["black", "coverage", "isort", "pre-commit", "pyenchant", "pylint"] + [[package]] name = "appnope" version = "0.1.2" @@ -137,20 +148,17 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "click" -version = "8.0.1" +version = "7.1.2" description = "Composable command line interface toolkit" category = "main" optional = false -python-versions = ">=3.6" - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "colorama" version = "0.4.4" description = "Cross-platform colored terminal text." -category = "main" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -252,20 +260,21 @@ python-versions = "*" [[package]] name = "flask" -version = "2.0.1" +version = "1.1.4" description = "A simple framework for building complex web applications." category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.dependencies] -click = ">=7.1.2" -itsdangerous = ">=2.0" -Jinja2 = ">=3.0" -Werkzeug = ">=2.0" +click = ">=5.1,<8.0" +itsdangerous = ">=0.24,<2.0" +Jinja2 = ">=2.10.1,<3.0" +Werkzeug = ">=0.15,<2.0" [package.extras] -async = ["asgiref (>=3.2)"] +dev = ["pytest", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"] +docs = ["sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"] dotenv = ["python-dotenv"] [[package]] @@ -279,6 +288,27 @@ python-versions = "*" [package.dependencies] Flask = "*" +[[package]] +name = "flask-restx" +version = "0.4.0" +description = "Fully featured framework for fast, easy and documented API development with Flask" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +aniso8601 = {version = ">=0.82", markers = "python_version >= \"3.5\""} +Flask = ">=0.8,<2.0.0" +jsonschema = "*" +pytz = "*" +six = ">=1.3.0" +werkzeug = "<2.0.0" + +[package.extras] +dev = ["blinker", "Faker (==2.0.0)", "mock (==3.0.5)", "pytest-benchmark (==3.2.2)", "pytest-cov (==2.7.1)", "pytest-flask (==0.15.1)", "pytest-mock (==1.10.4)", "pytest-profiling (==1.7.0)", "tzlocal", "invoke (==1.3.0)", "readme-renderer (==24.0)", "twine (==1.15.0)", "tox", "pytest (==4.6.5)", "pytest (==5.4.1)", "ossaudit", "black"] +doc = ["alabaster (==0.7.12)", "Sphinx (==2.1.2)", "sphinx-issues (==1.2.0)"] +test = ["blinker", "Faker (==2.0.0)", "mock (==3.0.5)", "pytest-benchmark (==3.2.2)", "pytest-cov (==2.7.1)", "pytest-flask (==0.15.1)", "pytest-mock (==1.10.4)", "pytest-profiling (==1.7.0)", "tzlocal", "invoke (==1.3.0)", "readme-renderer (==24.0)", "twine (==1.15.0)", "pytest (==4.6.5)", "pytest (==5.4.1)", "ossaudit"] + [[package]] name = "gunicorn" version = "20.1.0" @@ -437,11 +467,11 @@ w3lib = ">=1.17.0" [[package]] name = "itsdangerous" -version = "2.0.1" -description = "Safely pass data to untrusted environments and back." +version = "1.1.0" +description = "Various helpers to pass data to untrusted environments and back." category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "jedi" @@ -460,17 +490,17 @@ testing = ["Django (<3.1)", "colorama", "docopt", "pytest (<6.0.0)"] [[package]] name = "jinja2" -version = "3.0.1" +version = "2.11.3" description = "A very fast and expressive template engine." category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.dependencies] -MarkupSafe = ">=2.0" +MarkupSafe = ">=0.23" [package.extras] -i18n = ["Babel (>=2.7)"] +i18n = ["Babel (>=0.8)"] [[package]] name = "jmespath" @@ -860,6 +890,14 @@ category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "pytz" +version = "2021.1" +description = "World timezone definitions, modern and historical" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "queuelib" version = "1.6.1" @@ -1102,13 +1140,14 @@ python-versions = "*" [[package]] name = "werkzeug" -version = "2.0.1" +version = "1.0.1" description = "The comprehensive WSGI web application library." category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.extras] +dev = ["pytest", "pytest-timeout", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinx-issues"] watchdog = ["watchdog"] [[package]] @@ -1150,7 +1189,7 @@ misp = ["python-magic", "pydeep"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "e8e3df6d57b274c350ed4df996c608cdf1ecb06ffd3f985c3ed0912a42da310b" +content-hash = "27c09f95d480f1517a425ed98847d3096d0ec22b7cccf0727189a31e4206a9ed" [metadata.files] aiohttp = [ @@ -1192,6 +1231,10 @@ aiohttp = [ {file = "aiohttp-3.7.4.post0-cp39-cp39-win_amd64.whl", hash = "sha256:02f46fc0e3c5ac58b80d4d56eb0a7c7d97fcef69ace9326289fb9f1955e65cfe"}, {file = "aiohttp-3.7.4.post0.tar.gz", hash = "sha256:493d3299ebe5f5a7c66b9819eacdcfbbaaf1a8e84911ddffcdc48888497afecf"}, ] +aniso8601 = [ + {file = "aniso8601-9.0.1-py2.py3-none-any.whl", hash = "sha256:1d2b7ef82963909e93c4f24ce48d4de9e66009a21bf1c1e1c85bdd0812fe412f"}, + {file = "aniso8601-9.0.1.tar.gz", hash = "sha256:72e3117667eedf66951bb2d93f4296a56b94b078a8a95905a052611fb3f1b973"}, +] appnope = [ {file = "appnope-0.1.2-py2.py3-none-any.whl", hash = "sha256:93aa393e9d6c54c5cd570ccadd8edad61ea0c4b9ea7a01409020c9aa019eb442"}, {file = "appnope-0.1.2.tar.gz", hash = "sha256:dd83cd4b5b460958838f6eb3000c660b1f9caf2a5b1de4264e941512f603258a"}, @@ -1300,8 +1343,8 @@ chardet = [ {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"}, ] click = [ - {file = "click-8.0.1-py3-none-any.whl", hash = "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6"}, - {file = "click-8.0.1.tar.gz", hash = "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a"}, + {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, + {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, ] colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, @@ -1351,13 +1394,17 @@ filetype = [ {file = "filetype-1.0.7.tar.gz", hash = "sha256:da393ece8d98b47edf2dd5a85a2c8733e44b769e32c71af4cd96ed8d38d96aa7"}, ] flask = [ - {file = "Flask-2.0.1-py3-none-any.whl", hash = "sha256:a6209ca15eb63fc9385f38e452704113d679511d9574d09b2cf9183ae7d20dc9"}, - {file = "Flask-2.0.1.tar.gz", hash = "sha256:1c4c257b1892aec1398784c63791cbaa43062f1f7aeb555c4da961b20ee68f55"}, + {file = "Flask-1.1.4-py2.py3-none-any.whl", hash = "sha256:c34f04500f2cbbea882b1acb02002ad6fe6b7ffa64a6164577995657f50aed22"}, + {file = "Flask-1.1.4.tar.gz", hash = "sha256:0fbeb6180d383a9186d0d6ed954e0042ad9f18e0e8de088b2b419d526927d196"}, ] flask-login = [ {file = "Flask-Login-0.5.0.tar.gz", hash = "sha256:6d33aef15b5bcead780acc339464aae8a6e28f13c90d8b1cf9de8b549d1c0b4b"}, {file = "Flask_Login-0.5.0-py2.py3-none-any.whl", hash = "sha256:7451b5001e17837ba58945aead261ba425fdf7b4f0448777e597ddab39f4fba0"}, ] +flask-restx = [ + {file = "flask-restx-0.4.0.tar.gz", hash = "sha256:8bdbfd8ad2949383c490b3b180481c0e9f70163230a5b567a56592c82f7c5cb0"}, + {file = "flask_restx-0.4.0-py2.py3-none-any.whl", hash = "sha256:780ac57ebda25e9f17ffccf3ff1b47a2c4efe5787cebddbbba1a91f53560d79f"}, +] gunicorn = [ {file = "gunicorn-20.1.0.tar.gz", hash = "sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8"}, ] @@ -1406,16 +1453,16 @@ itemloaders = [ {file = "itemloaders-1.0.4.tar.gz", hash = "sha256:1277cd8ca3e4c02dcdfbc1bcae9134ad89acfa6041bd15b4561c6290203a0c96"}, ] itsdangerous = [ - {file = "itsdangerous-2.0.1-py3-none-any.whl", hash = "sha256:5174094b9637652bdb841a3029700391451bd092ba3db90600dea710ba28e97c"}, - {file = "itsdangerous-2.0.1.tar.gz", hash = "sha256:9e724d68fc22902a1435351f84c3fb8623f303fffcc566a4cb952df8c572cff0"}, + {file = "itsdangerous-1.1.0-py2.py3-none-any.whl", hash = "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749"}, + {file = "itsdangerous-1.1.0.tar.gz", hash = "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19"}, ] jedi = [ {file = "jedi-0.18.0-py2.py3-none-any.whl", hash = "sha256:18456d83f65f400ab0c2d3319e48520420ef43b23a086fdc05dff34132f0fb93"}, {file = "jedi-0.18.0.tar.gz", hash = "sha256:92550a404bad8afed881a137ec9a461fed49eca661414be45059329614ed0707"}, ] jinja2 = [ - {file = "Jinja2-3.0.1-py3-none-any.whl", hash = "sha256:1f06f2da51e7b56b8f238affdd6b4e2c61e39598a378cc49345bc1bd42a978a4"}, - {file = "Jinja2-3.0.1.tar.gz", hash = "sha256:703f484b47a6af502e743c9122595cc812b0271f661722403114f71a79d0f5a4"}, + {file = "Jinja2-2.11.3-py2.py3-none-any.whl", hash = "sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419"}, + {file = "Jinja2-2.11.3.tar.gz", hash = "sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6"}, ] jmespath = [ {file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"}, @@ -1778,6 +1825,10 @@ python-magic = [ {file = "python-magic-0.4.24.tar.gz", hash = "sha256:de800df9fb50f8ec5974761054a708af6e4246b03b4bdaee993f948947b0ebcf"}, {file = "python_magic-0.4.24-py2.py3-none-any.whl", hash = "sha256:4fec8ee805fea30c07afccd1592c0f17977089895bdfaae5fec870a84e997626"}, ] +pytz = [ + {file = "pytz-2021.1-py2.py3-none-any.whl", hash = "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"}, + {file = "pytz-2021.1.tar.gz", hash = "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da"}, +] queuelib = [ {file = "queuelib-1.6.1-py2.py3-none-any.whl", hash = "sha256:90ee30ebb0b57112606358b63c09a681bbb9a7dd1120af09c836b475504cea85"}, {file = "queuelib-1.6.1.tar.gz", hash = "sha256:631d067c9be57e395c382d680d3653ca1452cd29e8da25c5e8d94b5c0c528c31"}, @@ -1893,8 +1944,8 @@ wcwidth = [ {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, ] werkzeug = [ - {file = "Werkzeug-2.0.1-py3-none-any.whl", hash = "sha256:6c1ec500dcdba0baa27600f6a22f6333d8b662d22027ff9f6202e3367413caa8"}, - {file = "Werkzeug-2.0.1.tar.gz", hash = "sha256:1de1db30d010ff1af14a009224ec49ab2329ad2cde454c8a708130642d579c42"}, + {file = "Werkzeug-1.0.1-py2.py3-none-any.whl", hash = "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43"}, + {file = "Werkzeug-1.0.1.tar.gz", hash = "sha256:6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c"}, ] wrapt = [ {file = "wrapt-1.12.1.tar.gz", hash = "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7"}, diff --git a/pyproject.toml b/pyproject.toml index d85ea309..8cf6acbc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,6 @@ background_indexer = "bin.background_indexer:main" [tool.poetry.dependencies] python = "^3.8" requests = "^2.25.1" -flask = "^2.0.0" gunicorn = "^20.1.0" cchardet = "^2.1.7" redis = "^3.5.3" @@ -58,6 +57,7 @@ pydeep = {version = "^0.4", optional = true} Pillow = "^8.2.0" lief = "^0.11.4" Flask-Login = "^0.5.0" +flask-restx = "^0.4.0" [tool.poetry.extras] misp = ['python-magic', 'pydeep'] diff --git a/website/web/__init__.py b/website/web/__init__.py index d6258952..0a7b81c1 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -14,10 +14,12 @@ import logging import hashlib from urllib.parse import quote_plus, unquote_plus import time +import pkg_resources from flask import Flask, render_template, request, send_file, redirect, url_for, Response, flash, jsonify from flask_bootstrap import Bootstrap # type: ignore import flask_login # type: ignore +from flask_restx import Resource, Api, fields, abort # type: ignore from werkzeug.security import generate_password_hash, check_password_hash from pymisp import MISPEvent, MISPServerError @@ -577,7 +579,7 @@ def tree(tree_uuid: str, node_uuid: Optional[str]=None): @flask_login.login_required def mark_as_legitimate(tree_uuid: str): if request.data: - legitimate_entries: Dict = request.get_json(force=True) # type: ignore + legitimate_entries: Dict = request.get_json(force=True) lookyloo.add_to_legitimate(tree_uuid, **legitimate_entries) else: lookyloo.add_to_legitimate(tree_uuid) @@ -696,17 +698,6 @@ def rebuild_cache(): return redirect(url_for('index')) -@app.route('/submit', methods=['POST']) -def submit(): - if flask_login.current_user.is_authenticated: - user = flask_login.current_user.get_id() - else: - user = src_request_ip(request) - to_query: Dict = request.get_json(force=True) # type: ignore - perma_uuid = lookyloo.enqueue_capture(to_query, source='api', user=user, authenticated=flask_login.current_user.is_authenticated) - return Response(perma_uuid, mimetype='text/text') - - @app.route('/search', methods=['GET', 'POST']) def search(): if request.form.get('url'): @@ -1021,122 +1012,255 @@ def web_misp_push_view(tree_uuid: str): default_tags=lookyloo.misp.default_tags) -# Query API - -@app.route('/json/get_token', methods=['POST']) -def json_get_token(): - auth: Dict = request.get_json(force=True) # type: ignore - if 'username' in auth and 'password' in auth: # Expected keys in json - if (auth['username'] in users_table - and check_password_hash(users_table[auth['username']]['password'], auth['password'])): - return jsonify({'authkey': users_table[auth['username']]['authkey']}) - return jsonify({'error': 'User/Password invalid.'}) - - -@app.route('/json//status', methods=['GET']) -def get_capture_status(tree_uuid: str): - return jsonify({'status_code': lookyloo.get_capture_status(tree_uuid)}) - - -@app.route('/json//redirects', methods=['GET']) -def json_redirects(tree_uuid: str): - cache = lookyloo.capture_cache(tree_uuid) - if not cache: - return {'error': 'UUID missing in cache, try again later.'} - - to_return: Dict[str, Any] = {'response': {'url': cache.url, 'redirects': []}} - if not cache.redirects: - to_return['response']['info'] = 'No redirects' - return to_return - if cache.incomplete_redirects: - # Trigger tree build, get all redirects - lookyloo.get_crawled_tree(tree_uuid) - cache = lookyloo.capture_cache(tree_uuid) - if cache: - to_return['response']['redirects'] = cache.redirects - else: - to_return['response']['redirects'] = cache.redirects - - return jsonify(to_return) - - -@app.route('/json//misp_export', methods=['GET']) -def misp_export(tree_uuid: str): - with_parents = request.args.get('with_parents') - event = lookyloo.misp_export(tree_uuid, True if with_parents else False) - if isinstance(event, dict): - return jsonify(event) - - to_return = [] - for e in event: - to_return.append(e.to_json(indent=2)) - return jsonify(to_return) - - -@app.route('/json//misp_push', methods=['GET', 'POST']) -@flask_login.login_required -def misp_push(tree_uuid: str): - if request.method == 'POST': - parameters: Dict = request.get_json(force=True) # type: ignore - with_parents = True if 'with_parents' in parameters else False - allow_duplicates = True if 'allow_duplicates' in parameters else False - else: - with_parents = False - allow_duplicates = False - to_return: Dict = {} - if not lookyloo.misp.available: - to_return['error'] = 'MISP module not available.' - elif not lookyloo.misp.enable_push: - to_return['error'] = 'Push not enabled in MISP module.' - else: - event = lookyloo.misp_export(tree_uuid, with_parents) - if isinstance(event, dict): - to_return['error'] = event - else: - new_events = lookyloo.misp.push(event, allow_duplicates) - if isinstance(new_events, dict): - to_return['error'] = new_events - else: - events_to_return = [] - for e in new_events: - events_to_return.append(e.to_json(indent=2)) - jsonify(events_to_return) - - return jsonify(to_return) - - -@app.route('/json/hash_info/', methods=['GET']) -def json_hash_info(h: str): - details, body = lookyloo.get_body_hash_full(h) - if not details: - return {'error': 'Unknown Hash.'} - to_return: Dict[str, Any] = {'response': {'hash': h, 'details': details, - 'body': base64.b64encode(body.getvalue()).decode()}} - return jsonify(to_return) - - -@app.route('/json/url_info', methods=['POST']) -def json_url_info(): - to_query: Dict = request.get_json(force=True) # type: ignore - occurrences = lookyloo.get_url_occurrences(to_query.pop('url'), **to_query) - return jsonify(occurrences) - - -@app.route('/json/hostname_info', methods=['POST']) -def json_hostname_info(): - to_query: Dict = request.get_json(force=True) # type: ignore - occurrences = lookyloo.get_hostname_occurrences(to_query.pop('hostname'), **to_query) - return jsonify(occurrences) - - -@app.route('/json/stats', methods=['GET']) -def json_stats(): - to_return = lookyloo.get_stats() - return Response(json.dumps(to_return), mimetype='application/json') - - @app.route('/whois/', methods=['GET']) def whois(query: str): to_return = lookyloo.uwhois.whois(query) return send_file(BytesIO(to_return.encode()), mimetype='test/plain', as_attachment=True, attachment_filename=f'whois.{query}.txt') + + +# Query API + +authorizations = { + 'apikey': { + 'type': 'apiKey', + 'in': 'header', + 'name': 'Authorization' + } +} + +api = Api(app, title='Lookyloo API', + description='API to submit captures and query a lookyloo instance.', + doc='/doc/', + authorizations=authorizations, + version=pkg_resources.get_distribution('lookyloo').version) + + +def api_auth_check(method): + if flask_login.current_user.is_authenticated or load_user_from_request(request): + return method + abort(403, 'Authentication required.') + + +token_request_fields = api.model('AuthTokenFields', { + 'username': fields.String(description="Your username", required=True), + 'password': fields.String(description="Your password", required=True), +}) + + +@api.route('/json/get_token') +@api.doc(description='Get the API token required for authenticated calls') +class AuthToken(Resource): + + @api.param('username', 'Your username') + @api.param('password', 'Your password') + def get(self): + username = request.args['username'] if request.args.get('username') else False + password = request.args['password'] if request.args.get('password') else False + if username in users_table and check_password_hash(users_table[username]['password'], password): + return {'authkey': users_table[username]['authkey']} + return {'error': 'User/Password invalid.'} + + @api.doc(body=token_request_fields) + def post(self): + auth: Dict = request.get_json(force=True) + if 'username' in auth and 'password' in auth: # Expected keys in json + if (auth['username'] in users_table + and check_password_hash(users_table[auth['username']]['password'], auth['password'])): + return {'authkey': users_table[auth['username']]['authkey']} + return {'error': 'User/Password invalid.'} + + +@api.route('/json//status') +@api.doc(description='Get the status of a capture', + params={'tree_uuid': 'The UUID of the capture'}) +class CaptureStatusQuery(Resource): + def get(self, tree_uuid: str): + return {'status_code': lookyloo.get_capture_status(tree_uuid)} + + +@api.route('/json//redirects') +@api.doc(description='Get all the redirects of a capture', + params={'tree_uuid': 'The UUID of the capture'}) +class CaptureRedirects(Resource): + def get(self, tree_uuid: str): + cache = lookyloo.capture_cache(tree_uuid) + if not cache: + return {'error': 'UUID missing in cache, try again later.'} + + to_return: Dict[str, Any] = {'response': {'url': cache.url, 'redirects': []}} + if not cache.redirects: + to_return['response']['info'] = 'No redirects' + return to_return + if cache.incomplete_redirects: + # Trigger tree build, get all redirects + lookyloo.get_crawled_tree(tree_uuid) + cache = lookyloo.capture_cache(tree_uuid) + if cache: + to_return['response']['redirects'] = cache.redirects + else: + to_return['response']['redirects'] = cache.redirects + + return to_return + + +@api.route('/json//misp_export') +@api.doc(description='Get an export of the capture in MISP format', + params={'tree_uuid': 'The UUID of the capture'}) +class MISPExport(Resource): + def get(self, tree_uuid: str): + with_parents = request.args.get('with_parents') + event = lookyloo.misp_export(tree_uuid, True if with_parents else False) + if isinstance(event, dict): + return event + + to_return = [] + for e in event: + to_return.append(e.to_json(indent=2)) + return to_return + + +misp_push_fields = api.model('MISPPushFields', { + 'allow_duplicates': fields.Integer(description="Push the event even if it is already present on the MISP instance", + example=0, min=0, max=1), + 'with_parents': fields.Integer(description="Also push the parents of the capture (if any)", + example=0, min=0, max=1), +}) + + +@api.route('/json//misp_push') +@api.doc(description='Push an event to a pre-configured MISP instance', + params={'tree_uuid': 'The UUID of the capture'}, + security='apikey') +class MISPPush(Resource): + method_decorators = [api_auth_check] + + @api.param('with_parents', 'Also push the parents of the capture (if any)') + @api.param('allow_duplicates', 'Push the event even if it is already present on the MISP instance') + def get(self, tree_uuid: str): + with_parents = True if request.args.get('with_parents') else False + allow_duplicates = True if request.args.get('allow_duplicates') else False + to_return: Dict = {} + if not lookyloo.misp.available: + to_return['error'] = 'MISP module not available.' + elif not lookyloo.misp.enable_push: + to_return['error'] = 'Push not enabled in MISP module.' + else: + event = lookyloo.misp_export(tree_uuid, with_parents) + if isinstance(event, dict): + to_return['error'] = event + else: + new_events = lookyloo.misp.push(event, allow_duplicates) + if isinstance(new_events, dict): + to_return['error'] = new_events + else: + events_to_return = [] + for e in new_events: + events_to_return.append(e.to_json(indent=2)) + return events_to_return + + return to_return + + @api.doc(body=misp_push_fields) + def post(self, tree_uuid: str): + parameters: Dict = request.get_json(force=True) + with_parents = True if parameters.get('with_parents') else False + allow_duplicates = True if parameters.get('allow_duplicates') else False + + to_return: Dict = {} + if not lookyloo.misp.available: + to_return['error'] = 'MISP module not available.' + elif not lookyloo.misp.enable_push: + to_return['error'] = 'Push not enabled in MISP module.' + else: + event = lookyloo.misp_export(tree_uuid, with_parents) + if isinstance(event, dict): + to_return['error'] = event + else: + new_events = lookyloo.misp.push(event, allow_duplicates) + if isinstance(new_events, dict): + to_return['error'] = new_events + else: + events_to_return = [] + for e in new_events: + events_to_return.append(e.to_json(indent=2)) + return events_to_return + + return to_return + + +@api.route('/json/hash_info/') +@api.doc(description='Search for a ressource with a specific hash (sha512)', + params={'h': 'The hash (sha512)'}) +class HashInfo(Resource): + def get(self, h: str): + details, body = lookyloo.get_body_hash_full(h) + if not details: + return {'error': 'Unknown Hash.'} + to_return: Dict[str, Any] = {'response': {'hash': h, 'details': details, + 'body': base64.b64encode(body.getvalue()).decode()}} + return to_return + + +url_info_fields = api.model('URLInfoFields', { + 'url': fields.String(description="The URL to search", required=True), + 'limit': fields.Integer(description="The maximal amount of captures to return", example=20), +}) + + +@api.route('/json/url_info') +@api.doc(description='Search for a URL') +class URLInfo(Resource): + + @api.doc(body=url_info_fields) + def post(self): + to_query: Dict = request.get_json(force=True) + occurrences = lookyloo.get_url_occurrences(to_query.pop('url'), **to_query) + return occurrences + + +hostname_info_fields = api.model('HostnameInfoFields', { + 'hostname': fields.String(description="The hostname to search", required=True), + 'limit': fields.Integer(description="The maximal amount of captures to return", example=20), +}) + + +@api.route('/json/hostname_info') +@api.doc(description='Search for a hostname') +class HostnameInfo(Resource): + + @api.doc(body=hostname_info_fields) + def post(self): + to_query: Dict = request.get_json(force=True) + occurrences = lookyloo.get_hostname_occurrences(to_query.pop('hostname'), **to_query) + return occurrences + + +@api.route('/json/stats') +@api.doc(description='Get the statistics of the lookyloo instance.') +class InstanceStats(Resource): + def get(self): + return lookyloo.get_stats() + + +submit_fields = api.model('SubmitFields', { + 'url': fields.String(description="The URL to capture", required=True), + 'listing': fields.Integer(description="Display the capture on the index", min=0, max=1, example=1), + 'user_agent': fields.String(description="User agent to use for the capture", example=''), + 'referer': fields.String(description="Referer to pass to the capture", example=''), + 'cookies': fields.String(description="JSON export of a list of cookies as exported from an other capture", example='') +}) + + +@api.route('/submit') +class SubmitCapture(Resource): + + @api.doc(body=submit_fields) + def post(self): + if flask_login.current_user.is_authenticated: + user = flask_login.current_user.get_id() + else: + user = src_request_ip(request) + to_query: Dict = request.get_json(force=True) + perma_uuid = lookyloo.enqueue_capture(to_query, source='api', user=user, authenticated=flask_login.current_user.is_authenticated) + return Response(perma_uuid, mimetype='text/text') From e5b76b3a2f2fdccc6fdb0825a0c83286442834b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 7 Jun 2021 13:12:23 -0700 Subject: [PATCH 02/18] chg: Move API into a new file, cleanup --- website/web/__init__.py | 463 ++++++++------------------------------ website/web/genericapi.py | 258 +++++++++++++++++++++ website/web/helpers.py | 91 ++++++++ 3 files changed, 438 insertions(+), 374 deletions(-) create mode 100644 website/web/genericapi.py create mode 100644 website/web/helpers.py diff --git a/website/web/__init__.py b/website/web/__init__.py index 0a7b81c1..f9237b51 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -1,17 +1,13 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import base64 from io import BytesIO, StringIO -import os -from pathlib import Path from datetime import datetime, timedelta, timezone import json import http import calendar from typing import Optional, Dict, Any, Union, List import logging -import hashlib from urllib.parse import quote_plus, unquote_plus import time import pkg_resources @@ -19,28 +15,24 @@ import pkg_resources from flask import Flask, render_template, request, send_file, redirect, url_for, Response, flash, jsonify from flask_bootstrap import Bootstrap # type: ignore import flask_login # type: ignore -from flask_restx import Resource, Api, fields, abort # type: ignore -from werkzeug.security import generate_password_hash, check_password_hash +from flask_restx import Api # type: ignore +from .genericapi import api as generic_api +from werkzeug.security import check_password_hash from pymisp import MISPEvent, MISPServerError -from lookyloo.helpers import (get_homedir, update_user_agents, get_user_agents, get_config, +from lookyloo.helpers import (update_user_agents, get_user_agents, get_config, get_taxonomies, load_cookies, CaptureStatus) from lookyloo.lookyloo import Lookyloo, Indexing from lookyloo.exceptions import NoValidHarFile, MissingUUID + from .proxied import ReverseProxied +from .helpers import src_request_ip, User, load_user_from_request, build_users_table, get_secret_key app: Flask = Flask(__name__) app.wsgi_app = ReverseProxied(app.wsgi_app) # type: ignore -secret_file_path: Path = get_homedir() / 'secret_key' - -if not secret_file_path.exists() or secret_file_path.stat().st_size < 64: - with secret_file_path.open('wb') as f: - f.write(os.urandom(64)) - -with secret_file_path.open('rb') as f: - app.config['SECRET_KEY'] = f.read() +app.config['SECRET_KEY'] = get_secret_key() Bootstrap(app) app.config['BOOTSTRAP_SERVE_LOCAL'] = True @@ -51,45 +43,11 @@ app.debug = False # Auth stuff login_manager = flask_login.LoginManager() login_manager.init_app(app) -try: - # Use legacy user mgmt, no need to print a warning, and it will fail on new install. - users = get_config('generic', 'cache_clean_user', quiet=True) -except Exception: - users = get_config('generic', 'users') - -users_table: Dict[str, Dict[str, str]] = {} -for username, authstuff in users.items(): - if isinstance(authstuff, str): - # just a password, make a key - users_table[username] = {} - users_table[username]['password'] = generate_password_hash(authstuff) - users_table[username]['authkey'] = hashlib.pbkdf2_hmac('sha256', - app.config['SECRET_KEY'], - authstuff.encode(), - 100000).hex() - - elif isinstance(authstuff, list) and len(authstuff) == 2: - if isinstance(authstuff[0], str) and isinstance(authstuff[1], str) and len(authstuff[1]) == 64: - users_table[username] = {} - users_table[username]['password'] = generate_password_hash(authstuff[0]) - users_table[username]['authkey'] = authstuff[1] - - if username not in users_table: - raise Exception('User setup invalid. Must be "username": "password" or "username": ["password", "token 64 chars (sha256)"]') - -keys_table = {} -for username, authstuff in users_table.items(): - if 'authkey' in authstuff: - keys_table[authstuff['authkey']] = username - - -class User(flask_login.UserMixin): - pass @login_manager.user_loader def user_loader(username): - if username not in users_table: + if username not in build_users_table(): return None user = User() user.id = username @@ -97,16 +55,8 @@ def user_loader(username): @login_manager.request_loader -def load_user_from_request(request): - api_key = request.headers.get('Authorization') - if not api_key: - return None - user = User() - api_key = api_key.strip() - if api_key in keys_table: - user.id = keys_table[api_key] - return user - return None +def _load_user_from_request(request): + return load_user_from_request(request) @app.route('/login', methods=['GET', 'POST']) @@ -121,6 +71,7 @@ def login(): ''' username = request.form['username'] + users_table = build_users_table() if username in users_table and check_password_hash(users_table[username]['password'], request.form['password']): user = User() user.id = username @@ -196,14 +147,6 @@ app.jinja_env.globals.update(month_name=month_name) # ##### Generic/configuration methods ##### -def src_request_ip(request) -> str: - # NOTE: X-Real-IP is the IP passed by the reverse proxy in the headers. - real_ip = request.headers.get('X-Real-IP') - if not real_ip: - real_ip = request.remote_addr - return real_ip - - @app.after_request def after_request(response): # We keep a list user agents in order to build a list to use in the capture @@ -351,6 +294,76 @@ def web_misp_lookup_view(tree_uuid: str): return render_template('misp_lookup.html', uuid=tree_uuid, hits=hits, misp_root_url=misp_root_url) +@app.route('/tree//misp_push', methods=['GET', 'POST']) +@flask_login.login_required +def web_misp_push_view(tree_uuid: str): + error = False + if not lookyloo.misp.available: + flash('MISP module not available.', 'error') + return redirect(url_for('tree', tree_uuid=tree_uuid)) + elif not lookyloo.misp.enable_push: + flash('Push not enabled in MISP module.', 'error') + return redirect(url_for('tree', tree_uuid=tree_uuid)) + else: + event = lookyloo.misp_export(tree_uuid) + if isinstance(event, dict): + flash(f'Unable to generate the MISP export: {event}', 'error') + return redirect(url_for('tree', tree_uuid=tree_uuid)) + + if request.method == 'POST': + # event is a MISPEvent at this point + # Submit the event + tags = request.form.getlist('tags') + error = False + events: List[MISPEvent] = [] + with_parents = request.form.get('with_parents') + if with_parents: + exports = lookyloo.misp_export(tree_uuid, True) + if isinstance(exports, dict): + flash(f'Unable to create event: {exports}', 'error') + error = True + else: + events = exports + else: + events = event + + if error: + return redirect(url_for('tree', tree_uuid=tree_uuid)) + + for e in events: + for tag in tags: + e.add_tag(tag) + + # Change the event info field of the last event in the chain + events[-1].info = request.form.get('event_info') + + try: + new_events = lookyloo.misp.push(events, True if request.form.get('force_push') else False, + True if request.form.get('auto_publish') else False) + except MISPServerError: + flash(f'MISP returned an error, the event(s) might still have been created on {lookyloo.misp.client.root_url}', 'error') + else: + if isinstance(new_events, dict): + flash(f'Unable to create event(s): {new_events}', 'error') + else: + for e in new_events: + flash(f'MISP event {e.id} created on {lookyloo.misp.client.root_url}', 'success') + return redirect(url_for('tree', tree_uuid=tree_uuid)) + else: + # the 1st attribute in the event is the link to lookyloo + existing_misp_url = lookyloo.misp.get_existing_event_url(event[-1].attributes[0].value) + + fav_tags = lookyloo.misp.get_fav_tags() + cache = lookyloo.capture_cache(tree_uuid) + + return render_template('misp_push_view.html', tree_uuid=tree_uuid, + event=event[0], fav_tags=fav_tags, + existing_event=existing_misp_url, + auto_publish=lookyloo.misp.auto_publish, + has_parent=True if cache and cache.parent else False, + default_tags=lookyloo.misp.default_tags) + + @app.route('/tree//modules', methods=['GET']) def modules(tree_uuid: str): modules_responses = lookyloo.get_modules_responses(tree_uuid) @@ -791,6 +804,13 @@ def statsfull(): return render_template('stats.html', stats=stats) +@app.route('/whois/', methods=['GET']) +def whois(query: str): + to_return = lookyloo.uwhois.whois(query) + return send_file(BytesIO(to_return.encode()), + mimetype='test/plain', as_attachment=True, attachment_filename=f'whois.{query}.txt') + + # ##### Methods related to a specific URLNode ##### @app.route('/tree//url//request_cookies', methods=['GET']) @@ -942,83 +962,6 @@ def add_context(tree_uuid: str, node_uuid: str): return redirect(url_for('ressources')) -@app.route('/tree//misp_push', methods=['GET', 'POST']) -@flask_login.login_required -def web_misp_push_view(tree_uuid: str): - error = False - if not lookyloo.misp.available: - flash('MISP module not available.', 'error') - return redirect(url_for('tree', tree_uuid=tree_uuid)) - elif not lookyloo.misp.enable_push: - flash('Push not enabled in MISP module.', 'error') - return redirect(url_for('tree', tree_uuid=tree_uuid)) - else: - event = lookyloo.misp_export(tree_uuid) - if isinstance(event, dict): - flash(f'Unable to generate the MISP export: {event}', 'error') - return redirect(url_for('tree', tree_uuid=tree_uuid)) - - if request.method == 'POST': - # event is a MISPEvent at this point - # Submit the event - tags = request.form.getlist('tags') - error = False - events: List[MISPEvent] = [] - with_parents = request.form.get('with_parents') - if with_parents: - exports = lookyloo.misp_export(tree_uuid, True) - if isinstance(exports, dict): - flash(f'Unable to create event: {exports}', 'error') - error = True - else: - events = exports - else: - events = event - - if error: - return redirect(url_for('tree', tree_uuid=tree_uuid)) - - for e in events: - for tag in tags: - e.add_tag(tag) - - # Change the event info field of the last event in the chain - events[-1].info = request.form.get('event_info') - - try: - new_events = lookyloo.misp.push(events, True if request.form.get('force_push') else False, - True if request.form.get('auto_publish') else False) - except MISPServerError: - flash(f'MISP returned an error, the event(s) might still have been created on {lookyloo.misp.client.root_url}', 'error') - else: - if isinstance(new_events, dict): - flash(f'Unable to create event(s): {new_events}', 'error') - else: - for e in new_events: - flash(f'MISP event {e.id} created on {lookyloo.misp.client.root_url}', 'success') - return redirect(url_for('tree', tree_uuid=tree_uuid)) - else: - # the 1st attribute in the event is the link to lookyloo - existing_misp_url = lookyloo.misp.get_existing_event_url(event[-1].attributes[0].value) - - fav_tags = lookyloo.misp.get_fav_tags() - cache = lookyloo.capture_cache(tree_uuid) - - return render_template('misp_push_view.html', tree_uuid=tree_uuid, - event=event[0], fav_tags=fav_tags, - existing_event=existing_misp_url, - auto_publish=lookyloo.misp.auto_publish, - has_parent=True if cache and cache.parent else False, - default_tags=lookyloo.misp.default_tags) - - -@app.route('/whois/', methods=['GET']) -def whois(query: str): - to_return = lookyloo.uwhois.whois(query) - return send_file(BytesIO(to_return.encode()), - mimetype='test/plain', as_attachment=True, attachment_filename=f'whois.{query}.txt') - - # Query API authorizations = { @@ -1035,232 +978,4 @@ api = Api(app, title='Lookyloo API', authorizations=authorizations, version=pkg_resources.get_distribution('lookyloo').version) - -def api_auth_check(method): - if flask_login.current_user.is_authenticated or load_user_from_request(request): - return method - abort(403, 'Authentication required.') - - -token_request_fields = api.model('AuthTokenFields', { - 'username': fields.String(description="Your username", required=True), - 'password': fields.String(description="Your password", required=True), -}) - - -@api.route('/json/get_token') -@api.doc(description='Get the API token required for authenticated calls') -class AuthToken(Resource): - - @api.param('username', 'Your username') - @api.param('password', 'Your password') - def get(self): - username = request.args['username'] if request.args.get('username') else False - password = request.args['password'] if request.args.get('password') else False - if username in users_table and check_password_hash(users_table[username]['password'], password): - return {'authkey': users_table[username]['authkey']} - return {'error': 'User/Password invalid.'} - - @api.doc(body=token_request_fields) - def post(self): - auth: Dict = request.get_json(force=True) - if 'username' in auth and 'password' in auth: # Expected keys in json - if (auth['username'] in users_table - and check_password_hash(users_table[auth['username']]['password'], auth['password'])): - return {'authkey': users_table[auth['username']]['authkey']} - return {'error': 'User/Password invalid.'} - - -@api.route('/json//status') -@api.doc(description='Get the status of a capture', - params={'tree_uuid': 'The UUID of the capture'}) -class CaptureStatusQuery(Resource): - def get(self, tree_uuid: str): - return {'status_code': lookyloo.get_capture_status(tree_uuid)} - - -@api.route('/json//redirects') -@api.doc(description='Get all the redirects of a capture', - params={'tree_uuid': 'The UUID of the capture'}) -class CaptureRedirects(Resource): - def get(self, tree_uuid: str): - cache = lookyloo.capture_cache(tree_uuid) - if not cache: - return {'error': 'UUID missing in cache, try again later.'} - - to_return: Dict[str, Any] = {'response': {'url': cache.url, 'redirects': []}} - if not cache.redirects: - to_return['response']['info'] = 'No redirects' - return to_return - if cache.incomplete_redirects: - # Trigger tree build, get all redirects - lookyloo.get_crawled_tree(tree_uuid) - cache = lookyloo.capture_cache(tree_uuid) - if cache: - to_return['response']['redirects'] = cache.redirects - else: - to_return['response']['redirects'] = cache.redirects - - return to_return - - -@api.route('/json//misp_export') -@api.doc(description='Get an export of the capture in MISP format', - params={'tree_uuid': 'The UUID of the capture'}) -class MISPExport(Resource): - def get(self, tree_uuid: str): - with_parents = request.args.get('with_parents') - event = lookyloo.misp_export(tree_uuid, True if with_parents else False) - if isinstance(event, dict): - return event - - to_return = [] - for e in event: - to_return.append(e.to_json(indent=2)) - return to_return - - -misp_push_fields = api.model('MISPPushFields', { - 'allow_duplicates': fields.Integer(description="Push the event even if it is already present on the MISP instance", - example=0, min=0, max=1), - 'with_parents': fields.Integer(description="Also push the parents of the capture (if any)", - example=0, min=0, max=1), -}) - - -@api.route('/json//misp_push') -@api.doc(description='Push an event to a pre-configured MISP instance', - params={'tree_uuid': 'The UUID of the capture'}, - security='apikey') -class MISPPush(Resource): - method_decorators = [api_auth_check] - - @api.param('with_parents', 'Also push the parents of the capture (if any)') - @api.param('allow_duplicates', 'Push the event even if it is already present on the MISP instance') - def get(self, tree_uuid: str): - with_parents = True if request.args.get('with_parents') else False - allow_duplicates = True if request.args.get('allow_duplicates') else False - to_return: Dict = {} - if not lookyloo.misp.available: - to_return['error'] = 'MISP module not available.' - elif not lookyloo.misp.enable_push: - to_return['error'] = 'Push not enabled in MISP module.' - else: - event = lookyloo.misp_export(tree_uuid, with_parents) - if isinstance(event, dict): - to_return['error'] = event - else: - new_events = lookyloo.misp.push(event, allow_duplicates) - if isinstance(new_events, dict): - to_return['error'] = new_events - else: - events_to_return = [] - for e in new_events: - events_to_return.append(e.to_json(indent=2)) - return events_to_return - - return to_return - - @api.doc(body=misp_push_fields) - def post(self, tree_uuid: str): - parameters: Dict = request.get_json(force=True) - with_parents = True if parameters.get('with_parents') else False - allow_duplicates = True if parameters.get('allow_duplicates') else False - - to_return: Dict = {} - if not lookyloo.misp.available: - to_return['error'] = 'MISP module not available.' - elif not lookyloo.misp.enable_push: - to_return['error'] = 'Push not enabled in MISP module.' - else: - event = lookyloo.misp_export(tree_uuid, with_parents) - if isinstance(event, dict): - to_return['error'] = event - else: - new_events = lookyloo.misp.push(event, allow_duplicates) - if isinstance(new_events, dict): - to_return['error'] = new_events - else: - events_to_return = [] - for e in new_events: - events_to_return.append(e.to_json(indent=2)) - return events_to_return - - return to_return - - -@api.route('/json/hash_info/') -@api.doc(description='Search for a ressource with a specific hash (sha512)', - params={'h': 'The hash (sha512)'}) -class HashInfo(Resource): - def get(self, h: str): - details, body = lookyloo.get_body_hash_full(h) - if not details: - return {'error': 'Unknown Hash.'} - to_return: Dict[str, Any] = {'response': {'hash': h, 'details': details, - 'body': base64.b64encode(body.getvalue()).decode()}} - return to_return - - -url_info_fields = api.model('URLInfoFields', { - 'url': fields.String(description="The URL to search", required=True), - 'limit': fields.Integer(description="The maximal amount of captures to return", example=20), -}) - - -@api.route('/json/url_info') -@api.doc(description='Search for a URL') -class URLInfo(Resource): - - @api.doc(body=url_info_fields) - def post(self): - to_query: Dict = request.get_json(force=True) - occurrences = lookyloo.get_url_occurrences(to_query.pop('url'), **to_query) - return occurrences - - -hostname_info_fields = api.model('HostnameInfoFields', { - 'hostname': fields.String(description="The hostname to search", required=True), - 'limit': fields.Integer(description="The maximal amount of captures to return", example=20), -}) - - -@api.route('/json/hostname_info') -@api.doc(description='Search for a hostname') -class HostnameInfo(Resource): - - @api.doc(body=hostname_info_fields) - def post(self): - to_query: Dict = request.get_json(force=True) - occurrences = lookyloo.get_hostname_occurrences(to_query.pop('hostname'), **to_query) - return occurrences - - -@api.route('/json/stats') -@api.doc(description='Get the statistics of the lookyloo instance.') -class InstanceStats(Resource): - def get(self): - return lookyloo.get_stats() - - -submit_fields = api.model('SubmitFields', { - 'url': fields.String(description="The URL to capture", required=True), - 'listing': fields.Integer(description="Display the capture on the index", min=0, max=1, example=1), - 'user_agent': fields.String(description="User agent to use for the capture", example=''), - 'referer': fields.String(description="Referer to pass to the capture", example=''), - 'cookies': fields.String(description="JSON export of a list of cookies as exported from an other capture", example='') -}) - - -@api.route('/submit') -class SubmitCapture(Resource): - - @api.doc(body=submit_fields) - def post(self): - if flask_login.current_user.is_authenticated: - user = flask_login.current_user.get_id() - else: - user = src_request_ip(request) - to_query: Dict = request.get_json(force=True) - perma_uuid = lookyloo.enqueue_capture(to_query, source='api', user=user, authenticated=flask_login.current_user.is_authenticated) - return Response(perma_uuid, mimetype='text/text') +api.add_namespace(generic_api) diff --git a/website/web/genericapi.py b/website/web/genericapi.py new file mode 100644 index 00000000..434069eb --- /dev/null +++ b/website/web/genericapi.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import base64 +from typing import Dict, Any + +from flask import request, Response +import flask_login # type: ignore +from flask_restx import Namespace, Resource, fields, abort # type: ignore +from werkzeug.security import check_password_hash + +from lookyloo.lookyloo import Lookyloo + +from .helpers import src_request_ip, load_user_from_request, build_users_table + +api = Namespace('GenericAPI', description='Generic Lookyloo API', path='/') + + +lookyloo: Lookyloo = Lookyloo() + + +def api_auth_check(method): + if flask_login.current_user.is_authenticated or load_user_from_request(request): + return method + abort(403, 'Authentication required.') + + +token_request_fields = api.model('AuthTokenFields', { + 'username': fields.String(description="Your username", required=True), + 'password': fields.String(description="Your password", required=True), +}) + + +@api.route('/json/get_token') +@api.doc(description='Get the API token required for authenticated calls') +class AuthToken(Resource): + + users_table = build_users_table() + + @api.param('username', 'Your username') + @api.param('password', 'Your password') + def get(self): + username = request.args['username'] if request.args.get('username') else False + password = request.args['password'] if request.args.get('password') else False + if username in self.users_table and check_password_hash(self.users_table[username]['password'], password): + return {'authkey': self.users_table[username]['authkey']} + return {'error': 'User/Password invalid.'} + + @api.doc(body=token_request_fields) + def post(self): + auth: Dict = request.get_json(force=True) + if 'username' in auth and 'password' in auth: # Expected keys in json + if (auth['username'] in self.users_table + and check_password_hash(self.users_table[auth['username']]['password'], auth['password'])): + return {'authkey': self.users_table[auth['username']]['authkey']} + return {'error': 'User/Password invalid.'} + + +@api.route('/json//status') +@api.doc(description='Get the status of a capture', + params={'tree_uuid': 'The UUID of the capture'}) +class CaptureStatusQuery(Resource): + def get(self, tree_uuid: str): + return {'status_code': lookyloo.get_capture_status(tree_uuid)} + + +@api.route('/json//redirects') +@api.doc(description='Get all the redirects of a capture', + params={'tree_uuid': 'The UUID of the capture'}) +class CaptureRedirects(Resource): + def get(self, tree_uuid: str): + cache = lookyloo.capture_cache(tree_uuid) + if not cache: + return {'error': 'UUID missing in cache, try again later.'} + + to_return: Dict[str, Any] = {'response': {'url': cache.url, 'redirects': []}} + if not cache.redirects: + to_return['response']['info'] = 'No redirects' + return to_return + if cache.incomplete_redirects: + # Trigger tree build, get all redirects + lookyloo.get_crawled_tree(tree_uuid) + cache = lookyloo.capture_cache(tree_uuid) + if cache: + to_return['response']['redirects'] = cache.redirects + else: + to_return['response']['redirects'] = cache.redirects + + return to_return + + +@api.route('/json//misp_export') +@api.doc(description='Get an export of the capture in MISP format', + params={'tree_uuid': 'The UUID of the capture'}) +class MISPExport(Resource): + def get(self, tree_uuid: str): + with_parents = request.args.get('with_parents') + event = lookyloo.misp_export(tree_uuid, True if with_parents else False) + if isinstance(event, dict): + return event + + to_return = [] + for e in event: + to_return.append(e.to_json(indent=2)) + return to_return + + +misp_push_fields = api.model('MISPPushFields', { + 'allow_duplicates': fields.Integer(description="Push the event even if it is already present on the MISP instance", + example=0, min=0, max=1), + 'with_parents': fields.Integer(description="Also push the parents of the capture (if any)", + example=0, min=0, max=1), +}) + + +@api.route('/json//misp_push') +@api.doc(description='Push an event to a pre-configured MISP instance', + params={'tree_uuid': 'The UUID of the capture'}, + security='apikey') +class MISPPush(Resource): + method_decorators = [api_auth_check] + + @api.param('with_parents', 'Also push the parents of the capture (if any)') + @api.param('allow_duplicates', 'Push the event even if it is already present on the MISP instance') + def get(self, tree_uuid: str): + with_parents = True if request.args.get('with_parents') else False + allow_duplicates = True if request.args.get('allow_duplicates') else False + to_return: Dict = {} + if not lookyloo.misp.available: + to_return['error'] = 'MISP module not available.' + elif not lookyloo.misp.enable_push: + to_return['error'] = 'Push not enabled in MISP module.' + else: + event = lookyloo.misp_export(tree_uuid, with_parents) + if isinstance(event, dict): + to_return['error'] = event + else: + new_events = lookyloo.misp.push(event, allow_duplicates) + if isinstance(new_events, dict): + to_return['error'] = new_events + else: + events_to_return = [] + for e in new_events: + events_to_return.append(e.to_json(indent=2)) + return events_to_return + + return to_return + + @api.doc(body=misp_push_fields) + def post(self, tree_uuid: str): + parameters: Dict = request.get_json(force=True) + with_parents = True if parameters.get('with_parents') else False + allow_duplicates = True if parameters.get('allow_duplicates') else False + + to_return: Dict = {} + if not lookyloo.misp.available: + to_return['error'] = 'MISP module not available.' + elif not lookyloo.misp.enable_push: + to_return['error'] = 'Push not enabled in MISP module.' + else: + event = lookyloo.misp_export(tree_uuid, with_parents) + if isinstance(event, dict): + to_return['error'] = event + else: + new_events = lookyloo.misp.push(event, allow_duplicates) + if isinstance(new_events, dict): + to_return['error'] = new_events + else: + events_to_return = [] + for e in new_events: + events_to_return.append(e.to_json(indent=2)) + return events_to_return + + return to_return + + +@api.route('/json/hash_info/') +@api.doc(description='Search for a ressource with a specific hash (sha512)', + params={'h': 'The hash (sha512)'}) +class HashInfo(Resource): + def get(self, h: str): + details, body = lookyloo.get_body_hash_full(h) + if not details: + return {'error': 'Unknown Hash.'} + to_return: Dict[str, Any] = {'response': {'hash': h, 'details': details, + 'body': base64.b64encode(body.getvalue()).decode()}} + return to_return + + +url_info_fields = api.model('URLInfoFields', { + 'url': fields.String(description="The URL to search", required=True), + 'limit': fields.Integer(description="The maximal amount of captures to return", example=20), +}) + + +@api.route('/json/url_info') +@api.doc(description='Search for a URL') +class URLInfo(Resource): + + @api.doc(body=url_info_fields) + def post(self): + to_query: Dict = request.get_json(force=True) + occurrences = lookyloo.get_url_occurrences(to_query.pop('url'), **to_query) + return occurrences + + +hostname_info_fields = api.model('HostnameInfoFields', { + 'hostname': fields.String(description="The hostname to search", required=True), + 'limit': fields.Integer(description="The maximal amount of captures to return", example=20), +}) + + +@api.route('/json/hostname_info') +@api.doc(description='Search for a hostname') +class HostnameInfo(Resource): + + @api.doc(body=hostname_info_fields) + def post(self): + to_query: Dict = request.get_json(force=True) + occurrences = lookyloo.get_hostname_occurrences(to_query.pop('hostname'), **to_query) + return occurrences + + +@api.route('/json/stats') +@api.doc(description='Get the statistics of the lookyloo instance.') +class InstanceStats(Resource): + def get(self): + return lookyloo.get_stats() + + +submit_fields = api.model('SubmitFields', { + 'url': fields.String(description="The URL to capture", required=True), + 'listing': fields.Integer(description="Display the capture on the index", min=0, max=1, example=1), + 'user_agent': fields.String(description="User agent to use for the capture", example=''), + 'referer': fields.String(description="Referer to pass to the capture", example=''), + 'cookies': fields.String(description="JSON export of a list of cookies as exported from an other capture", example='') +}) + + +@api.route('/submit') +class SubmitCapture(Resource): + + @api.doc(body=submit_fields) + def post(self): + if flask_login.current_user.is_authenticated: + user = flask_login.current_user.get_id() + else: + user = src_request_ip(request) + to_query: Dict = request.get_json(force=True) + perma_uuid = lookyloo.enqueue_capture(to_query, source='api', user=user, authenticated=flask_login.current_user.is_authenticated) + return Response(perma_uuid, mimetype='text/text') + + +@api.route('/json//stats') +@api.doc(description='Get the statistics of the capture.') +class CaptureStats(Resource): + def get(self, tree_uuid: str): + return lookyloo.get_statistics(tree_uuid) diff --git a/website/web/helpers.py b/website/web/helpers.py new file mode 100644 index 00000000..74a083e4 --- /dev/null +++ b/website/web/helpers.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import hashlib +import os + +from functools import lru_cache +from pathlib import Path +from typing import Dict, List, Union + +import flask_login # type: ignore + +from werkzeug.security import generate_password_hash + +from lookyloo.helpers import get_homedir, get_config + + +def src_request_ip(request) -> str: + # NOTE: X-Real-IP is the IP passed by the reverse proxy in the headers. + real_ip = request.headers.get('X-Real-IP') + if not real_ip: + real_ip = request.remote_addr + return real_ip + + +class User(flask_login.UserMixin): + pass + + +def load_user_from_request(request): + api_key = request.headers.get('Authorization') + if not api_key: + return None + user = User() + api_key = api_key.strip() + keys_table = build_keys_table() + if api_key in keys_table: + user.id = keys_table[api_key] + return user + return None + + +@lru_cache(64) +def build_keys_table() -> Dict[str, str]: + keys_table = {} + for username, authstuff in build_users_table().items(): + if 'authkey' in authstuff: + keys_table[authstuff['authkey']] = username + return keys_table + + +@lru_cache(64) +def get_users() -> Dict[str, Union[str, List[str]]]: + try: + # Use legacy user mgmt, no need to print a warning, and it will fail on new install. + return get_config('generic', 'cache_clean_user', quiet=True) + except Exception: + return get_config('generic', 'users') + + +@lru_cache(64) +def build_users_table() -> Dict[str, Dict[str, str]]: + users_table: Dict[str, Dict[str, str]] = {} + for username, authstuff in get_users().items(): + if isinstance(authstuff, str): + # just a password, make a key + users_table[username] = {} + users_table[username]['password'] = generate_password_hash(authstuff) + users_table[username]['authkey'] = hashlib.pbkdf2_hmac('sha256', get_secret_key(), + authstuff.encode(), + 100000).hex() + + elif isinstance(authstuff, list) and len(authstuff) == 2: + if isinstance(authstuff[0], str) and isinstance(authstuff[1], str) and len(authstuff[1]) == 64: + users_table[username] = {} + users_table[username]['password'] = generate_password_hash(authstuff[0]) + users_table[username]['authkey'] = authstuff[1] + else: + raise Exception('User setup invalid. Must be "username": "password" or "username": ["password", "token 64 chars (sha256)"]') + return users_table + + +@lru_cache(64) +def get_secret_key() -> bytes: + secret_file_path: Path = get_homedir() / 'secret_key' + if not secret_file_path.exists() or secret_file_path.stat().st_size < 64: + if not secret_file_path.exists() or secret_file_path.stat().st_size < 64: + with secret_file_path.open('wb') as f: + f.write(os.urandom(64)) + with secret_file_path.open('rb') as f: + return f.read() From 810cceb26378158dba4ee492f26b0fa76c3c81f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 7 Jun 2021 15:37:11 -0700 Subject: [PATCH 03/18] new: Add more calls to the API, cleanup --- website/web/__init__.py | 20 ++--- website/web/genericapi.py | 154 +++++++++++++++++++++++++++++++------- 2 files changed, 137 insertions(+), 37 deletions(-) diff --git a/website/web/__init__.py b/website/web/__init__.py index f9237b51..6638cd13 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -223,16 +223,6 @@ def hostnode_popup(tree_uuid: str, node_uuid: str): # ##### Tree level Methods ##### -@app.route('/tree//rebuild') -@flask_login.login_required -def rebuild_tree(tree_uuid: str): - try: - lookyloo.remove_pickle(tree_uuid) - return redirect(url_for('tree', tree_uuid=tree_uuid)) - except Exception: - return redirect(url_for('index')) - - @app.route('/tree//trigger_modules', methods=['GET']) def trigger_modules(tree_uuid: str): force = True if request.args.get('force') else False @@ -495,6 +485,16 @@ def hide_capture(tree_uuid: str): return redirect(url_for('tree', tree_uuid=tree_uuid)) +@app.route('/tree//rebuild') +@flask_login.login_required +def rebuild_tree(tree_uuid: str): + try: + lookyloo.remove_pickle(tree_uuid) + return redirect(url_for('tree', tree_uuid=tree_uuid)) + except Exception: + return redirect(url_for('index')) + + @app.route('/tree//cache', methods=['GET']) def cache_tree(tree_uuid: str): lookyloo.capture_cache(tree_uuid) diff --git a/website/web/genericapi.py b/website/web/genericapi.py index 434069eb..7bb7bc24 100644 --- a/website/web/genericapi.py +++ b/website/web/genericapi.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import json import base64 from typing import Dict, Any -from flask import request, Response +from flask import request, send_file import flask_login # type: ignore from flask_restx import Namespace, Resource, fields, abort # type: ignore from werkzeug.security import check_password_hash @@ -56,20 +57,20 @@ class AuthToken(Resource): return {'error': 'User/Password invalid.'} -@api.route('/json//status') +@api.route('/json//status') @api.doc(description='Get the status of a capture', - params={'tree_uuid': 'The UUID of the capture'}) + params={'capture_uuid': 'The UUID of the capture'}) class CaptureStatusQuery(Resource): - def get(self, tree_uuid: str): - return {'status_code': lookyloo.get_capture_status(tree_uuid)} + def get(self, capture_uuid: str): + return {'status_code': lookyloo.get_capture_status(capture_uuid)} -@api.route('/json//redirects') +@api.route('/json//redirects') @api.doc(description='Get all the redirects of a capture', - params={'tree_uuid': 'The UUID of the capture'}) + params={'capture_uuid': 'The UUID of the capture'}) class CaptureRedirects(Resource): - def get(self, tree_uuid: str): - cache = lookyloo.capture_cache(tree_uuid) + def get(self, capture_uuid: str): + cache = lookyloo.capture_cache(capture_uuid) if not cache: return {'error': 'UUID missing in cache, try again later.'} @@ -79,8 +80,8 @@ class CaptureRedirects(Resource): return to_return if cache.incomplete_redirects: # Trigger tree build, get all redirects - lookyloo.get_crawled_tree(tree_uuid) - cache = lookyloo.capture_cache(tree_uuid) + lookyloo.get_crawled_tree(capture_uuid) + cache = lookyloo.capture_cache(capture_uuid) if cache: to_return['response']['redirects'] = cache.redirects else: @@ -89,13 +90,13 @@ class CaptureRedirects(Resource): return to_return -@api.route('/json//misp_export') +@api.route('/json//misp_export') @api.doc(description='Get an export of the capture in MISP format', - params={'tree_uuid': 'The UUID of the capture'}) + params={'capture_uuid': 'The UUID of the capture'}) class MISPExport(Resource): - def get(self, tree_uuid: str): + def get(self, capture_uuid: str): with_parents = request.args.get('with_parents') - event = lookyloo.misp_export(tree_uuid, True if with_parents else False) + event = lookyloo.misp_export(capture_uuid, True if with_parents else False) if isinstance(event, dict): return event @@ -113,16 +114,16 @@ misp_push_fields = api.model('MISPPushFields', { }) -@api.route('/json//misp_push') +@api.route('/json//misp_push') @api.doc(description='Push an event to a pre-configured MISP instance', - params={'tree_uuid': 'The UUID of the capture'}, + params={'capture_uuid': 'The UUID of the capture'}, security='apikey') class MISPPush(Resource): method_decorators = [api_auth_check] @api.param('with_parents', 'Also push the parents of the capture (if any)') @api.param('allow_duplicates', 'Push the event even if it is already present on the MISP instance') - def get(self, tree_uuid: str): + def get(self, capture_uuid: str): with_parents = True if request.args.get('with_parents') else False allow_duplicates = True if request.args.get('allow_duplicates') else False to_return: Dict = {} @@ -131,7 +132,7 @@ class MISPPush(Resource): elif not lookyloo.misp.enable_push: to_return['error'] = 'Push not enabled in MISP module.' else: - event = lookyloo.misp_export(tree_uuid, with_parents) + event = lookyloo.misp_export(capture_uuid, with_parents) if isinstance(event, dict): to_return['error'] = event else: @@ -147,7 +148,7 @@ class MISPPush(Resource): return to_return @api.doc(body=misp_push_fields) - def post(self, tree_uuid: str): + def post(self, capture_uuid: str): parameters: Dict = request.get_json(force=True) with_parents = True if parameters.get('with_parents') else False allow_duplicates = True if parameters.get('allow_duplicates') else False @@ -158,7 +159,7 @@ class MISPPush(Resource): elif not lookyloo.misp.enable_push: to_return['error'] = 'Push not enabled in MISP module.' else: - event = lookyloo.misp_export(tree_uuid, with_parents) + event = lookyloo.misp_export(capture_uuid, with_parents) if isinstance(event, dict): to_return['error'] = event else: @@ -237,10 +238,29 @@ submit_fields = api.model('SubmitFields', { }) +@api.route('/json//stats') +@api.doc(description='Get the statistics of the capture.', + params={'capture_uuid': 'The UUID of the capture'}) +class CaptureStats(Resource): + def get(self, capture_uuid: str): + return lookyloo.get_statistics(capture_uuid) + + +@api.route('/json//cookies') +@api.doc(description='Get the complete cookie jar created during the capture.', + params={'capture_uuid': 'The UUID of the capture'}) +class CaptureCookies(Resource): + def get(self, capture_uuid: str): + return json.loads(lookyloo.get_cookies(capture_uuid).read()) + + +# Just text + @api.route('/submit') class SubmitCapture(Resource): @api.doc(body=submit_fields) + @api.produces(['text/text']) def post(self): if flask_login.current_user.is_authenticated: user = flask_login.current_user.get_id() @@ -248,11 +268,91 @@ class SubmitCapture(Resource): user = src_request_ip(request) to_query: Dict = request.get_json(force=True) perma_uuid = lookyloo.enqueue_capture(to_query, source='api', user=user, authenticated=flask_login.current_user.is_authenticated) - return Response(perma_uuid, mimetype='text/text') + return perma_uuid -@api.route('/json//stats') -@api.doc(description='Get the statistics of the capture.') -class CaptureStats(Resource): - def get(self, tree_uuid: str): - return lookyloo.get_statistics(tree_uuid) +# Binary stuff + +@api.route('/bin//screenshot') +@api.doc(description='Get the screenshot associated to the capture.', + params={'capture_uuid': 'The UUID of the capture'}) +class CaptureScreenshot(Resource): + + @api.produces(['image/png']) + def get(self, capture_uuid: str): + return send_file(lookyloo.get_screenshot(capture_uuid), mimetype='image/png') + + +@api.route('/bin//export') +@api.doc(description='Get all the files generated by the capture, except the pickle.', + params={'capture_uuid': 'The UUID of the capture'}) +class CaptureExport(Resource): + + @api.produces(['application/zip']) + def get(self, capture_uuid: str): + return send_file(lookyloo.get_capture(capture_uuid), mimetype='application/zip') + + +# Admin stuff + +@api.route('/admin/rebuild_all') +@api.doc(description='Rebuild all the trees. WARNING: IT IS GOING TO TAKE A VERY LONG TIME.', + security='apikey') +class RebuildAll(Resource): + method_decorators = [api_auth_check] + + def post(self): + try: + lookyloo.rebuild_all() + except Exception as e: + return {'error': f'Unable to rebuild all captures: {e}.'} + else: + return {'info': 'Captures successfully rebuilt.'} + + +@api.route('/admin/rebuild_all_cache') +@api.doc(description='Rebuild all the caches. It will take a while, but less that rebuild all.', + security='apikey') +class RebuildAllCache(Resource): + method_decorators = [api_auth_check] + + def post(self): + try: + lookyloo.rebuild_cache() + except Exception as e: + return {'error': f'Unable to rebuild all the caches: {e}.'} + else: + return {'info': 'All caches successfully rebuilt.'} + + +@api.route('/admin//rebuild') +@api.doc(description='Rebuild the tree.', + params={'capture_uuid': 'The UUID of the capture'}, + security='apikey') +class CaptureRebuildTree(Resource): + method_decorators = [api_auth_check] + + def post(self, capture_uuid): + try: + lookyloo.remove_pickle(capture_uuid) + lookyloo.get_crawled_tree(capture_uuid) + except Exception as e: + return {'error': f'Unable to rebuild tree: {e}.'} + else: + return {'info': f'Tree {capture_uuid} successfully rebuilt.'} + + +@api.route('/admin//hide') +@api.doc(description='Hide the capture from the index.', + params={'capture_uuid': 'The UUID of the capture'}, + security='apikey') +class CaptureHide(Resource): + method_decorators = [api_auth_check] + + def post(self, capture_uuid): + try: + lookyloo.hide_capture(capture_uuid) + except Exception as e: + return {'error': f'Unable to hide the tree: {e}.'} + else: + return {'info': f'Capture {capture_uuid} successfully hidden.'} From befcfbbe1dae7e3772d88b3b740a48d4abf74c04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 8 Jun 2021 08:46:24 -0700 Subject: [PATCH 04/18] chg: Bump deps --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 773ff23f..bd683642 100644 --- a/poetry.lock +++ b/poetry.lock @@ -793,7 +793,7 @@ docs = ["Sphinx (>=3.5.4,<4.0.0)", "myst-parser (>=0.14.0,<0.15.0)"] [[package]] name = "pymisp" -version = "2.4.143" +version = "2.4.144" description = "Python API for MISP." category = "main" optional = false @@ -1796,8 +1796,8 @@ pylookyloo = [ {file = "pylookyloo-1.6.0.tar.gz", hash = "sha256:3daa751f3400c6365a9c145bebc58f56b2f0605ee8eaf28a6ed6042bcc3800ed"}, ] pymisp = [ - {file = "pymisp-2.4.143-py3-none-any.whl", hash = "sha256:ceb6029045cfd2ab803902dc82e761909ca5337c2f6df484aef4e067e2f08d82"}, - {file = "pymisp-2.4.143.tar.gz", hash = "sha256:a51cddc4f274ec8daf7ccd0dd1fd599ee755ed77d168e72dfad01a41823877e3"}, + {file = "pymisp-2.4.144-py3-none-any.whl", hash = "sha256:81be4569199c117513d35d06f3cd46d5b412c1e8509ec1cfa0e6bfd19751ef4f"}, + {file = "pymisp-2.4.144.tar.gz", hash = "sha256:cf04fe19391fed4251cef1f267d80a19e37ed4b88602fb5f9790a3cd814d9d00"}, ] pyopenssl = [ {file = "pyOpenSSL-20.0.1-py2.py3-none-any.whl", hash = "sha256:818ae18e06922c066f777a33f1fca45786d85edfe71cd043de6379337a7f274b"}, From 3d2c7420a46ba7301e480f24c291ba9f08b6bad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 9 Jun 2021 12:04:26 -0700 Subject: [PATCH 05/18] chg: Bump mypy --- lookyloo/indexing.py | 18 +++---- lookyloo/lookyloo.py | 2 +- poetry.lock | 114 +++++++++++++++++++++---------------------- pyproject.toml | 4 +- 4 files changed, 68 insertions(+), 70 deletions(-) diff --git a/lookyloo/indexing.py b/lookyloo/indexing.py index fae26a37..c4df4196 100644 --- a/lookyloo/indexing.py +++ b/lookyloo/indexing.py @@ -37,7 +37,7 @@ class Indexing(): return self.redis.zrevrange(f'cn|{cookie_name}', 0, -1, withscores=True) def get_cookies_names_captures(self, cookie_name: str) -> List[Tuple[str, str]]: - return [uuids.split('|') for uuids in self.redis.smembers(f'cn|{cookie_name}|captures')] # type: ignore + return [uuids.split('|') for uuids in self.redis.smembers(f'cn|{cookie_name}|captures')] def index_cookies_capture(self, crawled_tree: CrawledTree) -> None: if self.redis.sismember('indexed_cookies', crawled_tree.uuid): @@ -74,8 +74,8 @@ class Indexing(): pipeline.zincrby('aggregate_domains_cn', cn_freq, f'{main_domain_part}|{cn}') pipeline.zincrby('aggregate_cn_domains', d_freq, f'{cn}|{main_domain_part}') pipeline.execute() - aggregate_domains_cn = self.redis.zrevrange('aggregate_domains_cn', 0, -1, withscores=True) - aggregate_cn_domains = self.redis.zrevrange('aggregate_cn_domains', 0, -1, withscores=True) + aggregate_domains_cn: List[Tuple[str, float]] = self.redis.zrevrange('aggregate_domains_cn', 0, -1, withscores=True) + aggregate_cn_domains: List[Tuple[str, float]] = self.redis.zrevrange('aggregate_cn_domains', 0, -1, withscores=True) self.redis.delete('aggregate_domains_cn') self.redis.delete('aggregate_cn_domains') return {'domains': aggregate_domains_cn, 'cookies': aggregate_cn_domains} @@ -129,7 +129,7 @@ class Indexing(): filter_capture_uuid: Optional[str]=None, limit: int=20) -> Tuple[int, List[Tuple[str, str, str, bool]]]: to_return: List[Tuple[str, str, str, bool]] = [] - all_captures: Set[str] = self.redis.smembers(f'bh|{body_hash}|captures') # type: ignore + all_captures: Set[str] = self.redis.smembers(f'bh|{body_hash}|captures') len_captures = len(all_captures) for capture_uuid in list(all_captures)[:limit]: if capture_uuid == filter_capture_uuid: @@ -149,7 +149,7 @@ class Indexing(): return self.redis.zrevrange(f'bh|{body_hash}', 0, -1, withscores=True) def get_body_hash_urls(self, body_hash: str) -> Dict[str, List[Dict[str, str]]]: - all_captures: Set[str] = self.redis.smembers(f'bh|{body_hash}|captures') # type: ignore + all_captures: Set[str] = self.redis.smembers(f'bh|{body_hash}|captures') urls = defaultdict(list) for capture_uuid in list(all_captures): for entry in self.redis.zrevrange(f'bh|{body_hash}|captures|{capture_uuid}', 0, -1): @@ -189,16 +189,16 @@ class Indexing(): def get_captures_url(self, url: str) -> Set[str]: m = hashlib.md5() m.update(url.encode()) - return self.redis.smembers(f'urls|{m.hexdigest()}|captures') # type: ignore + return self.redis.smembers(f'urls|{m.hexdigest()}|captures') def get_captures_hostname(self, hostname: str) -> Set[str]: - return self.redis.smembers(f'hostnames|{hostname}|captures') # type: ignore + return self.redis.smembers(f'hostnames|{hostname}|captures') # ###### Categories ###### @property def categories(self) -> List[Tuple[str, int]]: - return [(c, int(score)) + return [(c, int(score)) # type: ignore for c, score in self.redis.zrevrange('categories', 0, 200, withscores=True)] def index_categories_capture(self, capture_uuid: str, categories: Iterable[str]): @@ -217,4 +217,4 @@ class Indexing(): pipeline.execute() def get_captures_category(self, category: str) -> Set[str]: - return self.redis.smembers(category) # type: ignore + return self.redis.smembers(category) diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index 287fc39d..0f74c57e 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -657,7 +657,7 @@ class Lookyloo(): if not value or not value[0]: return None uuid, score = value[0] - queue = self.redis.get(f'{uuid}_mgmt') + queue: str = self.redis.get(f'{uuid}_mgmt') # type: ignore self.redis.sadd('ongoing', uuid) lazy_cleanup = self.redis.pipeline() diff --git a/poetry.lock b/poetry.lock index bd683642..6658e81f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -578,7 +578,7 @@ python-versions = ">=3.6" [[package]] name = "mypy" -version = "0.812" +version = "0.901" description = "Optional static typing for Python" category = "dev" optional = false @@ -586,11 +586,12 @@ python-versions = ">=3.5" [package.dependencies] mypy-extensions = ">=0.4.3,<0.5.0" -typed-ast = ">=1.4.0,<1.5.0" +toml = "*" typing-extensions = ">=3.7.4" [package.extras] dmypy = ["psutil (>=4.0)"] +python2 = ["typed-ast (>=1.4.0,<1.5.0)"] [[package]] name = "mypy-extensions" @@ -1024,6 +1025,14 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + [[package]] name = "traitlets" version = "5.0.5" @@ -1080,9 +1089,17 @@ optional = false python-versions = "*" [[package]] -name = "typed-ast" -version = "1.4.3" -description = "a fork of Python 2 and 3 ast modules with type comment support" +name = "types-redis" +version = "3.5.0" +description = "Typing stubs for redis" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "types-requests" +version = "0.1.9" +description = "Typing stubs for requests" category = "dev" optional = false python-versions = "*" @@ -1189,7 +1206,7 @@ misp = ["python-magic", "pydeep"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "27c09f95d480f1517a425ed98847d3096d0ec22b7cccf0727189a31e4206a9ed" +content-hash = "8ab8e928500dbdec94ae776c840a95e5b21b1f385eca361a75601677e9ac855d" [metadata.files] aiohttp = [ @@ -1615,28 +1632,29 @@ multidict = [ {file = "multidict-5.1.0.tar.gz", hash = "sha256:25b4e5f22d3a37ddf3effc0710ba692cfc792c2b9edfb9c05aefe823256e84d5"}, ] mypy = [ - {file = "mypy-0.812-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:a26f8ec704e5a7423c8824d425086705e381b4f1dfdef6e3a1edab7ba174ec49"}, - {file = "mypy-0.812-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:28fb5479c494b1bab244620685e2eb3c3f988d71fd5d64cc753195e8ed53df7c"}, - {file = "mypy-0.812-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:9743c91088d396c1a5a3c9978354b61b0382b4e3c440ce83cf77994a43e8c521"}, - {file = "mypy-0.812-cp35-cp35m-win_amd64.whl", hash = "sha256:d7da2e1d5f558c37d6e8c1246f1aec1e7349e4913d8fb3cb289a35de573fe2eb"}, - {file = "mypy-0.812-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:4eec37370483331d13514c3f55f446fc5248d6373e7029a29ecb7b7494851e7a"}, - {file = "mypy-0.812-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:d65cc1df038ef55a99e617431f0553cd77763869eebdf9042403e16089fe746c"}, - {file = "mypy-0.812-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:61a3d5b97955422964be6b3baf05ff2ce7f26f52c85dd88db11d5e03e146a3a6"}, - {file = "mypy-0.812-cp36-cp36m-win_amd64.whl", hash = "sha256:25adde9b862f8f9aac9d2d11971f226bd4c8fbaa89fb76bdadb267ef22d10064"}, - {file = "mypy-0.812-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:552a815579aa1e995f39fd05dde6cd378e191b063f031f2acfe73ce9fb7f9e56"}, - {file = "mypy-0.812-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:499c798053cdebcaa916eef8cd733e5584b5909f789de856b482cd7d069bdad8"}, - {file = "mypy-0.812-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:5873888fff1c7cf5b71efbe80e0e73153fe9212fafdf8e44adfe4c20ec9f82d7"}, - {file = "mypy-0.812-cp37-cp37m-win_amd64.whl", hash = "sha256:9f94aac67a2045ec719ffe6111df543bac7874cee01f41928f6969756e030564"}, - {file = "mypy-0.812-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d23e0ea196702d918b60c8288561e722bf437d82cb7ef2edcd98cfa38905d506"}, - {file = "mypy-0.812-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:674e822aa665b9fd75130c6c5f5ed9564a38c6cea6a6432ce47eafb68ee578c5"}, - {file = "mypy-0.812-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:abf7e0c3cf117c44d9285cc6128856106183938c68fd4944763003decdcfeb66"}, - {file = "mypy-0.812-cp38-cp38-win_amd64.whl", hash = "sha256:0d0a87c0e7e3a9becdfbe936c981d32e5ee0ccda3e0f07e1ef2c3d1a817cf73e"}, - {file = "mypy-0.812-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7ce3175801d0ae5fdfa79b4f0cfed08807af4d075b402b7e294e6aa72af9aa2a"}, - {file = "mypy-0.812-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:b09669bcda124e83708f34a94606e01b614fa71931d356c1f1a5297ba11f110a"}, - {file = "mypy-0.812-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:33f159443db0829d16f0a8d83d94df3109bb6dd801975fe86bacb9bf71628e97"}, - {file = "mypy-0.812-cp39-cp39-win_amd64.whl", hash = "sha256:3f2aca7f68580dc2508289c729bd49ee929a436208d2b2b6aab15745a70a57df"}, - {file = "mypy-0.812-py3-none-any.whl", hash = "sha256:2f9b3407c58347a452fc0736861593e105139b905cca7d097e413453a1d650b4"}, - {file = "mypy-0.812.tar.gz", hash = "sha256:cd07039aa5df222037005b08fbbfd69b3ab0b0bd7a07d7906de75ae52c4e3119"}, + {file = "mypy-0.901-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:91211acf1485a1db0b1261bc5f9ed450cba3c0dfd8da0a6680e94827591e34d7"}, + {file = "mypy-0.901-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:c8bc628961cca4335ac7d1f2ed59b7125d9252fe4c78c3d66d30b50162359c99"}, + {file = "mypy-0.901-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:4a622faa3be76114cdce009f8ec173401494cf9e8f22713e7ae75fee9d906ab3"}, + {file = "mypy-0.901-cp35-cp35m-win_amd64.whl", hash = "sha256:8183561bfd950e93eeab8379ae5ec65873c856f5b58498d23aa8691f74c86030"}, + {file = "mypy-0.901-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:da914faaa80c25f463913da6db12adba703822a768f452f29f75b40bb4357139"}, + {file = "mypy-0.901-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:307a6c047596d768c3d689734307e47a91596eb9dbb67cfdf7d1fd9117b27f13"}, + {file = "mypy-0.901-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:a85c6759dcc6a9884131fa06a037bd34352aa3947e7f5d9d5a35652cc3a44bcd"}, + {file = "mypy-0.901-cp36-cp36m-win_amd64.whl", hash = "sha256:9941b685807b60c58020bb67b3217c9df47820dcd00425f55cdf71f31d3c42d9"}, + {file = "mypy-0.901-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:08cf1f31029612e1008a9432337ca4b1fbac989ff7c8200e2c9ec42705cd4c7b"}, + {file = "mypy-0.901-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:bc61153eb4df769538bb4a6e1045f59c2e6119339690ec719feeacbfc3809e89"}, + {file = "mypy-0.901-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:1cd241966a35036f936d4739bd71a1c64e15f02bf7d12bb2815cccfb2993a9de"}, + {file = "mypy-0.901-cp37-cp37m-win_amd64.whl", hash = "sha256:97be0e8ed116f7f79472a49cf06dd45dd806771142401f684d4f13ee652a63c0"}, + {file = "mypy-0.901-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:79beb6741df15395908ecc706b3a593a98804c1d5b5b6bd0c5b03b67c7ac03a0"}, + {file = "mypy-0.901-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:bf347c327c48d963bdef5bf365215d3e98b5fddbe5069fc796cec330e8235a20"}, + {file = "mypy-0.901-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:053b92ebae901fc7954677949049f70133f2f63e3e83dc100225c26d6a46fe95"}, + {file = "mypy-0.901-cp38-cp38-win_amd64.whl", hash = "sha256:f208cc967e566698c4e30a1f65843fc88d8da05a8693bac8b975417e0aee9ced"}, + {file = "mypy-0.901-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c86e3f015bfe7958646825d41c0691c6e5a5cd4015e3409b5c29c18a3c712534"}, + {file = "mypy-0.901-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e8577d30daf1b7b6582020f539f76e78ee1ed64a0323b28c8e0333c45db9369f"}, + {file = "mypy-0.901-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:5ddd8f4096d5fc2e7d7bb3924ac22758862163ad2c1cdc902c4b85568160e90a"}, + {file = "mypy-0.901-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:4b54518e399c3f4dc53380d4252c83276b2e60623cfc5274076eb8aae57572ac"}, + {file = "mypy-0.901-cp39-cp39-win_amd64.whl", hash = "sha256:7845ad3a31407bfbd64c76d032c16ab546d282930f747023bf07c17b054bebc5"}, + {file = "mypy-0.901-py3-none-any.whl", hash = "sha256:61b10ba18a01d05fc46adbf4f18b0e92178f6b5fd0f45926ffc2a408b5419728"}, + {file = "mypy-0.901.tar.gz", hash = "sha256:18753a8bb9bcf031ff10009852bd48d781798ecbccf45be5449892e6af4e3f9f"}, ] mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, @@ -1865,6 +1883,10 @@ soupsieve = [ {file = "soupsieve-2.2.1-py3-none-any.whl", hash = "sha256:c2c1c2d44f158cdbddab7824a9af8c4f83c76b1e23e049479aa432feb6c4c23b"}, {file = "soupsieve-2.2.1.tar.gz", hash = "sha256:052774848f448cf19c7e959adf5566904d525f33a3f8b6ba6f6f8f26ec7de0cc"}, ] +toml = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] traitlets = [ {file = "traitlets-5.0.5-py3-none-any.whl", hash = "sha256:69ff3f9d5351f31a7ad80443c2674b7099df13cc41fc5fa6e2f6d3b0330b0426"}, {file = "traitlets-5.0.5.tar.gz", hash = "sha256:178f4ce988f69189f7e523337a3e11d91c786ded9360174a3d9ca83e79bc5396"}, @@ -1891,37 +1913,11 @@ twisted-iocpsupport = [ {file = "twisted_iocpsupport-1.0.1-pp36-pypy36_pp73-win32.whl", hash = "sha256:ed8cb959d7dce287419e766267a280139fedf14217d85695d7b6384d8ccd3353"}, {file = "twisted_iocpsupport-1.0.1-pp37-pypy37_pp73-win32.whl", hash = "sha256:1535e537b6d60d0cfd2b4785f1c7a021beb62d8df6bac3dc6c45bb866ce648e4"}, ] -typed-ast = [ - {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6"}, - {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075"}, - {file = "typed_ast-1.4.3-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:1b3ead4a96c9101bef08f9f7d1217c096f31667617b58de957f690c92378b528"}, - {file = "typed_ast-1.4.3-cp35-cp35m-win32.whl", hash = "sha256:dde816ca9dac1d9c01dd504ea5967821606f02e510438120091b84e852367428"}, - {file = "typed_ast-1.4.3-cp35-cp35m-win_amd64.whl", hash = "sha256:777a26c84bea6cd934422ac2e3b78863a37017618b6e5c08f92ef69853e765d3"}, - {file = "typed_ast-1.4.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f8afcf15cc511ada719a88e013cec87c11aff7b91f019295eb4530f96fe5ef2f"}, - {file = "typed_ast-1.4.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:52b1eb8c83f178ab787f3a4283f68258525f8d70f778a2f6dd54d3b5e5fb4341"}, - {file = "typed_ast-1.4.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:01ae5f73431d21eead5015997ab41afa53aa1fbe252f9da060be5dad2c730ace"}, - {file = "typed_ast-1.4.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:c190f0899e9f9f8b6b7863debfb739abcb21a5c054f911ca3596d12b8a4c4c7f"}, - {file = "typed_ast-1.4.3-cp36-cp36m-win32.whl", hash = "sha256:398e44cd480f4d2b7ee8d98385ca104e35c81525dd98c519acff1b79bdaac363"}, - {file = "typed_ast-1.4.3-cp36-cp36m-win_amd64.whl", hash = "sha256:bff6ad71c81b3bba8fa35f0f1921fb24ff4476235a6e94a26ada2e54370e6da7"}, - {file = "typed_ast-1.4.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0fb71b8c643187d7492c1f8352f2c15b4c4af3f6338f21681d3681b3dc31a266"}, - {file = "typed_ast-1.4.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:760ad187b1041a154f0e4d0f6aae3e40fdb51d6de16e5c99aedadd9246450e9e"}, - {file = "typed_ast-1.4.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5feca99c17af94057417d744607b82dd0a664fd5e4ca98061480fd8b14b18d04"}, - {file = "typed_ast-1.4.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:95431a26309a21874005845c21118c83991c63ea800dd44843e42a916aec5899"}, - {file = "typed_ast-1.4.3-cp37-cp37m-win32.whl", hash = "sha256:aee0c1256be6c07bd3e1263ff920c325b59849dc95392a05f258bb9b259cf39c"}, - {file = "typed_ast-1.4.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9ad2c92ec681e02baf81fdfa056fe0d818645efa9af1f1cd5fd6f1bd2bdfd805"}, - {file = "typed_ast-1.4.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b36b4f3920103a25e1d5d024d155c504080959582b928e91cb608a65c3a49e1a"}, - {file = "typed_ast-1.4.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:067a74454df670dcaa4e59349a2e5c81e567d8d65458d480a5b3dfecec08c5ff"}, - {file = "typed_ast-1.4.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7538e495704e2ccda9b234b82423a4038f324f3a10c43bc088a1636180f11a41"}, - {file = "typed_ast-1.4.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:af3d4a73793725138d6b334d9d247ce7e5f084d96284ed23f22ee626a7b88e39"}, - {file = "typed_ast-1.4.3-cp38-cp38-win32.whl", hash = "sha256:f2362f3cb0f3172c42938946dbc5b7843c2a28aec307c49100c8b38764eb6927"}, - {file = "typed_ast-1.4.3-cp38-cp38-win_amd64.whl", hash = "sha256:dd4a21253f42b8d2b48410cb31fe501d32f8b9fbeb1f55063ad102fe9c425e40"}, - {file = "typed_ast-1.4.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f328adcfebed9f11301eaedfa48e15bdece9b519fb27e6a8c01aa52a17ec31b3"}, - {file = "typed_ast-1.4.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:2c726c276d09fc5c414693a2de063f521052d9ea7c240ce553316f70656c84d4"}, - {file = "typed_ast-1.4.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:cae53c389825d3b46fb37538441f75d6aecc4174f615d048321b716df2757fb0"}, - {file = "typed_ast-1.4.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:b9574c6f03f685070d859e75c7f9eeca02d6933273b5e69572e5ff9d5e3931c3"}, - {file = "typed_ast-1.4.3-cp39-cp39-win32.whl", hash = "sha256:209596a4ec71d990d71d5e0d312ac935d86930e6eecff6ccc7007fe54d703808"}, - {file = "typed_ast-1.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c"}, - {file = "typed_ast-1.4.3.tar.gz", hash = "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65"}, +types-redis = [ + {file = "types_redis-3.5.0-py2.py3-none-any.whl", hash = "sha256:cf29f55ccc3b44c78259cc22e4c0030f77b7a47ec5656211402be0fcf1a98536"}, +] +types-requests = [ + {file = "types_requests-0.1.9-py2.py3-none-any.whl", hash = "sha256:24a51b692d36101e5b81f08733d74b944ccae3e90589fb712e4dfc6f36965c35"}, ] typing-extensions = [ {file = "typing_extensions-3.10.0.0-py2-none-any.whl", hash = "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497"}, diff --git a/pyproject.toml b/pyproject.toml index 8cf6acbc..ace74f20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,8 +63,10 @@ flask-restx = "^0.4.0" misp = ['python-magic', 'pydeep'] [tool.poetry.dev-dependencies] -mypy = "^0.812" +mypy = "^0.901" ipython = "^7.23.1" +types-redis = "^3.5.0" +types-requests = "^0.1.8" [build-system] requires = ["poetry_core>=1.0", "setuptools"] From c52509614e11a15f613dfb2bbabdc48277c4ad9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 9 Jun 2021 17:59:24 -0700 Subject: [PATCH 06/18] new: get capture info API call --- lookyloo/lookyloo.py | 7 +++++++ website/web/__init__.py | 4 ++-- website/web/genericapi.py | 8 ++++++++ website/web/templates/tree.html | 12 ++++++------ 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index 0f74c57e..3efc2200 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -324,6 +324,13 @@ class Lookyloo(): ct = self.get_crawled_tree(capture_uuid) return ct.root_hartree.stats + def get_info(self, capture_uuid: str, /) -> Dict[str, Any]: + '''Get basic information about the capture.''' + ct = self.get_crawled_tree(capture_uuid) + to_return = {'url': ct.root_url, 'title': ct.root_hartree.har.initial_title, + 'capture_time': ct.start_time.isoformat(), 'user_agent': ct.user_agent} + return to_return + def get_meta(self, capture_uuid: str, /) -> Dict[str, str]: '''Get the meta informations from a capture (mostly, details about the User Agent used.)''' capture_dir = self._get_capture_dir(capture_uuid) diff --git a/website/web/__init__.py b/website/web/__init__.py index 6638cd13..4f6df541 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -550,6 +550,7 @@ def tree(tree_uuid: str, node_uuid: Optional[str]=None): ct = lookyloo.get_crawled_tree(tree_uuid) b64_thumbnail = lookyloo.get_screenshot_thumbnail(tree_uuid, for_datauri=True) screenshot_size = lookyloo.get_screenshot(tree_uuid).getbuffer().nbytes + info = lookyloo.get_info(tree_uuid) meta = lookyloo.get_meta(tree_uuid) hostnode_to_highlight = None if node_uuid: @@ -567,8 +568,7 @@ def tree(tree_uuid: str, node_uuid: Optional[str]=None): print(e) pass return render_template('tree.html', tree_json=ct.to_json(), - start_time=ct.start_time.isoformat(), - user_agent=ct.user_agent, root_url=ct.root_url, + info=info, tree_uuid=tree_uuid, public_domain=lookyloo.public_domain, screenshot_thumbnail=b64_thumbnail, page_title=cache.title, screenshot_size=screenshot_size, diff --git a/website/web/genericapi.py b/website/web/genericapi.py index 7bb7bc24..6232020f 100644 --- a/website/web/genericapi.py +++ b/website/web/genericapi.py @@ -246,6 +246,14 @@ class CaptureStats(Resource): return lookyloo.get_statistics(capture_uuid) +@api.route('/json//info') +@api.doc(description='Get basic information about the capture.', + params={'capture_uuid': 'The UUID of the capture'}) +class CaptureInfo(Resource): + def get(self, capture_uuid: str): + return lookyloo.get_info(capture_uuid) + + @api.route('/json//cookies') @api.doc(description='Get the complete cookie jar created during the capture.', params={'capture_uuid': 'The UUID of the capture'}) diff --git a/website/web/templates/tree.html b/website/web/templates/tree.html index 856b8ad0..f4dd7663 100644 --- a/website/web/templates/tree.html +++ b/website/web/templates/tree.html @@ -3,14 +3,14 @@ {% from 'bootstrap/utils.html' import render_messages %} {% from "macros.html" import shorten_string %} -{% block title %}Capture of {{root_url}}{% endblock %} +{% block title %}Capture of {{info['url']}}{% endblock %} {% block card %} { document.getElementById("start_time").innerHTML = `${capture_starttime.getFullYear()}-${("0" + (capture_starttime.getMonth() + 1)).slice(-2)}-${("0" + capture_starttime.getDate()).slice(-2)} ${capture_starttime.toLocaleTimeString()}`; @@ -342,16 +342,16 @@
URL captured
-
{{ shorten_string(root_url, 1000) }}
+
{{ shorten_string(info['url'], 1000) }}
Page title
-
{{ page_title }}
+
{{ info['title'] }}
Capture time
User Agent
-
{{ user_agent }}
+
{{ info['user_agent'] }}
{% if meta %} {% for k, v in meta.items() if k not in ['user_agent'] %} From c312f0039fc069d3e5bfcb202735a6e1370b2326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 10 Jun 2021 10:32:02 -0700 Subject: [PATCH 07/18] chg: Bump deps --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 6658e81f..54cf6c02 100644 --- a/poetry.lock +++ b/poetry.lock @@ -98,7 +98,7 @@ lxml = ["lxml"] [[package]] name = "bootstrap-flask" -version = "1.6.0" +version = "1.7.0" description = "Bootstrap helper for Flask/Jinja2." category = "main" optional = false @@ -1206,7 +1206,7 @@ misp = ["python-magic", "pydeep"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "8ab8e928500dbdec94ae776c840a95e5b21b1f385eca361a75601677e9ac855d" +content-hash = "88aca18cb48671e942b1add1ca07f9a390c97871d822637c299738c1103aa37b" [metadata.files] aiohttp = [ @@ -1278,8 +1278,8 @@ beautifulsoup4 = [ {file = "beautifulsoup4-4.9.3.tar.gz", hash = "sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25"}, ] bootstrap-flask = [ - {file = "Bootstrap-Flask-1.6.0.tar.gz", hash = "sha256:30c0d21952000b3bb4f51da9550737870ff040d3b4436515e54e1a1e2782dbd6"}, - {file = "Bootstrap_Flask-1.6.0-py2.py3-none-any.whl", hash = "sha256:7959838820db59ca94b5ae47bea7f4acd848e8d86107e4de01f16fc2b04691d6"}, + {file = "Bootstrap-Flask-1.7.0.tar.gz", hash = "sha256:dfb2a892e0dad47eb4160008e93eb65b16c988b8702f1b9d8d791043f7c0d023"}, + {file = "Bootstrap_Flask-1.7.0-py2.py3-none-any.whl", hash = "sha256:38452d68405d9d2d2f8e4ae3432411bf36043cbab017963d6b4cf1524875afe9"}, ] cchardet = [ {file = "cchardet-2.1.7-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:58005cd94e4bbb25fb4cee3ad75a0fe121384cb2b99ac5619aa45aecedf43c7b"}, diff --git a/pyproject.toml b/pyproject.toml index ace74f20..ca22f3af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ gunicorn = "^20.1.0" cchardet = "^2.1.7" redis = "^3.5.3" beautifulsoup4 = "^4.9.3" -bootstrap-flask = "^1.5.3" +bootstrap-flask = "^1.7.0" defang = "^0.5.3" vt-py = "^0.6.2" pyeupi = "^1.1" From 4786352e1849e281e73facc7fea4b092b083aafa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 11 Jun 2021 10:18:27 -0700 Subject: [PATCH 08/18] chg: Bump deps --- poetry.lock | 54 +++++++++++++++++++++++++------------------------- pyproject.toml | 4 ++-- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/poetry.lock b/poetry.lock index 54cf6c02..a11a5645 100644 --- a/poetry.lock +++ b/poetry.lock @@ -578,7 +578,7 @@ python-versions = ">=3.6" [[package]] name = "mypy" -version = "0.901" +version = "0.902" description = "Optional static typing for Python" category = "dev" optional = false @@ -1098,7 +1098,7 @@ python-versions = "*" [[package]] name = "types-requests" -version = "0.1.9" +version = "0.1.10" description = "Typing stubs for requests" category = "dev" optional = false @@ -1206,7 +1206,7 @@ misp = ["python-magic", "pydeep"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "88aca18cb48671e942b1add1ca07f9a390c97871d822637c299738c1103aa37b" +content-hash = "73b9e35eb289bc39d626f4ed32b4b4a6c496a00eea350401e0f49a8d521a08c3" [metadata.files] aiohttp = [ @@ -1632,29 +1632,29 @@ multidict = [ {file = "multidict-5.1.0.tar.gz", hash = "sha256:25b4e5f22d3a37ddf3effc0710ba692cfc792c2b9edfb9c05aefe823256e84d5"}, ] mypy = [ - {file = "mypy-0.901-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:91211acf1485a1db0b1261bc5f9ed450cba3c0dfd8da0a6680e94827591e34d7"}, - {file = "mypy-0.901-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:c8bc628961cca4335ac7d1f2ed59b7125d9252fe4c78c3d66d30b50162359c99"}, - {file = "mypy-0.901-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:4a622faa3be76114cdce009f8ec173401494cf9e8f22713e7ae75fee9d906ab3"}, - {file = "mypy-0.901-cp35-cp35m-win_amd64.whl", hash = "sha256:8183561bfd950e93eeab8379ae5ec65873c856f5b58498d23aa8691f74c86030"}, - {file = "mypy-0.901-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:da914faaa80c25f463913da6db12adba703822a768f452f29f75b40bb4357139"}, - {file = "mypy-0.901-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:307a6c047596d768c3d689734307e47a91596eb9dbb67cfdf7d1fd9117b27f13"}, - {file = "mypy-0.901-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:a85c6759dcc6a9884131fa06a037bd34352aa3947e7f5d9d5a35652cc3a44bcd"}, - {file = "mypy-0.901-cp36-cp36m-win_amd64.whl", hash = "sha256:9941b685807b60c58020bb67b3217c9df47820dcd00425f55cdf71f31d3c42d9"}, - {file = "mypy-0.901-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:08cf1f31029612e1008a9432337ca4b1fbac989ff7c8200e2c9ec42705cd4c7b"}, - {file = "mypy-0.901-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:bc61153eb4df769538bb4a6e1045f59c2e6119339690ec719feeacbfc3809e89"}, - {file = "mypy-0.901-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:1cd241966a35036f936d4739bd71a1c64e15f02bf7d12bb2815cccfb2993a9de"}, - {file = "mypy-0.901-cp37-cp37m-win_amd64.whl", hash = "sha256:97be0e8ed116f7f79472a49cf06dd45dd806771142401f684d4f13ee652a63c0"}, - {file = "mypy-0.901-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:79beb6741df15395908ecc706b3a593a98804c1d5b5b6bd0c5b03b67c7ac03a0"}, - {file = "mypy-0.901-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:bf347c327c48d963bdef5bf365215d3e98b5fddbe5069fc796cec330e8235a20"}, - {file = "mypy-0.901-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:053b92ebae901fc7954677949049f70133f2f63e3e83dc100225c26d6a46fe95"}, - {file = "mypy-0.901-cp38-cp38-win_amd64.whl", hash = "sha256:f208cc967e566698c4e30a1f65843fc88d8da05a8693bac8b975417e0aee9ced"}, - {file = "mypy-0.901-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c86e3f015bfe7958646825d41c0691c6e5a5cd4015e3409b5c29c18a3c712534"}, - {file = "mypy-0.901-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e8577d30daf1b7b6582020f539f76e78ee1ed64a0323b28c8e0333c45db9369f"}, - {file = "mypy-0.901-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:5ddd8f4096d5fc2e7d7bb3924ac22758862163ad2c1cdc902c4b85568160e90a"}, - {file = "mypy-0.901-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:4b54518e399c3f4dc53380d4252c83276b2e60623cfc5274076eb8aae57572ac"}, - {file = "mypy-0.901-cp39-cp39-win_amd64.whl", hash = "sha256:7845ad3a31407bfbd64c76d032c16ab546d282930f747023bf07c17b054bebc5"}, - {file = "mypy-0.901-py3-none-any.whl", hash = "sha256:61b10ba18a01d05fc46adbf4f18b0e92178f6b5fd0f45926ffc2a408b5419728"}, - {file = "mypy-0.901.tar.gz", hash = "sha256:18753a8bb9bcf031ff10009852bd48d781798ecbccf45be5449892e6af4e3f9f"}, + {file = "mypy-0.902-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:3f12705eabdd274b98f676e3e5a89f247ea86dc1af48a2d5a2b080abac4e1243"}, + {file = "mypy-0.902-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:2f9fedc1f186697fda191e634ac1d02f03d4c260212ccb018fabbb6d4b03eee8"}, + {file = "mypy-0.902-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:0756529da2dd4d53d26096b7969ce0a47997123261a5432b48cc6848a2cb0bd4"}, + {file = "mypy-0.902-cp35-cp35m-win_amd64.whl", hash = "sha256:68a098c104ae2b75e946b107ef69dd8398d54cb52ad57580dfb9fc78f7f997f0"}, + {file = "mypy-0.902-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cd01c599cf9f897b6b6c6b5d8b182557fb7d99326bcdf5d449a0fbbb4ccee4b9"}, + {file = "mypy-0.902-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:e89880168c67cf4fde4506b80ee42f1537ad66ad366c101d388b3fd7d7ce2afd"}, + {file = "mypy-0.902-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:ebe2bc9cb638475f5d39068d2dbe8ae1d605bb8d8d3ff281c695df1670ab3987"}, + {file = "mypy-0.902-cp36-cp36m-win_amd64.whl", hash = "sha256:f89bfda7f0f66b789792ab64ce0978e4a991a0e4dd6197349d0767b0f1095b21"}, + {file = "mypy-0.902-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:746e0b0101b8efec34902810047f26a8c80e1efbb4fc554956d848c05ef85d76"}, + {file = "mypy-0.902-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:0190fb77e93ce971954c9e54ea61de2802065174e5e990c9d4c1d0f54fbeeca2"}, + {file = "mypy-0.902-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:b5dfcd22c6bab08dfeded8d5b44bdcb68c6f1ab261861e35c470b89074f78a70"}, + {file = "mypy-0.902-cp37-cp37m-win_amd64.whl", hash = "sha256:b5ba1f0d5f9087e03bf5958c28d421a03a4c1ad260bf81556195dffeccd979c4"}, + {file = "mypy-0.902-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9ef5355eaaf7a23ab157c21a44c614365238a7bdb3552ec3b80c393697d974e1"}, + {file = "mypy-0.902-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:517e7528d1be7e187a5db7f0a3e479747307c1b897d9706b1c662014faba3116"}, + {file = "mypy-0.902-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:fd634bc17b1e2d6ce716f0e43446d0d61cdadb1efcad5c56ca211c22b246ebc8"}, + {file = "mypy-0.902-cp38-cp38-win_amd64.whl", hash = "sha256:fc4d63da57ef0e8cd4ab45131f3fe5c286ce7dd7f032650d0fbc239c6190e167"}, + {file = "mypy-0.902-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:353aac2ce41ddeaf7599f1c73fed2b75750bef3b44b6ad12985a991bc002a0da"}, + {file = "mypy-0.902-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ae94c31bb556ddb2310e4f913b706696ccbd43c62d3331cd3511caef466871d2"}, + {file = "mypy-0.902-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:8be7bbd091886bde9fcafed8dd089a766fa76eb223135fe5c9e9798f78023a20"}, + {file = "mypy-0.902-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:4efc67b9b3e2fddbe395700f91d5b8deb5980bfaaccb77b306310bd0b9e002eb"}, + {file = "mypy-0.902-cp39-cp39-win_amd64.whl", hash = "sha256:9f1d74eeb3f58c7bd3f3f92b8f63cb1678466a55e2c4612bf36909105d0724ab"}, + {file = "mypy-0.902-py3-none-any.whl", hash = "sha256:a26d0e53e90815c765f91966442775cf03b8a7514a4e960de7b5320208b07269"}, + {file = "mypy-0.902.tar.gz", hash = "sha256:9236c21194fde5df1b4d8ebc2ef2c1f2a5dc7f18bcbea54274937cae2e20a01c"}, ] mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, @@ -1917,7 +1917,7 @@ types-redis = [ {file = "types_redis-3.5.0-py2.py3-none-any.whl", hash = "sha256:cf29f55ccc3b44c78259cc22e4c0030f77b7a47ec5656211402be0fcf1a98536"}, ] types-requests = [ - {file = "types_requests-0.1.9-py2.py3-none-any.whl", hash = "sha256:24a51b692d36101e5b81f08733d74b944ccae3e90589fb712e4dfc6f36965c35"}, + {file = "types_requests-0.1.10-py2.py3-none-any.whl", hash = "sha256:5b1864f1564adb810123c563b4de86236c14058978ec1294b07a988c532cb079"}, ] typing-extensions = [ {file = "typing_extensions-3.10.0.0-py2-none-any.whl", hash = "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497"}, diff --git a/pyproject.toml b/pyproject.toml index ca22f3af..3483fd20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,10 +63,10 @@ flask-restx = "^0.4.0" misp = ['python-magic', 'pydeep'] [tool.poetry.dev-dependencies] -mypy = "^0.901" +mypy = "^0.902" ipython = "^7.23.1" types-redis = "^3.5.0" -types-requests = "^0.1.8" +types-requests = "^0.1.10" [build-system] requires = ["poetry_core>=1.0", "setuptools"] From d0ec858b7ebba6b3e59a10ae4da06462149695b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 11 Jun 2021 10:47:01 -0700 Subject: [PATCH 09/18] chg: Add missing types --- poetry.lock | 65 +++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index a11a5645..4c53e0cd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1088,6 +1088,46 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "types-click" +version = "7.1.0" +description = "Typing stubs for click" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "types-flask" +version = "1.1.0" +description = "Typing stubs for Flask" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +types-click = "*" +types-Jinja2 = "*" +types-Werkzeug = "*" + +[[package]] +name = "types-jinja2" +version = "2.11.0" +description = "Typing stubs for Jinja2" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +types-MarkupSafe = "*" + +[[package]] +name = "types-markupsafe" +version = "1.1.0" +description = "Typing stubs for MarkupSafe" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "types-redis" version = "3.5.0" @@ -1104,6 +1144,14 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "types-werkzeug" +version = "1.0.0" +description = "Typing stubs for Werkzeug" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "typing-extensions" version = "3.10.0.0" @@ -1206,7 +1254,7 @@ misp = ["python-magic", "pydeep"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "73b9e35eb289bc39d626f4ed32b4b4a6c496a00eea350401e0f49a8d521a08c3" +content-hash = "e1bf704e07b3d70cb89f7ff6b67ca045da7b6c1dc59983f6c116e7812787575d" [metadata.files] aiohttp = [ @@ -1913,12 +1961,27 @@ twisted-iocpsupport = [ {file = "twisted_iocpsupport-1.0.1-pp36-pypy36_pp73-win32.whl", hash = "sha256:ed8cb959d7dce287419e766267a280139fedf14217d85695d7b6384d8ccd3353"}, {file = "twisted_iocpsupport-1.0.1-pp37-pypy37_pp73-win32.whl", hash = "sha256:1535e537b6d60d0cfd2b4785f1c7a021beb62d8df6bac3dc6c45bb866ce648e4"}, ] +types-click = [ + {file = "types_click-7.1.0-py2.py3-none-any.whl", hash = "sha256:1b37e9e63ec1f4e67bb8016afac882ec94bcb98a09b92bebde8f1e0f0484e6ed"}, +] +types-flask = [ + {file = "types_Flask-1.1.0-py2.py3-none-any.whl", hash = "sha256:0d16fbd554b274d003761b10bb06e86aa535b122cd98b0a8e45082750c54e35b"}, +] +types-jinja2 = [ + {file = "types_Jinja2-2.11.0-py2.py3-none-any.whl", hash = "sha256:f9d182ec241bdbeb8759c65693c5f95b638a0dfd9e3d91606b3d48e58b0b5caf"}, +] +types-markupsafe = [ + {file = "types_MarkupSafe-1.1.0-py2.py3-none-any.whl", hash = "sha256:9b7a9142ff5e97af10235fe504b823a4462b71cbd8189d6dde585aacf4cc9331"}, +] types-redis = [ {file = "types_redis-3.5.0-py2.py3-none-any.whl", hash = "sha256:cf29f55ccc3b44c78259cc22e4c0030f77b7a47ec5656211402be0fcf1a98536"}, ] types-requests = [ {file = "types_requests-0.1.10-py2.py3-none-any.whl", hash = "sha256:5b1864f1564adb810123c563b4de86236c14058978ec1294b07a988c532cb079"}, ] +types-werkzeug = [ + {file = "types_Werkzeug-1.0.0-py2.py3-none-any.whl", hash = "sha256:4beb9df90a2ab27fc473b64a17b49b3088dc8ce761d8708b5fe2bb317d779189"}, +] typing-extensions = [ {file = "typing_extensions-3.10.0.0-py2-none-any.whl", hash = "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497"}, {file = "typing_extensions-3.10.0.0-py3-none-any.whl", hash = "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"}, diff --git a/pyproject.toml b/pyproject.toml index 3483fd20..1782d531 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ mypy = "^0.902" ipython = "^7.23.1" types-redis = "^3.5.0" types-requests = "^0.1.10" +types-Flask = "^1.1.0" [build-system] requires = ["poetry_core>=1.0", "setuptools"] From e62d1c3219e9702a323cb8468fc83354063e1c98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 11 Jun 2021 12:15:27 -0700 Subject: [PATCH 10/18] fix: Missing typing dep --- poetry.lock | 13 ++++++++++++- pyproject.toml | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 4c53e0cd..52f670d6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1128,6 +1128,14 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "types-pkg-resources" +version = "0.1.2" +description = "Typing stubs for pkg_resources" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "types-redis" version = "3.5.0" @@ -1254,7 +1262,7 @@ misp = ["python-magic", "pydeep"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "e1bf704e07b3d70cb89f7ff6b67ca045da7b6c1dc59983f6c116e7812787575d" +content-hash = "993ff285e6d4726ce390cc9e90ec43422e277981f9c00cd5843352852c287b15" [metadata.files] aiohttp = [ @@ -1973,6 +1981,9 @@ types-jinja2 = [ types-markupsafe = [ {file = "types_MarkupSafe-1.1.0-py2.py3-none-any.whl", hash = "sha256:9b7a9142ff5e97af10235fe504b823a4462b71cbd8189d6dde585aacf4cc9331"}, ] +types-pkg-resources = [ + {file = "types_pkg_resources-0.1.2-py2.py3-none-any.whl", hash = "sha256:42d640500de564f1ccc21f918117afadf78039e4fa7f513c647ccf742d609aeb"}, +] types-redis = [ {file = "types_redis-3.5.0-py2.py3-none-any.whl", hash = "sha256:cf29f55ccc3b44c78259cc22e4c0030f77b7a47ec5656211402be0fcf1a98536"}, ] diff --git a/pyproject.toml b/pyproject.toml index 990287aa..643810f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ ipython = "^7.23.1" types-redis = "^3.5.0" types-requests = "^0.1.10" types-Flask = "^1.1.0" +types-pkg-resources = "^0.1.2" [build-system] requires = ["poetry_core>=1.0", "setuptools"] From 62c5a49a720f5d4562ab4fdd4091315fca82d20c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 11 Jun 2021 12:19:49 -0700 Subject: [PATCH 11/18] Update dockerimage.yml --- .github/workflows/dockerimage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 51f21830..e7afd979 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -12,7 +12,7 @@ jobs: uses: actions/checkout@v2 - name: Build and push Docker images - uses: docker/build-push-action@v1 + uses: docker/build-push-action@v2 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} From 04e5e240338716d30628c66d701a004dbe5c82c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 14 Jun 2021 10:33:27 -0700 Subject: [PATCH 12/18] chg: Bump deps --- poetry.lock | 26 +++++++++++++------------- pyproject.toml | 4 ++-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/poetry.lock b/poetry.lock index 52f670d6..1e3eadfd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1090,7 +1090,7 @@ python-versions = "*" [[package]] name = "types-click" -version = "7.1.0" +version = "7.1.1" description = "Typing stubs for click" category = "dev" optional = false @@ -1111,7 +1111,7 @@ types-Werkzeug = "*" [[package]] name = "types-jinja2" -version = "2.11.0" +version = "2.11.1" description = "Typing stubs for Jinja2" category = "dev" optional = false @@ -1122,7 +1122,7 @@ types-MarkupSafe = "*" [[package]] name = "types-markupsafe" -version = "1.1.0" +version = "1.1.1" description = "Typing stubs for MarkupSafe" category = "dev" optional = false @@ -1138,7 +1138,7 @@ python-versions = "*" [[package]] name = "types-redis" -version = "3.5.0" +version = "3.5.1" description = "Typing stubs for redis" category = "dev" optional = false @@ -1146,7 +1146,7 @@ python-versions = "*" [[package]] name = "types-requests" -version = "0.1.10" +version = "0.1.11" description = "Typing stubs for requests" category = "dev" optional = false @@ -1154,7 +1154,7 @@ python-versions = "*" [[package]] name = "types-werkzeug" -version = "1.0.0" +version = "1.0.1" description = "Typing stubs for Werkzeug" category = "dev" optional = false @@ -1262,7 +1262,7 @@ misp = ["python-magic", "pydeep"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "993ff285e6d4726ce390cc9e90ec43422e277981f9c00cd5843352852c287b15" +content-hash = "2b987bbd9e75ee3ffd6a4db33f2d37a59bba102771c61c50a83a44f30278ed86" [metadata.files] aiohttp = [ @@ -1970,28 +1970,28 @@ twisted-iocpsupport = [ {file = "twisted_iocpsupport-1.0.1-pp37-pypy37_pp73-win32.whl", hash = "sha256:1535e537b6d60d0cfd2b4785f1c7a021beb62d8df6bac3dc6c45bb866ce648e4"}, ] types-click = [ - {file = "types_click-7.1.0-py2.py3-none-any.whl", hash = "sha256:1b37e9e63ec1f4e67bb8016afac882ec94bcb98a09b92bebde8f1e0f0484e6ed"}, + {file = "types_click-7.1.1-py2.py3-none-any.whl", hash = "sha256:b1db0fb31a6d4a31c4282c231827cbdab628b0d504b016937822d943cd5e40a5"}, ] types-flask = [ {file = "types_Flask-1.1.0-py2.py3-none-any.whl", hash = "sha256:0d16fbd554b274d003761b10bb06e86aa535b122cd98b0a8e45082750c54e35b"}, ] types-jinja2 = [ - {file = "types_Jinja2-2.11.0-py2.py3-none-any.whl", hash = "sha256:f9d182ec241bdbeb8759c65693c5f95b638a0dfd9e3d91606b3d48e58b0b5caf"}, + {file = "types_Jinja2-2.11.1-py2.py3-none-any.whl", hash = "sha256:84f66a2612376ff3c1656198b10a04c7711b6d917d4f5f7f2c2f7ecec0afa040"}, ] types-markupsafe = [ - {file = "types_MarkupSafe-1.1.0-py2.py3-none-any.whl", hash = "sha256:9b7a9142ff5e97af10235fe504b823a4462b71cbd8189d6dde585aacf4cc9331"}, + {file = "types_MarkupSafe-1.1.1-py2.py3-none-any.whl", hash = "sha256:a2c32269a26b4a7205f6f1581bd37cfbcd390297352b828a9643978392239516"}, ] types-pkg-resources = [ {file = "types_pkg_resources-0.1.2-py2.py3-none-any.whl", hash = "sha256:42d640500de564f1ccc21f918117afadf78039e4fa7f513c647ccf742d609aeb"}, ] types-redis = [ - {file = "types_redis-3.5.0-py2.py3-none-any.whl", hash = "sha256:cf29f55ccc3b44c78259cc22e4c0030f77b7a47ec5656211402be0fcf1a98536"}, + {file = "types_redis-3.5.1-py2.py3-none-any.whl", hash = "sha256:b27d7f4b8e8858b1549cba14527d3c4d953089ea1538f37698d0da8eb968fab9"}, ] types-requests = [ - {file = "types_requests-0.1.10-py2.py3-none-any.whl", hash = "sha256:5b1864f1564adb810123c563b4de86236c14058978ec1294b07a988c532cb079"}, + {file = "types_requests-0.1.11-py2.py3-none-any.whl", hash = "sha256:e79c09e5866c6520cdf23dc260573a0a190444f36b112116fadc5542656c8309"}, ] types-werkzeug = [ - {file = "types_Werkzeug-1.0.0-py2.py3-none-any.whl", hash = "sha256:4beb9df90a2ab27fc473b64a17b49b3088dc8ce761d8708b5fe2bb317d779189"}, + {file = "types_Werkzeug-1.0.1-py2.py3-none-any.whl", hash = "sha256:51dca6aa5be250e5b2e3150c8ab40e531408e4684db7054d02b5b8dcd2a193cd"}, ] typing-extensions = [ {file = "typing_extensions-3.10.0.0-py2-none-any.whl", hash = "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497"}, diff --git a/pyproject.toml b/pyproject.toml index 643810f9..98c185ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,8 +65,8 @@ misp = ['python-magic', 'pydeep'] [tool.poetry.dev-dependencies] mypy = "^0.902" ipython = "^7.23.1" -types-redis = "^3.5.0" -types-requests = "^0.1.10" +types-redis = "^3.5.1" +types-requests = "^0.1.11" types-Flask = "^1.1.0" types-pkg-resources = "^0.1.2" From 7b32454f9aa615a008575618c27b9738f39f3ae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 15 Jun 2021 18:26:41 -0700 Subject: [PATCH 13/18] new: get all the ressources hashes in API --- website/web/genericapi.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/website/web/genericapi.py b/website/web/genericapi.py index 6232020f..e31fe5f4 100644 --- a/website/web/genericapi.py +++ b/website/web/genericapi.py @@ -65,6 +65,18 @@ class CaptureStatusQuery(Resource): return {'status_code': lookyloo.get_capture_status(capture_uuid)} +@api.route('/json//hashes') +@api.doc(description='Get all the hashes of all the resources of a capture', + params={'capture_uuid': 'The UUID of the capture'}) +class CaptureHashes(Resource): + def get(self, capture_uuid: str): + cache = lookyloo.capture_cache(capture_uuid) + if not cache: + return {'error': 'UUID missing in cache, try again later.'} + to_return: Dict[str, Any] = {'response': {'hashes': list(lookyloo.get_hashes(capture_uuid))}} + return to_return + + @api.route('/json//redirects') @api.doc(description='Get all the redirects of a capture', params={'capture_uuid': 'The UUID of the capture'}) From a44b7b30b59018c5ac4afc50ee255e6d9442e3fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 16 Jun 2021 14:57:14 -0700 Subject: [PATCH 14/18] new: Get URLs and Hostnames from captures --- lookyloo/lookyloo.py | 32 ++++++++++++++++++++++++++++++++ website/web/genericapi.py | 24 ++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index 6406360f..e1f95c70 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -1208,6 +1208,38 @@ class Lookyloo(): container = self.get_crawled_tree(tree_uuid) return get_resources_hashes(container) + def get_hostnames(self, tree_uuid: str, /, hostnode_uuid: Optional[str]=None, urlnode_uuid: Optional[str]=None) -> Set[str]: + """Return all the unique hostnames: + * of a complete tree if no hostnode_uuid and urlnode_uuid are given + * of a HostNode if hostnode_uuid is given + * of a URLNode if urlnode_uuid is given + """ + if urlnode_uuid: + node = self.get_urlnode_from_tree(tree_uuid, urlnode_uuid) + return {node.hostname} + elif hostnode_uuid: + node = self.get_hostnode_from_tree(tree_uuid, hostnode_uuid) + return {node.name} + else: + ct = self.get_crawled_tree(tree_uuid) + return {node.name for node in ct.root_hartree.hostname_tree.traverse()} + + def get_urls(self, tree_uuid: str, /, hostnode_uuid: Optional[str]=None, urlnode_uuid: Optional[str]=None) -> Set[str]: + """Return all the unique URLs: + * of a complete tree if no hostnode_uuid and urlnode_uuid are given + * of a HostNode if hostnode_uuid is given + * of a URLNode if urlnode_uuid is given + """ + if urlnode_uuid: + node = self.get_urlnode_from_tree(tree_uuid, urlnode_uuid) + return {node.name} + elif hostnode_uuid: + node = self.get_hostnode_from_tree(tree_uuid, hostnode_uuid) + return {urlnode.name for urlnode in node.urls} + else: + ct = self.get_crawled_tree(tree_uuid) + return {node.name for node in ct.root_hartree.url_tree.traverse()} + def get_hostnode_investigator(self, capture_uuid: str, /, node_uuid: str) -> Tuple[HostNode, List[Dict[str, Any]]]: '''Gather all the informations needed to display the Hostnode investigator popup.''' ct = self.get_crawled_tree(capture_uuid) diff --git a/website/web/genericapi.py b/website/web/genericapi.py index e31fe5f4..02ee2fce 100644 --- a/website/web/genericapi.py +++ b/website/web/genericapi.py @@ -65,6 +65,30 @@ class CaptureStatusQuery(Resource): return {'status_code': lookyloo.get_capture_status(capture_uuid)} +@api.route('/json//hostnames') +@api.doc(description='Get all the hostnames of all the resources of a capture', + params={'capture_uuid': 'The UUID of the capture'}) +class CaptureHostnames(Resource): + def get(self, capture_uuid: str): + cache = lookyloo.capture_cache(capture_uuid) + if not cache: + return {'error': 'UUID missing in cache, try again later.'} + to_return: Dict[str, Any] = {'response': {'hostnames': list(lookyloo.get_hostnames(capture_uuid))}} + return to_return + + +@api.route('/json//urls') +@api.doc(description='Get all the URLs of all the resources of a capture', + params={'capture_uuid': 'The UUID of the capture'}) +class CaptureURLs(Resource): + def get(self, capture_uuid: str): + cache = lookyloo.capture_cache(capture_uuid) + if not cache: + return {'error': 'UUID missing in cache, try again later.'} + to_return: Dict[str, Any] = {'response': {'urls': list(lookyloo.get_urls(capture_uuid))}} + return to_return + + @api.route('/json//hashes') @api.doc(description='Get all the hashes of all the resources of a capture', params={'capture_uuid': 'The UUID of the capture'}) From cf3d39728ca67a8dbc881d6c1103163b1d57245a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 17 Jun 2021 14:08:14 -0700 Subject: [PATCH 15/18] chg: Bump typessubs --- poetry.lock | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7468b04a..96a9fb2a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1090,7 +1090,7 @@ python-versions = "*" [[package]] name = "types-click" -version = "7.1.1" +version = "7.1.2" description = "Typing stubs for click" category = "dev" optional = false @@ -1098,7 +1098,7 @@ python-versions = "*" [[package]] name = "types-flask" -version = "1.1.0" +version = "1.1.1" description = "Typing stubs for Flask" category = "dev" optional = false @@ -1111,7 +1111,7 @@ types-Werkzeug = "*" [[package]] name = "types-jinja2" -version = "2.11.1" +version = "2.11.2" description = "Typing stubs for Jinja2" category = "dev" optional = false @@ -1122,7 +1122,7 @@ types-MarkupSafe = "*" [[package]] name = "types-markupsafe" -version = "1.1.2" +version = "1.1.3" description = "Typing stubs for MarkupSafe" category = "dev" optional = false @@ -1130,7 +1130,7 @@ python-versions = "*" [[package]] name = "types-pkg-resources" -version = "0.1.2" +version = "0.1.3" description = "Typing stubs for pkg_resources" category = "dev" optional = false @@ -1138,7 +1138,7 @@ python-versions = "*" [[package]] name = "types-redis" -version = "3.5.1" +version = "3.5.2" description = "Typing stubs for redis" category = "dev" optional = false @@ -1146,7 +1146,7 @@ python-versions = "*" [[package]] name = "types-requests" -version = "0.1.11" +version = "0.1.12" description = "Typing stubs for requests" category = "dev" optional = false @@ -1154,7 +1154,7 @@ python-versions = "*" [[package]] name = "types-werkzeug" -version = "1.0.1" +version = "1.0.2" description = "Typing stubs for Werkzeug" category = "dev" optional = false @@ -1970,28 +1970,36 @@ twisted-iocpsupport = [ {file = "twisted_iocpsupport-1.0.1-pp37-pypy37_pp73-win32.whl", hash = "sha256:1535e537b6d60d0cfd2b4785f1c7a021beb62d8df6bac3dc6c45bb866ce648e4"}, ] types-click = [ - {file = "types_click-7.1.1-py2.py3-none-any.whl", hash = "sha256:b1db0fb31a6d4a31c4282c231827cbdab628b0d504b016937822d943cd5e40a5"}, + {file = "types-click-7.1.2.tar.gz", hash = "sha256:040897284e4f9466825c3865f708a985a8e7ba4d8e22cb9198ffb7b522160850"}, + {file = "types_click-7.1.2-py2.py3-none-any.whl", hash = "sha256:4722746f1ec9fd3fc8b1d7fb8c840604dc22f9e32bcc7a31a36d6d85cc2bce24"}, ] types-flask = [ - {file = "types_Flask-1.1.0-py2.py3-none-any.whl", hash = "sha256:0d16fbd554b274d003761b10bb06e86aa535b122cd98b0a8e45082750c54e35b"}, + {file = "types-Flask-1.1.1.tar.gz", hash = "sha256:90afe8bd050cf8830cdc620bb9aa4471836b86af88e4d33fccc8789242a661e2"}, + {file = "types_Flask-1.1.1-py2.py3-none-any.whl", hash = "sha256:4885fd9c64756b0901d3de0a197d0ede60df114d0fbab500aa1bd73bb2c1651c"}, ] types-jinja2 = [ - {file = "types_Jinja2-2.11.1-py2.py3-none-any.whl", hash = "sha256:84f66a2612376ff3c1656198b10a04c7711b6d917d4f5f7f2c2f7ecec0afa040"}, + {file = "types-Jinja2-2.11.2.tar.gz", hash = "sha256:5b53d2b8bc6dd6dfbc0ae3e33e346fbe343cbeba1ed528858749e2a2ffe0e143"}, + {file = "types_Jinja2-2.11.2-py2.py3-none-any.whl", hash = "sha256:d27e112a8add449407de235f4533239149056327c8bddc6b0d6bf80cd7280c16"}, ] types-markupsafe = [ - {file = "types_MarkupSafe-1.1.2-py2.py3-none-any.whl", hash = "sha256:b5e311cb6aad7f6da0bb1455494305e2bb7941b04c3c8cf9ed7bbd33cf8ba374"}, + {file = "types-MarkupSafe-1.1.3.tar.gz", hash = "sha256:be8975ba91bd7e672f6f57753ca7ba2979ad9b6687a0e93dd2055926f8c71b0b"}, + {file = "types_MarkupSafe-1.1.3-py2.py3-none-any.whl", hash = "sha256:b1893d090c72204110c232d9b964d2612e15deff738bb75360030473a45cbc0e"}, ] types-pkg-resources = [ - {file = "types_pkg_resources-0.1.2-py2.py3-none-any.whl", hash = "sha256:42d640500de564f1ccc21f918117afadf78039e4fa7f513c647ccf742d609aeb"}, + {file = "types-pkg_resources-0.1.3.tar.gz", hash = "sha256:834a9b8d3dbea343562fd99d5d3359a726f6bf9d3733bccd2b4f3096fbab9dae"}, + {file = "types_pkg_resources-0.1.3-py2.py3-none-any.whl", hash = "sha256:0cb9972cee992249f93fff1a491bf2dc3ce674e5a1926e27d4f0866f7d9b6d9c"}, ] types-redis = [ - {file = "types_redis-3.5.1-py2.py3-none-any.whl", hash = "sha256:b27d7f4b8e8858b1549cba14527d3c4d953089ea1538f37698d0da8eb968fab9"}, + {file = "types-redis-3.5.2.tar.gz", hash = "sha256:eaf57c717c5c63e9b3a8e812aec2cf2115f5c97e9596157187af05ae485ee1dc"}, + {file = "types_redis-3.5.2-py2.py3-none-any.whl", hash = "sha256:0c4aa5c2f8b78093303d3d4e64c0cf9882ec8ad4e9e301b9a5fb35039eeac6c0"}, ] types-requests = [ - {file = "types_requests-0.1.11-py2.py3-none-any.whl", hash = "sha256:e79c09e5866c6520cdf23dc260573a0a190444f36b112116fadc5542656c8309"}, + {file = "types-requests-0.1.12.tar.gz", hash = "sha256:08323ce2be816c4c107ef605e66369cb26dfda99d278dd502ee32bbf37f9a285"}, + {file = "types_requests-0.1.12-py2.py3-none-any.whl", hash = "sha256:59b218da05de6eb58e0ab9d096af5e258359365cdf52cf3a1206a8ea87244301"}, ] types-werkzeug = [ - {file = "types_Werkzeug-1.0.1-py2.py3-none-any.whl", hash = "sha256:51dca6aa5be250e5b2e3150c8ab40e531408e4684db7054d02b5b8dcd2a193cd"}, + {file = "types-Werkzeug-1.0.2.tar.gz", hash = "sha256:7f6d4c8771a67d44e83134d56e59b482bf81ebd28e6557015fdfcc81e3d11b53"}, + {file = "types_Werkzeug-1.0.2-py2.py3-none-any.whl", hash = "sha256:1a4e551955e6fc608cf6b93a2749963b9cce5ff56cddfc90404af1c919afa937"}, ] typing-extensions = [ {file = "typing_extensions-3.10.0.0-py2-none-any.whl", hash = "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497"}, From a9a529f8a0c5274c9d81a8f39471abd0fff4ad77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 6 Jul 2021 06:40:40 +0200 Subject: [PATCH 16/18] chg: Use restx that supports flask >2.0 --- poetry.lock | 20 ++++++++++++-------- pyproject.toml | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7ad9a275..d6636f36 100644 --- a/poetry.lock +++ b/poetry.lock @@ -290,25 +290,32 @@ Flask = "*" [[package]] name = "flask-restx" -version = "0.4.0" +version = "0.4.1.dev0" description = "Fully featured framework for fast, easy and documented API development with Flask" category = "main" optional = false python-versions = "*" +develop = false [package.dependencies] aniso8601 = {version = ">=0.82", markers = "python_version >= \"3.5\""} -Flask = ">=0.8,<2.0.0" +Flask = ">=0.8,<2.0.0 || >2.0.0" jsonschema = "*" pytz = "*" six = ">=1.3.0" -werkzeug = "<2.0.0" +werkzeug = "!=2.0.0" [package.extras] dev = ["blinker", "Faker (==2.0.0)", "mock (==3.0.5)", "pytest-benchmark (==3.2.2)", "pytest-cov (==2.7.1)", "pytest-flask (==0.15.1)", "pytest-mock (==1.10.4)", "pytest-profiling (==1.7.0)", "tzlocal", "invoke (==1.3.0)", "readme-renderer (==24.0)", "twine (==1.15.0)", "tox", "pytest (==4.6.5)", "pytest (==5.4.1)", "ossaudit", "black"] doc = ["alabaster (==0.7.12)", "Sphinx (==2.1.2)", "sphinx-issues (==1.2.0)"] test = ["blinker", "Faker (==2.0.0)", "mock (==3.0.5)", "pytest-benchmark (==3.2.2)", "pytest-cov (==2.7.1)", "pytest-flask (==0.15.1)", "pytest-mock (==1.10.4)", "pytest-profiling (==1.7.0)", "tzlocal", "invoke (==1.3.0)", "readme-renderer (==24.0)", "twine (==1.15.0)", "pytest (==4.6.5)", "pytest (==5.4.1)", "ossaudit"] +[package.source] +type = "git" +url = "https://github.com/python-restx/flask-restx.git" +reference = "master" +resolved_reference = "0dc1c3c966d0394da4b807e491225e90f46b27b8" + [[package]] name = "gunicorn" version = "20.1.0" @@ -1262,7 +1269,7 @@ misp = ["python-magic", "pydeep"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "7cb24042a7ab8e1b97c9804c34ff2a28115d7963844eef796385798ac8ff33eb" +content-hash = "436048496998de03e68f4796ded142f84c9ce81935e1670db54b136ef062ba0a" [metadata.files] aiohttp = [ @@ -1474,10 +1481,7 @@ flask-login = [ {file = "Flask-Login-0.5.0.tar.gz", hash = "sha256:6d33aef15b5bcead780acc339464aae8a6e28f13c90d8b1cf9de8b549d1c0b4b"}, {file = "Flask_Login-0.5.0-py2.py3-none-any.whl", hash = "sha256:7451b5001e17837ba58945aead261ba425fdf7b4f0448777e597ddab39f4fba0"}, ] -flask-restx = [ - {file = "flask-restx-0.4.0.tar.gz", hash = "sha256:8bdbfd8ad2949383c490b3b180481c0e9f70163230a5b567a56592c82f7c5cb0"}, - {file = "flask_restx-0.4.0-py2.py3-none-any.whl", hash = "sha256:780ac57ebda25e9f17ffccf3ff1b47a2c4efe5787cebddbbba1a91f53560d79f"}, -] +flask-restx = [] gunicorn = [ {file = "gunicorn-20.1.0.tar.gz", hash = "sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8"}, ] diff --git a/pyproject.toml b/pyproject.toml index ebd88537..7ca05882 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ pydeep = {version = "^0.4", optional = true} Pillow = "^8.3.0" lief = "^0.11.4" Flask-Login = "^0.5.0" -flask-restx = "^0.4.0" +flask-restx = {git = "https://github.com/python-restx/flask-restx.git"} [tool.poetry.extras] misp = ['python-magic', 'pydeep'] From d60c1cc13c81ea8c489023b9823d3e6264457f4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 7 Jul 2021 11:27:25 +0200 Subject: [PATCH 17/18] chg: Use latest update on python-restx Until the release, install restx this way: https://github.com/python-restx/flask-restx/issues/313#issuecomment-874489319 --- poetry.lock | 133 +++++++++++++++++++++++++------------------------ pyproject.toml | 2 +- setup.py | 2 +- 3 files changed, 69 insertions(+), 68 deletions(-) diff --git a/poetry.lock b/poetry.lock index d6636f36..919461f4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -148,17 +148,20 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "click" -version = "7.1.2" +version = "8.0.1" description = "Composable command line interface toolkit" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.6" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "colorama" version = "0.4.4" description = "Cross-platform colored terminal text." -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -260,21 +263,20 @@ python-versions = "*" [[package]] name = "flask" -version = "1.1.4" +version = "2.0.1" description = "A simple framework for building complex web applications." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.6" [package.dependencies] -click = ">=5.1,<8.0" -itsdangerous = ">=0.24,<2.0" -Jinja2 = ">=2.10.1,<3.0" -Werkzeug = ">=0.15,<2.0" +click = ">=7.1.2" +itsdangerous = ">=2.0" +Jinja2 = ">=3.0" +Werkzeug = ">=2.0" [package.extras] -dev = ["pytest", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"] -docs = ["sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"] +async = ["asgiref (>=3.2)"] dotenv = ["python-dotenv"] [[package]] @@ -474,11 +476,11 @@ w3lib = ">=1.17.0" [[package]] name = "itsdangerous" -version = "1.1.0" -description = "Various helpers to pass data to untrusted environments and back." +version = "2.0.1" +description = "Safely pass data to untrusted environments and back." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.6" [[package]] name = "jedi" @@ -497,17 +499,17 @@ testing = ["Django (<3.1)", "colorama", "docopt", "pytest (<6.0.0)"] [[package]] name = "jinja2" -version = "2.11.3" +version = "3.0.1" description = "A very fast and expressive template engine." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.6" [package.dependencies] -MarkupSafe = ">=0.23" +MarkupSafe = ">=2.0" [package.extras] -i18n = ["Babel (>=0.8)"] +i18n = ["Babel (>=2.7)"] [[package]] name = "jmespath" @@ -663,7 +665,7 @@ python-versions = "*" [[package]] name = "pillow" -version = "8.3.0" +version = "8.3.1" description = "Python Imaging Library (Fork)" category = "main" optional = false @@ -1220,14 +1222,13 @@ python-versions = "*" [[package]] name = "werkzeug" -version = "1.0.1" +version = "2.0.1" description = "The comprehensive WSGI web application library." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.6" [package.extras] -dev = ["pytest", "pytest-timeout", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinx-issues"] watchdog = ["watchdog"] [[package]] @@ -1423,8 +1424,8 @@ chardet = [ {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"}, ] click = [ - {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, - {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, + {file = "click-8.0.1-py3-none-any.whl", hash = "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6"}, + {file = "click-8.0.1.tar.gz", hash = "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a"}, ] colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, @@ -1474,8 +1475,8 @@ filetype = [ {file = "filetype-1.0.7.tar.gz", hash = "sha256:da393ece8d98b47edf2dd5a85a2c8733e44b769e32c71af4cd96ed8d38d96aa7"}, ] flask = [ - {file = "Flask-1.1.4-py2.py3-none-any.whl", hash = "sha256:c34f04500f2cbbea882b1acb02002ad6fe6b7ffa64a6164577995657f50aed22"}, - {file = "Flask-1.1.4.tar.gz", hash = "sha256:0fbeb6180d383a9186d0d6ed954e0042ad9f18e0e8de088b2b419d526927d196"}, + {file = "Flask-2.0.1-py3-none-any.whl", hash = "sha256:a6209ca15eb63fc9385f38e452704113d679511d9574d09b2cf9183ae7d20dc9"}, + {file = "Flask-2.0.1.tar.gz", hash = "sha256:1c4c257b1892aec1398784c63791cbaa43062f1f7aeb555c4da961b20ee68f55"}, ] flask-login = [ {file = "Flask-Login-0.5.0.tar.gz", hash = "sha256:6d33aef15b5bcead780acc339464aae8a6e28f13c90d8b1cf9de8b549d1c0b4b"}, @@ -1530,16 +1531,16 @@ itemloaders = [ {file = "itemloaders-1.0.4.tar.gz", hash = "sha256:1277cd8ca3e4c02dcdfbc1bcae9134ad89acfa6041bd15b4561c6290203a0c96"}, ] itsdangerous = [ - {file = "itsdangerous-1.1.0-py2.py3-none-any.whl", hash = "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749"}, - {file = "itsdangerous-1.1.0.tar.gz", hash = "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19"}, + {file = "itsdangerous-2.0.1-py3-none-any.whl", hash = "sha256:5174094b9637652bdb841a3029700391451bd092ba3db90600dea710ba28e97c"}, + {file = "itsdangerous-2.0.1.tar.gz", hash = "sha256:9e724d68fc22902a1435351f84c3fb8623f303fffcc566a4cb952df8c572cff0"}, ] jedi = [ {file = "jedi-0.18.0-py2.py3-none-any.whl", hash = "sha256:18456d83f65f400ab0c2d3319e48520420ef43b23a086fdc05dff34132f0fb93"}, {file = "jedi-0.18.0.tar.gz", hash = "sha256:92550a404bad8afed881a137ec9a461fed49eca661414be45059329614ed0707"}, ] jinja2 = [ - {file = "Jinja2-2.11.3-py2.py3-none-any.whl", hash = "sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419"}, - {file = "Jinja2-2.11.3.tar.gz", hash = "sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6"}, + {file = "Jinja2-3.0.1-py3-none-any.whl", hash = "sha256:1f06f2da51e7b56b8f238affdd6b4e2c61e39598a378cc49345bc1bd42a978a4"}, + {file = "Jinja2-3.0.1.tar.gz", hash = "sha256:703f484b47a6af502e743c9122595cc812b0271f661722403114f71a79d0f5a4"}, ] jmespath = [ {file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"}, @@ -1767,40 +1768,40 @@ pickleshare = [ {file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"}, ] pillow = [ - {file = "Pillow-8.3.0-cp36-cp36m-macosx_10_10_x86_64.whl", hash = "sha256:333313bcc53a8a7359e98d5458dfe37bfa301da2fd0e0dc41f585ae0cede9181"}, - {file = "Pillow-8.3.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bccd0d604d814e9494f3bf3f077a23835580ed1743c5175581882e7dd1f178c3"}, - {file = "Pillow-8.3.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a7beda44f177ee602aa27e0a297da1657d9572679522c8fb8b336b734653516e"}, - {file = "Pillow-8.3.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:94db5ea640330de0945b41dc77fb4847b4ab6e87149126c71b36b112e8400898"}, - {file = "Pillow-8.3.0-cp36-cp36m-win32.whl", hash = "sha256:856fcbc3201a6cabf0478daa0c0a1a8a175af7e5173e2084ddb91cc707a09dd1"}, - {file = "Pillow-8.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:34ce3d993cb4ca840b1e31165b38cb19c64f64f822a8bc5565bde084baff3bdb"}, - {file = "Pillow-8.3.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:778a819c2d194e08d39d67ddb15ef0d32eba17bf7d0c2773e97bd221b2613a3e"}, - {file = "Pillow-8.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b42ea77f4e7374a67e1f27aaa9c62627dff681f67890e5b8f0c1e21b1500d9d2"}, - {file = "Pillow-8.3.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:53f6e4b73b3899015ac4aa95d99da0f48ea18a6d7c8db672e8bead3fb9570ef5"}, - {file = "Pillow-8.3.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:fb91deb5121b6dde88599bcb3db3fdad9cf33ff3d4ccc5329ee1fe9655a2f7ff"}, - {file = "Pillow-8.3.0-cp37-cp37m-win32.whl", hash = "sha256:8f65d2a98f198e904dbe89ecb10862d5f0511367d823689039e17c4d011de11e"}, - {file = "Pillow-8.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:25f6564df21d15bcac142b4ed92b6c02e53557539f535f31c1f3bcc985484753"}, - {file = "Pillow-8.3.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:c2d78c8230bda5fc9f6b1d457c7f8f3432f4fe85bed86f80ba3ed73d59775a88"}, - {file = "Pillow-8.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:950e873ceefbd283cbe7bc5b648b832d1dcf89eeded6726ebec42bc7d67966c0"}, - {file = "Pillow-8.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1037288a22cc8ec9d2918a24ded733a1cc4342fd7f21d15d37e6bbe5fb4a7306"}, - {file = "Pillow-8.3.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:063d17a02a0170c2f880fbd373b2738b089c6adcbd1f7418667bc9e97524c11b"}, - {file = "Pillow-8.3.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:561339ed7c324bbcb29b5e4f4705c97df950785394b3ac181f5bf6a08088a672"}, - {file = "Pillow-8.3.0-cp38-cp38-win32.whl", hash = "sha256:331f8321418682386e4f0d0e6369f732053f95abddd2af4e1b1ef74a9537ef37"}, - {file = "Pillow-8.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:eccaefbd646022b5313ca4b0c5f1ae6e0d3a52ef66de64970ecf3f9b2a1be751"}, - {file = "Pillow-8.3.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:6f7517a220aca8b822e25b08b0df9546701a606a328da5bc057e5f32a3f9b07c"}, - {file = "Pillow-8.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cc8e926d6ffa65d0dddb871b7afe117f17bc045951e66afde60eb0eba923db9e"}, - {file = "Pillow-8.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:519b3b24dedc81876d893475bade1b92c4ce7c24b9b82224f0bd8daae682e039"}, - {file = "Pillow-8.3.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:72858a27dd7bd1c40f91c4f85db3b9f121c8412fd66573121febb00d074d0530"}, - {file = "Pillow-8.3.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:3251557c53c1ed0c345559afc02d2b0a0aa5788042e161366ed90b27bc322d3d"}, - {file = "Pillow-8.3.0-cp39-cp39-win32.whl", hash = "sha256:ce90aad0a3dc0f13a9ff0ab1f43bcbea436089b83c3fadbe37c6f1733b938bf1"}, - {file = "Pillow-8.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:490c9236ef4762733b6c2e1f1fcb37793cb9c57d860aa84d6994c990461882e5"}, - {file = "Pillow-8.3.0-pp36-pypy36_pp73-macosx_10_10_x86_64.whl", hash = "sha256:aef0838f28328523e9e5f2c1852dd96fb85768deb0eb8f908c54dad0f44d2f6f"}, - {file = "Pillow-8.3.0-pp36-pypy36_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:713b762892efa8cd5d8dac24d16ac2d2dbf981963ed1b3297e79755f03f8cbb8"}, - {file = "Pillow-8.3.0-pp36-pypy36_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cec702974f162026bf8de47f6f4b7ce9584a63c50002b38f195ee797165fea77"}, - {file = "Pillow-8.3.0-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:d9ef8119ce44f90d2f8ac7c58f7da480ada5151f217dc8da03681b73fc91dec3"}, - {file = "Pillow-8.3.0-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fc25d59ecf23ea19571065306806a29c43c67f830f0e8a121303916ba257f484"}, - {file = "Pillow-8.3.0-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:28f184c0a65be098af412f78b0b6f3bbafd1614e1dc896e810d8357342a794b7"}, - {file = "Pillow-8.3.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:c3529fb98a40f89269175442c5ff4ef81d22e91b2bdcbd33833a350709b5130c"}, - {file = "Pillow-8.3.0.tar.gz", hash = "sha256:803606e206f3e366eea46b1e7ab4dac74cfac770d04de9c35319814e11e47c46"}, + {file = "Pillow-8.3.1-cp36-cp36m-macosx_10_10_x86_64.whl", hash = "sha256:196560dba4da7a72c5e7085fccc5938ab4075fd37fe8b5468869724109812edd"}, + {file = "Pillow-8.3.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c9569049d04aaacd690573a0398dbd8e0bf0255684fee512b413c2142ab723"}, + {file = "Pillow-8.3.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c088a000dfdd88c184cc7271bfac8c5b82d9efa8637cd2b68183771e3cf56f04"}, + {file = "Pillow-8.3.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:fc214a6b75d2e0ea7745488da7da3c381f41790812988c7a92345978414fad37"}, + {file = "Pillow-8.3.1-cp36-cp36m-win32.whl", hash = "sha256:a17ca41f45cf78c2216ebfab03add7cc350c305c38ff34ef4eef66b7d76c5229"}, + {file = "Pillow-8.3.1-cp36-cp36m-win_amd64.whl", hash = "sha256:67b3666b544b953a2777cb3f5a922e991be73ab32635666ee72e05876b8a92de"}, + {file = "Pillow-8.3.1-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:ff04c373477723430dce2e9d024c708a047d44cf17166bf16e604b379bf0ca14"}, + {file = "Pillow-8.3.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9364c81b252d8348e9cc0cb63e856b8f7c1b340caba6ee7a7a65c968312f7dab"}, + {file = "Pillow-8.3.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a2f381932dca2cf775811a008aa3027671ace723b7a38838045b1aee8669fdcf"}, + {file = "Pillow-8.3.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:d0da39795049a9afcaadec532e7b669b5ebbb2a9134576ebcc15dd5bdae33cc0"}, + {file = "Pillow-8.3.1-cp37-cp37m-win32.whl", hash = "sha256:2b6dfa068a8b6137da34a4936f5a816aba0ecc967af2feeb32c4393ddd671cba"}, + {file = "Pillow-8.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a4eef1ff2d62676deabf076f963eda4da34b51bc0517c70239fafed1d5b51500"}, + {file = "Pillow-8.3.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:660a87085925c61a0dcc80efb967512ac34dbb256ff7dd2b9b4ee8dbdab58cf4"}, + {file = "Pillow-8.3.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:15a2808e269a1cf2131930183dcc0419bc77bb73eb54285dde2706ac9939fa8e"}, + {file = "Pillow-8.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:969cc558cca859cadf24f890fc009e1bce7d7d0386ba7c0478641a60199adf79"}, + {file = "Pillow-8.3.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2ee77c14a0299d0541d26f3d8500bb57e081233e3fa915fa35abd02c51fa7fae"}, + {file = "Pillow-8.3.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c11003197f908878164f0e6da15fce22373ac3fc320cda8c9d16e6bba105b844"}, + {file = "Pillow-8.3.1-cp38-cp38-win32.whl", hash = "sha256:3f08bd8d785204149b5b33e3b5f0ebbfe2190ea58d1a051c578e29e39bfd2367"}, + {file = "Pillow-8.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:70af7d222df0ff81a2da601fab42decb009dc721545ed78549cb96e3a1c5f0c8"}, + {file = "Pillow-8.3.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:37730f6e68bdc6a3f02d2079c34c532330d206429f3cee651aab6b66839a9f0e"}, + {file = "Pillow-8.3.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4bc3c7ef940eeb200ca65bd83005eb3aae8083d47e8fcbf5f0943baa50726856"}, + {file = "Pillow-8.3.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c35d09db702f4185ba22bb33ef1751ad49c266534339a5cebeb5159d364f6f82"}, + {file = "Pillow-8.3.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0b2efa07f69dc395d95bb9ef3299f4ca29bcb2157dc615bae0b42c3c20668ffc"}, + {file = "Pillow-8.3.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:cc866706d56bd3a7dbf8bac8660c6f6462f2f2b8a49add2ba617bc0c54473d83"}, + {file = "Pillow-8.3.1-cp39-cp39-win32.whl", hash = "sha256:9a211b663cf2314edbdb4cf897beeb5c9ee3810d1d53f0e423f06d6ebbf9cd5d"}, + {file = "Pillow-8.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:c2a5ff58751670292b406b9f06e07ed1446a4b13ffced6b6cab75b857485cbc8"}, + {file = "Pillow-8.3.1-pp36-pypy36_pp73-macosx_10_10_x86_64.whl", hash = "sha256:c379425c2707078dfb6bfad2430728831d399dc95a7deeb92015eb4c92345eaf"}, + {file = "Pillow-8.3.1-pp36-pypy36_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:114f816e4f73f9ec06997b2fde81a92cbf0777c9e8f462005550eed6bae57e63"}, + {file = "Pillow-8.3.1-pp36-pypy36_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8960a8a9f4598974e4c2aeb1bff9bdd5db03ee65fd1fce8adf3223721aa2a636"}, + {file = "Pillow-8.3.1-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:147bd9e71fb9dcf08357b4d530b5167941e222a6fd21f869c7911bac40b9994d"}, + {file = "Pillow-8.3.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1fd5066cd343b5db88c048d971994e56b296868766e461b82fa4e22498f34d77"}, + {file = "Pillow-8.3.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f4ebde71785f8bceb39dcd1e7f06bcc5d5c3cf48b9f69ab52636309387b097c8"}, + {file = "Pillow-8.3.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:1c03e24be975e2afe70dfc5da6f187eea0b49a68bb2b69db0f30a61b7031cee4"}, + {file = "Pillow-8.3.1.tar.gz", hash = "sha256:2cac53839bfc5cece8fdbe7f084d5e3ee61e1303cccc86511d351adcb9e2c792"}, ] priority = [ {file = "priority-1.3.0-py2.py3-none-any.whl", hash = "sha256:be4fcb94b5e37cdeb40af5533afe6dd603bd665fe9c8b3052610fc1001d5d1eb"}, @@ -2051,8 +2052,8 @@ wcwidth = [ {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, ] werkzeug = [ - {file = "Werkzeug-1.0.1-py2.py3-none-any.whl", hash = "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43"}, - {file = "Werkzeug-1.0.1.tar.gz", hash = "sha256:6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c"}, + {file = "Werkzeug-2.0.1-py3-none-any.whl", hash = "sha256:6c1ec500dcdba0baa27600f6a22f6333d8b662d22027ff9f6202e3367413caa8"}, + {file = "Werkzeug-2.0.1.tar.gz", hash = "sha256:1de1db30d010ff1af14a009224ec49ab2329ad2cde454c8a708130642d579c42"}, ] wrapt = [ {file = "wrapt-1.12.1.tar.gz", hash = "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7"}, diff --git a/pyproject.toml b/pyproject.toml index 7ca05882..042c4ce4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "lookyloo" -version = "1.6.0" +version = "1.7.0-dev" description = "Web interface to track the trackers." authors = ["Raphaël Vinot "] license = "BSD-3-Clause" diff --git a/setup.py b/setup.py index 214139f3..8e17aeac 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from setuptools import setup # type: ignore setup( name='lookyloo', - version='1.0-dev', + version='1.7-dev', author='Raphaël Vinot', author_email='raphael.vinot@circl.lu', maintainer='Raphaël Vinot', From f9cdb62bbb6e1b0721d87a7d0def6b12141c6548 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 21 Jul 2021 11:04:06 +0200 Subject: [PATCH 18/18] chg: Use flask-restx from pypi --- poetry.lock | 220 ++++++++++++++++++++++++++----------------------- pyproject.toml | 2 +- 2 files changed, 118 insertions(+), 104 deletions(-) diff --git a/poetry.lock b/poetry.lock index 919461f4..7b6692b4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -129,7 +129,7 @@ python-versions = "*" [[package]] name = "cffi" -version = "1.14.5" +version = "1.14.6" description = "Foreign Function Interface for Python calling C code." category = "main" optional = false @@ -146,6 +146,17 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "charset-normalizer" +version = "2.0.3" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = ">=3.5.0" + +[package.extras] +unicode_backport = ["unicodedata2"] + [[package]] name = "click" version = "8.0.1" @@ -292,12 +303,11 @@ Flask = "*" [[package]] name = "flask-restx" -version = "0.4.1.dev0" +version = "0.5.0" description = "Fully featured framework for fast, easy and documented API development with Flask" category = "main" optional = false python-versions = "*" -develop = false [package.dependencies] aniso8601 = {version = ">=0.82", markers = "python_version >= \"3.5\""} @@ -312,12 +322,6 @@ dev = ["blinker", "Faker (==2.0.0)", "mock (==3.0.5)", "pytest-benchmark (==3.2. doc = ["alabaster (==0.7.12)", "Sphinx (==2.1.2)", "sphinx-issues (==1.2.0)"] test = ["blinker", "Faker (==2.0.0)", "mock (==3.0.5)", "pytest-benchmark (==3.2.2)", "pytest-cov (==2.7.1)", "pytest-flask (==0.15.1)", "pytest-mock (==1.10.4)", "pytest-profiling (==1.7.0)", "tzlocal", "invoke (==1.3.0)", "readme-renderer (==24.0)", "twine (==1.15.0)", "pytest (==4.6.5)", "pytest (==5.4.1)", "ossaudit"] -[package.source] -type = "git" -url = "https://github.com/python-restx/flask-restx.git" -reference = "master" -resolved_reference = "0dc1c3c966d0394da4b807e491225e90f46b27b8" - [[package]] name = "gunicorn" version = "20.1.0" @@ -346,7 +350,7 @@ hyperframe = ">=5.2.0,<6" [[package]] name = "har2tree" -version = "1.6.4" +version = "1.6.5" description = "HTTP Archive (HAR) to ETE Toolkit generator" category = "main" optional = false @@ -364,7 +368,7 @@ six = ">=1.16.0,<2.0.0" w3lib = ">=1.22.0,<2.0.0" [package.extras] -docs = ["Sphinx (>=4.0,<5.0)"] +docs = ["Sphinx (>=4.1.0,<5.0.0)"] [[package]] name = "hpack" @@ -395,11 +399,11 @@ idna = ">=2.5" [[package]] name = "idna" -version = "2.10" +version = "3.2" description = "Internationalized Domain Names in Applications (IDNA)" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.5" [[package]] name = "incremental" @@ -454,7 +458,7 @@ python-versions = "*" [[package]] name = "itemadapter" -version = "0.2.0" +version = "0.3.0" description = "Common interface for data container classes" category = "main" optional = false @@ -612,7 +616,7 @@ python-versions = "*" [[package]] name = "numpy" -version = "1.21.0" +version = "1.21.1" description = "NumPy is the fundamental package for array computing with Python." category = "main" optional = false @@ -860,14 +864,14 @@ python-versions = ">=3.6" [[package]] name = "pysanejs" -version = "1.4" +version = "1.5" description = "Python client for SaneJS" category = "main" optional = false -python-versions = ">=3.6,<4.0" +python-versions = ">=3.8,<4.0" [package.dependencies] -requests = ">=2.25.0,<3.0.0" +requests = ">=2.26.0,<3.0.0" [[package]] name = "pytaxonomies" @@ -883,7 +887,7 @@ webui = ["flask-nav (>=0.6,<0.7)", "Flask (>=1.1.1,<2.0.0)", "Flask-Bootstrap (> [[package]] name = "python-dateutil" -version = "2.8.1" +version = "2.8.2" description = "Extensions to the standard Python datetime module" category = "main" optional = false @@ -929,21 +933,21 @@ hiredis = ["hiredis (>=0.1.3)"] [[package]] name = "requests" -version = "2.25.1" +version = "2.26.0" description = "Python HTTP for Humans." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [package.dependencies] certifi = ">=2017.4.17" -chardet = ">=3.0.2,<5" -idna = ">=2.5,<3" +charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} +idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} urllib3 = ">=1.21.1,<1.27" [package.extras] -security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] [[package]] name = "scrapy" @@ -1270,7 +1274,7 @@ misp = ["python-magic", "pydeep"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "436048496998de03e68f4796ded142f84c9ce81935e1670db54b136ef062ba0a" +content-hash = "84076d77474193846f7fac6a881b94dc937ad7570b0bfa104fbec87432241353" [metadata.files] aiohttp = [ @@ -1381,48 +1385,55 @@ certifi = [ {file = "certifi-2021.5.30.tar.gz", hash = "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee"}, ] cffi = [ - {file = "cffi-1.14.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:bb89f306e5da99f4d922728ddcd6f7fcebb3241fc40edebcb7284d7514741991"}, - {file = "cffi-1.14.5-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:34eff4b97f3d982fb93e2831e6750127d1355a923ebaeeb565407b3d2f8d41a1"}, - {file = "cffi-1.14.5-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:99cd03ae7988a93dd00bcd9d0b75e1f6c426063d6f03d2f90b89e29b25b82dfa"}, - {file = "cffi-1.14.5-cp27-cp27m-win32.whl", hash = "sha256:65fa59693c62cf06e45ddbb822165394a288edce9e276647f0046e1ec26920f3"}, - {file = "cffi-1.14.5-cp27-cp27m-win_amd64.whl", hash = "sha256:51182f8927c5af975fece87b1b369f722c570fe169f9880764b1ee3bca8347b5"}, - {file = "cffi-1.14.5-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:43e0b9d9e2c9e5d152946b9c5fe062c151614b262fda2e7b201204de0b99e482"}, - {file = "cffi-1.14.5-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:cbde590d4faaa07c72bf979734738f328d239913ba3e043b1e98fe9a39f8b2b6"}, - {file = "cffi-1.14.5-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:5de7970188bb46b7bf9858eb6890aad302577a5f6f75091fd7cdd3ef13ef3045"}, - {file = "cffi-1.14.5-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:a465da611f6fa124963b91bf432d960a555563efe4ed1cc403ba5077b15370aa"}, - {file = "cffi-1.14.5-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:d42b11d692e11b6634f7613ad8df5d6d5f8875f5d48939520d351007b3c13406"}, - {file = "cffi-1.14.5-cp35-cp35m-win32.whl", hash = "sha256:72d8d3ef52c208ee1c7b2e341f7d71c6fd3157138abf1a95166e6165dd5d4369"}, - {file = "cffi-1.14.5-cp35-cp35m-win_amd64.whl", hash = "sha256:29314480e958fd8aab22e4a58b355b629c59bf5f2ac2492b61e3dc06d8c7a315"}, - {file = "cffi-1.14.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:3d3dd4c9e559eb172ecf00a2a7517e97d1e96de2a5e610bd9b68cea3925b4892"}, - {file = "cffi-1.14.5-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:48e1c69bbacfc3d932221851b39d49e81567a4d4aac3b21258d9c24578280058"}, - {file = "cffi-1.14.5-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:69e395c24fc60aad6bb4fa7e583698ea6cc684648e1ffb7fe85e3c1ca131a7d5"}, - {file = "cffi-1.14.5-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:9e93e79c2551ff263400e1e4be085a1210e12073a31c2011dbbda14bda0c6132"}, - {file = "cffi-1.14.5-cp36-cp36m-win32.whl", hash = "sha256:58e3f59d583d413809d60779492342801d6e82fefb89c86a38e040c16883be53"}, - {file = "cffi-1.14.5-cp36-cp36m-win_amd64.whl", hash = "sha256:005a36f41773e148deac64b08f233873a4d0c18b053d37da83f6af4d9087b813"}, - {file = "cffi-1.14.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2894f2df484ff56d717bead0a5c2abb6b9d2bf26d6960c4604d5c48bbc30ee73"}, - {file = "cffi-1.14.5-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:0857f0ae312d855239a55c81ef453ee8fd24136eaba8e87a2eceba644c0d4c06"}, - {file = "cffi-1.14.5-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:cd2868886d547469123fadc46eac7ea5253ea7fcb139f12e1dfc2bbd406427d1"}, - {file = "cffi-1.14.5-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:35f27e6eb43380fa080dccf676dece30bef72e4a67617ffda586641cd4508d49"}, - {file = "cffi-1.14.5-cp37-cp37m-win32.whl", hash = "sha256:9ff227395193126d82e60319a673a037d5de84633f11279e336f9c0f189ecc62"}, - {file = "cffi-1.14.5-cp37-cp37m-win_amd64.whl", hash = "sha256:9cf8022fb8d07a97c178b02327b284521c7708d7c71a9c9c355c178ac4bbd3d4"}, - {file = "cffi-1.14.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b198cec6c72df5289c05b05b8b0969819783f9418e0409865dac47288d2a053"}, - {file = "cffi-1.14.5-cp38-cp38-manylinux1_i686.whl", hash = "sha256:ad17025d226ee5beec591b52800c11680fca3df50b8b29fe51d882576e039ee0"}, - {file = "cffi-1.14.5-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:6c97d7350133666fbb5cf4abdc1178c812cb205dc6f41d174a7b0f18fb93337e"}, - {file = "cffi-1.14.5-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8ae6299f6c68de06f136f1f9e69458eae58f1dacf10af5c17353eae03aa0d827"}, - {file = "cffi-1.14.5-cp38-cp38-win32.whl", hash = "sha256:b85eb46a81787c50650f2392b9b4ef23e1f126313b9e0e9013b35c15e4288e2e"}, - {file = "cffi-1.14.5-cp38-cp38-win_amd64.whl", hash = "sha256:1f436816fc868b098b0d63b8920de7d208c90a67212546d02f84fe78a9c26396"}, - {file = "cffi-1.14.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1071534bbbf8cbb31b498d5d9db0f274f2f7a865adca4ae429e147ba40f73dea"}, - {file = "cffi-1.14.5-cp39-cp39-manylinux1_i686.whl", hash = "sha256:9de2e279153a443c656f2defd67769e6d1e4163952b3c622dcea5b08a6405322"}, - {file = "cffi-1.14.5-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:6e4714cc64f474e4d6e37cfff31a814b509a35cb17de4fb1999907575684479c"}, - {file = "cffi-1.14.5-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:158d0d15119b4b7ff6b926536763dc0714313aa59e320ddf787502c70c4d4bee"}, - {file = "cffi-1.14.5-cp39-cp39-win32.whl", hash = "sha256:afb29c1ba2e5a3736f1c301d9d0abe3ec8b86957d04ddfa9d7a6a42b9367e396"}, - {file = "cffi-1.14.5-cp39-cp39-win_amd64.whl", hash = "sha256:f2d45f97ab6bb54753eab54fffe75aaf3de4ff2341c9daee1987ee1837636f1d"}, - {file = "cffi-1.14.5.tar.gz", hash = "sha256:fd78e5fee591709f32ef6edb9a015b4aa1a5022598e36227500c8f4e02328d9c"}, + {file = "cffi-1.14.6-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:22b9c3c320171c108e903d61a3723b51e37aaa8c81255b5e7ce102775bd01e2c"}, + {file = "cffi-1.14.6-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:f0c5d1acbfca6ebdd6b1e3eded8d261affb6ddcf2186205518f1428b8569bb99"}, + {file = "cffi-1.14.6-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:99f27fefe34c37ba9875f224a8f36e31d744d8083e00f520f133cab79ad5e819"}, + {file = "cffi-1.14.6-cp27-cp27m-win32.whl", hash = "sha256:55af55e32ae468e9946f741a5d51f9896da6b9bf0bbdd326843fec05c730eb20"}, + {file = "cffi-1.14.6-cp27-cp27m-win_amd64.whl", hash = "sha256:7bcac9a2b4fdbed2c16fa5681356d7121ecabf041f18d97ed5b8e0dd38a80224"}, + {file = "cffi-1.14.6-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:ed38b924ce794e505647f7c331b22a693bee1538fdf46b0222c4717b42f744e7"}, + {file = "cffi-1.14.6-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:e22dcb48709fc51a7b58a927391b23ab37eb3737a98ac4338e2448bef8559b33"}, + {file = "cffi-1.14.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e8c6a99be100371dbb046880e7a282152aa5d6127ae01783e37662ef73850d8f"}, + {file = "cffi-1.14.6-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:19ca0dbdeda3b2615421d54bef8985f72af6e0c47082a8d26122adac81a95872"}, + {file = "cffi-1.14.6-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:d950695ae4381ecd856bcaf2b1e866720e4ab9a1498cba61c602e56630ca7195"}, + {file = "cffi-1.14.6-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9dc245e3ac69c92ee4c167fbdd7428ec1956d4e754223124991ef29eb57a09d"}, + {file = "cffi-1.14.6-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a8661b2ce9694ca01c529bfa204dbb144b275a31685a075ce123f12331be790b"}, + {file = "cffi-1.14.6-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b315d709717a99f4b27b59b021e6207c64620790ca3e0bde636a6c7f14618abb"}, + {file = "cffi-1.14.6-cp36-cp36m-win32.whl", hash = "sha256:80b06212075346b5546b0417b9f2bf467fea3bfe7352f781ffc05a8ab24ba14a"}, + {file = "cffi-1.14.6-cp36-cp36m-win_amd64.whl", hash = "sha256:a9da7010cec5a12193d1af9872a00888f396aba3dc79186604a09ea3ee7c029e"}, + {file = "cffi-1.14.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4373612d59c404baeb7cbd788a18b2b2a8331abcc84c3ba40051fcd18b17a4d5"}, + {file = "cffi-1.14.6-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:f10afb1004f102c7868ebfe91c28f4a712227fe4cb24974350ace1f90e1febbf"}, + {file = "cffi-1.14.6-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:fd4305f86f53dfd8cd3522269ed7fc34856a8ee3709a5e28b2836b2db9d4cd69"}, + {file = "cffi-1.14.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d6169cb3c6c2ad50db5b868db6491a790300ade1ed5d1da29289d73bbe40b56"}, + {file = "cffi-1.14.6-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5d4b68e216fc65e9fe4f524c177b54964af043dde734807586cf5435af84045c"}, + {file = "cffi-1.14.6-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33791e8a2dc2953f28b8d8d300dde42dd929ac28f974c4b4c6272cb2955cb762"}, + {file = "cffi-1.14.6-cp37-cp37m-win32.whl", hash = "sha256:0c0591bee64e438883b0c92a7bed78f6290d40bf02e54c5bf0978eaf36061771"}, + {file = "cffi-1.14.6-cp37-cp37m-win_amd64.whl", hash = "sha256:8eb687582ed7cd8c4bdbff3df6c0da443eb89c3c72e6e5dcdd9c81729712791a"}, + {file = "cffi-1.14.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ba6f2b3f452e150945d58f4badd92310449876c4c954836cfb1803bdd7b422f0"}, + {file = "cffi-1.14.6-cp38-cp38-manylinux1_i686.whl", hash = "sha256:64fda793737bc4037521d4899be780534b9aea552eb673b9833b01f945904c2e"}, + {file = "cffi-1.14.6-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:9f3e33c28cd39d1b655ed1ba7247133b6f7fc16fa16887b120c0c670e35ce346"}, + {file = "cffi-1.14.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26bb2549b72708c833f5abe62b756176022a7b9a7f689b571e74c8478ead51dc"}, + {file = "cffi-1.14.6-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb687a11f0a7a1839719edd80f41e459cc5366857ecbed383ff376c4e3cc6afd"}, + {file = "cffi-1.14.6-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d2ad4d668a5c0645d281dcd17aff2be3212bc109b33814bbb15c4939f44181cc"}, + {file = "cffi-1.14.6-cp38-cp38-win32.whl", hash = "sha256:487d63e1454627c8e47dd230025780e91869cfba4c753a74fda196a1f6ad6548"}, + {file = "cffi-1.14.6-cp38-cp38-win_amd64.whl", hash = "sha256:c33d18eb6e6bc36f09d793c0dc58b0211fccc6ae5149b808da4a62660678b156"}, + {file = "cffi-1.14.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:06c54a68935738d206570b20da5ef2b6b6d92b38ef3ec45c5422c0ebaf338d4d"}, + {file = "cffi-1.14.6-cp39-cp39-manylinux1_i686.whl", hash = "sha256:f174135f5609428cc6e1b9090f9268f5c8935fddb1b25ccb8255a2d50de6789e"}, + {file = "cffi-1.14.6-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:f3ebe6e73c319340830a9b2825d32eb6d8475c1dac020b4f0aa774ee3b898d1c"}, + {file = "cffi-1.14.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c8d896becff2fa653dc4438b54a5a25a971d1f4110b32bd3068db3722c80202"}, + {file = "cffi-1.14.6-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4922cd707b25e623b902c86188aca466d3620892db76c0bdd7b99a3d5e61d35f"}, + {file = "cffi-1.14.6-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c9e005e9bd57bc987764c32a1bee4364c44fdc11a3cc20a40b93b444984f2b87"}, + {file = "cffi-1.14.6-cp39-cp39-win32.whl", hash = "sha256:eb9e2a346c5238a30a746893f23a9535e700f8192a68c07c0258e7ece6ff3728"}, + {file = "cffi-1.14.6-cp39-cp39-win_amd64.whl", hash = "sha256:818014c754cd3dba7229c0f5884396264d51ffb87ec86e927ef0be140bfdb0d2"}, + {file = "cffi-1.14.6.tar.gz", hash = "sha256:c9a875ce9d7fe32887784274dd533c57909b7b1dcadcc128a2ac21331a9765dd"}, ] chardet = [ {file = "chardet-4.0.0-py2.py3-none-any.whl", hash = "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"}, {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"}, ] +charset-normalizer = [ + {file = "charset-normalizer-2.0.3.tar.gz", hash = "sha256:c46c3ace2d744cfbdebceaa3c19ae691f53ae621b39fd7570f59d14fb7f2fd12"}, + {file = "charset_normalizer-2.0.3-py3-none-any.whl", hash = "sha256:88fce3fa5b1a84fdcb3f603d889f723d1dd89b26059d0123ca435570e848d5e1"}, +] click = [ {file = "click-8.0.1-py3-none-any.whl", hash = "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6"}, {file = "click-8.0.1.tar.gz", hash = "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a"}, @@ -1482,7 +1493,10 @@ flask-login = [ {file = "Flask-Login-0.5.0.tar.gz", hash = "sha256:6d33aef15b5bcead780acc339464aae8a6e28f13c90d8b1cf9de8b549d1c0b4b"}, {file = "Flask_Login-0.5.0-py2.py3-none-any.whl", hash = "sha256:7451b5001e17837ba58945aead261ba425fdf7b4f0448777e597ddab39f4fba0"}, ] -flask-restx = [] +flask-restx = [ + {file = "flask-restx-0.5.0.tar.gz", hash = "sha256:7e9f7cd5e843dd653a71fafb7c8ce9d7b4fef29f982a2254b1e0ebb3fac1fe12"}, + {file = "flask_restx-0.5.0-py2.py3-none-any.whl", hash = "sha256:c3c2b724e688c0a50ee5e78f2a508b7f0c34644f00f64170fa8a3d0cdc34f67a"}, +] gunicorn = [ {file = "gunicorn-20.1.0.tar.gz", hash = "sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8"}, ] @@ -1491,8 +1505,8 @@ h2 = [ {file = "h2-3.2.0.tar.gz", hash = "sha256:875f41ebd6f2c44781259005b157faed1a5031df3ae5aa7bcb4628a6c0782f14"}, ] har2tree = [ - {file = "har2tree-1.6.4-py3-none-any.whl", hash = "sha256:eef68d6835c67df14769d77fc4525c531ac545fc16f4671acd8744db81608922"}, - {file = "har2tree-1.6.4.tar.gz", hash = "sha256:3a2806a6917e5944e1d0751187345dc38b5c67d49de1da557b2ff24f9eb2b1a6"}, + {file = "har2tree-1.6.5-py3-none-any.whl", hash = "sha256:0d56154326d2f2d31be0bed5efc6049164fd1a7b9da3cc065118e393e0b28bbe"}, + {file = "har2tree-1.6.5.tar.gz", hash = "sha256:c07b2efaaae2fc2727a22345e5663dab90cf2f2aa23bc7f658939c2c3be5aa93"}, ] hpack = [ {file = "hpack-3.0.0-py2.py3-none-any.whl", hash = "sha256:0edd79eda27a53ba5be2dfabf3b15780928a0dff6eb0c60a3d6767720e970c89"}, @@ -1507,8 +1521,8 @@ hyperlink = [ {file = "hyperlink-21.0.0.tar.gz", hash = "sha256:427af957daa58bc909471c6c40f74c5450fa123dd093fc53efd2e91d2705a56b"}, ] idna = [ - {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, - {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, + {file = "idna-3.2-py3-none-any.whl", hash = "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a"}, + {file = "idna-3.2.tar.gz", hash = "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"}, ] incremental = [ {file = "incremental-21.3.0-py2.py3-none-any.whl", hash = "sha256:92014aebc6a20b78a8084cdd5645eeaa7f74b8933f70fa3ada2cfbd1e3b54321"}, @@ -1523,8 +1537,8 @@ ipython-genutils = [ {file = "ipython_genutils-0.2.0.tar.gz", hash = "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8"}, ] itemadapter = [ - {file = "itemadapter-0.2.0-py3-none-any.whl", hash = "sha256:5327c2136353cb965b6b4ba564af002fd458691b8e30d3bd6b14c474d92c6b25"}, - {file = "itemadapter-0.2.0.tar.gz", hash = "sha256:cb7aaa577fefe2aa6f229ccf4d058e05f44e0178a98c8fb70ee4d95acfabb423"}, + {file = "itemadapter-0.3.0-py3-none-any.whl", hash = "sha256:cfc7964518016412dfa23ade9d094ec3b5b3009f200117d4ce773aceff6efe5a"}, + {file = "itemadapter-0.3.0.tar.gz", hash = "sha256:ab2651ba20f5f6d0e15f041deba4c13ffc59270def2bd01518d13e94c4cd27d1"}, ] itemloaders = [ {file = "itemloaders-1.0.4-py3-none-any.whl", hash = "sha256:4cb46a0f8915e910c770242ae3b60b1149913ed37162804f1e40e8535d6ec497"}, @@ -1722,34 +1736,34 @@ mypy-extensions = [ {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] numpy = [ - {file = "numpy-1.21.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d5caa946a9f55511e76446e170bdad1d12d6b54e17a2afe7b189112ed4412bb8"}, - {file = "numpy-1.21.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ac4fd578322842dbda8d968e3962e9f22e862b6ec6e3378e7415625915e2da4d"}, - {file = "numpy-1.21.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:598fe100b2948465cf3ed64b1a326424b5e4be2670552066e17dfaa67246011d"}, - {file = "numpy-1.21.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c55407f739f0bfcec67d0df49103f9333edc870061358ac8a8c9e37ea02fcd2"}, - {file = "numpy-1.21.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:75579acbadbf74e3afd1153da6177f846212ea2a0cc77de53523ae02c9256513"}, - {file = "numpy-1.21.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:cc367c86eb87e5b7c9592935620f22d13b090c609f1b27e49600cd033b529f54"}, - {file = "numpy-1.21.0-cp37-cp37m-win32.whl", hash = "sha256:d89b0dc7f005090e32bb4f9bf796e1dcca6b52243caf1803fdd2b748d8561f63"}, - {file = "numpy-1.21.0-cp37-cp37m-win_amd64.whl", hash = "sha256:eda2829af498946c59d8585a9fd74da3f810866e05f8df03a86f70079c7531dd"}, - {file = "numpy-1.21.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:1a784e8ff7ea2a32e393cc53eb0003eca1597c7ca628227e34ce34eb11645a0e"}, - {file = "numpy-1.21.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bba474a87496d96e61461f7306fba2ebba127bed7836212c360f144d1e72ac54"}, - {file = "numpy-1.21.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fd0a359c1c17f00cb37de2969984a74320970e0ceef4808c32e00773b06649d9"}, - {file = "numpy-1.21.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e4d5a86a5257843a18fb1220c5f1c199532bc5d24e849ed4b0289fb59fbd4d8f"}, - {file = "numpy-1.21.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:620732f42259eb2c4642761bd324462a01cdd13dd111740ce3d344992dd8492f"}, - {file = "numpy-1.21.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9205711e5440954f861ceeea8f1b415d7dd15214add2e878b4d1cf2bcb1a914"}, - {file = "numpy-1.21.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ad09f55cc95ed8d80d8ab2052f78cc21cb231764de73e229140d81ff49d8145e"}, - {file = "numpy-1.21.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a1f2fb2da242568af0271455b89aee0f71e4e032086ee2b4c5098945d0e11cf6"}, - {file = "numpy-1.21.0-cp38-cp38-win32.whl", hash = "sha256:e58ddb53a7b4959932f5582ac455ff90dcb05fac3f8dcc8079498d43afbbde6c"}, - {file = "numpy-1.21.0-cp38-cp38-win_amd64.whl", hash = "sha256:d2910d0a075caed95de1a605df00ee03b599de5419d0b95d55342e9a33ad1fb3"}, - {file = "numpy-1.21.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a290989cd671cd0605e9c91a70e6df660f73ae87484218e8285c6522d29f6e38"}, - {file = "numpy-1.21.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3537b967b350ad17633b35c2f4b1a1bbd258c018910b518c30b48c8e41272717"}, - {file = "numpy-1.21.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ccc6c650f8700ce1e3a77668bb7c43e45c20ac06ae00d22bdf6760b38958c883"}, - {file = "numpy-1.21.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:709884863def34d72b183d074d8ba5cfe042bc3ff8898f1ffad0209161caaa99"}, - {file = "numpy-1.21.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:bebab3eaf0641bba26039fb0b2c5bf9b99407924b53b1ea86e03c32c64ef5aef"}, - {file = "numpy-1.21.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf680682ad0a3bef56dae200dbcbac2d57294a73e5b0f9864955e7dd7c2c2491"}, - {file = "numpy-1.21.0-cp39-cp39-win32.whl", hash = "sha256:d95d16204cd51ff1a1c8d5f9958ce90ae190be81d348b514f9be39f878b8044a"}, - {file = "numpy-1.21.0-cp39-cp39-win_amd64.whl", hash = "sha256:2ba579dde0563f47021dcd652253103d6fd66165b18011dce1a0609215b2791e"}, - {file = "numpy-1.21.0-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3c40e6b860220ed862e8097b8f81c9af6d7405b723f4a7af24a267b46f90e461"}, - {file = "numpy-1.21.0.zip", hash = "sha256:e80fe25cba41c124d04c662f33f6364909b985f2eb5998aaa5ae4b9587242cce"}, + {file = "numpy-1.21.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38e8648f9449a549a7dfe8d8755a5979b45b3538520d1e735637ef28e8c2dc50"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fd7d7409fa643a91d0a05c7554dd68aa9c9bb16e186f6ccfe40d6e003156e33a"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a75b4498b1e93d8b700282dc8e655b8bd559c0904b3910b144646dbbbc03e062"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1412aa0aec3e00bc23fbb8664d76552b4efde98fb71f60737c83efbac24112f1"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e46ceaff65609b5399163de5893d8f2a82d3c77d5e56d976c8b5fb01faa6b671"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c6a2324085dd52f96498419ba95b5777e40b6bcbc20088fddb9e8cbb58885e8e"}, + {file = "numpy-1.21.1-cp37-cp37m-win32.whl", hash = "sha256:73101b2a1fef16602696d133db402a7e7586654682244344b8329cdcbbb82172"}, + {file = "numpy-1.21.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7a708a79c9a9d26904d1cca8d383bf869edf6f8e7650d85dbc77b041e8c5a0f8"}, + {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95b995d0c413f5d0428b3f880e8fe1660ff9396dcd1f9eedbc311f37b5652e16"}, + {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:635e6bd31c9fb3d475c8f44a089569070d10a9ef18ed13738b03049280281267"}, + {file = "numpy-1.21.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a3d5fb89bfe21be2ef47c0614b9c9c707b7362386c9a3ff1feae63e0267ccb6"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a326af80e86d0e9ce92bcc1e65c8ff88297de4fa14ee936cb2293d414c9ec63"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:791492091744b0fe390a6ce85cc1bf5149968ac7d5f0477288f78c89b385d9af"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0318c465786c1f63ac05d7c4dbcecd4d2d7e13f0959b01b534ea1e92202235c5"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a513bd9c1551894ee3d31369f9b07460ef223694098cf27d399513415855b68"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:91c6f5fc58df1e0a3cc0c3a717bb3308ff850abdaa6d2d802573ee2b11f674a8"}, + {file = "numpy-1.21.1-cp38-cp38-win32.whl", hash = "sha256:978010b68e17150db8765355d1ccdd450f9fc916824e8c4e35ee620590e234cd"}, + {file = "numpy-1.21.1-cp38-cp38-win_amd64.whl", hash = "sha256:9749a40a5b22333467f02fe11edc98f022133ee1bfa8ab99bda5e5437b831214"}, + {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d7a4aeac3b94af92a9373d6e77b37691b86411f9745190d2c351f410ab3a791f"}, + {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9e7912a56108aba9b31df688a4c4f5cb0d9d3787386b87d504762b6754fbb1b"}, + {file = "numpy-1.21.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:25b40b98ebdd272bc3020935427a4530b7d60dfbe1ab9381a39147834e985eac"}, + {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a92c5aea763d14ba9d6475803fc7904bda7decc2a0a68153f587ad82941fec1"}, + {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:05a0f648eb28bae4bcb204e6fd14603de2908de982e761a2fc78efe0f19e96e1"}, + {file = "numpy-1.21.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f01f28075a92eede918b965e86e8f0ba7b7797a95aa8d35e1cc8821f5fc3ad6a"}, + {file = "numpy-1.21.1-cp39-cp39-win32.whl", hash = "sha256:88c0b89ad1cc24a5efbb99ff9ab5db0f9a86e9cc50240177a571fbe9c2860ac2"}, + {file = "numpy-1.21.1-cp39-cp39-win_amd64.whl", hash = "sha256:01721eefe70544d548425a07c80be8377096a54118070b8a62476866d5208e33"}, + {file = "numpy-1.21.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2d4d1de6e6fb3d28781c73fbde702ac97f03d79e4ffd6598b880b2d95d62ead4"}, + {file = "numpy-1.21.1.zip", hash = "sha256:dff4af63638afcc57a3dfb9e4b26d434a7a602d225b42d746ea7fe2edf1342fd"}, ] parsel = [ {file = "parsel-1.6.0-py2.py3-none-any.whl", hash = "sha256:9e1fa8db1c0b4a878bf34b35c043d89c9d1cbebc23b4d34dbc3c0ec33f2e087d"}, @@ -1914,16 +1928,16 @@ pyrsistent = [ {file = "pyrsistent-0.18.0.tar.gz", hash = "sha256:773c781216f8c2900b42a7b638d5b517bb134ae1acbebe4d1e8f1f41ea60eb4b"}, ] pysanejs = [ - {file = "pysanejs-1.4-py3-none-any.whl", hash = "sha256:55d1c14a4e0a8658cefb622b175c62901e33bea24fbc68f2aa420f592ed687e8"}, - {file = "pysanejs-1.4.tar.gz", hash = "sha256:c3c767acb6329c8c4ff888fb105e63ebc7825c1aba04b9ed44c03cbebc6e8246"}, + {file = "pysanejs-1.5-py3-none-any.whl", hash = "sha256:e36779030a0f1777a59912a5a830b2854eb580f6e78940b7ce901e7a6f98c7bc"}, + {file = "pysanejs-1.5.tar.gz", hash = "sha256:67f3e7b3a8b60ba5c9bcda84129dcea7180ee1db7bb96090f630be904b9e7ec1"}, ] pytaxonomies = [ {file = "pytaxonomies-1.3-py3-none-any.whl", hash = "sha256:c3c698d2b4030580a4f4ee251a56e816670d678ad433f059920a15792cdf9850"}, {file = "pytaxonomies-1.3.tar.gz", hash = "sha256:c03658aec15b8783f871634b95d8dfde204990ddd6c548f1fa18a1e1f3e3eb36"}, ] python-dateutil = [ - {file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"}, - {file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"}, + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, ] python-magic = [ {file = "python-magic-0.4.24.tar.gz", hash = "sha256:de800df9fb50f8ec5974761054a708af6e4246b03b4bdaee993f948947b0ebcf"}, @@ -1942,8 +1956,8 @@ redis = [ {file = "redis-3.5.3.tar.gz", hash = "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2"}, ] requests = [ - {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"}, - {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"}, + {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"}, + {file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"}, ] scrapy = [ {file = "Scrapy-2.5.0-py2.py3-none-any.whl", hash = "sha256:5f590fdc84b496e5a4bb5ef99836b0aa688a07cfcb4bc3bb7290f66486f27424"}, diff --git a/pyproject.toml b/pyproject.toml index 042c4ce4..b604b28b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ pydeep = {version = "^0.4", optional = true} Pillow = "^8.3.0" lief = "^0.11.4" Flask-Login = "^0.5.0" -flask-restx = {git = "https://github.com/python-restx/flask-restx.git"} +flask-restx = "^0.5.0" [tool.poetry.extras] misp = ['python-magic', 'pydeep']