diff --git a/client/poetry.lock b/client/poetry.lock index 8b1b976..0ea2ced 100644 --- a/client/poetry.lock +++ b/client/poetry.lock @@ -4,7 +4,7 @@ description = "Python package for providing Mozilla's CA Bundle." name = "certifi" optional = false python-versions = "*" -version = "2020.4.5.1" +version = "2020.6.20" [[package]] category = "main" @@ -28,7 +28,7 @@ description = "Python HTTP for Humans." name = "requests" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "2.23.0" +version = "2.24.0" [package.dependencies] certifi = ">=2017.4.17" @@ -59,8 +59,8 @@ python-versions = "^3.6" [metadata.files] certifi = [ - {file = "certifi-2020.4.5.1-py2.py3-none-any.whl", hash = "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304"}, - {file = "certifi-2020.4.5.1.tar.gz", hash = "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519"}, + {file = "certifi-2020.6.20-py2.py3-none-any.whl", hash = "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"}, + {file = "certifi-2020.6.20.tar.gz", hash = "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3"}, ] chardet = [ {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, @@ -71,8 +71,8 @@ idna = [ {file = "idna-2.9.tar.gz", hash = "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb"}, ] requests = [ - {file = "requests-2.23.0-py2.py3-none-any.whl", hash = "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee"}, - {file = "requests-2.23.0.tar.gz", hash = "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6"}, + {file = "requests-2.24.0-py2.py3-none-any.whl", hash = "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"}, + {file = "requests-2.24.0.tar.gz", hash = "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b"}, ] urllib3 = [ {file = "urllib3-1.25.9-py2.py3-none-any.whl", hash = "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115"}, diff --git a/client/pylookyloo/__init__.py b/client/pylookyloo/__init__.py index a951534..ef45532 100644 --- a/client/pylookyloo/__init__.py +++ b/client/pylookyloo/__init__.py @@ -1,13 +1,15 @@ -from .api import Lookyloo - import argparse +import json + +from .api import Lookyloo def main(): parser = argparse.ArgumentParser(description='Enqueue a URL on Lookyloo.', epilog='The response is the permanent URL where you can see the result of the capture.') parser.add_argument('--url', type=str, help='URL of the instance (defaults to https://lookyloo.circl.lu/, the public instance).') - parser.add_argument('--query', required=True, help='URL to enqueue.') + parser.add_argument('--query', help='URL to enqueue.') parser.add_argument('--listing', default=False, action='store_true', help='Should the report be publicly listed.') + parser.add_argument('--redirects', help='Get redirects for a given capture.') args = parser.parse_args() if args.url: @@ -16,7 +18,11 @@ def main(): lookyloo = Lookyloo() if lookyloo.is_up: - url = lookyloo.enqueue(args.query, listing=args.listing) - print(url) + if args.query: + url = lookyloo.enqueue(args.query, listing=args.listing) + print(url) + else: + response = lookyloo.get_redirects(args.redirects) + print(json.dumps(response)) else: print(f'Unable to reach {lookyloo.root_url}. Is the server up?') diff --git a/client/pylookyloo/api.py b/client/pylookyloo/api.py index 25d818b..8f916d4 100644 --- a/client/pylookyloo/api.py +++ b/client/pylookyloo/api.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from typing import Optional +from typing import Optional, Dict, Any from urllib.parse import urljoin +from pathlib import Path import requests @@ -20,7 +21,12 @@ class Lookyloo(): r = self.session.head(self.root_url) return r.status_code == 200 - def enqueue(self, url: Optional[str]=None, **kwargs) -> str: + def enqueue(self, url: Optional[str]=None, quiet: bool=False, **kwargs) -> str: + '''Enqueue an URL. + :param url: URL to enqueue + :param quiet: Returns the UUID only, instead of the whole URL + :param kwargs: accepts all the parameters supported by `Lookyloo.scrape` + ''' if not url and 'url' not in kwargs: raise Exception(f'url entry required: {kwargs}') @@ -29,4 +35,11 @@ class Lookyloo(): else: to_send = kwargs response = self.session.post(urljoin(self.root_url, 'submit'), json=to_send) - return urljoin(self.root_url, f'tree/{response.text}') + if quiet: + return response.text + else: + return urljoin(self.root_url, f'tree/{response.text}') + + def get_redirects(self, capture_uuid: str) -> Dict[str, Any]: + r = self.session.get(urljoin(self.root_url, str(Path('json', capture_uuid, 'redirects')))) + return r.json() diff --git a/poetry.lock b/poetry.lock index 25e0a0b..8668598 100644 --- a/poetry.lock +++ b/poetry.lock @@ -109,7 +109,7 @@ description = "Bootstrap helper for Flask/Jinja2." name = "bootstrap-flask" optional = false python-versions = "*" -version = "1.3.2" +version = "1.4" [package.dependencies] Flask = "*" @@ -132,7 +132,7 @@ description = "Python package for providing Mozilla's CA Bundle." name = "certifi" optional = false python-versions = "*" -version = "2020.4.5.2" +version = "2020.6.20" [[package]] category = "main" @@ -296,7 +296,7 @@ description = "HTTP Archive (HAR) to ETE Toolkit generator" name = "har2tree" optional = false python-versions = ">=3.6,<4.0" -version = "1.1" +version = "1.1.1" [package.dependencies] beautifulsoup4 = ">=4.8.2,<5.0.0" @@ -407,14 +407,14 @@ description = "An autocompletion tool for Python that can be used for text edito name = "jedi" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "0.17.0" +version = "0.17.1" [package.dependencies] -parso = ">=0.7.0" +parso = ">=0.7.0,<0.8.0" [package.extras] qa = ["flake8 (3.7.9)"] -testing = ["colorama", "docopt", "pytest (>=3.9.0,<5.0.0)"] +testing = ["Django (<3.1)", "colorama", "docopt", "pytest (>=3.9.0,<5.0.0)"] [[package]] category = "main" @@ -458,7 +458,7 @@ description = "More routines for operating on iterables, beyond itertools" name = "more-itertools" optional = false python-versions = ">=3.5" -version = "8.3.0" +version = "8.4.0" [[package]] category = "main" @@ -610,7 +610,7 @@ description = "library with cross-python path, ini-parsing, io, code, log facili name = "py" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "1.8.1" +version = "1.9.0" [[package]] category = "main" @@ -731,7 +731,7 @@ description = "Python client for SaneJS" name = "pysanejs" optional = false python-versions = ">=3.6,<4.0" -version = "1.2" +version = "1.3" [package.dependencies] requests = ">=2.22.0,<3.0.0" @@ -802,7 +802,7 @@ description = "Python HTTP for Humans." name = "requests" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "2.23.0" +version = "2.24.0" [package.dependencies] certifi = ">=2017.4.17" @@ -1014,7 +1014,7 @@ description = "Measures the displayed width of unicode strings in a terminal" name = "wcwidth" optional = false python-versions = "*" -version = "0.2.4" +version = "0.2.5" [[package]] category = "main" @@ -1124,8 +1124,8 @@ beautifulsoup4 = [ {file = "beautifulsoup4-4.9.1.tar.gz", hash = "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7"}, ] bootstrap-flask = [ - {file = "Bootstrap-Flask-1.3.2.tar.gz", hash = "sha256:26c61912fff4584e23472ae66f2d98a8d7340bc1c89b48d21e19acb900356b9b"}, - {file = "Bootstrap_Flask-1.3.2-py2.py3-none-any.whl", hash = "sha256:3df7044dc9075c01e0c198450da2bd2b84e21a72c8e7cf6157ba51edb8a08ad9"}, + {file = "Bootstrap-Flask-1.4.tar.gz", hash = "sha256:f5e75b7fb976ba2268eac88240e1a9b249c7848a0cb639f45a6be14d913c88c2"}, + {file = "Bootstrap_Flask-1.4-py2.py3-none-any.whl", hash = "sha256:a64368423512bff9e12b2d9f63be510bd3926f31e61b6be3760761d67a47a786"}, ] cchardet = [ {file = "cchardet-2.1.6-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:2aa1b008965c703ad6597361b0f6d427c8971fe94a2c99ec3724c228ae50d6a6"}, @@ -1159,8 +1159,8 @@ cchardet = [ {file = "cchardet-2.1.6.tar.gz", hash = "sha256:b76afb2059ad69eab576949980a17413c1e9e5a5624abf9e43542d8853f146b3"}, ] certifi = [ - {file = "certifi-2020.4.5.2-py2.py3-none-any.whl", hash = "sha256:9cd41137dc19af6a5e03b630eefe7d1f458d964d406342dd3edf625839b944cc"}, - {file = "certifi-2020.4.5.2.tar.gz", hash = "sha256:5ad7e9a056d25ffa5082862e36f119f7f7cec6457fa07ee2f8c339814b80c9b1"}, + {file = "certifi-2020.6.20-py2.py3-none-any.whl", hash = "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"}, + {file = "certifi-2020.6.20.tar.gz", hash = "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3"}, ] cffi = [ {file = "cffi-1.14.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1cae98a7054b5c9391eb3249b86e0e99ab1e02bb0cc0575da191aedadbdf4384"}, @@ -1260,8 +1260,8 @@ gunicorn = [ {file = "gunicorn-20.0.4.tar.gz", hash = "sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626"}, ] har2tree = [ - {file = "har2tree-1.1-py3-none-any.whl", hash = "sha256:d2e1053ac6fa52fc5e77d1c2b20d6bbdc56f70b02e61462617ac23a3269aaa6d"}, - {file = "har2tree-1.1.tar.gz", hash = "sha256:802b1f97fb3d80674d3ac8c28e0dac92d5a1da01af20bb93ce45a2584c80af6d"}, + {file = "har2tree-1.1.1-py3-none-any.whl", hash = "sha256:c001ee687c1b114c416404912fded4e82db67d891a3a658d88130b82a339f73c"}, + {file = "har2tree-1.1.1.tar.gz", hash = "sha256:ff80c721783c2c50fe68c9e9ec7800396a561e624b4856ed05f0002815c35d5c"}, ] hyperlink = [ {file = "hyperlink-19.0.0-py2.py3-none-any.whl", hash = "sha256:ab4a308feb039b04f855a020a6eda3b18ca5a68e6d8f8c899cbe9e653721d04f"}, @@ -1292,8 +1292,8 @@ itsdangerous = [ {file = "itsdangerous-1.1.0.tar.gz", hash = "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19"}, ] jedi = [ - {file = "jedi-0.17.0-py2.py3-none-any.whl", hash = "sha256:cd60c93b71944d628ccac47df9a60fec53150de53d42dc10a7fc4b5ba6aae798"}, - {file = "jedi-0.17.0.tar.gz", hash = "sha256:df40c97641cb943661d2db4c33c2e1ff75d491189423249e989bcea4464f3030"}, + {file = "jedi-0.17.1-py2.py3-none-any.whl", hash = "sha256:1ddb0ec78059e8e27ec9eb5098360b4ea0a3dd840bedf21415ea820c21b40a22"}, + {file = "jedi-0.17.1.tar.gz", hash = "sha256:807d5d4f96711a2bcfdd5dfa3b1ae6d09aa53832b182090b222b5efb81f52f63"}, ] jinja2 = [ {file = "Jinja2-2.11.2-py2.py3-none-any.whl", hash = "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035"}, @@ -1359,8 +1359,8 @@ markupsafe = [ {file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"}, ] more-itertools = [ - {file = "more-itertools-8.3.0.tar.gz", hash = "sha256:558bb897a2232f5e4f8e2399089e35aecb746e1f9191b6584a151647e89267be"}, - {file = "more_itertools-8.3.0-py3-none-any.whl", hash = "sha256:7818f596b1e87be009031c7653d01acc46ed422e6656b394b0f765ce66ed4982"}, + {file = "more-itertools-8.4.0.tar.gz", hash = "sha256:68c70cc7167bdf5c7c9d8f6954a7837089c6a36bf565383919bb595efb8a17e5"}, + {file = "more_itertools-8.4.0-py3-none-any.whl", hash = "sha256:b78134b2063dd214000685165d81c154522c3ee0a1c0d4d113c80361c234c5a2"}, ] multidict = [ {file = "multidict-4.7.6-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000"}, @@ -1441,8 +1441,8 @@ publicsuffix2 = [ {file = "publicsuffix2-2.20191221.tar.gz", hash = "sha256:00f8cc31aa8d0d5592a5ced19cccba7de428ebca985db26ac852d920ddd6fe7b"}, ] py = [ - {file = "py-1.8.1-py2.py3-none-any.whl", hash = "sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0"}, - {file = "py-1.8.1.tar.gz", hash = "sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa"}, + {file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"}, + {file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"}, ] pyasn1 = [ {file = "pyasn1-0.4.8-py2.4.egg", hash = "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3"}, @@ -1507,8 +1507,8 @@ pypydispatcher = [ {file = "PyPyDispatcher-2.1.2.tar.gz", hash = "sha256:b6bec5dfcff9d2535bca2b23c80eae367b1ac250a645106948d315fcfa9130f2"}, ] pysanejs = [ - {file = "pysanejs-1.2-py3-none-any.whl", hash = "sha256:2fc14edec435374eed3f699de6a5b5c826e322180d65d724af56ac9353dc4372"}, - {file = "pysanejs-1.2.tar.gz", hash = "sha256:371ef63fa2463daa34d736dce6a3e3ca0bdfd6c8599dab9ebcd987348550608e"}, + {file = "pysanejs-1.3-py3-none-any.whl", hash = "sha256:13de2fab01263b37e950c4b1acfe0c871cec99349303072d173d2f3a9cb099b8"}, + {file = "pysanejs-1.3.tar.gz", hash = "sha256:a15eefc1e4c83fe4f166518e562cd23ac75c846739f75727106bb1c02d37670d"}, ] pytest = [ {file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"}, @@ -1527,8 +1527,8 @@ redis = [ {file = "redis-3.5.3.tar.gz", hash = "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2"}, ] requests = [ - {file = "requests-2.23.0-py2.py3-none-any.whl", hash = "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee"}, - {file = "requests-2.23.0.tar.gz", hash = "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6"}, + {file = "requests-2.24.0-py2.py3-none-any.whl", hash = "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"}, + {file = "requests-2.24.0.tar.gz", hash = "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b"}, ] requests-toolbelt = [ {file = "requests-toolbelt-0.9.1.tar.gz", hash = "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0"}, @@ -1627,8 +1627,8 @@ w3lib = [ {file = "w3lib-1.22.0.tar.gz", hash = "sha256:0ad6d0203157d61149fd45aaed2e24f53902989c32fc1dccc2e2bfba371560df"}, ] wcwidth = [ - {file = "wcwidth-0.2.4-py2.py3-none-any.whl", hash = "sha256:79375666b9954d4a1a10739315816324c3e73110af9d0e102d906fdb0aec009f"}, - {file = "wcwidth-0.2.4.tar.gz", hash = "sha256:8c6b5b6ee1360b842645f336d9e5d68c55817c26d3050f46b235ef2bc650e48f"}, + {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, + {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, ] werkzeug = [ {file = "Werkzeug-1.0.1-py2.py3-none-any.whl", hash = "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43"}, diff --git a/website/web/__init__.py b/website/web/__init__.py index 1d4b8b4..a3938db 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -471,7 +471,7 @@ def json_redirects(tree_uuid: str): if not cache: return {'error': 'UUID missing in cache, try again later.'} - to_return: Dict[str, list] = {'response': {'url': cache['url'], 'redirects': []}} + to_return: Dict[str, Any] = {'response': {'url': cache['url'], 'redirects': []}} if not cache['redirects']: to_return['response']['info'] = 'No redirects' return to_return @@ -479,5 +479,9 @@ def json_redirects(tree_uuid: str): # Trigger tree build, get all redirects lookyloo.load_tree(capture_dir) cache = lookyloo.capture_cache(capture_dir) - to_return['response']['redirects'] = cache['redirects'] + if cache: + to_return['response']['redirects'] = cache['redirects'] + else: + to_return['response']['redirects'] = cache['redirects'] + return jsonify(to_return)