CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

new: Add query for redirects in client

pull/79/head
Raphaël Vinot 2020-06-26 18:11:22 +02:00
parent ee29a638dd
commit fa935a6773
5 changed files with 68 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
client/poetry.lock generated
View File

@ -4,7 +4,7 @@ description = "Python package for providing Mozilla's CA Bundle."
name = "certifi"
optional = false
python-versions = "*"
version = "2020.4.5.1"
version = "2020.6.20"
[[package]]
category = "main"
@ -28,7 +28,7 @@ description = "Python HTTP for Humans."
name = "requests"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "2.23.0"
version = "2.24.0"
[package.dependencies]
certifi = ">=2017.4.17"
@ -59,8 +59,8 @@ python-versions = "^3.6"
[metadata.files]
certifi = [
{file = "certifi-2020.4.5.1-py2.py3-none-any.whl", hash = "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304"},
{file = "certifi-2020.4.5.1.tar.gz", hash = "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519"},
{file = "certifi-2020.6.20-py2.py3-none-any.whl", hash = "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"},
{file = "certifi-2020.6.20.tar.gz", hash = "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3"},
]
chardet = [
{file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"},
@ -71,8 +71,8 @@ idna = [
{file = "idna-2.9.tar.gz", hash = "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb"},
]
requests = [
{file = "requests-2.23.0-py2.py3-none-any.whl", hash = "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee"},
{file = "requests-2.23.0.tar.gz", hash = "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6"},
{file = "requests-2.24.0-py2.py3-none-any.whl", hash = "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"},
{file = "requests-2.24.0.tar.gz", hash = "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b"},
]
urllib3 = [
{file = "urllib3-1.25.9-py2.py3-none-any.whl", hash = "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115"},

16
client/pylookyloo/__init__.py
View File

@ -1,13 +1,15 @@
from .api import Lookyloo
import argparse
import json
from .api import Lookyloo
def main():
parser = argparse.ArgumentParser(description='Enqueue a URL on Lookyloo.', epilog='The response is the permanent URL where you can see the result of the capture.')
parser.add_argument('--url', type=str, help='URL of the instance (defaults to https://lookyloo.circl.lu/, the public instance).')
parser.add_argument('--query', required=True, help='URL to enqueue.')
parser.add_argument('--query', help='URL to enqueue.')
parser.add_argument('--listing', default=False, action='store_true', help='Should the report be publicly listed.')
parser.add_argument('--redirects', help='Get redirects for a given capture.')
args = parser.parse_args()
if args.url:
@ -16,7 +18,11 @@ def main():
lookyloo = Lookyloo()
if lookyloo.is_up:
url = lookyloo.enqueue(args.query, listing=args.listing)
print(url)
if args.query:
url = lookyloo.enqueue(args.query, listing=args.listing)
print(url)
else:
response = lookyloo.get_redirects(args.redirects)
print(json.dumps(response))
else:
print(f'Unable to reach {lookyloo.root_url}. Is the server up?')

19
client/pylookyloo/api.py
View File

@ -1,8 +1,9 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from typing import Optional
from typing import Optional, Dict, Any
from urllib.parse import urljoin
from pathlib import Path
import requests
@ -20,7 +21,12 @@ class Lookyloo():
r = self.session.head(self.root_url)
return r.status_code == 200
def enqueue(self, url: Optional[str]=None, **kwargs) -> str:
def enqueue(self, url: Optional[str]=None, quiet: bool=False, **kwargs) -> str:
'''Enqueue an URL.
:param url: URL to enqueue
:param quiet: Returns the UUID only, instead of the whole URL
:param kwargs: accepts all the parameters supported by `Lookyloo.scrape`
'''
if not url and 'url' not in kwargs:
raise Exception(f'url entry required: {kwargs}')
@ -29,4 +35,11 @@ class Lookyloo():
else:
to_send = kwargs
response = self.session.post(urljoin(self.root_url, 'submit'), json=to_send)
return urljoin(self.root_url, f'tree/{response.text}')
if quiet:
return response.text
else:
return urljoin(self.root_url, f'tree/{response.text}')
def get_redirects(self, capture_uuid: str) -> Dict[str, Any]:
r = self.session.get(urljoin(self.root_url, str(Path('json', capture_uuid, 'redirects'))))
return r.json()

58
poetry.lock generated
View File

@ -109,7 +109,7 @@ description = "Bootstrap helper for Flask/Jinja2."
name = "bootstrap-flask"
optional = false
python-versions = "*"
version = "1.3.2"
version = "1.4"
[package.dependencies]
Flask = "*"
@ -132,7 +132,7 @@ description = "Python package for providing Mozilla's CA Bundle."
name = "certifi"
optional = false
python-versions = "*"
version = "2020.4.5.2"
version = "2020.6.20"
[[package]]
category = "main"
@ -296,7 +296,7 @@ description = "HTTP Archive (HAR) to ETE Toolkit generator"
name = "har2tree"
optional = false
python-versions = ">=3.6,<4.0"
version = "1.1"
version = "1.1.1"
[package.dependencies]
beautifulsoup4 = ">=4.8.2,<5.0.0"
@ -407,14 +407,14 @@ description = "An autocompletion tool for Python that can be used for text edito
name = "jedi"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "0.17.0"
version = "0.17.1"
[package.dependencies]
parso = ">=0.7.0"
parso = ">=0.7.0,<0.8.0"
[package.extras]
qa = ["flake8 (3.7.9)"]
testing = ["colorama", "docopt", "pytest (>=3.9.0,<5.0.0)"]
testing = ["Django (<3.1)", "colorama", "docopt", "pytest (>=3.9.0,<5.0.0)"]
[[package]]
category = "main"
@ -458,7 +458,7 @@ description = "More routines for operating on iterables, beyond itertools"
name = "more-itertools"
optional = false
python-versions = ">=3.5"
version = "8.3.0"
version = "8.4.0"
[[package]]
category = "main"
@ -610,7 +610,7 @@ description = "library with cross-python path, ini-parsing, io, code, log facili
name = "py"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "1.8.1"
version = "1.9.0"
[[package]]
category = "main"
@ -731,7 +731,7 @@ description = "Python client for SaneJS"
name = "pysanejs"
optional = false
python-versions = ">=3.6,<4.0"
version = "1.2"
version = "1.3"
[package.dependencies]
requests = ">=2.22.0,<3.0.0"
@ -802,7 +802,7 @@ description = "Python HTTP for Humans."
name = "requests"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "2.23.0"
version = "2.24.0"
[package.dependencies]
certifi = ">=2017.4.17"
@ -1014,7 +1014,7 @@ description = "Measures the displayed width of unicode strings in a terminal"
name = "wcwidth"
optional = false
python-versions = "*"
version = "0.2.4"
version = "0.2.5"
[[package]]
category = "main"
@ -1124,8 +1124,8 @@ beautifulsoup4 = [
{file = "beautifulsoup4-4.9.1.tar.gz", hash = "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7"},
]
bootstrap-flask = [
{file = "Bootstrap-Flask-1.3.2.tar.gz", hash = "sha256:26c61912fff4584e23472ae66f2d98a8d7340bc1c89b48d21e19acb900356b9b"},
{file = "Bootstrap_Flask-1.3.2-py2.py3-none-any.whl", hash = "sha256:3df7044dc9075c01e0c198450da2bd2b84e21a72c8e7cf6157ba51edb8a08ad9"},
{file = "Bootstrap-Flask-1.4.tar.gz", hash = "sha256:f5e75b7fb976ba2268eac88240e1a9b249c7848a0cb639f45a6be14d913c88c2"},
{file = "Bootstrap_Flask-1.4-py2.py3-none-any.whl", hash = "sha256:a64368423512bff9e12b2d9f63be510bd3926f31e61b6be3760761d67a47a786"},
]
cchardet = [
{file = "cchardet-2.1.6-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:2aa1b008965c703ad6597361b0f6d427c8971fe94a2c99ec3724c228ae50d6a6"},
@ -1159,8 +1159,8 @@ cchardet = [
{file = "cchardet-2.1.6.tar.gz", hash = "sha256:b76afb2059ad69eab576949980a17413c1e9e5a5624abf9e43542d8853f146b3"},
]
certifi = [
{file = "certifi-2020.4.5.2-py2.py3-none-any.whl", hash = "sha256:9cd41137dc19af6a5e03b630eefe7d1f458d964d406342dd3edf625839b944cc"},
{file = "certifi-2020.4.5.2.tar.gz", hash = "sha256:5ad7e9a056d25ffa5082862e36f119f7f7cec6457fa07ee2f8c339814b80c9b1"},
{file = "certifi-2020.6.20-py2.py3-none-any.whl", hash = "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"},
{file = "certifi-2020.6.20.tar.gz", hash = "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3"},
]
cffi = [
{file = "cffi-1.14.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1cae98a7054b5c9391eb3249b86e0e99ab1e02bb0cc0575da191aedadbdf4384"},
@ -1260,8 +1260,8 @@ gunicorn = [
{file = "gunicorn-20.0.4.tar.gz", hash = "sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626"},
]
har2tree = [
{file = "har2tree-1.1-py3-none-any.whl", hash = "sha256:d2e1053ac6fa52fc5e77d1c2b20d6bbdc56f70b02e61462617ac23a3269aaa6d"},
{file = "har2tree-1.1.tar.gz", hash = "sha256:802b1f97fb3d80674d3ac8c28e0dac92d5a1da01af20bb93ce45a2584c80af6d"},
{file = "har2tree-1.1.1-py3-none-any.whl", hash = "sha256:c001ee687c1b114c416404912fded4e82db67d891a3a658d88130b82a339f73c"},
{file = "har2tree-1.1.1.tar.gz", hash = "sha256:ff80c721783c2c50fe68c9e9ec7800396a561e624b4856ed05f0002815c35d5c"},
]
hyperlink = [
{file = "hyperlink-19.0.0-py2.py3-none-any.whl", hash = "sha256:ab4a308feb039b04f855a020a6eda3b18ca5a68e6d8f8c899cbe9e653721d04f"},
@ -1292,8 +1292,8 @@ itsdangerous = [
{file = "itsdangerous-1.1.0.tar.gz", hash = "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19"},
]
jedi = [
{file = "jedi-0.17.0-py2.py3-none-any.whl", hash = "sha256:cd60c93b71944d628ccac47df9a60fec53150de53d42dc10a7fc4b5ba6aae798"},
{file = "jedi-0.17.0.tar.gz", hash = "sha256:df40c97641cb943661d2db4c33c2e1ff75d491189423249e989bcea4464f3030"},
{file = "jedi-0.17.1-py2.py3-none-any.whl", hash = "sha256:1ddb0ec78059e8e27ec9eb5098360b4ea0a3dd840bedf21415ea820c21b40a22"},
{file = "jedi-0.17.1.tar.gz", hash = "sha256:807d5d4f96711a2bcfdd5dfa3b1ae6d09aa53832b182090b222b5efb81f52f63"},
]
jinja2 = [
{file = "Jinja2-2.11.2-py2.py3-none-any.whl", hash = "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035"},
@ -1359,8 +1359,8 @@ markupsafe = [
{file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"},
]
more-itertools = [
{file = "more-itertools-8.3.0.tar.gz", hash = "sha256:558bb897a2232f5e4f8e2399089e35aecb746e1f9191b6584a151647e89267be"},
{file = "more_itertools-8.3.0-py3-none-any.whl", hash = "sha256:7818f596b1e87be009031c7653d01acc46ed422e6656b394b0f765ce66ed4982"},
{file = "more-itertools-8.4.0.tar.gz", hash = "sha256:68c70cc7167bdf5c7c9d8f6954a7837089c6a36bf565383919bb595efb8a17e5"},
{file = "more_itertools-8.4.0-py3-none-any.whl", hash = "sha256:b78134b2063dd214000685165d81c154522c3ee0a1c0d4d113c80361c234c5a2"},
]
multidict = [
{file = "multidict-4.7.6-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000"},
@ -1441,8 +1441,8 @@ publicsuffix2 = [
{file = "publicsuffix2-2.20191221.tar.gz", hash = "sha256:00f8cc31aa8d0d5592a5ced19cccba7de428ebca985db26ac852d920ddd6fe7b"},
]
py = [
{file = "py-1.8.1-py2.py3-none-any.whl", hash = "sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0"},
{file = "py-1.8.1.tar.gz", hash = "sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa"},
{file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"},
{file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"},
]
pyasn1 = [
{file = "pyasn1-0.4.8-py2.4.egg", hash = "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3"},
@ -1507,8 +1507,8 @@ pypydispatcher = [
{file = "PyPyDispatcher-2.1.2.tar.gz", hash = "sha256:b6bec5dfcff9d2535bca2b23c80eae367b1ac250a645106948d315fcfa9130f2"},
]
pysanejs = [
{file = "pysanejs-1.2-py3-none-any.whl", hash = "sha256:2fc14edec435374eed3f699de6a5b5c826e322180d65d724af56ac9353dc4372"},
{file = "pysanejs-1.2.tar.gz", hash = "sha256:371ef63fa2463daa34d736dce6a3e3ca0bdfd6c8599dab9ebcd987348550608e"},
{file = "pysanejs-1.3-py3-none-any.whl", hash = "sha256:13de2fab01263b37e950c4b1acfe0c871cec99349303072d173d2f3a9cb099b8"},
{file = "pysanejs-1.3.tar.gz", hash = "sha256:a15eefc1e4c83fe4f166518e562cd23ac75c846739f75727106bb1c02d37670d"},
]
pytest = [
{file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"},
@ -1527,8 +1527,8 @@ redis = [
{file = "redis-3.5.3.tar.gz", hash = "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2"},
]
requests = [
{file = "requests-2.23.0-py2.py3-none-any.whl", hash = "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee"},
{file = "requests-2.23.0.tar.gz", hash = "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6"},
{file = "requests-2.24.0-py2.py3-none-any.whl", hash = "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"},
{file = "requests-2.24.0.tar.gz", hash = "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b"},
]
requests-toolbelt = [
{file = "requests-toolbelt-0.9.1.tar.gz", hash = "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0"},
@ -1627,8 +1627,8 @@ w3lib = [
{file = "w3lib-1.22.0.tar.gz", hash = "sha256:0ad6d0203157d61149fd45aaed2e24f53902989c32fc1dccc2e2bfba371560df"},
]
wcwidth = [
{file = "wcwidth-0.2.4-py2.py3-none-any.whl", hash = "sha256:79375666b9954d4a1a10739315816324c3e73110af9d0e102d906fdb0aec009f"},
{file = "wcwidth-0.2.4.tar.gz", hash = "sha256:8c6b5b6ee1360b842645f336d9e5d68c55817c26d3050f46b235ef2bc650e48f"},
{file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"},
{file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"},
]
werkzeug = [
{file = "Werkzeug-1.0.1-py2.py3-none-any.whl", hash = "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43"},

8
website/web/__init__.py
View File

@ -471,7 +471,7 @@ def json_redirects(tree_uuid: str):
if not cache:
return {'error': 'UUID missing in cache, try again later.'}
to_return: Dict[str, list] = {'response': {'url': cache['url'], 'redirects': []}}
to_return: Dict[str, Any] = {'response': {'url': cache['url'], 'redirects': []}}
if not cache['redirects']:
to_return['response']['info'] = 'No redirects'
return to_return
@ -479,5 +479,9 @@ def json_redirects(tree_uuid: str):
# Trigger tree build, get all redirects
lookyloo.load_tree(capture_dir)
cache = lookyloo.capture_cache(capture_dir)
to_return['response']['redirects'] = cache['redirects']
if cache:
to_return['response']['redirects'] = cache['redirects']
else:
to_return['response']['redirects'] = cache['redirects']
return jsonify(to_return)