mirror of https://github.com/CIRCL/lookyloo
new: Support for TZ, Geo, locale, color scheme
Raphaël Vinot
parent
5a61bdf5ee
commit
4ff9b7651d
|
|
@ -605,6 +605,10 @@ class Lookyloo():
|
|||
http_credentials=query.get('http_credentials', None),
|
||||
viewport=query.get('viewport', None),
|
||||
referer=query.get('referer', None),
|
||||
timezone_id=query.get('timezone_id', None),
|
||||
locale=query.get('locale', None),
|
||||
geolocation=query.get('geolocation', None),
|
||||
color_scheme=query.get('color_scheme', None),
|
||||
rendered_hostname_only=query.get('rendered_hostname_only', True),
|
||||
# force=query.get('force', False),
|
||||
# recapture_interval=query.get('recapture_interval', 300),
|
||||
|
|
|
|||
|
|
@ -1238,18 +1238,18 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-
|
|||
|
||||
[[package]]
|
||||
name = "lacuscore"
|
||||
version = "1.5.2"
|
||||
version = "1.5.4"
|
||||
description = "Core of Lacus, usable as a module"
|
||||
optional = false
|
||||
python-versions = ">=3.8,<4.0"
|
||||
files = [
|
||||
{file = "lacuscore-1.5.2-py3-none-any.whl", hash = "sha256:0ba3b4d99ced63824ab3274512dfd17f10f92b8f5eebf280bb779c81796e8967"},
|
||||
{file = "lacuscore-1.5.2.tar.gz", hash = "sha256:2c5d3a7c52130d934ecb3ae6b51d46e767f1959e56f5e71cd5ff573f2b1f887d"},
|
||||
{file = "lacuscore-1.5.4-py3-none-any.whl", hash = "sha256:bb118ee6e7603ecb18191e5eb60a74d2a9c700076d1c80ec581acefceb66def7"},
|
||||
{file = "lacuscore-1.5.4.tar.gz", hash = "sha256:6de799e9735a32950445225db9018e1f52d593c29e1f536059b541ac7d6fb47d"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
defang = ">=0.5.3,<0.6.0"
|
||||
playwrightcapture = {version = ">=1.20.2,<2.0.0", extras = ["recaptcha"]}
|
||||
playwrightcapture = {version = ">=1.20.3,<2.0.0", extras = ["recaptcha"]}
|
||||
redis = {version = ">=4.5.5,<5.0.0", extras = ["hiredis"]}
|
||||
requests = ">=2.31.0,<3.0.0"
|
||||
ua-parser = ">=0.16.1,<0.17.0"
|
||||
|
|
@ -1876,13 +1876,13 @@ typing-extensions = {version = "*", markers = "python_version <= \"3.8\""}
|
|||
|
||||
[[package]]
|
||||
name = "playwrightcapture"
|
||||
version = "1.20.2"
|
||||
version = "1.20.4"
|
||||
description = "A simple library to capture websites using playwright"
|
||||
optional = false
|
||||
python-versions = ">=3.8,<4.0"
|
||||
files = [
|
||||
{file = "playwrightcapture-1.20.2-py3-none-any.whl", hash = "sha256:49f1bba4df565a2e00489343b097bd7d4a4c62f1d6f7735a8f7917c6d7584a6b"},
|
||||
{file = "playwrightcapture-1.20.2.tar.gz", hash = "sha256:4b2eec6f511c45b5c24b4ba9eec2f34325f997b82cedd0c0b1ea708b87652343"},
|
||||
{file = "playwrightcapture-1.20.4-py3-none-any.whl", hash = "sha256:e943b5cc3700402cfe395ca61c42b7812bde9d9c1c30c71abf219e2f700ac83b"},
|
||||
{file = "playwrightcapture-1.20.4.tar.gz", hash = "sha256:ae8b6b680992b992290260fbac16a07bedd0d0e3259d8864cba70272481a2b0a"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
|
|
@ -1891,8 +1891,10 @@ dateparser = ">=1.1.8,<2.0.0"
|
|||
lxml = ">=4.9.2,<5.0.0"
|
||||
playwright = ">=1.34.0,<2.0.0"
|
||||
pydub = {version = ">=0.25.1,<0.26.0", optional = true, markers = "extra == \"recaptcha\""}
|
||||
pytz = {version = ">=2023.3,<2024.0", markers = "python_version < \"3.9\""}
|
||||
requests = {version = ">=2.31.0,<3.0.0", optional = true, markers = "extra == \"recaptcha\""}
|
||||
SpeechRecognition = {version = ">=3.10.0,<4.0.0", optional = true, markers = "extra == \"recaptcha\""}
|
||||
tzdata = ">=2023.3,<2024.0"
|
||||
w3lib = ">=2.1.1,<3.0.0"
|
||||
|
||||
[package.extras]
|
||||
|
|
@ -2086,13 +2088,13 @@ docs = ["Sphinx (>=5.3.0,<6.0.0)"]
|
|||
|
||||
[[package]]
|
||||
name = "pylacus"
|
||||
version = "1.5.0"
|
||||
version = "1.5.1"
|
||||
description = "Python CLI and module for lacus"
|
||||
optional = false
|
||||
python-versions = ">=3.8,<4.0"
|
||||
files = [
|
||||
{file = "pylacus-1.5.0-py3-none-any.whl", hash = "sha256:178c21708a76bdef95c49a74fa84d6cb682793583d1e261eb767f06e081d8dd4"},
|
||||
{file = "pylacus-1.5.0.tar.gz", hash = "sha256:e7738ff708a6900a789061e3edb92d5ed9221c7630bc36aef402414f3b835b50"},
|
||||
{file = "pylacus-1.5.1-py3-none-any.whl", hash = "sha256:abcbb2ea1d26f5807bfcb4d6cb3d973aba6a65b2024b2b30b83f5925a87c4196"},
|
||||
{file = "pylacus-1.5.1.tar.gz", hash = "sha256:4ab886e5acea4d50aa0a75db230797c5e0e91a937a598bbbe36a61db5bfd7f87"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
|
|
@ -2690,6 +2692,17 @@ files = [
|
|||
{file = "types_python_dateutil-2.8.19.13-py3-none-any.whl", hash = "sha256:0b0e7c68e7043b0354b26a1e0225cb1baea7abb1b324d02b50e2d08f1221043f"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "types-pytz"
|
||||
version = "2023.3.0.0"
|
||||
description = "Typing stubs for pytz"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
files = [
|
||||
{file = "types-pytz-2023.3.0.0.tar.gz", hash = "sha256:ecdc70d543aaf3616a7e48631543a884f74205f284cefd6649ddf44c6a820aac"},
|
||||
{file = "types_pytz-2023.3.0.0-py3-none-any.whl", hash = "sha256:4fc2a7fbbc315f0b6630e0b899fd6c743705abe1094d007b0e612d10da15e0f3"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "types-redis"
|
||||
version = "4.5.5.2"
|
||||
|
|
@ -2783,13 +2796,13 @@ files = [
|
|||
|
||||
[[package]]
|
||||
name = "urllib3"
|
||||
version = "2.0.2"
|
||||
version = "2.0.3"
|
||||
description = "HTTP library with thread-safe connection pooling, file post, and more."
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
files = [
|
||||
{file = "urllib3-2.0.2-py3-none-any.whl", hash = "sha256:d055c2f9d38dc53c808f6fdc8eab7360b6fdbbde02340ed25cfbcd817c62469e"},
|
||||
{file = "urllib3-2.0.2.tar.gz", hash = "sha256:61717a1095d7e155cdb737ac7bb2f4324a858a1e2e6466f6d03ff630ca68d3cc"},
|
||||
{file = "urllib3-2.0.3-py3-none-any.whl", hash = "sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1"},
|
||||
{file = "urllib3-2.0.3.tar.gz", hash = "sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
|
|
@ -3060,4 +3073,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more
|
|||
[metadata]
|
||||
lock-version = "2.0"
|
||||
python-versions = ">=3.8,<3.12"
|
||||
content-hash = "0f6b7f573feddda1fe25d45e964e3b08d54e1f910ac4288f08c1a492994249ae"
|
||||
content-hash = "7042d65a2d0849ded20d663781db236b254b64b7ae13a7a323c0e358b7d57009"
|
||||
|
|
|
|||
|
|
@ -65,14 +65,15 @@ passivetotal = "^2.5.9"
|
|||
werkzeug = "^2.3.4"
|
||||
filetype = "^1.2.0"
|
||||
pypandora = "^1.4.0"
|
||||
lacuscore = "^1.5.2"
|
||||
pylacus = "^1.5.0"
|
||||
lacuscore = "^1.5.4"
|
||||
pylacus = "^1.5.1"
|
||||
pyipasnhistory = "^2.1.2"
|
||||
publicsuffixlist = "^0.10.0.20230506"
|
||||
pyfaup = "^1.2"
|
||||
chardet = "^5.1.0"
|
||||
pysecuritytxt = "^1.1.1"
|
||||
pylookyloomonitoring = "^1.0.5"
|
||||
pytz = {"version" = "^2023.3", python = "<3.9"}
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
mypy = "^1.3.0"
|
||||
|
|
@ -87,6 +88,7 @@ types-Deprecated = "^1.2.9.2"
|
|||
types-python-dateutil = "^2.8.19.13"
|
||||
types-beautifulsoup4 = "^4.12.0.5"
|
||||
types-Pillow = "^9.5.0.4"
|
||||
types-pytz = "^2023.3.0.0"
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry_core"]
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import json
|
|||
import logging
|
||||
import logging.config
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import filetype # type: ignore
|
||||
|
|
@ -15,7 +16,7 @@ import filetype # type: ignore
|
|||
from datetime import date, datetime, timedelta, timezone
|
||||
from importlib.metadata import version
|
||||
from io import BytesIO, StringIO
|
||||
from typing import Any, Dict, List, Optional, Union, TypedDict
|
||||
from typing import Any, Dict, List, Optional, Union, TypedDict, Set, Tuple
|
||||
from urllib.parse import quote_plus, unquote_plus, urlparse
|
||||
from uuid import uuid4
|
||||
from zipfile import ZipFile
|
||||
|
|
@ -35,6 +36,12 @@ from lookyloo.exceptions import MissingUUID, NoValidHarFile
|
|||
from lookyloo.helpers import get_taxonomies, UserAgents, load_cookies
|
||||
from lookyloo.lookyloo import Indexing, Lookyloo, CaptureSettings
|
||||
|
||||
if sys.version_info < (3, 9):
|
||||
from pytz import all_timezones_set
|
||||
else:
|
||||
from zoneinfo import available_timezones
|
||||
all_timezones_set = available_timezones()
|
||||
|
||||
from .genericapi import api as generic_api
|
||||
from .helpers import (User, build_users_table, get_secret_key,
|
||||
load_user_from_request, src_request_ip, sri_load)
|
||||
|
|
@ -200,6 +207,16 @@ def get_icon(icon_id: str) -> Optional[Icon]:
|
|||
app.jinja_env.globals.update(get_icon=get_icon)
|
||||
|
||||
|
||||
def get_tz_info() -> Tuple[Optional[str], str, Set[str]]:
|
||||
now = datetime.now().astimezone()
|
||||
local_TZ = now.tzname()
|
||||
local_UTC_offset = f'UTC{now.strftime("%z")}'
|
||||
return local_TZ, local_UTC_offset, all_timezones_set
|
||||
|
||||
|
||||
app.jinja_env.globals.update(tz_info=get_tz_info)
|
||||
|
||||
|
||||
# ##### Generic/configuration methods #####
|
||||
|
||||
@app.after_request
|
||||
|
|
@ -1067,6 +1084,23 @@ def capture_web():
|
|||
if request.form.get('headers'):
|
||||
capture_query['headers'] = request.form['headers']
|
||||
|
||||
if request.form.get('timezone_id'):
|
||||
capture_query['timezone_id'] = request.form['timezone_id']
|
||||
|
||||
if request.form.get('locale'):
|
||||
capture_query['locale'] = request.form['locale']
|
||||
|
||||
if request.form.get('geo_longitude') and request.form.get('geo_latitude'):
|
||||
capture_query['geolocation'] = {'longitude': float(request.form['geo_longitude']),
|
||||
'latitude': float(request.form['geo_latitude'])}
|
||||
|
||||
if request.form.get('http_auth_username') and request.form.get('http_auth_password'):
|
||||
capture_query['http_credentials'] = {'username': request.form['http_auth_username'],
|
||||
'password': request.form['http_auth_password']}
|
||||
|
||||
if request.form.get('color_scheme'):
|
||||
capture_query['color_scheme'] = request.form['color_scheme']
|
||||
|
||||
if request.form.get('proxy'):
|
||||
parsed_proxy = urlparse(request.form['proxy'])
|
||||
if parsed_proxy.scheme and parsed_proxy.hostname and parsed_proxy.port:
|
||||
|
|
|
|||
|
|
@ -251,6 +251,52 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row mb-3">
|
||||
{% set local_TZ, local_UTC_offset, all_timezones = tz_info() %}
|
||||
<label for="timezone_id" class="col-sm-2 col-form-label">Timezone (defaults to {{local_TZ}} - {{local_UTC_offset}}):</label>
|
||||
<div class="col-sm-10">
|
||||
<input class="form-control" list="tzOptions" name="timezone_id" id="timezone_id" aria-label="Pick the timezone for the capture">
|
||||
<datalist id="tzOptions">
|
||||
{% for tz in all_timezones %}
|
||||
<option value="{{tz}}">{{tz}}</option>
|
||||
{%endfor%}
|
||||
</datalist>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row mb-3">
|
||||
<label for="locale" class="col-sm-2 col-form-label">Locale:</label>
|
||||
<div class="col-sm-10">
|
||||
<input type="text" class="form-control" name="locale" id="locale" placeholder="Pass a language locale to the capture">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row mb-3">
|
||||
<label for="color_scheme" class="col-sm-2 col-form-label">Color scheme:</label>
|
||||
<div class="col-sm-10">
|
||||
<select class="form-select" id="color_scheme" name="color_scheme" aria-label="Select a prefered color scheme">
|
||||
<option selected>Select a color scheme</option>
|
||||
<option value="light">Light</option>
|
||||
<option value="dark">Dark</option>
|
||||
<option value="no-preference">No preference</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row mb-3">
|
||||
<label for="geoloc" class="col-sm-2 col-form-label">Geolocation:</label>
|
||||
<div class="col-sm-10" id="geoloc">
|
||||
<div class="row g-3">
|
||||
<div class="col">
|
||||
<input type="number" step="any" class="form-control" name="geo_longitude" id="geo_longitude" placeholder="Longitude">
|
||||
</div>
|
||||
<div class="col">
|
||||
<input type="number" step="any" class="form-control" name="geo_latitude" id="geo_latitude" placeholder="Latitude">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row mb-3">
|
||||
<label for="proxy" class="col-sm-2 col-form-label">Proxy:</label>
|
||||
<div class="col-sm-10">
|
||||
|
|
@ -258,6 +304,27 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row mb-3">
|
||||
<label for="httpauth" class="col-sm-2 col-form-label">HTTP Authentication</label>
|
||||
<div class="col-sm-10" id="httpauth">
|
||||
<div class="row g-3">
|
||||
<div class="col">
|
||||
<input type="text" class="form-control" name="http_auth_username" id="http_auth_username" placeholder="Username">
|
||||
</div>
|
||||
<div class="col">
|
||||
<input type="text" class="form-control" name="http_auth_password" id="http_auth_password" placeholder="Password">
|
||||
</div>
|
||||
</div>
|
||||
<div class="alert alert-danger" role="alert">
|
||||
The authentication credentials will be stored on the lookyloo instance and potentially
|
||||
accessed by third parties (either because the Lookyloo instance is public,
|
||||
or people other than you have access to the instance).
|
||||
If that's the case, please make sure none of them can be used to login as yourself
|
||||
on websites.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row mb-3">
|
||||
<label for="cookies" class="col-sm-2 col-form-label">Cookies:</label>
|
||||
<div class="col-sm-10">
|
||||
|
|
|
|||
Loading…
Reference in New Issue