new: Support for TZ, Geo, locale, color scheme

lookyloo/lookyloo.py

@@ -605,6 +605,10 @@ class Lookyloo():
                 http_credentials=query.get('http_credentials', None),
                 viewport=query.get('viewport', None),
                 referer=query.get('referer', None),
+                timezone_id=query.get('timezone_id', None),
+                locale=query.get('locale', None),
+                geolocation=query.get('geolocation', None),
+                color_scheme=query.get('color_scheme', None),
                 rendered_hostname_only=query.get('rendered_hostname_only', True),
                 # force=query.get('force', False),
                 # recapture_interval=query.get('recapture_interval', 300),

poetry.lock

@@ -1238,18 +1238,18 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-
 
 [[package]]
 name = "lacuscore"
-version = "1.5.2"
+version = "1.5.4"
 description = "Core of Lacus, usable as a module"
 optional = false
 python-versions = ">=3.8,<4.0"
 files = [
-    {file = "lacuscore-1.5.2-py3-none-any.whl", hash = "sha256:0ba3b4d99ced63824ab3274512dfd17f10f92b8f5eebf280bb779c81796e8967"},
+    {file = "lacuscore-1.5.4-py3-none-any.whl", hash = "sha256:bb118ee6e7603ecb18191e5eb60a74d2a9c700076d1c80ec581acefceb66def7"},
-    {file = "lacuscore-1.5.2.tar.gz", hash = "sha256:2c5d3a7c52130d934ecb3ae6b51d46e767f1959e56f5e71cd5ff573f2b1f887d"},
+    {file = "lacuscore-1.5.4.tar.gz", hash = "sha256:6de799e9735a32950445225db9018e1f52d593c29e1f536059b541ac7d6fb47d"},
 ]
 
 [package.dependencies]
 defang = ">=0.5.3,<0.6.0"
-playwrightcapture = {version = ">=1.20.2,<2.0.0", extras = ["recaptcha"]}
+playwrightcapture = {version = ">=1.20.3,<2.0.0", extras = ["recaptcha"]}
 redis = {version = ">=4.5.5,<5.0.0", extras = ["hiredis"]}
 requests = ">=2.31.0,<3.0.0"
 ua-parser = ">=0.16.1,<0.17.0"
@@ -1876,13 +1876,13 @@ typing-extensions = {version = "*", markers = "python_version <= \"3.8\""}
 
 [[package]]
 name = "playwrightcapture"
-version = "1.20.2"
+version = "1.20.4"
 description = "A simple library to capture websites using playwright"
 optional = false
 python-versions = ">=3.8,<4.0"
 files = [
-    {file = "playwrightcapture-1.20.2-py3-none-any.whl", hash = "sha256:49f1bba4df565a2e00489343b097bd7d4a4c62f1d6f7735a8f7917c6d7584a6b"},
+    {file = "playwrightcapture-1.20.4-py3-none-any.whl", hash = "sha256:e943b5cc3700402cfe395ca61c42b7812bde9d9c1c30c71abf219e2f700ac83b"},
-    {file = "playwrightcapture-1.20.2.tar.gz", hash = "sha256:4b2eec6f511c45b5c24b4ba9eec2f34325f997b82cedd0c0b1ea708b87652343"},
+    {file = "playwrightcapture-1.20.4.tar.gz", hash = "sha256:ae8b6b680992b992290260fbac16a07bedd0d0e3259d8864cba70272481a2b0a"},
 ]
 
 [package.dependencies]
@@ -1891,8 +1891,10 @@ dateparser = ">=1.1.8,<2.0.0"
 lxml = ">=4.9.2,<5.0.0"
 playwright = ">=1.34.0,<2.0.0"
 pydub = {version = ">=0.25.1,<0.26.0", optional = true, markers = "extra == \"recaptcha\""}
+pytz = {version = ">=2023.3,<2024.0", markers = "python_version < \"3.9\""}
 requests = {version = ">=2.31.0,<3.0.0", optional = true, markers = "extra == \"recaptcha\""}
 SpeechRecognition = {version = ">=3.10.0,<4.0.0", optional = true, markers = "extra == \"recaptcha\""}
+tzdata = ">=2023.3,<2024.0"
 w3lib = ">=2.1.1,<3.0.0"
 
 [package.extras]
@@ -2086,13 +2088,13 @@ docs = ["Sphinx (>=5.3.0,<6.0.0)"]
 
 [[package]]
 name = "pylacus"
-version = "1.5.0"
+version = "1.5.1"
 description = "Python CLI and module for lacus"
 optional = false
 python-versions = ">=3.8,<4.0"
 files = [
-    {file = "pylacus-1.5.0-py3-none-any.whl", hash = "sha256:178c21708a76bdef95c49a74fa84d6cb682793583d1e261eb767f06e081d8dd4"},
+    {file = "pylacus-1.5.1-py3-none-any.whl", hash = "sha256:abcbb2ea1d26f5807bfcb4d6cb3d973aba6a65b2024b2b30b83f5925a87c4196"},
-    {file = "pylacus-1.5.0.tar.gz", hash = "sha256:e7738ff708a6900a789061e3edb92d5ed9221c7630bc36aef402414f3b835b50"},
+    {file = "pylacus-1.5.1.tar.gz", hash = "sha256:4ab886e5acea4d50aa0a75db230797c5e0e91a937a598bbbe36a61db5bfd7f87"},
 ]
 
 [package.dependencies]
@@ -2690,6 +2692,17 @@ files = [
     {file = "types_python_dateutil-2.8.19.13-py3-none-any.whl", hash = "sha256:0b0e7c68e7043b0354b26a1e0225cb1baea7abb1b324d02b50e2d08f1221043f"},
 ]
 
+[[package]]
+name = "types-pytz"
+version = "2023.3.0.0"
+description = "Typing stubs for pytz"
+optional = false
+python-versions = "*"
+files = [
+    {file = "types-pytz-2023.3.0.0.tar.gz", hash = "sha256:ecdc70d543aaf3616a7e48631543a884f74205f284cefd6649ddf44c6a820aac"},
+    {file = "types_pytz-2023.3.0.0-py3-none-any.whl", hash = "sha256:4fc2a7fbbc315f0b6630e0b899fd6c743705abe1094d007b0e612d10da15e0f3"},
+]
+
 [[package]]
 name = "types-redis"
 version = "4.5.5.2"
@@ -2783,13 +2796,13 @@ files = [
 
 [[package]]
 name = "urllib3"
-version = "2.0.2"
+version = "2.0.3"
 description = "HTTP library with thread-safe connection pooling, file post, and more."
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "urllib3-2.0.2-py3-none-any.whl", hash = "sha256:d055c2f9d38dc53c808f6fdc8eab7360b6fdbbde02340ed25cfbcd817c62469e"},
+    {file = "urllib3-2.0.3-py3-none-any.whl", hash = "sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1"},
-    {file = "urllib3-2.0.2.tar.gz", hash = "sha256:61717a1095d7e155cdb737ac7bb2f4324a858a1e2e6466f6d03ff630ca68d3cc"},
+    {file = "urllib3-2.0.3.tar.gz", hash = "sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825"},
 ]
 
 [package.extras]
@@ -3060,4 +3073,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.8,<3.12"
-content-hash = "0f6b7f573feddda1fe25d45e964e3b08d54e1f910ac4288f08c1a492994249ae"
+content-hash = "7042d65a2d0849ded20d663781db236b254b64b7ae13a7a323c0e358b7d57009"

pyproject.toml

@@ -65,14 +65,15 @@ passivetotal = "^2.5.9"
 werkzeug = "^2.3.4"
 filetype = "^1.2.0"
 pypandora = "^1.4.0"
-lacuscore = "^1.5.2"
+lacuscore = "^1.5.4"
-pylacus = "^1.5.0"
+pylacus = "^1.5.1"
 pyipasnhistory = "^2.1.2"
 publicsuffixlist = "^0.10.0.20230506"
 pyfaup = "^1.2"
 chardet = "^5.1.0"
 pysecuritytxt = "^1.1.1"
 pylookyloomonitoring = "^1.0.5"
+pytz = {"version" = "^2023.3", python = "<3.9"}
 
 [tool.poetry.group.dev.dependencies]
 mypy = "^1.3.0"
@@ -87,6 +88,7 @@ types-Deprecated = "^1.2.9.2"
 types-python-dateutil = "^2.8.19.13"
 types-beautifulsoup4 = "^4.12.0.5"
 types-Pillow = "^9.5.0.4"
+types-pytz = "^2023.3.0.0"
 
 [build-system]
 requires = ["poetry_core"]

website/web/__init__.py

@@ -8,6 +8,7 @@ import json
 import logging
 import logging.config
 import os
+import sys
 import time
 
 import filetype  # type: ignore
@@ -15,7 +16,7 @@ import filetype  # type: ignore
 from datetime import date, datetime, timedelta, timezone
 from importlib.metadata import version
 from io import BytesIO, StringIO
-from typing import Any, Dict, List, Optional, Union, TypedDict
+from typing import Any, Dict, List, Optional, Union, TypedDict, Set, Tuple
 from urllib.parse import quote_plus, unquote_plus, urlparse
 from uuid import uuid4
 from zipfile import ZipFile
@@ -35,6 +36,12 @@ from lookyloo.exceptions import MissingUUID, NoValidHarFile
 from lookyloo.helpers import get_taxonomies, UserAgents, load_cookies
 from lookyloo.lookyloo import Indexing, Lookyloo, CaptureSettings
 
+if sys.version_info < (3, 9):
+    from pytz import all_timezones_set
+else:
+    from zoneinfo import available_timezones
+    all_timezones_set = available_timezones()
+
 from .genericapi import api as generic_api
 from .helpers import (User, build_users_table, get_secret_key,
                       load_user_from_request, src_request_ip, sri_load)
@@ -200,6 +207,16 @@ def get_icon(icon_id: str) -> Optional[Icon]:
 app.jinja_env.globals.update(get_icon=get_icon)
 
 
+def get_tz_info() -> Tuple[Optional[str], str, Set[str]]:
+    now = datetime.now().astimezone()
+    local_TZ = now.tzname()
+    local_UTC_offset = f'UTC{now.strftime("%z")}'
+    return local_TZ, local_UTC_offset, all_timezones_set
+
+
+app.jinja_env.globals.update(tz_info=get_tz_info)
+
+
 # ##### Generic/configuration methods #####
 
 @app.after_request
@@ -1067,6 +1084,23 @@ def capture_web():
         if request.form.get('headers'):
             capture_query['headers'] = request.form['headers']
 
+        if request.form.get('timezone_id'):
+            capture_query['timezone_id'] = request.form['timezone_id']
+
+        if request.form.get('locale'):
+            capture_query['locale'] = request.form['locale']
+
+        if request.form.get('geo_longitude') and request.form.get('geo_latitude'):
+            capture_query['geolocation'] = {'longitude': float(request.form['geo_longitude']),
+                                            'latitude': float(request.form['geo_latitude'])}
+
+        if request.form.get('http_auth_username') and request.form.get('http_auth_password'):
+            capture_query['http_credentials'] = {'username': request.form['http_auth_username'],
+                                                 'password': request.form['http_auth_password']}
+
+        if request.form.get('color_scheme'):
+            capture_query['color_scheme'] = request.form['color_scheme']
+
         if request.form.get('proxy'):
             parsed_proxy = urlparse(request.form['proxy'])
             if parsed_proxy.scheme and parsed_proxy.hostname and parsed_proxy.port:

website/web/templates/capture.html

@@ -251,6 +251,52 @@
           </div>
         </div>
 
+        <div class="row mb-3">
+          {% set local_TZ, local_UTC_offset, all_timezones = tz_info() %}
+          <label for="timezone_id" class="col-sm-2 col-form-label">Timezone (defaults to {{local_TZ}} - {{local_UTC_offset}}):</label>
+          <div class="col-sm-10">
+            <input class="form-control" list="tzOptions" name="timezone_id" id="timezone_id" aria-label="Pick the timezone for the capture">
+            <datalist id="tzOptions">
+              {% for tz in all_timezones %}
+              <option value="{{tz}}">{{tz}}</option>
+              {%endfor%}
+            </datalist>
+          </div>
+        </div>
+
+        <div class="row mb-3">
+          <label for="locale" class="col-sm-2 col-form-label">Locale:</label>
+          <div class="col-sm-10">
+            <input type="text" class="form-control" name="locale" id="locale" placeholder="Pass a language locale to the capture">
+          </div>
+        </div>
+
+        <div class="row mb-3">
+          <label for="color_scheme" class="col-sm-2 col-form-label">Color scheme:</label>
+          <div class="col-sm-10">
+            <select class="form-select" id="color_scheme" name="color_scheme" aria-label="Select a prefered color scheme">
+              <option selected>Select a color scheme</option>
+              <option value="light">Light</option>
+              <option value="dark">Dark</option>
+              <option value="no-preference">No preference</option>
+            </select>
+          </div>
+        </div>
+
+        <div class="row mb-3">
+          <label for="geoloc" class="col-sm-2 col-form-label">Geolocation:</label>
+          <div class="col-sm-10" id="geoloc">
+            <div class="row g-3">
+              <div class="col">
+                <input type="number" step="any" class="form-control" name="geo_longitude" id="geo_longitude" placeholder="Longitude">
+              </div>
+              <div class="col">
+                <input type="number" step="any" class="form-control" name="geo_latitude" id="geo_latitude" placeholder="Latitude">
+              </div>
+            </div>
+          </div>
+        </div>
+
         <div class="row mb-3">
           <label for="proxy" class="col-sm-2 col-form-label">Proxy:</label>
           <div class="col-sm-10">
@@ -258,6 +304,27 @@
           </div>
         </div>
 
+        <div class="row mb-3">
+          <label for="httpauth" class="col-sm-2 col-form-label">HTTP Authentication</label>
+          <div class="col-sm-10" id="httpauth">
+            <div class="row g-3">
+              <div class="col">
+                <input type="text" class="form-control" name="http_auth_username" id="http_auth_username" placeholder="Username">
+              </div>
+              <div class="col">
+                  <input type="text" class="form-control" name="http_auth_password" id="http_auth_password" placeholder="Password">
+              </div>
+            </div>
+            <div class="alert alert-danger" role="alert">
+              The authentication credentials will be stored on the lookyloo instance and potentially
+              accessed by third parties (either because the Lookyloo instance is public,
+              or people other than you have access to the instance).
+              If that's the case, please make sure none of them can be used to login as yourself
+              on websites.
+            </div>
+          </div>
+        </div>
+
         <div class="row mb-3">
           <label for="cookies" class="col-sm-2 col-form-label">Cookies:</label>
           <div class="col-sm-10">