new: Update script.

bin/start_website.py

@@ -15,7 +15,6 @@ if __name__ == '__main__':
     ip = get_config('generic', 'website_listen_ip')
     port = get_config('generic', 'website_listen_port')
     try:
-        pid_libs = Popen([str(website_dir / '3rdparty.sh')], cwd=website_dir)
         p = Popen(['gunicorn', '-w', '10',
                    '--graceful-timeout', '2', '--timeout', '300',
                    '-b', f'{ip}:{port}',
@@ -34,9 +33,7 @@ if __name__ == '__main__':
         try:
             # Killing everything if possible.
             p.send_signal(signal.SIGWINCH)
-            pid_libs.send_signal(signal.SIGWINCH)
             p.send_signal(signal.SIGTERM)
-            pid_libs.send_signal(signal.SIGTERM)
         except Exception:
             pass
         unset_running('website')

bin/update.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import argparse
+import logging
+import subprocess
+import shlex
+import sys
+
+from lookyloo.helpers import get_homedir
+
+logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
+                    level=logging.INFO, datefmt='%I:%M:%S')
+
+
+def keep_going(ignore=False):
+    if ignore:
+        return
+    keep_going = input('Continue? (y/N) ')
+    if keep_going.lower() != 'y':
+        print('Okay, quitting.')
+        sys.exit()
+
+
+def run_command(command):
+    args = shlex.split(command)
+    process = subprocess.run(args, cwd=homedir, capture_output=True)
+    print(process.stdout.decode())
+    if process.returncode:
+        print(process.stderr.decode())
+        sys.exit()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Pull latest release, update dependencies, update and validate the config files, update 3rd deps for the website.')
+    parser.add_argument('--yes', default=False, action='store_true', help='Run all commands without asking.')
+    args = parser.parse_args()
+
+    homedir = get_homedir()
+
+    print('* Update repository.')
+    keep_going(args.yes)
+    run_command('git pull')
+
+    print('* Install/update dependencies.')
+    keep_going(args.yes)
+    run_command('poetry install')
+
+    print('* Validate configuration files.')
+    keep_going(args.yes)
+    run_command('tools/validate_config_files.py --check')
+
+    print('* Update configuration files.')
+    keep_going(args.yes)
+    run_command('tools/validate_config_files.py --update')
+
+    print('* Update third party dependencies for the website.')
+    keep_going(args.yes)
+    run_command('website/3rdparty.sh')

config/generic.json.sample

@@ -34,7 +34,7 @@
     "website_listen_ip": "IP Flask will listen on. Defaults to 0.0.0.0, meaning all interfaces.",
     "website_listen_port": "Port Flask will listen on.",
     "splash_url": "URL to connect to splash",
-    "default_user_agent": "Ultimate fallback if the capture form, or the asynchronous submission, don't provide a UA",
+    "default_user_agent": "Ultimate fallback if the capture form, or the asynchronous submission, doesn't provide a user agent.",
     "cache_clean_user": "Format: {username: password}",
     "time_delta_on_index": "Time interval of the capture displayed on the index",
     "max_depth": "Maximum depth for scraping. Anything > 1 will be exponentially bigger.",

pyproject.toml

@@ -27,6 +27,7 @@ async_scrape = "bin/async_scrape.py"
 shutdown = "bin/shutdown.py"
 stop = "bin/stop.py"
 rebuild_caches = "bin/rebuild_caches.py"
+update = "bin/update.py"
 
 
 [tool.poetry.dependencies]

setup.py

@@ -13,7 +13,7 @@ setup(
     description='Web interface to track the trackers.',
     packages=['lookyloo'],
     scripts=['bin/start_website.py', 'bin/start.py', 'bin/run_backend.py', 'bin/async_scrape.py',
-             'bin/shutdown.py', 'bin/stop.py', 'bin/rebuild_caches.py'],
+             'bin/shutdown.py', 'bin/stop.py', 'bin/rebuild_caches.py', 'bin/update.py'],
     include_package_data=True,
     classifiers=[
         'License :: OSI Approved :: BSD License',

tools/validate_config_files.py

@@ -9,7 +9,8 @@ from lookyloo.helpers import get_homedir
 
 
 def validate_generic_config_file():
-    with (get_homedir() / 'config' / 'generic.json').open() as f:
+    user_config = get_homedir() / 'config' / 'generic.json'
+    with user_config.open() as f:
         generic_config = json.load(f)
     with (get_homedir() / 'config' / 'generic.json.sample').open() as f:
         generic_config_sample = json.load(f)
@@ -41,6 +42,8 @@ def validate_generic_config_file():
         if key not in generic_config_sample:
             raise Exception(f'{key} is missing in the sample config file')
 
+    return True
+
 
 def validate_modules_config_file():
     with (get_homedir() / 'config' / 'modules.json').open() as f:
@@ -55,6 +58,8 @@ def validate_modules_config_file():
             logger.warning(f'Entry missing in user config file: {key}. Will default to: {json.dumps(modules_config_sample[key], indent=2)}')
             continue
 
+    return True
+
 
 def update_user_configs():
     for file_name in ['generic', 'modules']:
@@ -72,11 +77,13 @@ def update_user_configs():
                 continue
             if generic_config.get(key) is None:
                 print(f'{key} was missing in {file_name}, adding it.')
+                print(f"Description: {generic_config_sample['_notes'][key]}")
                 generic_config[key] = generic_config_sample[key]
                 has_new_entry = True
         if has_new_entry:
             with (get_homedir() / 'config' / f'{file_name}.json').open('w') as fw:
                 json.dump(generic_config, fw, indent=2, sort_keys=True)
+    return has_new_entry
 
 
 if __name__ == '__main__':
@@ -87,8 +94,11 @@ if __name__ == '__main__':
     args = parser.parse_args()
 
     if args.check:
-        validate_generic_config_file()
+        if validate_generic_config_file():
-        validate_modules_config_file()
+            print(f"The entries in {get_homedir() / 'config' / 'generic.json'} are valid.")
+        if validate_modules_config_file():
+            print(f"The entries in {get_homedir() / 'config' / 'modules.json'} are valid.")
 
     if args.update:
-        update_user_configs()
+        if not update_user_configs():
+            print(f"No updates needed in {get_homedir() / 'config' / 'generic.json'}.")

website/3rdparty.sh

@@ -11,3 +11,5 @@ datatables="1.10.22"
 
 wget -q https://cdn.datatables.net/v/bs4/dt-${datatables}/datatables.min.css -O web/static/datatables.min.css
 wget -q https://cdn.datatables.net/v/bs4/dt-${datatables}/datatables.min.js -O web/static/datatables.min.js
+
+echo 'Done'