mirror of https://github.com/CIRCL/lookyloo
new: Update script.
parent
3f4fdb937b
commit
d33698357c
|
@ -15,7 +15,6 @@ if __name__ == '__main__':
|
|||
ip = get_config('generic', 'website_listen_ip')
|
||||
port = get_config('generic', 'website_listen_port')
|
||||
try:
|
||||
pid_libs = Popen([str(website_dir / '3rdparty.sh')], cwd=website_dir)
|
||||
p = Popen(['gunicorn', '-w', '10',
|
||||
'--graceful-timeout', '2', '--timeout', '300',
|
||||
'-b', f'{ip}:{port}',
|
||||
|
@ -34,9 +33,7 @@ if __name__ == '__main__':
|
|||
try:
|
||||
# Killing everything if possible.
|
||||
p.send_signal(signal.SIGWINCH)
|
||||
pid_libs.send_signal(signal.SIGWINCH)
|
||||
p.send_signal(signal.SIGTERM)
|
||||
pid_libs.send_signal(signal.SIGTERM)
|
||||
except Exception:
|
||||
pass
|
||||
unset_running('website')
|
||||
|
|
|
@ -0,0 +1,59 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import subprocess
|
||||
import shlex
|
||||
import sys
|
||||
|
||||
from lookyloo.helpers import get_homedir
|
||||
|
||||
logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
|
||||
level=logging.INFO, datefmt='%I:%M:%S')
|
||||
|
||||
|
||||
def keep_going(ignore=False):
|
||||
if ignore:
|
||||
return
|
||||
keep_going = input('Continue? (y/N) ')
|
||||
if keep_going.lower() != 'y':
|
||||
print('Okay, quitting.')
|
||||
sys.exit()
|
||||
|
||||
|
||||
def run_command(command):
|
||||
args = shlex.split(command)
|
||||
process = subprocess.run(args, cwd=homedir, capture_output=True)
|
||||
print(process.stdout.decode())
|
||||
if process.returncode:
|
||||
print(process.stderr.decode())
|
||||
sys.exit()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='Pull latest release, update dependencies, update and validate the config files, update 3rd deps for the website.')
|
||||
parser.add_argument('--yes', default=False, action='store_true', help='Run all commands without asking.')
|
||||
args = parser.parse_args()
|
||||
|
||||
homedir = get_homedir()
|
||||
|
||||
print('* Update repository.')
|
||||
keep_going(args.yes)
|
||||
run_command('git pull')
|
||||
|
||||
print('* Install/update dependencies.')
|
||||
keep_going(args.yes)
|
||||
run_command('poetry install')
|
||||
|
||||
print('* Validate configuration files.')
|
||||
keep_going(args.yes)
|
||||
run_command('tools/validate_config_files.py --check')
|
||||
|
||||
print('* Update configuration files.')
|
||||
keep_going(args.yes)
|
||||
run_command('tools/validate_config_files.py --update')
|
||||
|
||||
print('* Update third party dependencies for the website.')
|
||||
keep_going(args.yes)
|
||||
run_command('website/3rdparty.sh')
|
|
@ -34,7 +34,7 @@
|
|||
"website_listen_ip": "IP Flask will listen on. Defaults to 0.0.0.0, meaning all interfaces.",
|
||||
"website_listen_port": "Port Flask will listen on.",
|
||||
"splash_url": "URL to connect to splash",
|
||||
"default_user_agent": "Ultimate fallback if the capture form, or the asynchronous submission, don't provide a UA",
|
||||
"default_user_agent": "Ultimate fallback if the capture form, or the asynchronous submission, doesn't provide a user agent.",
|
||||
"cache_clean_user": "Format: {username: password}",
|
||||
"time_delta_on_index": "Time interval of the capture displayed on the index",
|
||||
"max_depth": "Maximum depth for scraping. Anything > 1 will be exponentially bigger.",
|
||||
|
|
|
@ -27,6 +27,7 @@ async_scrape = "bin/async_scrape.py"
|
|||
shutdown = "bin/shutdown.py"
|
||||
stop = "bin/stop.py"
|
||||
rebuild_caches = "bin/rebuild_caches.py"
|
||||
update = "bin/update.py"
|
||||
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
|
|
2
setup.py
2
setup.py
|
@ -13,7 +13,7 @@ setup(
|
|||
description='Web interface to track the trackers.',
|
||||
packages=['lookyloo'],
|
||||
scripts=['bin/start_website.py', 'bin/start.py', 'bin/run_backend.py', 'bin/async_scrape.py',
|
||||
'bin/shutdown.py', 'bin/stop.py', 'bin/rebuild_caches.py'],
|
||||
'bin/shutdown.py', 'bin/stop.py', 'bin/rebuild_caches.py', 'bin/update.py'],
|
||||
include_package_data=True,
|
||||
classifiers=[
|
||||
'License :: OSI Approved :: BSD License',
|
||||
|
|
|
@ -9,7 +9,8 @@ from lookyloo.helpers import get_homedir
|
|||
|
||||
|
||||
def validate_generic_config_file():
|
||||
with (get_homedir() / 'config' / 'generic.json').open() as f:
|
||||
user_config = get_homedir() / 'config' / 'generic.json'
|
||||
with user_config.open() as f:
|
||||
generic_config = json.load(f)
|
||||
with (get_homedir() / 'config' / 'generic.json.sample').open() as f:
|
||||
generic_config_sample = json.load(f)
|
||||
|
@ -41,6 +42,8 @@ def validate_generic_config_file():
|
|||
if key not in generic_config_sample:
|
||||
raise Exception(f'{key} is missing in the sample config file')
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def validate_modules_config_file():
|
||||
with (get_homedir() / 'config' / 'modules.json').open() as f:
|
||||
|
@ -55,6 +58,8 @@ def validate_modules_config_file():
|
|||
logger.warning(f'Entry missing in user config file: {key}. Will default to: {json.dumps(modules_config_sample[key], indent=2)}')
|
||||
continue
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def update_user_configs():
|
||||
for file_name in ['generic', 'modules']:
|
||||
|
@ -72,11 +77,13 @@ def update_user_configs():
|
|||
continue
|
||||
if generic_config.get(key) is None:
|
||||
print(f'{key} was missing in {file_name}, adding it.')
|
||||
print(f"Description: {generic_config_sample['_notes'][key]}")
|
||||
generic_config[key] = generic_config_sample[key]
|
||||
has_new_entry = True
|
||||
if has_new_entry:
|
||||
with (get_homedir() / 'config' / f'{file_name}.json').open('w') as fw:
|
||||
json.dump(generic_config, fw, indent=2, sort_keys=True)
|
||||
return has_new_entry
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
@ -87,8 +94,11 @@ if __name__ == '__main__':
|
|||
args = parser.parse_args()
|
||||
|
||||
if args.check:
|
||||
validate_generic_config_file()
|
||||
validate_modules_config_file()
|
||||
if validate_generic_config_file():
|
||||
print(f"The entries in {get_homedir() / 'config' / 'generic.json'} are valid.")
|
||||
if validate_modules_config_file():
|
||||
print(f"The entries in {get_homedir() / 'config' / 'modules.json'} are valid.")
|
||||
|
||||
if args.update:
|
||||
update_user_configs()
|
||||
if not update_user_configs():
|
||||
print(f"No updates needed in {get_homedir() / 'config' / 'generic.json'}.")
|
||||
|
|
|
@ -11,3 +11,5 @@ datatables="1.10.22"
|
|||
|
||||
wget -q https://cdn.datatables.net/v/bs4/dt-${datatables}/datatables.min.css -O web/static/datatables.min.css
|
||||
wget -q https://cdn.datatables.net/v/bs4/dt-${datatables}/datatables.min.js -O web/static/datatables.min.js
|
||||
|
||||
echo 'Done'
|
||||
|
|
Loading…
Reference in New Issue