From 35f4292ab07f39c5771708eff1f2e63288416fda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 5 Apr 2019 14:01:36 +0200 Subject: [PATCH] fix: Systemd service, add proper stop script --- bin/async_scrape.py | 4 +++- bin/shutdown.py | 17 +++++++++++++++ bin/start.py | 1 - bin/start_website.py | 25 ++++++++++++++++++---- bin/stop.py | 14 ++++++++++++ cache/cache.conf | 2 +- etc/systemd/system/lookyloo.service | 15 ------------- etc/systemd/system/lookyloo.service.sample | 18 ++++++++++++++++ setup.py | 3 ++- website/3rdparty.sh | 6 +++--- 10 files changed, 79 insertions(+), 26 deletions(-) create mode 100755 bin/shutdown.py create mode 100755 bin/stop.py delete mode 100644 etc/systemd/system/lookyloo.service create mode 100644 etc/systemd/system/lookyloo.service.sample diff --git a/bin/async_scrape.py b/bin/async_scrape.py index b08c10c..a7cd5ba 100755 --- a/bin/async_scrape.py +++ b/bin/async_scrape.py @@ -5,7 +5,7 @@ from pathlib import Path import logging from lookyloo.abstractmanager import AbstractManager -from lookyloo.helpers import get_homedir +from lookyloo.helpers import get_homedir, set_running, unset_running from lookyloo.lookyloo import Lookyloo logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s', @@ -21,7 +21,9 @@ class AsyncScraper(AbstractManager): self.lookyloo = Lookyloo(loglevel=loglevel) def _to_run_forever(self): + set_running('async_scrape') self.lookyloo.process_scrape_queue() + unset_running('async_scrape') if __name__ == '__main__': diff --git a/bin/shutdown.py b/bin/shutdown.py new file mode 100755 index 0000000..794a02d --- /dev/null +++ b/bin/shutdown.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from lookyloo.helpers import is_running, get_socket_path +import time +from redis import StrictRedis + +if __name__ == '__main__': + r = StrictRedis(unix_socket_path=get_socket_path('cache'), db=1) + r.set('shutdown', 1) + time.sleep(5) + while True: + running = is_running() + if not running: + break + print(running) + time.sleep(5) diff --git a/bin/start.py b/bin/start.py index 45eaadf..23523df 100755 --- a/bin/start.py +++ b/bin/start.py @@ -3,7 +3,6 @@ from subprocess import Popen from lookyloo.helpers import get_homedir -import time if __name__ == '__main__': # Just fail if the env isn't set. diff --git a/bin/start_website.py b/bin/start_website.py index a470078..c28265e 100755 --- a/bin/start_website.py +++ b/bin/start_website.py @@ -1,14 +1,31 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import time +import signal from subprocess import Popen -from lookyloo.helpers import get_homedir +from lookyloo.helpers import get_homedir, shutdown_requested, set_running, unset_running if __name__ == '__main__': website_dir = get_homedir() / 'website' Popen([str(website_dir / '3rdparty.sh')], cwd=website_dir) try: - Popen(['gunicorn', '--worker-class', 'eventlet', '-w', '10', '-b', '0.0.0.0:5100', 'web:app'], - cwd=website_dir).communicate() + p = Popen(['gunicorn', '--worker-class', 'eventlet', '-w', '10', '--graceful-timeout', '2', + '-b', '0.0.0.0:5100', 'web:app'], + cwd=website_dir) + set_running('website') + while True: + if shutdown_requested() or p.poll() is not None: + break + time.sleep(1) except KeyboardInterrupt: - print('Stopping gunicorn.') + print('Website killed by user.') + finally: + print('Shutting down website.') + try: + # Killing everything if possible. + p.send_signal(signal.SIGWINCH) + p.send_signal(signal.SIGTERM) + except Exception: + pass + unset_running('website') diff --git a/bin/stop.py b/bin/stop.py new file mode 100755 index 0000000..fbe1f6d --- /dev/null +++ b/bin/stop.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from subprocess import Popen +from lookyloo.helpers import get_homedir, get_socket_path +from redis import Redis + +if __name__ == '__main__': + get_homedir() + p = Popen(['shutdown.py']) + p.wait() + r = Redis(unix_socket_path=get_socket_path('cache'), db=1) + r.delete('shutdown') + Popen(['run_backend.py', '--stop']) diff --git a/cache/cache.conf b/cache/cache.conf index f27d27d..b2c3637 100644 --- a/cache/cache.conf +++ b/cache/cache.conf @@ -168,7 +168,7 @@ loglevel notice # Specify the log file name. Also the empty string can be used to force # Redis to log on the standard output. Note that if you use standard # output for logging but daemonize, logs will be sent to /dev/null -logfile "cache.log" +# logfile "cache.log" # To enable logging to the system logger, just set 'syslog-enabled' to yes, # and optionally update the other syslog parameters to suit your needs. diff --git a/etc/systemd/system/lookyloo.service b/etc/systemd/system/lookyloo.service deleted file mode 100644 index a807619..0000000 --- a/etc/systemd/system/lookyloo.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=uWSGI instance to serve lookyloo -After=network.target - -[Service] -User=www-data -Group=www-data -WorkingDirectory=/home//lookyloo -Environment="PATH=/home///bin" -ExecStart=start.py -Environment=LOOKYLOO_HOME=/home//lookyloo - -[Install] -WantedBy=multi-user.target - diff --git a/etc/systemd/system/lookyloo.service.sample b/etc/systemd/system/lookyloo.service.sample new file mode 100644 index 0000000..0a5d8c3 --- /dev/null +++ b/etc/systemd/system/lookyloo.service.sample @@ -0,0 +1,18 @@ +[Unit] +Description=uWSGI instance to serve lookyloo +After=network.target + +[Service] +User=www-data +Group=www-data +Type=forking +WorkingDirectory= +Environment=LOOKYLOO_HOME= +Environment=PATH=/bin:$PATH +ExecStart=/bin/start.py +ExecStop=/bin/stop.py + + +[Install] +WantedBy=multi-user.target + diff --git a/setup.py b/setup.py index 055a34d..c346f11 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,8 @@ setup( url='https://github.com/CIRCL/lookyloo', description='Web interface to track the trackers.', packages=['lookyloo'], - scripts=['bin/start_website.py', 'bin/start.py', 'bin/run_backend.py', 'bin/async_scrape.py'], + scripts=['bin/start_website.py', 'bin/start.py', 'bin/run_backend.py', 'bin/async_scrape.py', + 'bin/shutdown.py', 'bin/stop.py'], include_package_data=True, classifiers=[ 'License :: OSI Approved :: BSD License', diff --git a/website/3rdparty.sh b/website/3rdparty.sh index 5ed3625..39ae6ec 100755 --- a/website/3rdparty.sh +++ b/website/3rdparty.sh @@ -5,8 +5,8 @@ set -x mkdir -p web/static/ -wget https://d3js.org/d3.v5.min.js -O web/static/d3.v5.min.js +wget -q https://d3js.org/d3.v5.min.js -O web/static/d3.v5.min.js -FileSaver="5733e40e5af936eb3f48554cf6a8a7075d71d18a" +FileSaver="2.0.0" -wget https://cdn.rawgit.com/eligrey/FileSaver.js/${FileSaver}/FileSaver.js -O web/static/FileSaver.js +wget -q https://raw.githubusercontent.com/eligrey/FileSaver.js/${FileSaver}/src/FileSaver.js -O web/static/FileSaver.js