fix: Systemd service, add proper stop script

bin/async_scrape.py

@@ -5,7 +5,7 @@ from pathlib import Path
 import logging
 
 from lookyloo.abstractmanager import AbstractManager
-from lookyloo.helpers import get_homedir
+from lookyloo.helpers import get_homedir, set_running, unset_running
 from lookyloo.lookyloo import Lookyloo
 
 logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
@@ -21,7 +21,9 @@ class AsyncScraper(AbstractManager):
         self.lookyloo = Lookyloo(loglevel=loglevel)
 
     def _to_run_forever(self):
+        set_running('async_scrape')
         self.lookyloo.process_scrape_queue()
+        unset_running('async_scrape')
 
 
 if __name__ == '__main__':

bin/shutdown.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from lookyloo.helpers import is_running, get_socket_path
+import time
+from redis import StrictRedis
+
+if __name__ == '__main__':
+    r = StrictRedis(unix_socket_path=get_socket_path('cache'), db=1)
+    r.set('shutdown', 1)
+    time.sleep(5)
+    while True:
+        running = is_running()
+        if not running:
+            break
+        print(running)
+        time.sleep(5)

bin/start.py

@@ -3,7 +3,6 @@
 
 from subprocess import Popen
 from lookyloo.helpers import get_homedir
-import time
 
 if __name__ == '__main__':
     # Just fail if the env isn't set.

bin/start_website.py

@@ -1,14 +1,31 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+import time
+import signal
 from subprocess import Popen
-from lookyloo.helpers import get_homedir
+from lookyloo.helpers import get_homedir, shutdown_requested, set_running, unset_running
 
 if __name__ == '__main__':
     website_dir = get_homedir() / 'website'
     Popen([str(website_dir / '3rdparty.sh')], cwd=website_dir)
     try:
-        Popen(['gunicorn', '--worker-class', 'eventlet', '-w', '10', '-b', '0.0.0.0:5100', 'web:app'],
-              cwd=website_dir).communicate()
+        p = Popen(['gunicorn', '--worker-class', 'eventlet', '-w', '10', '--graceful-timeout', '2',
+                   '-b', '0.0.0.0:5100', 'web:app'],
+                  cwd=website_dir)
+        set_running('website')
+        while True:
+            if shutdown_requested() or p.poll() is not None:
+                break
+            time.sleep(1)
     except KeyboardInterrupt:
-        print('Stopping gunicorn.')
+        print('Website killed by user.')
+    finally:
+        print('Shutting down website.')
+        try:
+            # Killing everything if possible.
+            p.send_signal(signal.SIGWINCH)
+            p.send_signal(signal.SIGTERM)
+        except Exception:
+            pass
+        unset_running('website')

bin/stop.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from subprocess import Popen
+from lookyloo.helpers import get_homedir, get_socket_path
+from redis import Redis
+
+if __name__ == '__main__':
+    get_homedir()
+    p = Popen(['shutdown.py'])
+    p.wait()
+    r = Redis(unix_socket_path=get_socket_path('cache'), db=1)
+    r.delete('shutdown')
+    Popen(['run_backend.py', '--stop'])

cache/cache.conf

@@ -168,7 +168,7 @@ loglevel notice
 # Specify the log file name. Also the empty string can be used to force
 # Redis to log on the standard output. Note that if you use standard
 # output for logging but daemonize, logs will be sent to /dev/null
-logfile "cache.log"
+# logfile "cache.log"
 
 # To enable logging to the system logger, just set 'syslog-enabled' to yes,
 # and optionally update the other syslog parameters to suit your needs.

etc/systemd/system/lookyloo.service

@@ -1,15 +0,0 @@
-[Unit]
-Description=uWSGI instance to serve lookyloo
-After=network.target
-
-[Service]
-User=www-data
-Group=www-data
-WorkingDirectory=/home/<CHANGE_ME>/lookyloo
-Environment="PATH=/home/<CHANGE_ME>/<MY_VIRTUALENV_PATH>/bin"
-ExecStart=start.py
-Environment=LOOKYLOO_HOME=/home/<CHANGE_ME>/lookyloo
-
-[Install]
-WantedBy=multi-user.target
-

etc/systemd/system/lookyloo.service.sample

@@ -0,0 +1,18 @@
+[Unit]
+Description=uWSGI instance to serve lookyloo
+After=network.target
+
+[Service]
+User=www-data
+Group=www-data
+Type=forking
+WorkingDirectory=</git/home/directory>
+Environment=LOOKYLOO_HOME=</git/home/directory>
+Environment=PATH=</path/to/virtualenv>/bin:$PATH
+ExecStart=</path/to/virtualenv>/bin/start.py
+ExecStop=</path/to/virtualenv>/bin/stop.py
+
+
+[Install]
+WantedBy=multi-user.target
+

setup.py

@@ -12,7 +12,8 @@ setup(
     url='https://github.com/CIRCL/lookyloo',
     description='Web interface to track the trackers.',
     packages=['lookyloo'],
-    scripts=['bin/start_website.py', 'bin/start.py', 'bin/run_backend.py', 'bin/async_scrape.py'],
+    scripts=['bin/start_website.py', 'bin/start.py', 'bin/run_backend.py', 'bin/async_scrape.py',
+             'bin/shutdown.py', 'bin/stop.py'],
     include_package_data=True,
     classifiers=[
         'License :: OSI Approved :: BSD License',

website/3rdparty.sh

@@ -5,8 +5,8 @@ set -x
 
 mkdir -p web/static/
 
-wget https://d3js.org/d3.v5.min.js -O web/static/d3.v5.min.js
+wget -q https://d3js.org/d3.v5.min.js -O web/static/d3.v5.min.js
 
-FileSaver="5733e40e5af936eb3f48554cf6a8a7075d71d18a"
+FileSaver="2.0.0"
 
-wget https://cdn.rawgit.com/eligrey/FileSaver.js/${FileSaver}/FileSaver.js -O web/static/FileSaver.js
+wget -q https://raw.githubusercontent.com/eligrey/FileSaver.js/${FileSaver}/src/FileSaver.js -O web/static/FileSaver.js