fix: Docker, capture form, error message.

bin/rebuild_caches.py

@@ -1,16 +1,21 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+import argparse
 import logging
+
 from lookyloo.lookyloo import Lookyloo
 
 logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
                     level=logging.INFO, datefmt='%I:%M:%S')
 
 if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Rebuild the redis cache.')
+    parser.add_argument('--rebuild_pickles', default=False, action='store_true', help='Delete and rebuild the pickles. Count 20s/pickle, it can take a very long time.')
+    args = parser.parse_args()
+
     lookyloo = Lookyloo()
-    remove_pickles = input('Do you want to remove the pickles? Rebuilding will take a very long time. (y/N)')
-    if remove_pickles == 'y':
+    if args.rebuild_pickles:
         lookyloo.rebuild_all()
     else:
         lookyloo.rebuild_cache()

docker-compose.yml

@@ -9,7 +9,7 @@ services:
         - ./cache:/cache
 
   splash:
-    image: "scrapinghub/splash"
+    image: "scrapinghub/splash:3.5.0"
     ports:
         - "8050:8050"
         - "5023:5023"
@@ -25,6 +25,7 @@ services:
         - /bin/sh
         - -c
         - |
+            poetry run rebuild_caches.py
             poetry run async_scrape.py &
             poetry run start_website.py
     volumes:

lookyloo/lookyloo.py

@@ -272,7 +272,7 @@ class Lookyloo():
                 except json.decoder.JSONDecodeError:
                     # old format
                     error_to_cache = content
-                error_cache['error'] = f'The capture has an error: {error_to_cache}'
+                error_cache['error'] = f'The capture {capture_dir.name} has an error: {error_to_cache}'
 
         fatal_error = False
         if har_files:

website/web/templates/scrape.html

@@ -44,7 +44,7 @@
       <div class="form-group row">
         <label for="referer" class="col-sm-2 col-form-label">Referer:</label>
         <div class="col-sm-10">
-          <input type="text" class="form-control" name="referer" id=referer placeholder="Referer of the URL to capture" required>
+          <input type="text" class="form-control" name="referer" id=referer placeholder="Referer of the URL to capture">
         </div>
       </div>