diff --git a/bin/rebuild_caches.py b/bin/rebuild_caches.py index 53ad52b..5f8ecf0 100755 --- a/bin/rebuild_caches.py +++ b/bin/rebuild_caches.py @@ -1,16 +1,21 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import argparse import logging + from lookyloo.lookyloo import Lookyloo logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s', level=logging.INFO, datefmt='%I:%M:%S') if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Rebuild the redis cache.') + parser.add_argument('--rebuild_pickles', default=False, action='store_true', help='Delete and rebuild the pickles. Count 20s/pickle, it can take a very long time.') + args = parser.parse_args() + lookyloo = Lookyloo() - remove_pickles = input('Do you want to remove the pickles? Rebuilding will take a very long time. (y/N)') - if remove_pickles == 'y': + if args.rebuild_pickles: lookyloo.rebuild_all() else: lookyloo.rebuild_cache() diff --git a/docker-compose.yml b/docker-compose.yml index 1aba4cb..5f9b6b2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,7 @@ services: - ./cache:/cache splash: - image: "scrapinghub/splash" + image: "scrapinghub/splash:3.5.0" ports: - "8050:8050" - "5023:5023" @@ -25,6 +25,7 @@ services: - /bin/sh - -c - | + poetry run rebuild_caches.py poetry run async_scrape.py & poetry run start_website.py volumes: diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index 0f8bf7c..f809861 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -272,7 +272,7 @@ class Lookyloo(): except json.decoder.JSONDecodeError: # old format error_to_cache = content - error_cache['error'] = f'The capture has an error: {error_to_cache}' + error_cache['error'] = f'The capture {capture_dir.name} has an error: {error_to_cache}' fatal_error = False if har_files: diff --git a/website/web/templates/scrape.html b/website/web/templates/scrape.html index b9d8b09..7216cf2 100644 --- a/website/web/templates/scrape.html +++ b/website/web/templates/scrape.html @@ -44,7 +44,7 @@
- +