mirror of https://github.com/CIRCL/lookyloo
fix: Docker, capture form, error message.
parent
33fb74ae83
commit
d18f5f4f88
|
@ -1,16 +1,21 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
import argparse
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from lookyloo.lookyloo import Lookyloo
|
from lookyloo.lookyloo import Lookyloo
|
||||||
|
|
||||||
logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
|
logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
|
||||||
level=logging.INFO, datefmt='%I:%M:%S')
|
level=logging.INFO, datefmt='%I:%M:%S')
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
parser = argparse.ArgumentParser(description='Rebuild the redis cache.')
|
||||||
|
parser.add_argument('--rebuild_pickles', default=False, action='store_true', help='Delete and rebuild the pickles. Count 20s/pickle, it can take a very long time.')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
lookyloo = Lookyloo()
|
lookyloo = Lookyloo()
|
||||||
remove_pickles = input('Do you want to remove the pickles? Rebuilding will take a very long time. (y/N)')
|
if args.rebuild_pickles:
|
||||||
if remove_pickles == 'y':
|
|
||||||
lookyloo.rebuild_all()
|
lookyloo.rebuild_all()
|
||||||
else:
|
else:
|
||||||
lookyloo.rebuild_cache()
|
lookyloo.rebuild_cache()
|
||||||
|
|
|
@ -9,7 +9,7 @@ services:
|
||||||
- ./cache:/cache
|
- ./cache:/cache
|
||||||
|
|
||||||
splash:
|
splash:
|
||||||
image: "scrapinghub/splash"
|
image: "scrapinghub/splash:3.5.0"
|
||||||
ports:
|
ports:
|
||||||
- "8050:8050"
|
- "8050:8050"
|
||||||
- "5023:5023"
|
- "5023:5023"
|
||||||
|
@ -25,6 +25,7 @@ services:
|
||||||
- /bin/sh
|
- /bin/sh
|
||||||
- -c
|
- -c
|
||||||
- |
|
- |
|
||||||
|
poetry run rebuild_caches.py
|
||||||
poetry run async_scrape.py &
|
poetry run async_scrape.py &
|
||||||
poetry run start_website.py
|
poetry run start_website.py
|
||||||
volumes:
|
volumes:
|
||||||
|
|
|
@ -272,7 +272,7 @@ class Lookyloo():
|
||||||
except json.decoder.JSONDecodeError:
|
except json.decoder.JSONDecodeError:
|
||||||
# old format
|
# old format
|
||||||
error_to_cache = content
|
error_to_cache = content
|
||||||
error_cache['error'] = f'The capture has an error: {error_to_cache}'
|
error_cache['error'] = f'The capture {capture_dir.name} has an error: {error_to_cache}'
|
||||||
|
|
||||||
fatal_error = False
|
fatal_error = False
|
||||||
if har_files:
|
if har_files:
|
||||||
|
|
|
@ -44,7 +44,7 @@
|
||||||
<div class="form-group row">
|
<div class="form-group row">
|
||||||
<label for="referer" class="col-sm-2 col-form-label">Referer:</label>
|
<label for="referer" class="col-sm-2 col-form-label">Referer:</label>
|
||||||
<div class="col-sm-10">
|
<div class="col-sm-10">
|
||||||
<input type="text" class="form-control" name="referer" id=referer placeholder="Referer of the URL to capture" required>
|
<input type="text" class="form-control" name="referer" id=referer placeholder="Referer of the URL to capture">
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue