CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: Better handling of error messages, display them.

pull/67/head
Raphaël Vinot 2020-03-17 14:17:18 +01:00
parent 1c5483317f
commit af98c2b075
4 changed files with 54 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
lookyloo/lookyloo.py
View File

@ -63,20 +63,26 @@ class Lookyloo():
def _set_report_cache(self, report_dir: Path) -> None:
if self.redis.exists(str(report_dir)):
return
if (report_dir / 'error.txt').exists():
# Something went wrong
return
har_files = sorted(report_dir.glob('*.har'))
if not har_files:
self.logger.warning(f'No har files in {report_dir}')
# if (report_dir / 'uuid').exists():
# (report_dir / 'uuid').unlink()
# if (report_dir / 'no_index').exists():
# (report_dir / 'no_index').unlink()
# report_dir.rmdir()
return
with (report_dir / 'uuid').open() as f:
uuid = f.read().strip()
error_cache: Dict[str, str] = {}
if (report_dir / 'error.txt').exists():
# Something went wrong
with (Path(report_dir) / 'error.txt').open() as _error:
error_cache['error'] = f'Capture in ({report_dir}) has an error: {_error.read()}, see https://splash.readthedocs.io/en/stable/scripting-ref.html#splash-go and https://doc.qt.io/qt-5/qnetworkreply.html#NetworkError-enum'
har_files = sorted(report_dir.glob('*.har'))
if not har_files:
error_cache['error'] = f'No har files in {report_dir}'
if error_cache:
self.logger.warning(error_cache['error'])
self.redis.hmset(str(report_dir), error_cache)
self.redis.hset('lookup_dirs', uuid, str(report_dir))
return
har = HarFile(har_files[0])
cache: Dict[str, Union[str, int]] = {'uuid': uuid,
@ -93,16 +99,15 @@ class Lookyloo():
def report_cache(self, report_dir: Union[str, Path]) -> Optional[Dict[str, Union[str, int]]]:
if isinstance(report_dir, Path):
report_dir = str(report_dir)
if (Path(report_dir) / 'error.txt').exists():
with (Path(report_dir) / 'error.txt').open() as _error:
self.logger.warning(f'Capture in ({report_dir}) has an error: {_error.read()}, see https://splash.readthedocs.io/en/stable/scripting-ref.html#splash-go')
cached = self.redis.hgetall(report_dir)
if all(key in cached.keys() for key in ['uuid', 'title', 'timestamp', 'url', 'redirects']):
cached['redirects'] = json.loads(cached['redirects'])
return cached
self.logger.warning(f'Cache ({report_dir}) is invalid: {json.dumps(cached, indent=2)}')
return None
elif 'error' in cached:
return cached
else:
self.logger.warning(f'Cache ({report_dir}) is invalid: {json.dumps(cached, indent=2)}')
return None
def _init_existing_dumps(self) -> None:
for report_dir in self.report_dirs:

11
website/web/__init__.py
View File

@ -9,7 +9,7 @@ import os
import logging
from pathlib import Path
from flask import Flask, render_template, request, session, send_file, redirect, url_for, Response
from flask import Flask, render_template, request, session, send_file, redirect, url_for, Response, flash
from flask_bootstrap import Bootstrap # type: ignore
from lookyloo.helpers import get_homedir, update_user_agents, get_user_agents
@ -149,6 +149,12 @@ def image(tree_uuid):
def tree(tree_uuid):
report_dir = lookyloo.lookup_report_dir(tree_uuid)
if not report_dir:
flash(f'Unable to find this UUID ({tree_uuid}). The capture may still be ongoing, try again later.', 'error')
return redirect(url_for('index'))
cache = lookyloo.report_cache(report_dir)
if 'error' in cache:
flash(cache['error'], 'error')
return redirect(url_for('index'))
try:
@ -167,11 +173,10 @@ def index():
return 'Ack'
lookyloo.cleanup_old_tmpfiles()
update_user_agents()
session.clear()
titles = []
for report_dir in lookyloo.report_dirs:
cached = lookyloo.report_cache(report_dir)
if not cached or 'no_index' in cached:
if not cached or 'no_index' in cached or 'error' in cached:
continue
date, time = cached['timestamp'].split('T')
time, _ = time.split('.', 1)

7
website/web/templates/index.html
View File

@ -1,5 +1,7 @@
{% extends "main.html" %}
{% from 'bootstrap/utils.html' import render_messages %}
{% block title %}Lookyloo{% endblock %}
{% block content %}
@ -11,7 +13,8 @@
</center>
<center>
<h2><a href="{{ url_for('scrape_web') }}">Start a new capture</a></h2>
</br></br>
<br><br>
{{ render_messages(container=True, dismissible=True) }}
</center>
<center>
@ -29,7 +32,7 @@
<tr>
<td>
<p title="{{ page_title }}"><a href="{{ url_for('tree', tree_uuid=uuid) }}">{{ page_title }}</a></p>
<p title="{{ url }}">{{ url }}</div>
<p title="{{ url }}">{{ url }}</p>
</td>
<td>{{ date }} {{ time }}</td>
<td>

36
website/web/templates/tree.html
View File

@ -41,43 +41,43 @@
</div>
<hr/>
<img src="{{ url_for('static', filename='javascript.png') }}"
alt="JavaScript" height="20" width="20"> Javascript </br>
alt="JavaScript" height="20" width="20"> Javascript <br>
<img src="{{ url_for('static', filename='cookie_received.png') }}"
alt="Cookie received" height="20" width="20"> Cookie received</br>
alt="Cookie received" height="20" width="20"> Cookie received<br>
<img src="{{ url_for('static', filename='cookie_read.png') }}"
alt="Cookie read" height="20" width="20"> Cookie read</br>
alt="Cookie read" height="20" width="20"> Cookie read<br>
<img src="{{ url_for('static', filename='redirect.png') }}"
alt="Redirect" height="20" width="20"> Redirect</br>
alt="Redirect" height="20" width="20"> Redirect<br>
<img src="{{ url_for('static', filename='font.png') }}"
alt="Font" height="20" width="20"> Font</br>
alt="Font" height="20" width="20"> Font<br>
<img src="{{ url_for('static', filename='html.png') }}"
alt="HTML" height="20" width="20"> HTML</br>
alt="HTML" height="20" width="20"> HTML<br>
<img src="{{ url_for('static', filename='json.png') }}"
alt="JSON" height="20" width="20"> JSON</br>
alt="JSON" height="20" width="20"> JSON<br>
<img src="{{ url_for('static', filename='css.png') }}"
alt="CSS" height="20" width="20"> CSS</br>
alt="CSS" height="20" width="20"> CSS<br>
<img src="{{ url_for('static', filename='exe.png') }}"
alt="EXE" height="20" width="20"> EXE</br>
alt="EXE" height="20" width="20"> EXE<br>
<img src="{{ url_for('static', filename='img.png') }}"
alt="Image" height="20" width="20"> Image</br>
alt="Image" height="20" width="20"> Image<br>
<img src="{{ url_for('static', filename='video.png') }}"
alt="Video" height="20" width="20"> Video</br>
alt="Video" height="20" width="20"> Video<br>
<img src="{{ url_for('static', filename='ifr.png') }}"
alt="iFrame" height="20" width="20"> iFrame</br>
alt="iFrame" height="20" width="20"> iFrame<br>
<img src="{{ url_for('static', filename='wtf.png') }}"
alt="Content type not set/unknown" height="20" width="20"> Content type not set/unknown</br>
alt="Content type not set/unknown" height="20" width="20"> Content type not set/unknown<br>
</div>
<div style="width: 100px">
<div style="display: inline;">
@ -107,19 +107,19 @@
</center>
</div>
<hr/>
<b>Root URL</b>: {{ root_url }}</br>
<b>Start time</b>: {{ start_time }}</br>
<b>User Agent</b>: {{ user_agent }}</br>
<b>Root URL</b>: {{ root_url }}<br>
<b>Start time</b>: {{ start_time }}<br>
<b>User Agent</b>: {{ user_agent }}<br>
{% if meta %}
{%for k, v in meta.items()%}
<b>{{k.title()}}</b>: {{ v }}</br>
<b>{{k.title()}}</b>: {{ v }}<br>
{%endfor%}
{%endif%}
<b>Screenshot</b>:
<a href="{{ url_for('image', tree_uuid=tree_uuid) }}" class="btn btn-info" role="button">Download</a>
<button id="screenshot_view_button" type="button" class="btn btn-info"
data-toggle="collapse" data-target="#screenshot">View</button>
</br>
<br>
</div>
<div style="width: 100px;float: right;">
<div style="display: inline;">