mirror of https://github.com/CIRCL/lookyloo
new: Add initial redirects columns
parent
89edef68ab
commit
4234e44470
|
@ -30,7 +30,7 @@ import logging
|
|||
|
||||
from pysanejs import SaneJS
|
||||
from scrapysplashwrapper import crawl
|
||||
from har2tree import CrawledTree, Har2TreeError
|
||||
from har2tree import CrawledTree, Har2TreeError, HarFile
|
||||
|
||||
|
||||
class Lookyloo():
|
||||
|
@ -72,17 +72,13 @@ class Lookyloo():
|
|||
return
|
||||
with (report_dir / 'uuid').open() as f:
|
||||
uuid = f.read().strip()
|
||||
with har_files[0].open() as f:
|
||||
j = json.load(f)
|
||||
title = j['log']['pages'][0]['title']
|
||||
timestamp = j['log']['pages'][0]['startedDateTime']
|
||||
if j['log']['entries']:
|
||||
first_url = j['log']['entries'][0]['request']['url']
|
||||
else:
|
||||
first_url = '-'
|
||||
if not title:
|
||||
title = '!! No title found !! '
|
||||
cache = {'uuid': uuid, 'title': title, 'timestamp': timestamp, 'url': first_url}
|
||||
har = HarFile(har_files[0])
|
||||
|
||||
cache: Dict[str, Union[str, int]] = {'uuid': uuid,
|
||||
'title': har.initial_title,
|
||||
'timestamp': har.initial_start_time,
|
||||
'url': har.first_url,
|
||||
'redirects': json.dumps(har.initial_redirects)}
|
||||
if (report_dir / 'no_index').exists(): # If the folders claims anonymity
|
||||
cache['no_index'] = 1
|
||||
if uuid and not self.redis.exists(str(report_dir)):
|
||||
|
@ -92,7 +88,9 @@ class Lookyloo():
|
|||
def report_cache(self, report_dir: Union[str, Path]) -> Dict:
|
||||
if isinstance(report_dir, Path):
|
||||
report_dir = str(report_dir)
|
||||
return self.redis.hgetall(report_dir)
|
||||
cached = self.redis.hgetall(report_dir)
|
||||
cached['redirects'] = json.loads(cached['redirects'])
|
||||
return cached
|
||||
|
||||
def _init_existing_dumps(self) -> None:
|
||||
for report_dir in self.report_dirs:
|
||||
|
|
|
@ -208,7 +208,7 @@ lxml = "^4.4.2"
|
|||
six = "^1.14.0"
|
||||
|
||||
[package.source]
|
||||
reference = "12c88d6298e52a8458e220b6910597cb5539daa9"
|
||||
reference = "147ac0f014249358af7b54cf02d5c85644a60645"
|
||||
type = "git"
|
||||
url = "https://github.com/viper-framework/har2tree.git"
|
||||
[[package]]
|
||||
|
|
|
@ -175,6 +175,6 @@ def index():
|
|||
continue
|
||||
date, time = cached['timestamp'].split('T')
|
||||
time, _ = time.split('.', 1)
|
||||
titles.append((cached['uuid'], cached['title'], date, time, cached['url']))
|
||||
titles.append((cached['uuid'], cached['title'], date, time, cached['url'], cached['redirects']))
|
||||
titles = sorted(titles, key=lambda x: (x[2], x[3]), reverse=True)
|
||||
return render_template('index.html', titles=titles)
|
||||
|
|
|
@ -8,23 +8,35 @@
|
|||
</br></br>
|
||||
</center>
|
||||
|
||||
<table class="table table-hover" data-toggle="table" data-search="true">
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col">Page title</th>
|
||||
<th scope="col">Initial URL</th>
|
||||
<tr>
|
||||
<table class="table" data-toggle="table" data-search="true" data-show-columns="true">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Page title</th>
|
||||
<th>Initial URL</th>
|
||||
<th>Initial redirects</th>
|
||||
<tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for uuid, page_title, date, time, url in titles %}
|
||||
<tr>
|
||||
<td>
|
||||
<a href="{{ url_for('tree', tree_uuid=uuid) }}">{{ page_title }}</a>
|
||||
<div id="timestamp"> <b>{{ date }}</b> {{ time }}</div>
|
||||
</td>
|
||||
<td>{{ url }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
{% for uuid, page_title, date, time, url, redirects in titles %}
|
||||
<tr>
|
||||
<td>
|
||||
<a href="{{ url_for('tree', tree_uuid=uuid) }}">{{ page_title }}</a>
|
||||
<div id="timestamp"> <b>{{ date }}</b> {{ time }}</div>
|
||||
</td>
|
||||
<td>{{ url }}</td>
|
||||
<td>
|
||||
{% if redirects %}
|
||||
<ul>
|
||||
{% for r in redirects %}
|
||||
<li>{{ r }}</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
{% else%}
|
||||
No redirect
|
||||
{%endif%}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% endblock %}
|
||||
|
|
|
@ -21,9 +21,13 @@
|
|||
</head>
|
||||
<body>
|
||||
<!-- Your page contont -->
|
||||
<!--
|
||||
<div class="container">
|
||||
-->
|
||||
{% block content %}{% endblock%}
|
||||
<!--
|
||||
</div>
|
||||
-->
|
||||
|
||||
{% block scripts %}
|
||||
<!-- Optional JavaScript -->
|
||||
|
|
Loading…
Reference in New Issue