CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

new: Pagination on Favicon index

pull/1002/head
Raphaël Vinot 2024-11-27 15:37:03 +01:00
parent 556b7a4ac3
commit 18da251589
5 changed files with 50 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
lookyloo/indexing.py
View File

@ -502,7 +502,7 @@ class Indexing():
self.redis.delete(f'urls|{md5}|captures')
return 0, []
total = self.redis.zcard(f'urls|{md5}|captures')
return total, self.redis.zrevrangebyscore(f'urls|{md5}|captures', max_score, min_score, withscores=True)
return total, self.redis.zrevrangebyscore(f'urls|{md5}|captures', max_score, min_score, withscores=True, start=offset, num=limit)
def get_captures_url_count(self, url: str) -> int:
md5 = hashlib.md5(url.encode()).hexdigest()
@ -663,16 +663,18 @@ class Indexing():
pipeline.execute()
def get_captures_favicon(self, favicon_sha512: str, most_recent_capture: datetime | None=None,
oldest_capture: datetime | None = None) -> list[tuple[str, float]]:
oldest_capture: datetime | None = None,
offset: int | None=None, limit: int | None=None) -> tuple[int, list[tuple[str, float]]]:
"""Get all the captures for a specific favicon, on a time interval starting from the most recent one.
:param favicon_sha512: The favicon hash
:param most_recent_capture: The capture time of the most recent capture to consider
:param oldest_capture: The capture time of the oldest capture to consider, defaults to 30 days ago.
:param oldest_capture: The capture time of the oldest capture to consider.
"""
max_score: str | float = most_recent_capture.timestamp() if most_recent_capture else '+Inf'
min_score: str | float = oldest_capture.timestamp() if oldest_capture else (datetime.now() - timedelta(days=30)).timestamp()
return self.redis.zrevrangebyscore(f'favicons|{favicon_sha512}|captures', max_score, min_score, withscores=True)
min_score: str | float = oldest_capture.timestamp() if oldest_capture else '-Inf'
total = self.redis.zcard(f'favicons|{favicon_sha512}|captures')
return total, self.redis.zrevrangebyscore(f'favicons|{favicon_sha512}|captures', max_score, min_score, withscores=True, start=offset, num=limit)
def get_captures_favicon_count(self, favicon_sha512: str) -> int:
if self.redis.type(f'favicons|{favicon_sha512}|captures') == 'set': # type: ignore[no-untyped-call]

47
website/web/__init__.py
View File

@ -441,23 +441,12 @@ def get_capture_hash_investigator(hash_type: str, h: str) -> list[tuple[str, str
return [(cache.uuid, cache.title, cache.redirects[-1], cache.timestamp) for cache in cached_captures]
def get_favicon_investigator(favicon_sha512: str,
/) -> tuple[list[tuple[str, str, str, datetime]],
tuple[str, str, str]]:
def get_favicon_investigator(favicon_sha512: str, offset: int | None=None, limit: int | None=None) -> tuple[int, list[tuple[str, str, str, datetime]]]:
'''Returns all the captures related to a cookie name entry, used in the web interface.'''
cached_captures = lookyloo.sorted_capture_cache([uuid for uuid, _ in get_indexing(flask_login.current_user).get_captures_favicon(favicon_sha512)])
total, entries = get_indexing(flask_login.current_user).get_captures_favicon(favicon_sha512=favicon_sha512, offset=offset, limit=limit)
cached_captures = lookyloo.sorted_capture_cache([uuid for uuid, _ in entries])
captures = [(cache.uuid, cache.title, cache.redirects[-1], cache.timestamp) for cache in cached_captures]
favicon = get_indexing(flask_login.current_user).get_favicon(favicon_sha512)
if favicon:
mimetype = from_string(favicon, mime=True)
b64_favicon = base64.b64encode(favicon).decode()
mmh3_shodan = lookyloo.compute_mmh3_shodan(favicon)
else:
mimetype = ''
b64_favicon = ''
mmh3_shodan = ''
return captures, (mimetype, b64_favicon, mmh3_shodan)
return total, captures
def get_hhh_investigator(hhh: str, /) -> tuple[list[tuple[str, str, str, str]], list[tuple[str, str]]]:
@ -1773,11 +1762,19 @@ def capture_hash_details(hash_type: str, h: str) -> str:
@app.route('/favicon_details/<string:favicon_sha512>', methods=['GET'])
def favicon_detail(favicon_sha512: str) -> str:
from_popup = True if (request.args.get('from_popup') and request.args.get('from_popup') == 'True') else False
captures, favicon = get_favicon_investigator(favicon_sha512.strip())
mimetype, b64_favicon, mmh3_shodan = favicon
favicon = get_indexing(flask_login.current_user).get_favicon(favicon_sha512)
if favicon:
mimetype = from_string(favicon, mime=True)
b64_favicon = base64.b64encode(favicon).decode()
mmh3_shodan = lookyloo.compute_mmh3_shodan(favicon)
else:
mimetype = ''
b64_favicon = ''
mmh3_shodan = ''
return render_template('favicon_details.html',
captures=captures, mimetype=mimetype, b64_favicon=b64_favicon,
mimetype=mimetype, b64_favicon=b64_favicon,
mmh3_shodan=mmh3_shodan,
favicon_sha512=favicon_sha512,
from_popup=from_popup)
@ -1993,6 +1990,19 @@ def post_table(table_name: str, value: str) -> Response:
draw = request.form.get('draw', type=int)
start = request.form.get('start', type=int)
length = request.form.get('length', type=int)
captures: list[tuple[str, str, str, datetime, set[str]]] | list[tuple[str, str, str, datetime]]
if table_name == 'faviconDetailsTable':
total, captures = get_favicon_investigator(value.strip(), offset=start, limit=length)
prepared_captures = []
for capture_uuid, title, landing_page, capture_time in captures:
to_append = {
'capture_time': capture_time.isoformat(),
'capture_title': f"""<a href="{url_for('tree', tree_uuid=capture_uuid)}">{title}</a>""",
'landing_page': f"""<span class="d-inline-block text-break" style="max-width: 400px;">{landing_page}</span>"""
}
prepared_captures.append(to_append)
return jsonify({'draw': draw, 'recordsTotal': total, 'recordsFiltered': total, 'data': prepared_captures})
if table_name == 'hostnameTable':
total, captures = get_hostname_investigator(value.strip(), offset=start, limit=length)
prepared_captures = []
@ -2005,6 +2015,7 @@ def post_table(table_name: str, value: str) -> Response:
}
prepared_captures.append(to_append)
return jsonify({'draw': draw, 'recordsTotal': total, 'recordsFiltered': total, 'data': prepared_captures})
if table_name == 'urlTable':
url = base64.b64decode(value.strip()).decode()
total, captures = get_url_investigator(url, offset=start, limit=length)

2
website/web/sri.txt
View File

@ -33,7 +33,7 @@
"loader.gif": "ZZKD5vLSKBWKeUpa2KI9qheUJ49iTI/UULmVU/AX28fBfH00K3lLc2v5pVJZ4qXG1BbB13LTXzRKKU35H2XfNg==",
"lookyloo.jpeg": "i6wBj8CsIM5YAQLEMQfhs3CNOSKkErF8AMqqM6ZygSwCyQgv9CU8xt94veMZhM/ufBWoz7kAXmR+yywmxsTxug==",
"redirect.png": "PAjzlPV97rEFvH55mG1ZC9wRl98be3yMeX/nENuFkJcds6/AXgSR2ig/QyPULgobSnNgiYieLVWY/oqsgeywrQ==",
"render_tables.js": "Su4TaO5d7pZ+hZM3b6JQAhtW5vJJ9vUIsOatzJR/wZOLykpPe9HuzNV48AxH4wumWKsK5HVt0wW7njOjHl8p7A==",
"render_tables.js": "L8K+7SHzG6c4kddX4WgUTYIh9qeTPK3k16NWxMiHJt5KFtR2KiC9e9dpMPl+8m3LwNnhlzRwaIYYb/gxuuJbqw==",
"secure.svg": "H8ni7t0d60nCJDVGuZpuxC+RBy/ipAjWT627D12HlZGg6LUmjSwPTQTUekm3UJupEP7TUkhXyq6WHc5gy7QBjg==",
"stats.css": "/kY943FwWBTne4IIyf7iBROSfbGd82TeBicEXqKkRwawMVRIvM/Pk5MRa7okUyGIxaDjFQGmV/U1vy+PhN6Jbw==",
"stats_graph.js": "S/sMNQK1UMMLD0xQeEa7sq3ce8o6oPxwxGlyKVtaHOODjair86dbBDm7cu6pa/elMRDJT1j09jEFjWp+5GbhTw==",

12
website/web/static/render_tables.js
View File

@ -183,8 +183,20 @@
}
if (document.getElementById('faviconDetailsTable')) {
favicon = document.getElementById('faviconDetailsTable').dataset.favicon;
new DataTable('#faviconDetailsTable', {
processing: true,
serverSide: true,
retrieve: true,
ajax: {
url: `/tables/faviconDetailsTable/${favicon}`,
type: 'POST'
},
columns : [
{ data: 'capture_time' },
{ data: 'capture_title' },
{ data: 'landing_page' }
],
order: [[ 0, "desc" ]],
columnDefs: [{ width: '30%',
targets: 0,

21
website/web/templates/favicon_details.html
View File

@ -21,7 +21,7 @@
<img src="data:{{mimetype}};base64,{{ b64_favicon }}" style="width:32px;height:32px;"/>
<h5>Shodan MMH3 Hash: <a href="https://www.shodan.io/search?query=http.favicon.hash%3A{{ mmh3_shodan }}" target="_blank">{{ mmh3_shodan }}</a></h5>
</center>
<table id="faviconDetailsTable" class="table table-striped" style="width:100%">
<table id="faviconDetailsTable" class="table table-striped" style="width:100%" data-favicon="{{favicon_sha512}}">
<thead>
<tr>
<th>Capture Time</th>
@ -29,25 +29,6 @@
<th>Landing page</th>
</tr>
</thead>
<tbody>
{% for capture_uuid, title, landing_page, capture_time in captures %}
<tr>
<td>
{{capture_time}}
</td>
<td>
<a href="{{ url_for('tree', tree_uuid=capture_uuid) }}">
{{ title }}
</a>
</td>
<td>
<span class="d-inline-block text-break" style="max-width: 400px;">
{{ landing_page }}
</span>
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endblock %}