From 18da251589cfe186e3134f9f79ce5c2d57d0bb08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 27 Nov 2024 15:37:03 +0100 Subject: [PATCH] new: Pagination on Favicon index --- lookyloo/indexing.py | 12 +++--- website/web/__init__.py | 47 +++++++++++++--------- website/web/sri.txt | 2 +- website/web/static/render_tables.js | 12 ++++++ website/web/templates/favicon_details.html | 21 +--------- 5 files changed, 50 insertions(+), 44 deletions(-) diff --git a/lookyloo/indexing.py b/lookyloo/indexing.py index e2fd3905..5925c471 100644 --- a/lookyloo/indexing.py +++ b/lookyloo/indexing.py @@ -502,7 +502,7 @@ class Indexing(): self.redis.delete(f'urls|{md5}|captures') return 0, [] total = self.redis.zcard(f'urls|{md5}|captures') - return total, self.redis.zrevrangebyscore(f'urls|{md5}|captures', max_score, min_score, withscores=True) + return total, self.redis.zrevrangebyscore(f'urls|{md5}|captures', max_score, min_score, withscores=True, start=offset, num=limit) def get_captures_url_count(self, url: str) -> int: md5 = hashlib.md5(url.encode()).hexdigest() @@ -663,16 +663,18 @@ class Indexing(): pipeline.execute() def get_captures_favicon(self, favicon_sha512: str, most_recent_capture: datetime | None=None, - oldest_capture: datetime | None = None) -> list[tuple[str, float]]: + oldest_capture: datetime | None = None, + offset: int | None=None, limit: int | None=None) -> tuple[int, list[tuple[str, float]]]: """Get all the captures for a specific favicon, on a time interval starting from the most recent one. :param favicon_sha512: The favicon hash :param most_recent_capture: The capture time of the most recent capture to consider - :param oldest_capture: The capture time of the oldest capture to consider, defaults to 30 days ago. + :param oldest_capture: The capture time of the oldest capture to consider. """ max_score: str | float = most_recent_capture.timestamp() if most_recent_capture else '+Inf' - min_score: str | float = oldest_capture.timestamp() if oldest_capture else (datetime.now() - timedelta(days=30)).timestamp() - return self.redis.zrevrangebyscore(f'favicons|{favicon_sha512}|captures', max_score, min_score, withscores=True) + min_score: str | float = oldest_capture.timestamp() if oldest_capture else '-Inf' + total = self.redis.zcard(f'favicons|{favicon_sha512}|captures') + return total, self.redis.zrevrangebyscore(f'favicons|{favicon_sha512}|captures', max_score, min_score, withscores=True, start=offset, num=limit) def get_captures_favicon_count(self, favicon_sha512: str) -> int: if self.redis.type(f'favicons|{favicon_sha512}|captures') == 'set': # type: ignore[no-untyped-call] diff --git a/website/web/__init__.py b/website/web/__init__.py index 5df196c2..530da0c0 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -441,23 +441,12 @@ def get_capture_hash_investigator(hash_type: str, h: str) -> list[tuple[str, str return [(cache.uuid, cache.title, cache.redirects[-1], cache.timestamp) for cache in cached_captures] -def get_favicon_investigator(favicon_sha512: str, - /) -> tuple[list[tuple[str, str, str, datetime]], - tuple[str, str, str]]: +def get_favicon_investigator(favicon_sha512: str, offset: int | None=None, limit: int | None=None) -> tuple[int, list[tuple[str, str, str, datetime]]]: '''Returns all the captures related to a cookie name entry, used in the web interface.''' - cached_captures = lookyloo.sorted_capture_cache([uuid for uuid, _ in get_indexing(flask_login.current_user).get_captures_favicon(favicon_sha512)]) + total, entries = get_indexing(flask_login.current_user).get_captures_favicon(favicon_sha512=favicon_sha512, offset=offset, limit=limit) + cached_captures = lookyloo.sorted_capture_cache([uuid for uuid, _ in entries]) captures = [(cache.uuid, cache.title, cache.redirects[-1], cache.timestamp) for cache in cached_captures] - favicon = get_indexing(flask_login.current_user).get_favicon(favicon_sha512) - if favicon: - mimetype = from_string(favicon, mime=True) - b64_favicon = base64.b64encode(favicon).decode() - mmh3_shodan = lookyloo.compute_mmh3_shodan(favicon) - else: - mimetype = '' - b64_favicon = '' - mmh3_shodan = '' - - return captures, (mimetype, b64_favicon, mmh3_shodan) + return total, captures def get_hhh_investigator(hhh: str, /) -> tuple[list[tuple[str, str, str, str]], list[tuple[str, str]]]: @@ -1773,11 +1762,19 @@ def capture_hash_details(hash_type: str, h: str) -> str: @app.route('/favicon_details/', methods=['GET']) def favicon_detail(favicon_sha512: str) -> str: from_popup = True if (request.args.get('from_popup') and request.args.get('from_popup') == 'True') else False - captures, favicon = get_favicon_investigator(favicon_sha512.strip()) - mimetype, b64_favicon, mmh3_shodan = favicon + favicon = get_indexing(flask_login.current_user).get_favicon(favicon_sha512) + if favicon: + mimetype = from_string(favicon, mime=True) + b64_favicon = base64.b64encode(favicon).decode() + mmh3_shodan = lookyloo.compute_mmh3_shodan(favicon) + else: + mimetype = '' + b64_favicon = '' + mmh3_shodan = '' return render_template('favicon_details.html', - captures=captures, mimetype=mimetype, b64_favicon=b64_favicon, + mimetype=mimetype, b64_favicon=b64_favicon, mmh3_shodan=mmh3_shodan, + favicon_sha512=favicon_sha512, from_popup=from_popup) @@ -1993,6 +1990,19 @@ def post_table(table_name: str, value: str) -> Response: draw = request.form.get('draw', type=int) start = request.form.get('start', type=int) length = request.form.get('length', type=int) + captures: list[tuple[str, str, str, datetime, set[str]]] | list[tuple[str, str, str, datetime]] + if table_name == 'faviconDetailsTable': + total, captures = get_favicon_investigator(value.strip(), offset=start, limit=length) + prepared_captures = [] + for capture_uuid, title, landing_page, capture_time in captures: + to_append = { + 'capture_time': capture_time.isoformat(), + 'capture_title': f"""{title}""", + 'landing_page': f"""{landing_page}""" + } + prepared_captures.append(to_append) + return jsonify({'draw': draw, 'recordsTotal': total, 'recordsFiltered': total, 'data': prepared_captures}) + if table_name == 'hostnameTable': total, captures = get_hostname_investigator(value.strip(), offset=start, limit=length) prepared_captures = [] @@ -2005,6 +2015,7 @@ def post_table(table_name: str, value: str) -> Response: } prepared_captures.append(to_append) return jsonify({'draw': draw, 'recordsTotal': total, 'recordsFiltered': total, 'data': prepared_captures}) + if table_name == 'urlTable': url = base64.b64decode(value.strip()).decode() total, captures = get_url_investigator(url, offset=start, limit=length) diff --git a/website/web/sri.txt b/website/web/sri.txt index f53ffdc7..69bcf2fc 100644 --- a/website/web/sri.txt +++ b/website/web/sri.txt @@ -33,7 +33,7 @@ "loader.gif": "ZZKD5vLSKBWKeUpa2KI9qheUJ49iTI/UULmVU/AX28fBfH00K3lLc2v5pVJZ4qXG1BbB13LTXzRKKU35H2XfNg==", "lookyloo.jpeg": "i6wBj8CsIM5YAQLEMQfhs3CNOSKkErF8AMqqM6ZygSwCyQgv9CU8xt94veMZhM/ufBWoz7kAXmR+yywmxsTxug==", "redirect.png": "PAjzlPV97rEFvH55mG1ZC9wRl98be3yMeX/nENuFkJcds6/AXgSR2ig/QyPULgobSnNgiYieLVWY/oqsgeywrQ==", - "render_tables.js": "Su4TaO5d7pZ+hZM3b6JQAhtW5vJJ9vUIsOatzJR/wZOLykpPe9HuzNV48AxH4wumWKsK5HVt0wW7njOjHl8p7A==", + "render_tables.js": "L8K+7SHzG6c4kddX4WgUTYIh9qeTPK3k16NWxMiHJt5KFtR2KiC9e9dpMPl+8m3LwNnhlzRwaIYYb/gxuuJbqw==", "secure.svg": "H8ni7t0d60nCJDVGuZpuxC+RBy/ipAjWT627D12HlZGg6LUmjSwPTQTUekm3UJupEP7TUkhXyq6WHc5gy7QBjg==", "stats.css": "/kY943FwWBTne4IIyf7iBROSfbGd82TeBicEXqKkRwawMVRIvM/Pk5MRa7okUyGIxaDjFQGmV/U1vy+PhN6Jbw==", "stats_graph.js": "S/sMNQK1UMMLD0xQeEa7sq3ce8o6oPxwxGlyKVtaHOODjair86dbBDm7cu6pa/elMRDJT1j09jEFjWp+5GbhTw==", diff --git a/website/web/static/render_tables.js b/website/web/static/render_tables.js index 92af129e..257759e9 100644 --- a/website/web/static/render_tables.js +++ b/website/web/static/render_tables.js @@ -183,8 +183,20 @@ } if (document.getElementById('faviconDetailsTable')) { + favicon = document.getElementById('faviconDetailsTable').dataset.favicon; new DataTable('#faviconDetailsTable', { + processing: true, + serverSide: true, retrieve: true, + ajax: { + url: `/tables/faviconDetailsTable/${favicon}`, + type: 'POST' + }, + columns : [ + { data: 'capture_time' }, + { data: 'capture_title' }, + { data: 'landing_page' } + ], order: [[ 0, "desc" ]], columnDefs: [{ width: '30%', targets: 0, diff --git a/website/web/templates/favicon_details.html b/website/web/templates/favicon_details.html index 19903bf6..e7fec1a4 100644 --- a/website/web/templates/favicon_details.html +++ b/website/web/templates/favicon_details.html @@ -21,7 +21,7 @@
Shodan MMH3 Hash: {{ mmh3_shodan }}
- +
@@ -29,25 +29,6 @@ - - {% for capture_uuid, title, landing_page, capture_time in captures %} - - - - - - {% endfor %} -
Capture TimeLanding page
- {{capture_time}} - - - {{ title }} - - - - {{ landing_page }} - -
{% endblock %}