From 49ae4490b21b4f14a5c53aaf429a67f82cabbb69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 18 Mar 2021 00:40:14 +0100 Subject: [PATCH] new: (hidden) interface to search for hostnames and URLs --- lookyloo/lookyloo.py | 6 ++- website/web/__init__.py | 31 +++++++++++- website/web/templates/hostname.html | 77 +++++++++++++++++++++++++++++ website/web/templates/search.html | 65 ++++++++++++++++++++++++ website/web/templates/url.html | 77 +++++++++++++++++++++++++++++ 5 files changed, 252 insertions(+), 4 deletions(-) create mode 100644 website/web/templates/hostname.html create mode 100644 website/web/templates/search.html create mode 100644 website/web/templates/url.html diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index 80fbb091..cebf304c 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -837,7 +837,8 @@ class Lookyloo(): for capture in captures[:limit]: ct = self.get_crawled_tree(capture.uuid) to_append: Dict[str, Union[str, Dict]] = {'capture_uuid': capture.uuid, - 'start_timestamp': capture.timestamp.isoformat()} + 'start_timestamp': capture.timestamp.isoformat(), + 'title': capture.title} urlnodes: Dict[str, Dict[str, str]] = {} for urlnode in ct.root_hartree.url_tree.search_nodes(name=url): urlnodes[urlnode.uuid] = {'start_time': urlnode.start_time.isoformat(), @@ -863,7 +864,8 @@ class Lookyloo(): for capture in captures[:limit]: ct = self.get_crawled_tree(capture.uuid) to_append: Dict[str, Union[str, List, Dict]] = {'capture_uuid': capture.uuid, - 'start_timestamp': capture.timestamp.isoformat()} + 'start_timestamp': capture.timestamp.isoformat(), + 'title': capture.title} hostnodes: List[str] = [] if with_urls_occurrences: urlnodes: Dict[str, Dict[str, str]] = {} diff --git a/website/web/__init__.py b/website/web/__init__.py index 1e51f478..f8357b7f 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -13,6 +13,7 @@ import calendar from typing import Optional, Dict, Any, Union import logging import hashlib +from urllib.parse import quote_plus, unquote_plus from flask import Flask, render_template, request, send_file, redirect, url_for, Response, flash, jsonify from flask_bootstrap import Bootstrap # type: ignore @@ -609,6 +610,19 @@ def submit(): return Response(perma_uuid, mimetype='text/text') +@app.route('/search', methods=['GET', 'POST']) +def search(): + if request.form.get('url'): + return redirect(url_for('url_details', url=quote_plus(request.form.get('url')))) + if request.form.get('hostname'): + return redirect(url_for('hostname_details', hostname=request.form.get('hostname'))) + if request.form.get('ressource'): + return redirect(url_for('body_hash_details', body_hash=request.form.get('ressource'))) + if request.form.get('cookie'): + return redirect(url_for('cookies_name_detail', cookie_name=request.form.get('cookie'))) + return render_template('search.html') + + @app.route('/capture', methods=['GET', 'POST']) def capture_web(): if request.form.get('url'): @@ -651,16 +665,29 @@ def capture_web(): @app.route('/cookies/', methods=['GET']) def cookies_name_detail(cookie_name: str): - captures, domains = lookyloo.get_cookie_name_investigator(cookie_name) + captures, domains = lookyloo.get_cookie_name_investigator(cookie_name.strip()) return render_template('cookie_name.html', cookie_name=cookie_name, domains=domains, captures=captures) @app.route('/body_hashes/', methods=['GET']) def body_hash_details(body_hash: str): - captures, domains = lookyloo.get_body_hash_investigator(body_hash) + captures, domains = lookyloo.get_body_hash_investigator(body_hash.strip()) return render_template('body_hash.html', body_hash=body_hash, domains=domains, captures=captures) +@app.route('/urls/', methods=['GET']) +def url_details(url: str): + url = unquote_plus(url).strip() + hits = lookyloo.get_url_occurrences(url=url, limit=50) + return render_template('url.html', url=url, hits=hits) + + +@app.route('/hostnames/', methods=['GET']) +def hostname_details(hostname: str): + hits = lookyloo.get_hostname_occurrences(hostname=hostname.strip(), with_urls_occurrences=True, limit=50) + return render_template('hostname.html', hostname=hostname, hits=hits) + + @app.route('/stats', methods=['GET']) def statsfull(): stats = lookyloo.get_stats() diff --git a/website/web/templates/hostname.html b/website/web/templates/hostname.html new file mode 100644 index 00000000..0a4e9c49 --- /dev/null +++ b/website/web/templates/hostname.html @@ -0,0 +1,77 @@ +{% extends "main.html" %} + +{% from 'bootstrap/utils.html' import render_messages %} + +{% block title %}{{ url }}{% endblock %} + +{% block scripts %} +{{ super() }} + + + + + +{% endblock %} + +{% block styles %} +{{ super() }} + +{% endblock %} + + +{% block content %} +
+

{{ url }}

+ +
+
+ + + + + + + + + {% for hit in hits %} + + + + + {% endfor %} + +
Start timestampCaptures
+ {{ hit['start_timestamp'] }} + {{ hit['title'] }} +
+ Nodes: + +
+
+

The same file was seen in these captures:

+
    + {% for capture_uuid, title in captures %} +
  • {{ title }}
    • + {% endfor %} +
+{% endblock %} diff --git a/website/web/templates/search.html b/website/web/templates/search.html new file mode 100644 index 00000000..92307499 --- /dev/null +++ b/website/web/templates/search.html @@ -0,0 +1,65 @@ +{% extends "main.html" %} +{% block title %}Capture{% endblock %} + +{% block card %} + + + + + + +{% endblock %} + +{% block content %} +
+
+ + Lookyloo + +
+
+
Please only search one of the following thing at a time.
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+ +
+
+{% endblock %} + +{% block scripts %} + {{ super() }} + +{% endblock %} diff --git a/website/web/templates/url.html b/website/web/templates/url.html new file mode 100644 index 00000000..0a4e9c49 --- /dev/null +++ b/website/web/templates/url.html @@ -0,0 +1,77 @@ +{% extends "main.html" %} + +{% from 'bootstrap/utils.html' import render_messages %} + +{% block title %}{{ url }}{% endblock %} + +{% block scripts %} +{{ super() }} + + + + + +{% endblock %} + +{% block styles %} +{{ super() }} + +{% endblock %} + + +{% block content %} +
+

{{ url }}

+ +
+
+ + + + + + + + + {% for hit in hits %} + + + + + {% endfor %} + +
Start timestampCaptures
+ {{ hit['start_timestamp'] }} + {{ hit['title'] }} +
+ Nodes: + +
+
+

The same file was seen in these captures:

+
    + {% for capture_uuid, title in captures %} +
  • {{ title }}
    • + {% endfor %} +
+{% endblock %}