CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: Improve popup, remove session.

pull/79/head
Raphaël Vinot 2020-05-20 19:11:15 +02:00
parent cd5eabafba
commit 3b631f2c92
6 changed files with 51 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
lookyloo/lookyloo.py
View File

@ -18,7 +18,7 @@ from uuid import uuid4
from zipfile import ZipFile
from defang import refang # type: ignore
from har2tree import CrawledTree, Har2TreeError, HarFile
from har2tree import CrawledTree, Har2TreeError, HarFile, HostNode, URLNode
from redis import Redis
from scrapysplashwrapper import crawl
@ -84,6 +84,14 @@ class Lookyloo():
sample_config = json.load(_c)
return sample_config[entry]
def get_urlnode_from_tree(self, capture_dir: Path, node_uuid: str) -> URLNode:
ct = self._load_pickle(capture_dir / 'tree.pickle')
return ct.root_hartree.get_url_node_by_uuid(node_uuid)
def get_hostnode_from_tree(self, capture_dir: Path, node_uuid: str) -> HostNode:
ct = self._load_pickle(capture_dir / 'tree.pickle')
return ct.root_hartree.get_host_node_by_uuid(node_uuid)
def get_statistics(self, capture_dir: Path) -> Dict[str, Any]:
# We need the pickle
ct = self._load_pickle(capture_dir / 'tree.pickle')
@ -280,7 +288,7 @@ class Lookyloo():
ct = CrawledTree(har_files, uuid)
with pickle_file.open('wb') as _p:
pickle.dump(ct, _p)
return str(pickle_file), ct.to_json(), ct.start_time.isoformat(), ct.user_agent, ct.root_url, meta
return ct.to_json(), ct.start_time.isoformat(), ct.user_agent, ct.root_url, meta
except Har2TreeError as e:
raise NoValidHarFile(e.message)

17
lookyloo/modules.py
View File

@ -18,6 +18,11 @@ from pysanejs import SaneJS
class SaneJavaScript():
skip_lookup: Dict[str, str] = {
"717ea0ff7f3f624c268eccb244e24ec1305ab21557abb3d6f1a7e183ff68a2d28f13d1d2af926c9ef6d1fb16dd8cbe34cd98cacf79091dddc7874dcee21ecfdc": "1*1px gif",
"cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e": "Empty file"
}
def __init__(self, config: Dict[str, Any]):
if not ('enabled' in config or config['enabled']):
self.available = False
@ -44,12 +49,12 @@ class SaneJavaScript():
with sanejs_unknowns.open() as f:
unknown_hashes = [line.strip() for line in f.readlines()]
if force:
to_lookup = hashes
else:
to_lookup = [h for h in sha512 if (h not in unknown_hashes
and not (today_dir / h).exists())]
to_return = {}
to_return = {h: details for h, details in self.skip_lookup.items() if h in sha512}
to_lookup = [h for h in hashes if h not in self.skip_lookup]
if not force:
to_lookup = [h for h in to_lookup if (h not in unknown_hashes
and not (today_dir / h).exists())]
for h in to_lookup:
response = self.client.sha512(h)
if 'error' in response:

63
website/web/__init__.py
View File

@ -1,15 +1,13 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import json
import pickle
from zipfile import ZipFile, ZIP_DEFLATED
from io import BytesIO
import os
from pathlib import Path
from datetime import datetime, timedelta
from flask import Flask, render_template, request, session, send_file, redirect, url_for, Response, flash
from flask import Flask, render_template, request, send_file, redirect, url_for, Response, flash
from flask_bootstrap import Bootstrap # type: ignore
from flask_httpauth import HTTPDigestAuth # type: ignore
@ -18,7 +16,7 @@ from lookyloo.lookyloo import Lookyloo
from lookyloo.exceptions import NoValidHarFile
from .proxied import ReverseProxied
from typing import Tuple, Optional, Dict, Any
from typing import Optional, Dict, Any
import logging
@ -79,14 +77,6 @@ def rebuild_tree(tree_uuid: str):
return redirect(url_for('index'))
# keep
def load_tree(capture_dir: Path) -> Tuple[str, str, str, str, Dict[str, Any]]:
session.clear()
temp_file_name, tree_json, tree_time, tree_ua, tree_root_url, meta = lookyloo.load_tree(capture_dir)
session["tree"] = temp_file_name
return tree_json, tree_time, tree_ua, tree_root_url, meta
@app.route('/submit', methods=['POST', 'GET'])
def submit():
to_query = request.get_json(force=True)
@ -116,11 +106,10 @@ def scrape_web():
return render_template('scrape.html', user_agents=user_agents)
@app.route('/tree/hostname/<string:node_uuid>/text', methods=['GET'])
def hostnode_details_text(node_uuid: str):
with open(session["tree"], 'rb') as f:
ct = pickle.load(f)
hostnode = ct.root_hartree.get_host_node_by_uuid(node_uuid)
@app.route('/tree/<string:tree_uuid>/hostname/<string:node_uuid>/text', methods=['GET'])
def hostnode_details_text(tree_uuid: str, node_uuid: str):
capture_dir = lookyloo.lookup_capture_dir(tree_uuid)
hostnode = lookyloo.get_hostnode_from_tree(capture_dir, node_uuid)
urls = []
for url in hostnode.urls:
urls.append(url.name)
@ -134,11 +123,10 @@ def hostnode_details_text(node_uuid: str):
as_attachment=True, attachment_filename='file.md')
@app.route('/tree/hostname_popup/<string:node_uuid>', methods=['GET'])
def hostnode_popup(node_uuid: str):
with open(session["tree"], 'rb') as f:
ct = pickle.load(f)
hostnode = ct.root_hartree.get_host_node_by_uuid(node_uuid)
@app.route('/tree/<string:tree_uuid>/hostname_popup/<string:node_uuid>', methods=['GET'])
def hostnode_popup(tree_uuid: str, node_uuid: str):
capture_dir = lookyloo.lookup_capture_dir(tree_uuid)
hostnode = lookyloo.get_hostnode_from_tree(capture_dir, node_uuid)
table_keys = {
'js': "/static/javascript.png",
'exe': "/static/exe.png",
@ -163,36 +151,21 @@ def hostnode_popup(node_uuid: str):
for url in hostnode.urls:
if lookyloo.sanejs.available and hasattr(url, 'body_hash') and url.body_hash in lookups:
url.add_feature('sane_js_details', lookups[url.body_hash])
# TODO: Do something with it.
if lookups[url.body_hash] and isinstance(lookups[url.body_hash], list):
url.add_feature('sane_js_details_to_print', f'{" ".join(lookups[url.body_hash][0].split("|"))} and {len(lookups[url.body_hash])-1} other files')
urls.append(url)
return render_template('hostname_popup.html',
tree_uuid=tree_uuid,
hostname_uuid=node_uuid,
hostname=hostnode.name,
urls=urls,
keys=table_keys)
@app.route('/tree/hostname/<string:node_uuid>', methods=['GET'])
def hostnode_details(node_uuid: str):
with open(session["tree"], 'rb') as f:
ct = pickle.load(f)
hostnode = ct.root_hartree.get_host_node_by_uuid(node_uuid)
urls = []
for url in hostnode.urls:
if hasattr(url, 'body_hash'):
sane_js_r = lookyloo.sane_js_query(url.body_hash)
if sane_js_r.get('response'):
url.add_feature('sane_js_details', sane_js_r['response'])
print('######## SANEJS ##### ', url.sane_js_details)
urls.append(url.to_json())
return json.dumps(urls)
@app.route('/tree/url/<string:node_uuid>', methods=['GET'])
def urlnode_details(node_uuid: str):
with open(session["tree"], 'rb') as f:
ct = pickle.load(f)
urlnode = ct.root_hartree.get_url_node_by_uuid(node_uuid)
@app.route('/tree/<string:tree_uuid>/url/<string:node_uuid>', methods=['GET'])
def urlnode_details(tree_uuid: str, node_uuid: str):
capture_dir = lookyloo.lookup_capture_dir(tree_uuid)
urlnode = lookyloo.get_urlnode_from_tree(capture_dir, node_uuid)
to_return = BytesIO()
got_content = False
if hasattr(urlnode, 'body'):
@ -338,7 +311,7 @@ def tree(tree_uuid: str):
enable_mail_notification = True
else:
enable_mail_notification = False
tree_json, start_time, user_agent, root_url, meta = load_tree(capture_dir)
tree_json, start_time, user_agent, root_url, meta = lookyloo.load_tree(capture_dir)
return render_template('tree.html', tree_json=tree_json, start_time=start_time,
user_agent=user_agent, root_url=root_url, tree_uuid=tree_uuid,
meta=meta, enable_mail_notification=enable_mail_notification)

2
website/web/static/tree.js
View File

@ -93,7 +93,7 @@ d3.selection.prototype.moveToBack = function() {
};
function hostnode_click_popup(d) {
window.open('/tree/hostname_popup/' + d.data.uuid, '_blank', 'width=700,height=500,left=200,top=100');
window.open('/tree/' + treeUUID + '/hostname_popup/' + d.data.uuid, '_blank', 'width=1024,height=768,left=200,top=100');
};
function ProcessChildMessage(message) {

12
website/web/templates/hostname_popup.html
View File

@ -14,7 +14,7 @@
<center>
<h3>{{ hostname }}</h3>
<button type="button" class="btn btn-secondary" onclick="whereAmI()">Locate node on tree</button>
<a href="{{ url_for('hostnode_details_text', node_uuid=hostname_uuid) }}" class="btn btn-info" role="button">Get URLs as text</a>
<a href="{{ url_for('hostnode_details_text', tree_uuid=tree_uuid, node_uuid=hostname_uuid) }}" class="btn btn-info" role="button">Get URLs as text</a>
</center>
<p>Click on the URL to get the content of the response</p>
<div class="table-responsive">
@ -22,6 +22,7 @@
<thead>
<tr>
<th>URL</th>
<th>Known file</th>
{% for alt, path in keys.items() %}
<th><img src="{{ path }}" alt="{{ alt }}" width="21" height="21"/></th>
{% endfor %}
@ -31,7 +32,14 @@
{% for url in urls %}
<tr>
<td style="white-space:nowrap">
<a href="{{ url_for('urlnode_details', node_uuid=url.uuid) }}">{{ url.name }}</a>
<a href="{{ url_for('urlnode_details', tree_uuid=tree_uuid, node_uuid=url.uuid) }}">{{ url.name }}</a>
</td>
<td><br/>
{% if url.sane_js_details_to_print %}
{{ url.sane_js_details_to_print }}
{% else %}
{{ url.sane_js_details }}
{% endif %}
</td>
{% for key in keys.keys() %}
<td><br/>

1
website/web/templates/tree.html
View File

@ -40,6 +40,7 @@
{% block content %}
{{super()}}
<script>
var treeUUID = "{{ tree_uuid }}";
var treeData = {{ tree_json | safe }};
</script>