chg: [HHHash] add HHHash object and correlation https://www.foo.be/2023/07/HTTP-Headers-Hashing_HHHash

pull/604/head
Terrtia 2023-07-17 15:47:17 +02:00
parent 008a065d5f
commit a9485928db
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
13 changed files with 1100 additions and 35 deletions

View File

@ -301,6 +301,7 @@ class Crawler(AbstractModule):
print(etag_content) print(etag_content)
etag = Etags.create(etag_content) etag = Etags.create(etag_content)
etag.add(self.date.replace('/', ''), self.domain.id) etag.add(self.date.replace('/', ''), self.domain.id)
crawlers.extract_hhhash(entries['har'], self.domain.id, self.date.replace('/', ''))
# Next Children # Next Children
entries_children = entries.get('children') entries_children = entries.get('children')

View File

@ -15,8 +15,8 @@ config_loader = ConfigLoader()
r_serv_db = config_loader.get_db_conn("Kvrocks_DB") r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
config_loader = None config_loader = None
AIL_OBJECTS = sorted({'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'item', 'pgp', AIL_OBJECTS = sorted({'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'hhhash', 'item',
'screenshot', 'title', 'username'}) 'pgp', 'screenshot', 'title', 'username'})
def get_ail_uuid(): def get_ail_uuid():
ail_uuid = r_serv_db.get('ail:uuid') ail_uuid = r_serv_db.get('ail:uuid')

View File

@ -45,9 +45,10 @@ CORRELATION_TYPES_BY_OBJ = {
"cryptocurrency": ["domain", "item"], "cryptocurrency": ["domain", "item"],
"cve": ["domain", "item"], "cve": ["domain", "item"],
"decoded": ["domain", "item"], "decoded": ["domain", "item"],
"domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "item", "pgp", "title", "screenshot", "username"], "domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "hhhash", "item", "pgp", "title", "screenshot", "username"],
"etag": ["domain"], "etag": ["domain"],
"favicon": ["domain", "item"], # TODO Decoded "favicon": ["domain", "item"], # TODO Decoded
"hhhash": ["domain"],
"item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], "item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"],
"pgp": ["domain", "item"], "pgp": ["domain", "item"],
"screenshot": ["domain", "item"], "screenshot": ["domain", "item"],

View File

@ -39,6 +39,7 @@ from packages import git_status
from packages import Date from packages import Date
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from lib.objects.Domains import Domain from lib.objects.Domains import Domain
from lib.objects import HHHashs
from lib.objects.Items import Item from lib.objects.Items import Item
config_loader = ConfigLoader() config_loader = ConfigLoader()
@ -335,7 +336,7 @@ def _reprocess_all_hars_cookie_name():
from lib.objects import CookiesNames from lib.objects import CookiesNames
for har_id in get_all_har_ids(): for har_id in get_all_har_ids():
domain = har_id.split('/')[-1] domain = har_id.split('/')[-1]
domain = domain[:-41] domain = domain[:-44]
date = har_id.split('/') date = har_id.split('/')
date = f'{date[-4]}{date[-3]}{date[-2]}' date = f'{date[-4]}{date[-3]}{date[-2]}'
for cookie_name in extract_cookies_names_from_har(get_har_content(har_id)): for cookie_name in extract_cookies_names_from_har(get_har_content(har_id)):
@ -358,7 +359,7 @@ def _reprocess_all_hars_etag():
from lib.objects import Etags from lib.objects import Etags
for har_id in get_all_har_ids(): for har_id in get_all_har_ids():
domain = har_id.split('/')[-1] domain = har_id.split('/')[-1]
domain = domain[:-41] domain = domain[:-44]
date = har_id.split('/') date = har_id.split('/')
date = f'{date[-4]}{date[-3]}{date[-2]}' date = f'{date[-4]}{date[-3]}{date[-2]}'
for etag_content in extract_etag_from_har(get_har_content(har_id)): for etag_content in extract_etag_from_har(get_har_content(har_id)):
@ -366,6 +367,56 @@ def _reprocess_all_hars_etag():
etag = Etags.create(etag_content) etag = Etags.create(etag_content)
etag.add(date, domain) etag.add(date, domain)
def extract_hhhash_by_id(har_id, domain, date):
return extract_hhhash(get_har_content(har_id), domain, date)
def extract_hhhash(har, domain, date):
hhhashs = set()
urls = set()
for entrie in har.get('log', {}).get('entries', []):
url = entrie.get('request').get('url')
if url not in urls:
# filter redirect
if entrie.get('response').get('status') == 200: # != 301:
# print(url, entrie.get('response').get('status'))
f = get_faup()
f.decode(url)
domain_url = f.get().get('domain')
if domain_url == domain:
headers = entrie.get('response').get('headers')
hhhash_header = HHHashs.build_hhhash_headers(headers)
hhhash = HHHashs.hhhash_headers(hhhash_header)
if hhhash not in hhhashs:
print('', url, hhhash)
# -----
obj = HHHashs.create(hhhash_header, hhhash)
obj.add(date, domain)
hhhashs.add(hhhash)
urls.add(url)
print()
print()
print('HHHASH:')
for hhhash in hhhashs:
print(hhhash)
return hhhashs
def _reprocess_all_hars_hhhashs():
for har_id in get_all_har_ids():
print()
print(har_id)
domain = har_id.split('/')[-1]
domain = domain[:-44]
date = har_id.split('/')
date = f'{date[-4]}{date[-3]}{date[-2]}'
extract_hhhash_by_id(har_id, domain, date)
def _gzip_har(har_id): def _gzip_har(har_id):
har_path = os.path.join(HAR_DIR, har_id) har_path = os.path.join(HAR_DIR, har_id)
@ -1957,15 +2008,16 @@ def test_ail_crawlers():
# TODO MOVE ME IN CRAWLER OR FLASK # TODO MOVE ME IN CRAWLER OR FLASK
load_blacklist() load_blacklist()
# if __name__ == '__main__': if __name__ == '__main__':
# delete_captures() # delete_captures()
# item_id = 'crawled/2023/02/20/data.gz' # item_id = 'crawled/2023/02/20/data.gz'
# item = Item(item_id) # item = Item(item_id)
# content = item.get_content() # content = item.get_content()
# temp_url = '' # temp_url = ''
# r = extract_favicon_from_html(content, temp_url) # r = extract_favicon_from_html(content, temp_url)
# print(r) # print(r)
# _reprocess_all_hars_cookie_name() # _reprocess_all_hars_cookie_name()
# _reprocess_all_hars_etag() # _reprocess_all_hars_etag()
# _gzip_all_hars() # _gzip_all_hars()
_reprocess_all_hars_hhhashs()

138
bin/lib/objects/HHHashs.py Executable file
View File

@ -0,0 +1,138 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import hashlib
import os
import sys
from flask import url_for
from pymisp import MISPObject
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib.ConfigLoader import ConfigLoader
from lib.objects.abstract_daterange_object import AbstractDaterangeObject, AbstractDaterangeObjects
config_loader = ConfigLoader()
r_objects = config_loader.get_db_conn("Kvrocks_Objects")
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
config_loader = None
class HHHash(AbstractDaterangeObject):
"""
AIL HHHash Object.
"""
def __init__(self, obj_id):
super(HHHash, self).__init__('hhhash', obj_id)
# def get_ail_2_ail_payload(self):
# payload = {'raw': self.get_gzip_content(b64=True),
# 'compress': 'gzip'}
# return payload
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
def delete(self):
# # TODO:
pass
def get_content(self, r_type='str'):
if r_type == 'str':
return self._get_field('content')
def get_link(self, flask_context=False):
if flask_context:
url = url_for('correlation.show_correlation', type=self.type, id=self.id)
else:
url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}'
return url
# TODO # CHANGE COLOR
def get_svg_icon(self):
return {'style': 'fas', 'icon': '\uf036', 'color': '#71D090', 'radius': 5}
def get_misp_object(self):
obj_attrs = []
obj = MISPObject('hhhash')
first_seen = self.get_first_seen()
last_seen = self.get_last_seen()
if first_seen:
obj.first_seen = first_seen
if last_seen:
obj.last_seen = last_seen
if not first_seen or not last_seen:
self.logger.warning(
f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
obj_attrs.append(obj.add_attribute('hhhash', value=self.get_id()))
obj_attrs.append(obj.add_attribute('hhhash-headers', value=self.get_content()))
obj_attrs.append(obj.add_attribute('hhhash-tool', value='lacus'))
for obj_attr in obj_attrs:
for tag in self.get_tags():
obj_attr.add_tag(tag)
return obj
def get_nb_seen(self):
return self.get_nb_correlation('domain')
def get_meta(self, options=set()):
meta = self._get_meta(options=options)
meta['id'] = self.id
meta['tags'] = self.get_tags(r_list=True)
meta['content'] = self.get_content()
return meta
def add(self, date, obj_id): # date = HAR Date
self._add(date, 'domain', '', obj_id)
def create(self, hhhash_header, _first_seen=None, _last_seen=None): # TODO CREATE ADD FUNCTION -> urls set
self._set_field('content', hhhash_header)
self._create()
def create(hhhash_header, hhhash=None):
if not hhhash:
hhhash = hhhash_headers(hhhash_header)
hhhash = HHHash(hhhash)
if not hhhash.exists():
hhhash.create(hhhash_header)
return hhhash
def build_hhhash_headers(dict_headers): # filter_dup=True
hhhash = ''
previous_header = ''
for header in dict_headers:
header_name = header.get('name')
if header_name:
if header_name != previous_header: # remove dup headers, filter playwright invalid splitting
hhhash = f'{hhhash}:{header_name}'
previous_header = header_name
hhhash = hhhash[1:]
# print(hhhash)
return hhhash
def hhhash_headers(header_hhhash):
m = hashlib.sha256()
m.update(header_hhhash.encode())
digest = m.hexdigest()
return f"hhh:1:{digest}"
class HHHashs(AbstractDaterangeObjects):
"""
HHHashs Objects
"""
def __init__(self):
super().__init__('hhhash', HHHash)
def sanitize_id_to_search(self, name_to_search):
return name_to_search # TODO
# if __name__ == '__main__':
# name_to_search = '98'
# print(search_cves_by_name(name_to_search))

View File

@ -20,6 +20,7 @@ from lib.objects.Decodeds import Decoded, get_all_decodeds_objects, get_nb_decod
from lib.objects.Domains import Domain from lib.objects.Domains import Domain
from lib.objects import Etags from lib.objects import Etags
from lib.objects.Favicons import Favicon from lib.objects.Favicons import Favicon
from lib.objects import HHHashs
from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects
from lib.objects import Pgps from lib.objects import Pgps
from lib.objects.Screenshots import Screenshot from lib.objects.Screenshots import Screenshot
@ -62,6 +63,8 @@ def get_object(obj_type, subtype, id):
return Etags.Etag(id) return Etags.Etag(id)
elif obj_type == 'favicon': elif obj_type == 'favicon':
return Favicon(id) return Favicon(id)
elif obj_type == 'hhhash':
return HHHashs.HHHash(id)
elif obj_type == 'screenshot': elif obj_type == 'screenshot':
return Screenshot(id) return Screenshot(id)
elif obj_type == 'cryptocurrency': elif obj_type == 'cryptocurrency':
@ -104,9 +107,12 @@ def get_obj_global_id(obj_type, subtype, obj_id):
obj = get_object(obj_type, subtype, obj_id) obj = get_object(obj_type, subtype, obj_id)
return obj.get_global_id() return obj.get_global_id()
def get_obj_type_subtype_id_from_global_id(global_id):
obj_type, subtype, obj_id = global_id.split(':', 2)
return obj_type, subtype, obj_id
def get_obj_from_global_id(global_id): def get_obj_from_global_id(global_id):
obj = global_id.split(':', 3) obj = get_obj_type_subtype_id_from_global_id(global_id)
return get_object(obj[0], obj[1], obj[2]) return get_object(obj[0], obj[1], obj[2])
@ -162,7 +168,7 @@ def get_objects_meta(objs, options=set(), flask_context=False):
subtype = obj[1] subtype = obj[1]
obj_id = obj[2] obj_id = obj[2]
else: else:
obj_type, subtype, obj_id = obj.split(':', 2) obj_type, subtype, obj_id = get_obj_type_subtype_id_from_global_id(obj)
metas.append(get_object_meta(obj_type, subtype, obj_id, options=options, flask_context=flask_context)) metas.append(get_object_meta(obj_type, subtype, obj_id, options=options, flask_context=flask_context))
return metas return metas
@ -171,7 +177,7 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False):
obj = get_object(obj_type, subtype, id) obj = get_object(obj_type, subtype, id)
meta = obj.get_meta() meta = obj.get_meta()
meta['icon'] = obj.get_svg_icon() meta['icon'] = obj.get_svg_icon()
if subtype or obj_type == 'cookie-name' or obj_type == 'cve' or obj_type == 'etag' or obj_type == 'title' or obj_type == 'favicon': if subtype or obj_type == 'cookie-name' or obj_type == 'cve' or obj_type == 'etag' or obj_type == 'title' or obj_type == 'favicon' or obj_type == 'hhhash':
meta['sparkline'] = obj.get_sparkline() meta['sparkline'] = obj.get_sparkline()
if obj_type == 'cve': if obj_type == 'cve':
meta['cve_search'] = obj.get_cve_search() meta['cve_search'] = obj.get_cve_search()
@ -402,7 +408,7 @@ def create_correlation_graph_links(links_set):
def create_correlation_graph_nodes(nodes_set, obj_str_id, flask_context=True): def create_correlation_graph_nodes(nodes_set, obj_str_id, flask_context=True):
graph_nodes_list = [] graph_nodes_list = []
for node_id in nodes_set: for node_id in nodes_set:
obj_type, subtype, obj_id = node_id.split(':', 2) obj_type, subtype, obj_id = get_obj_type_subtype_id_from_global_id(node_id)
dict_node = {'id': node_id} dict_node = {'id': node_id}
dict_node['style'] = get_object_svg(obj_type, subtype, obj_id) dict_node['style'] = get_object_svg(obj_type, subtype, obj_id)

View File

@ -52,6 +52,7 @@ from blueprints.objects_subtypes import objects_subtypes
from blueprints.objects_title import objects_title from blueprints.objects_title import objects_title
from blueprints.objects_cookie_name import objects_cookie_name from blueprints.objects_cookie_name import objects_cookie_name
from blueprints.objects_etag import objects_etag from blueprints.objects_etag import objects_etag
from blueprints.objects_hhhash import objects_hhhash
Flask_dir = os.environ['AIL_FLASK'] Flask_dir = os.environ['AIL_FLASK']
@ -108,6 +109,7 @@ app.register_blueprint(objects_subtypes, url_prefix=baseUrl)
app.register_blueprint(objects_title, url_prefix=baseUrl) app.register_blueprint(objects_title, url_prefix=baseUrl)
app.register_blueprint(objects_cookie_name, url_prefix=baseUrl) app.register_blueprint(objects_cookie_name, url_prefix=baseUrl)
app.register_blueprint(objects_etag, url_prefix=baseUrl) app.register_blueprint(objects_etag, url_prefix=baseUrl)
app.register_blueprint(objects_hhhash, url_prefix=baseUrl)
# ========= =========# # ========= =========#

View File

@ -99,6 +99,9 @@ def show_correlation():
correl_option = request.form.get('CryptocurrencyCheck') correl_option = request.form.get('CryptocurrencyCheck')
if correl_option: if correl_option:
filter_types.append('cryptocurrency') filter_types.append('cryptocurrency')
correl_option = request.form.get('HHHashCheck')
if correl_option:
filter_types.append('hhhash')
correl_option = request.form.get('PgpCheck') correl_option = request.form.get('PgpCheck')
if correl_option: if correl_option:
filter_types.append('pgp') filter_types.append('pgp')
@ -177,26 +180,15 @@ def show_correlation():
@login_read_only @login_read_only
def get_description(): def get_description():
object_id = request.args.get('object_id') object_id = request.args.get('object_id')
object_id = object_id.split(':') obj_type, subtype, obj_id = ail_objects.get_obj_type_subtype_id_from_global_id(object_id)
# unpack object_id # # TODO: put me in lib
if len(object_id) == 3:
object_type = object_id[0]
type_id = object_id[1]
correlation_id = object_id[2]
elif len(object_id) == 2:
object_type = object_id[0]
type_id = None
correlation_id = object_id[1]
else:
return jsonify({})
# check if correlation_id exist # check if obj exist
# # TODO: return error json # # TODO: return error json
if not ail_objects.exists_obj(object_type, type_id, correlation_id): if not ail_objects.exists_obj(obj_type, subtype, obj_id):
return Response(json.dumps({"status": "error", "reason": "404 Not Found"}, indent=2, sort_keys=True), mimetype='application/json'), 404 return Response(json.dumps({"status": "error", "reason": "404 Not Found"}, indent=2, sort_keys=True), mimetype='application/json'), 404
# object exist # object exist
else: else:
res = ail_objects.get_object_meta(object_type, type_id, correlation_id, options={'tags', 'tags_safe'}, res = ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags', 'tags_safe'},
flask_context=True) flask_context=True)
if 'tags' in res: if 'tags' in res:
res['tags'] = list(res['tags']) res['tags'] = list(res['tags'])

View File

@ -0,0 +1,86 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
'''
import os
import sys
from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort
from flask_login import login_required, current_user
# Import Role_Manager
from Role_Manager import login_admin, login_analyst, login_read_only
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib.objects import HHHashs
from packages import Date
# ============ BLUEPRINT ============
objects_hhhash = Blueprint('objects_hhhash', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/hhhash'))
# ============ VARIABLES ============
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
# ============ FUNCTIONS ============
@objects_hhhash.route("/objects/hhhashs", methods=['GET'])
@login_required
@login_read_only
def objects_hhhashs():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
show_objects = request.args.get('show_objects')
date = Date.sanitise_date_range(date_from, date_to)
date_from = date['date_from']
date_to = date['date_to']
if show_objects:
dict_objects = HHHashs.HHHashs().api_get_meta_by_daterange(date_from, date_to)
else:
dict_objects = {}
return render_template("HHHashDaterange.html", date_from=date_from, date_to=date_to,
dict_objects=dict_objects, show_objects=show_objects)
@objects_hhhash.route("/objects/hhhash/post", methods=['POST'])
@login_required
@login_read_only
def objects_hhhashs_post():
date_from = request.form.get('date_from')
date_to = request.form.get('date_to')
show_objects = request.form.get('show_objects')
return redirect(url_for('objects_hhhash.objects_hhhashs', date_from=date_from, date_to=date_to, show_objects=show_objects))
@objects_hhhash.route("/objects/hhhash/range/json", methods=['GET'])
@login_required
@login_read_only
def objects_hhhash_range_json():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
date = Date.sanitise_date_range(date_from, date_to)
date_from = date['date_from']
date_to = date['date_to']
return jsonify(HHHashs.HHHashs().api_get_chart_nb_by_daterange(date_from, date_to))
# @objects_hhhash.route("/objects/hhhash/search", methods=['POST'])
# @login_required
# @login_read_only
# def objects_hhhashs_names_search():
# to_search = request.form.get('object_id')
#
# # TODO SANITIZE ID
# # TODO Search all
# cve = Cves.Cve(to_search)
# if not cve.exists():
# abort(404)
# else:
# return redirect(cve.get_link(flask_context=True))
# ============= ROUTES ==============

View File

@ -0,0 +1,173 @@
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
{% with modal_add_tags=dict_object['metadata_card']['add_tags_modal']%}
{% include 'modals/add_tags.html' %}
{% endwith %}
{% include 'modals/edit_tag.html' %}
<div class="card my-3">
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
<h4>{{ dict_object["metadata"]["content"] }}</h4>
<div class="text-secondary">{{ dict_object["correlation_id"] }}</div>
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<div class="row">
<div class="col-md-10">
<table class="table">
<thead>
<tr>
<th>Object type</th>
<th>First seen</th>
<th>Last seen</th>
<th>Nb seen</th>
</tr>
</thead>
<tbody>
<tr>
<td>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text>
</g>
</svg>
{{ dict_object["object_type"] }}
</td>
<td>{{ dict_object["metadata"]['first_seen'] }}</td>
<td>{{ dict_object["metadata"]['last_seen'] }}</td>
<td>{{ dict_object["metadata"]['nb_seen'] }}</td>
</tr>
</tbody>
</table>
</div>
<div class="col-md-1">
<div id="sparkline"></div>
</div>
</div>
</li>
<li class="list-group-item py-0">
<br>
<div class="mb-3">
Tags:
{% for tag in dict_object["metadata"]['tags'] %}
<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
data-tagid="{{ tag }}" data-objtype="hhhash" data-objsubtype="" data-objid="{{ dict_object["correlation_id"] }}">
{{ tag }}
</button>
{% endfor %}
<button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
<i class="far fa-plus-square"></i>
</button>
</div>
</li>
</ul>
{% with obj_type='hhhash', obj_id=dict_object['correlation_id'], obj_subtype='' %}
{% include 'modals/investigations_register_obj.html' %}
{% endwith %}
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
<i class="fas fa-microscope"></i> Investigations
</button>
</div>
</div>
<script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
<script>
sparkline("sparkline", {{ dict_object["metadata_card"]["sparkline"] }}, {});
</script>
<script>
function create_line_chart(id, url){
var width = 900;
var height = Math.round(width / 4);
var margin = {top: 20, right: 55, bottom: 50, left: 40};
var x = d3.scaleTime().range([0, width]);
var y = d3.scaleLinear().rangeRound([height, 0]);
var xAxis = d3.axisBottom(x);
var yAxis = d3.axisLeft(y);
var parseTime = d3.timeParse("%Y-%m-%d");
var line = d3.line()
.x(function(d) {
return x(d.date);
}).y(function(d) {
return y(d.value);
});
var svg_line = d3.select('#'+id).append('svg')
.attr("id", "graph_div")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append('g')
.attr('transform', "translate("+ margin.left +","+ margin.top +")");
var div = d3.select('body').append('div')
.attr('class', 'tooltip')
.style('opacity', 0);
//add div tooltip
d3.json(url)
.then(function(data){
data.forEach(function(d) {
d.date_label = d.date;
d.date = parseTime(d.date);
d.value = +d.value;
});
// fit the data
x.domain(d3.extent(data, function(d) { return d.date; }));
//x.domain(data.map(function (d) { return d.date; })); //E
y.domain([0, d3.max(data, function(d){ return d.value ; })]);
//line
svg_line.append("path")
.data([data])
.attr("class", "line_graph")
.attr("d", line);
// add X axis
svg_line.append("g")
.attr("transform", "translate(0," + height + ")")
.call(d3.axisBottom(x))
.selectAll("text")
.style("text-anchor", "end")
.attr("transform", "rotate(-45)" );
// Add the Y Axis
svg_line.append("g")
.call(d3.axisLeft(y));
//add a dot circle
svg_line.selectAll('dot')
.data(data).enter()
.append('circle')
.attr('r', 2)
.attr('cx', function(d) { return x(d.date); })
.attr('cy', function(d) { return y(d.value); })
.on('mouseover', function(d) {
div.transition().style('opacity', .9);
div.html('' + d.date_label+ '<br/>' + d.value).style('left', (d3.event.pageX) + 'px')
.style("left", (d3.event.pageX) + "px")
.style("top", (d3.event.pageY - 28) + "px");
})
.on('mouseout', function(d)
{
div.transition().style('opacity', 0);
});
});
}
</script>

View File

@ -119,6 +119,8 @@
{% include 'correlation/metadata_card_cookie_name.html' %} {% include 'correlation/metadata_card_cookie_name.html' %}
{% elif dict_object["object_type"] == "etag" %} {% elif dict_object["object_type"] == "etag" %}
{% include 'correlation/metadata_card_etag.html' %} {% include 'correlation/metadata_card_etag.html' %}
{% elif dict_object["object_type"] == "hhhash" %}
{% include 'correlation/metadata_card_hhhash.html' %}
{% elif dict_object["object_type"] == "item" %} {% elif dict_object["object_type"] == "item" %}
{% include 'correlation/metadata_card_item.html' %} {% include 'correlation/metadata_card_item.html' %}
{% endif %} {% endif %}
@ -230,6 +232,10 @@
<input class="form-check-input" type="checkbox" value="True" id="EtagCheck" name="EtagCheck" {%if "etag" in dict_object["filter"]%}checked{%endif%}> <input class="form-check-input" type="checkbox" value="True" id="EtagCheck" name="EtagCheck" {%if "etag" in dict_object["filter"]%}checked{%endif%}>
<label class="form-check-label" for="EtagCheck">Etag</label> <label class="form-check-label" for="EtagCheck">Etag</label>
</div> </div>
<div class="form-check">
<input class="form-check-input" type="checkbox" value="True" id="HHHashCheck" name="HHHashCheck" {%if "hhhash" in dict_object["filter"]%}checked{%endif%}>
<label class="form-check-label" for="HHHashCheck">HHHash</label>
</div>
<div class="form-check"> <div class="form-check">
<input class="form-check-input" type="checkbox" value="True" id="ScreenshotCheck" name="ScreenshotCheck" {%if "screenshot" in dict_object["filter"]%}checked{%endif%}> <input class="form-check-input" type="checkbox" value="True" id="ScreenshotCheck" name="ScreenshotCheck" {%if "screenshot" in dict_object["filter"]%}checked{%endif%}>
<label class="form-check-label" for="ScreenshotCheck">Screenshot</label> <label class="form-check-label" for="ScreenshotCheck">Screenshot</label>

View File

@ -0,0 +1,602 @@
<!DOCTYPE html>
<html>
<head>
<title>HHHashs - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
<style>
.input-group .form-control {
position: unset;
}
.line {
fill: none;
stroke: #000;
stroke-width: 2.0px;
}
.bar {
fill: steelblue;
}
.bar:hover{
fill: brown;
cursor: pointer;
}
.bar_stack:hover{
cursor: pointer;
}
.pie_path:hover{
cursor: pointer;
}
.svgText {
pointer-events: none;
}
div.tooltip {
position: absolute;
text-align: center;
padding: 2px;
font: 12px sans-serif;
background: #ebf4fb;
border: 2px solid #b7ddf2;
border-radius: 8px;
pointer-events: none;
color: #000000;
}
</style>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'sidebars/sidebar_objects.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="row">
<div class="col-xl-10">
<div class="mt-1" id="barchart_type"></div>
{# {% include 'hhhash/block_hhhash_search.html' %}#}
</div>
<div class="col-xl-2">
<div class="card mb-3 mt-2" style="background-color:#d9edf7;">
<div class="card-body text-center py-2">
<h6 class="card-title" style="color:#286090;">Select a date range :</h6>
<form action="{{ url_for('objects_hhhash.objects_hhhashs_post') }}" id="hash_selector_form" method='post'>
<div class="input-group" id="date-range-from">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from" autocomplete="off">
</div>
<div class="input-group" id="date-range-to">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to" autocomplete="off">
</div>
<div class="form-check my-1">
<input class="form-check-input" type="checkbox" id="checkbox-input-show" name="show_objects" value="True" {% if show_objects %}checked{% endif %}>
<label class="form-check-label" for="checkbox-input-show">
<span style="color:#286090; font-size: 14px;">
Show HHHashs Names <i class="fas fa-key"></i>
</span>
</label>
</div>
<button class="btn btn-primary" style="text-align:center;">
<i class="fas fa-copy"></i> Search
</button>
</form>
</div>
</div>
<div id="pie_chart_encoded">
</div>
<div id="pie_chart_top5_types">
</div>
</div>
</div>
{% if dict_objects %}
{% if date_from|string == date_to|string %}
<h3> {{ date_from }} HHHashs Name: </h3>
{% else %}
<h3> {{ date_from }} to {{ date_to }} HHHashs Name: </h3>
{% endif %}
<table id="tableb64" class="table table-striped table-bordered">
<thead class="bg-dark text-white">
<tr>
<th></th>
<th>First Seen</th>
<th>Last Seen</th>
<th>Total</th>
<th>Last days</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
{% for obj_id in dict_objects %}
<tr>
<td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?type=hhhash&id={{ obj_id }}">{{ dict_objects[obj_id]['content'] }}</a></td>
<td>{{ dict_objects[obj_id]['first_seen'] }}</td>
<td>{{ dict_objects[obj_id]['last_seen'] }}</td>
<td>{{ dict_objects[obj_id]['nb_seen'] }}</td>
<td id="sparklines_{{ obj_id }}" style="text-align:center;"></td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
{% if show_objects %}
{% if date_from|string == date_to|string %}
<h3> {{ date_from }}, No HHHash Name</h3>
{% else %}
<h3> {{ date_from }} to {{ date_to }}, No HHHash Name</h3>
{% endif %}
{% endif %}
{% endif %}
</div>
</div>
</div>
<script>
var chart = {};
$(document).ready(function(){
$("#page-Decoded").addClass("active");
$("#nav_hhhash").addClass("active");
$('#date-range-from').dateRangePicker({
separator : ' to ',
getValue: function()
{
if ($('#date-range-from-input').val() && $('#date-range-to').val() )
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to').val();
else
return '';
},
setValue: function(s,s1,s2)
{
$('#date-range-from-input').val(s1);
$('#date-range-to-input').val(s2);
},
});
$('#date-range-to').dateRangePicker({
separator : ' to ',
getValue: function()
{
if ($('#date-range-from-input').val() && $('#date-range-to').val() )
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to').val();
else
return '';
},
setValue: function(s,s1,s2)
{
$('#date-range-from-input').val(s1);
$('#date-range-to-input').val(s2);
},
});
$('#date-range-from').data('dateRangePicker').setDateRange('{{date_from}}','{{date_to}}');
$('#date-range-to').data('dateRangePicker').setDateRange('{{date_from}}','{{date_to}}');
$('#tableb64').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 3, "desc" ]]
});
chart.stackBarChart = barchart_type_stack("{{ url_for('objects_hhhash.objects_hhhash_range_json') }}?date_from={{date_from}}&date_to={{date_to}}", 'id');
chart.onResize();
$(window).on("resize", function() {
chart.onResize();
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
<script>
{% for obj_id in dict_objects %}
sparkline("sparklines_{{ obj_id }}", {{ dict_objects[obj_id]['sparkline'] }}, {});
{% endfor %}
</script>
<script>
var margin = {top: 20, right: 100, bottom: 55, left: 45},
width = 1000 - margin.left - margin.right,
height = 500 - margin.top - margin.bottom;
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
var y = d3.scaleLinear().rangeRound([height, 0]);
var xAxis = d3.axisBottom(x);
var yAxis = d3.axisLeft(y);
var color = d3.scaleOrdinal(d3.schemeSet3);
var svg = d3.select("#barchart_type").append("svg")
.attr("id", "thesvg")
.attr("viewBox", "0 0 1000 500")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
function barchart_type_stack(url, id) {
d3.json(url)
.then(function(data){
var labelVar = 'date'; //A
var varNames = d3.keys(data[0])
.filter(function (key) { return key !== labelVar;}); //B
data.forEach(function (d) { //D
var y0 = 0;
d.mapping = varNames.map(function (name) {
return {
name: name,
label: d[labelVar],
y0: y0,
y1: y0 += +d[name]
};
});
d.total = d.mapping[d.mapping.length - 1].y1;
});
x.domain(data.map(function (d) { return (d.date); })); //E
y.domain([0, d3.max(data, function (d) { return d.total; })]);
svg.append("g")
.attr("class", "x axis")
.attr("transform", "translate(0," + height + ")")
.call(xAxis)
.selectAll("text")
.attr("class", "bar")
{% if date_from|string == date_to|string and type is none %}
.on("click", function (d) { window.location.href = "{{ url_for('objects_hhhash.objects_hhhashs') }}?date_from={{date_from}}&date_to={{date_to}}&type_id="+d })
.attr("transform", "rotate(-18)" )
{% elif date_from|string == date_to|string and type is not none %}
.on("click", function (d) { window.location.href = "{{ url_for('objects_hhhash.objects_hhhashs') }}?date_from="+d+'&date_to='+d })
.attr("transform", "rotate(-18)" )
{% else %}
.on("click", function (d) { window.location.href = "{{ url_for('objects_hhhash.objects_hhhashs') }}?date_from="+d+'&date_to='+d })
.attr("transform", "rotate(-40)" )
{% endif %}
.style("text-anchor", "end");
svg.append("g")
.attr("class", "y axis")
.call(yAxis)
.append("text")
.attr("transform", "rotate(-90)")
.attr("y", 6)
.attr("dy", ".71em")
.style("text-anchor", "end");
var selection = svg.selectAll(".series")
.data(data)
.enter().append("g")
.attr("class", "series")
.attr("transform", function (d) { return "translate(" + x((d.date)) + ",0)"; });
selection.selectAll("rect")
.data(function (d) { return d.mapping; })
.enter().append("rect")
.attr("class", "bar_stack")
.attr("width", x.bandwidth())
.attr("y", function (d) { return y(d.y1); })
.attr("height", function (d) { return y(d.y0) - y(d.y1); })
.style("fill", function (d) { return color(d.name); })
.style("stroke", "grey")
.on("mouseover", function (d) { showPopover.call(this, d); })
.on("mouseout", function (d) { removePopovers(); })
{% if date_from|string == date_to|string and type is none %}
.on("click", function(d){ window.location.href = "{{ url_for('objects_hhhash.objects_hhhashs') }}" +'?date_from={{date_from}}&date_to={{date_to}}&type_id='+d.label+'&encoding='+d.name; });
{% elif date_from|string == date_to|string and type is not none %}
.on("click", function(d){ window.location.href = "{{ url_for('objects_hhhash.objects_hhhashs') }}" +'?type_id={{type_id}}&date_from='+d.label+'&date_to='+d.label+'&encoding='+d.name; });
{% else %}
.on("click", function(d){ window.location.href = "{{ url_for('objects_hhhash.objects_hhhashs') }}" +'?type_id='+ d.name +'&date_from='+d.label+'&date_to='+d.label; });
{% endif %}
data.forEach(function(d) {
if(d.total !== 0){
svg.append("text")
.attr("class", "bar")
.attr("dy", "-.35em")
.attr('x', x(d.date) + x.bandwidth()/2)
.attr('y', y(d.total))
{% if date_from|string == date_to|string and type is none %}
.on("click", function () {window.location.href = "{{ url_for('objects_hhhash.objects_hhhashs') }}"+'?date_from={{date_from}}&date_to={{date_to}}&type_id='+d.date })
{% elif date_from|string == date_to|string and type is not none %}
.on("click", function () {window.location.href = "{{ url_for('objects_hhhash.objects_hhhashs') }}?type_id={{type_id}}&date_from="+d.date+'&date_to='+d.date })
{% else %}
.on("click", function () {window.location.href = "{{ url_for('objects_hhhash.objects_hhhashs') }}"+'?date_from='+d.date+'&date_to='+d.date })
{% endif %}
.style("text-anchor", "middle")
.text(d.total);
}
});
drawLegend(varNames);
});
}
function drawLegend (varNames) {
var legend = svg.selectAll(".legend")
.data(varNames.slice().reverse())
.enter().append("g")
.attr("class", "legend")
.attr("transform", function (d, i) { return "translate(0," + i * 20 + ")"; });
legend.append("rect")
.attr("x", 943)
.attr("width", 10)
.attr("height", 10)
.style("fill", color)
.style("stroke", "grey");
legend.append("text")
.attr("class", "svgText")
.attr("x", 941)
.attr("y", 6)
.attr("dy", ".35em")
.style("text-anchor", "end")
.text(function (d) { return d; });
}
function removePopovers () {
$('.popover').each(function() {
$(this).remove();
});
}
function showPopover (d) {
$(this).popover({
title: "<b><span id='tooltip-id-name-bar'></span></b>",
placement: 'top',
container: 'body',
trigger: 'manual',
html : true,
content: function() {
return "<span id='tooltip-id-label'></span>" +
"<br/>num: <span id='tooltip-id-value-bar'></span>"; }
});
$(this).popover('show');
$("#tooltip-id-name-bar").text(d.name);
$("#tooltip-id-label").text(d.label);
$("#tooltip-id-value-bar").text(d3.format(",")(d.value ? d.value: d.y1 - d.y0));
}
chart.onResize = function () {
var aspect = 1000 / 500, chart = $("#thesvg");
var targetWidth = chart.parent().width();
chart.attr("width", targetWidth);
chart.attr("height", targetWidth / aspect);
}
window.chart = chart;
</script>
<script>
function draw_pie_chart(id, url_json, pie_on_click_url) {
var width_pie = 200;
var height_pie = 200;
var padding_pie = 10;
var opacity_pie = .8;
var radius_pie = Math.min(width_pie - padding_pie, height_pie - padding_pie) / 2;
//var color_pie = d3.scaleOrdinal(d3.schemeCategory10);
var color_pie = d3.scaleOrdinal(d3.schemeSet3);
var div_pie = d3.select("body").append("div")
.attr("class", "tooltip")
.style("opacity", 0);
var svg_pie = d3.select("#"+id)
.append('svg')
.attr("width", '100%')
.attr("height", '100%')
.attr('viewBox','0 0 '+Math.min(width_pie,height_pie) +' '+Math.min(width_pie,height_pie) )
.attr('preserveAspectRatio','xMinYMin')
var g_pie = svg_pie.append('g')
.attr('transform', 'translate(' + (width_pie/2) + ',' + (height_pie/2) + ')');
var arc_pie = d3.arc()
.innerRadius(0)
.outerRadius(radius_pie);
d3.json(url_json)
.then(function(data){
var pie_pie = d3.pie()
.value(function(d) { return d.value; })
.sort(null);
var path_pie = g_pie.selectAll('path')
.data(pie_pie(data))
.enter()
.append("g")
.append('path')
.attr('d', arc_pie)
.attr('fill', (d,i) => color_pie(i))
.attr('class', 'pie_path')
.on("mouseover", mouseovered_pie)
.on("mouseout", mouseouted_pie)
.on("click", function (d) {window.location.href = pie_on_click_url+d.data.name })
.style('opacity', opacity_pie)
.style('stroke', 'white');
});
function mouseovered_pie(d) {
//remove old content
$("#tooltip-id-name").remove();
$("#tooltip-id-value").remove();
// tooltip
var content;
content = "<b><span id='tooltip-id-name'></span></b><br/>"+
"<br/>"+
"<i>Decoded</i>: <span id='tooltip-id-value'></span><br/>"
div_pie.transition()
.duration(200)
.style("opacity", .9);
div_pie.html(content)
.style("left", (d3.event.pageX) + "px")
.style("top", (d3.event.pageY - 28) + "px");
$("#tooltip-id-name").text(d.data.name);
$("#tooltip-id-value").text(d.data.value);
}
function mouseouted_pie() {
div_pie.transition()
.duration(500)
.style("opacity", 0);
}
}
</script>
<script>
function barchart_type(url, id) {
var margin = {top: 20, right: 20, bottom: 70, left: 40};
var width = 960 - margin.left - margin.right;
var height = 500 - margin.top - margin.bottom;
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
var y = d3.scaleLinear().rangeRound([height, 0]);
var xAxis = d3.axisBottom(x)
//.tickFormat(d3.time.format("%Y-%m"));
var yAxis = d3.axisLeft(y)
.ticks(10);
/*var svg = d3.select(id).append("svg")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.attr("id", "thesvg")
.append("g")
.attr("transform",
"translate(" + margin.left + "," + margin.top + ")");*/
d3.json(url)
.then(function(data){
data.forEach(function(d) {
d.value = +d.value;
});
x.domain(data.map(function(d) { return d.date; }));
y.domain([0, d3.max(data, function(d) { return d.value; })]);
var label = svg.append("g")
.attr("class", "x axis")
.attr("transform", "translate(0," + height + ")")
.call(xAxis)
.selectAll("text")
.style("text-anchor", "end")
.attr("dx", "-.8em")
.attr("dy", "-.55em")
{% if daily_type_chart %}
.attr("transform", "rotate(-20)" );
{% else %}
.attr("transform", "rotate(-70)" )
.attr("class", "bar")
{% endif %}
svg.append("g")
.attr("class", "y axis")
.call(yAxis)
.append("text")
.attr("transform", "rotate(-90)")
.attr("y", 6)
.attr("dy", ".71em")
.style("text-anchor", "end")
.text("Value ($)");
var bar = svg.selectAll("bar")
.data(data)
.enter().append("rect")
.attr("class", "bar")
//.style("fill", "steelblue")
.attr("x", function(d) { return x(d.date); })
.attr("width", x.bandwidth())
.attr("y", function(d) { return y(d.value); })
.attr("height", function(d) { return height - y(d.value); })
data.forEach(function(d) {
if(d.value != 0){
svg.append("text")
.attr("class", "bar")
.attr("dy", "-.35em")
//.text(function(d) { return d.value; });
.text(d.value)
.style("text-anchor", "middle")
.attr('x', x(d.date) + x.bandwidth()/2)
.attr('y', y(d.value));
}
});
});
}
</script>
</body>
</html>

View File

@ -46,6 +46,12 @@
<span>Etag</span> <span>Etag</span>
</a> </a>
</li> </li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('objects_hhhash.objects_hhhashs')}}" id="nav_hhhash">
<i class="fas fa-align-left"></i>
<span>HHHash</span>
</a>
</li>
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" href="{{url_for('objects_title.objects_titles')}}" id="nav_title"> <a class="nav-link" href="{{url_for('objects_title.objects_titles')}}" id="nav_title">
<i class="fas fa-heading"></i> <i class="fas fa-heading"></i>