mirror of https://github.com/CIRCL/AIL-framework
chg: [Domain + UI Crawler] refractor show domain
Terrtia
parent
44bb18a8be
commit
a2d6874417
|
|
@ -56,6 +56,7 @@ def get_link_tree():
|
|||
pass
|
||||
|
||||
|
||||
|
||||
def get_domain_tags(domain):
|
||||
'''
|
||||
Retun all tags of a given domain.
|
||||
|
|
@ -103,11 +104,119 @@ def get_domain_all_correlation(domain, correlation_type=None):
|
|||
domain_correl['pgp'] = res
|
||||
return domain_correl
|
||||
|
||||
# TODO: handle port
|
||||
def get_domain_history(domain, domain_type, port): # TODO: add date_range: from to + nb_elem
|
||||
'''
|
||||
Retun .
|
||||
|
||||
:param domain: crawled domain
|
||||
:type domain: str
|
||||
|
||||
:return:
|
||||
:rtype: list of tuple (item_core, epoch)
|
||||
'''
|
||||
return r_serv_onion.zrange('crawler_history_{}:{}:{}'.format(domain_type, domain, port), 0, -1, withscores=True)
|
||||
|
||||
def get_domain_history_with_status(domain, domain_type, port): # TODO: add date_range: from to + nb_elem
|
||||
'''
|
||||
Retun .
|
||||
|
||||
:param domain: crawled domain
|
||||
:type domain: str
|
||||
|
||||
:return:
|
||||
:rtype: list of dict (epoch, date: %Y/%m/%d - %H:%M.%S, boolean status)
|
||||
'''
|
||||
l_history = []
|
||||
history = get_domain_history(domain, domain_type, port)
|
||||
for root_item, epoch_val in history:
|
||||
epoch_val = int(epoch_val) # force int
|
||||
# domain down, root_item==epoch_val
|
||||
try:
|
||||
int(root_item)
|
||||
status = False
|
||||
# domain up, root_item=str
|
||||
except ValueError:
|
||||
status = True
|
||||
l_history.append({"epoch": epoch_val, "date": time.strftime('%Y/%m/%d - %H:%M.%S', time.gmtime(epoch_val)), "status": status})
|
||||
return l_history
|
||||
|
||||
|
||||
class Domain(object):
|
||||
"""docstring for Domain."""
|
||||
|
||||
def __init__(self, domain, port=80):
|
||||
self.domain = str(domain)
|
||||
## TODO: handle none port
|
||||
self.type = get_domain_type(domain)
|
||||
|
||||
def get_domain_first_seen(self):
|
||||
'''
|
||||
Get domain first seen date
|
||||
|
||||
:return: domain first seen date
|
||||
:rtype: str
|
||||
'''
|
||||
first_seen = r_serv_onion.hget('{}_metadata:{}'.format(self.type, self.domain), 'first_seen')
|
||||
if first_seen is not None:
|
||||
first_seen = '{}/{}/{}'.format(first_seen[0:4], first_seen[4:6], first_seen[6:8])
|
||||
return first_seen
|
||||
|
||||
def get_domain_last_check(self):# # TODO: add epoch ???
|
||||
'''
|
||||
Get domain last check date
|
||||
|
||||
:return: domain last check date
|
||||
:rtype: str
|
||||
'''
|
||||
last_check = r_serv_onion.hget('{}_metadata:{}'.format(self.type, self.domain), 'last_check')
|
||||
if last_check is not None:
|
||||
last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8])
|
||||
return last_check
|
||||
|
||||
#def get_domain_all_ports(self):
|
||||
# pass
|
||||
|
||||
def get_domain_metadata(self, first_seen=True, last_ckeck=True, ports=True):
|
||||
'''
|
||||
Get Domain basic metadata
|
||||
|
||||
:param first_seen: get domain first_seen
|
||||
:type first_seen: boolean
|
||||
:param last_ckeck: get domain last_check
|
||||
:type last_ckeck: boolean
|
||||
:param ports: get all domain ports
|
||||
:type ports: boolean
|
||||
|
||||
:return: a dict of all metadata for a given domain
|
||||
:rtype: dict
|
||||
'''
|
||||
dict_metadata = {}
|
||||
if first_seen:
|
||||
res = self.get_domain_first_seen()
|
||||
if res is not None:
|
||||
dict_metadata['first_seen'] = res
|
||||
if last_ckeck:
|
||||
res = self.get_domain_last_check()
|
||||
if res is not None:
|
||||
dict_metadata['last_check'] = res
|
||||
return dict_metadata
|
||||
|
||||
def get_domain_tags(self):
|
||||
'''
|
||||
Retun all tags of a given domain.
|
||||
|
||||
:param domain: crawled domain
|
||||
'''
|
||||
return get_domain_tags(self.domain)
|
||||
|
||||
def get_domain_correlation(self):
|
||||
'''
|
||||
Retun all cryptocurrencies of a given domain.
|
||||
'''
|
||||
return get_domain_all_correlation(self.domain)
|
||||
|
||||
def get_domain_history_with_status(self):
|
||||
'''
|
||||
Retun the full history of a given domain and port.
|
||||
'''
|
||||
return get_domain_history_with_status(self.domain, self.type, 80)
|
||||
|
|
|
|||
|
|
@ -108,6 +108,24 @@ class Correlation(object):
|
|||
else:
|
||||
return []
|
||||
|
||||
def _get_correlation_obj_domain(self, field_name, correlation_type):
|
||||
'''
|
||||
Return all domains that contain this correlation.
|
||||
|
||||
:param domain: field name
|
||||
:type domain: str
|
||||
:param correlation_type: correlation type
|
||||
:type correlation_type: str
|
||||
|
||||
:return: a list of correlation
|
||||
:rtype: list
|
||||
'''
|
||||
res = r_serv_metadata.smembers('set_domain_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
|
||||
if res:
|
||||
return list(res)
|
||||
else:
|
||||
return []
|
||||
|
||||
def get_domain_correlation_dict(self, domain, correlation_type=None):
|
||||
'''
|
||||
Return all correlation of a given domain.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,59 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
|
||||
'''
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
|
||||
from flask_login import login_required, current_user, login_user, logout_user
|
||||
|
||||
sys.path.append('modules')
|
||||
import Flask_config
|
||||
|
||||
# Import Role_Manager
|
||||
from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
|
||||
from Role_Manager import login_admin, login_analyst
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||
import Domain
|
||||
|
||||
r_cache = Flask_config.r_cache
|
||||
r_serv_db = Flask_config.r_serv_db
|
||||
r_serv_tags = Flask_config.r_serv_tags
|
||||
bootstrap_label = Flask_config.bootstrap_label
|
||||
|
||||
# ============ BLUEPRINT ============
|
||||
crawler_splash = Blueprint('crawler_splash', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/crawler/crawler_splash'))
|
||||
|
||||
# ============ VARIABLES ============
|
||||
|
||||
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
|
||||
|
||||
# ============= ROUTES ==============
|
||||
@crawler_splash.route('/crawlers/showDomain')
|
||||
#@login_required
|
||||
#@login_analyst
|
||||
def showDomain():
|
||||
domain_name = request.args.get('domain')
|
||||
epoch = request.args.get('epoch')
|
||||
port = request.args.get('port')
|
||||
|
||||
domain = Domain.Domain(domain_name)
|
||||
|
||||
dict_domain = domain.get_domain_metadata()
|
||||
dict_domain = {**dict_domain, **domain.get_domain_correlation()}
|
||||
dict_domain['domain'] = domain_name
|
||||
dict_domain['tags'] = domain.get_domain_tags()
|
||||
dict_domain['history'] = domain.get_domain_history_with_status()
|
||||
|
||||
print(dict_domain)
|
||||
|
||||
return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label, screenshot={'item': None, '':None}, dict_links={})
|
||||
|
|
@ -6,10 +6,10 @@
|
|||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
|
||||
|
|
@ -45,7 +45,7 @@
|
|||
</div>
|
||||
{% endif %}
|
||||
</span>
|
||||
<h3 class="card-title text-white">{{ domain }} :</h3>
|
||||
<h3 class="card-title text-white">{{ dict_domain['domain'] }} :</h3>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<table class="table table-responsive table-condensed">
|
||||
|
|
@ -58,23 +58,22 @@
|
|||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td class="panelText">{{ first_seen }}</td>
|
||||
<td class="panelText">{{ last_check }}</td>
|
||||
<td class="panelText">{%if "first_seen" in dict_domain%}{{ dict_domain['first_seen'] }}{%endif%}</td>
|
||||
<td class="panelText">{%if "last_check" in dict_domain%}{{ dict_domain['last_check'] }}{%endif%}</td>
|
||||
<td class="panelText">{{ ports }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
Origin Paste:
|
||||
{% if origin_paste_name=='manual' or origin_paste_name=='auto' %}
|
||||
<span class="badge badge-dark">{{ origin_paste_name }}</span>
|
||||
{%else%}
|
||||
<a class="badge badge-dark" target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste', paste=origin_paste) }}" />{{ origin_paste_name }}</a>
|
||||
{%endif%}
|
||||
<div>
|
||||
{% for tag in origin_paste_tags %}
|
||||
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag[1] }}">
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
|
||||
{% for tag in dict_domain['tags'] %}
|
||||
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
<br>
|
||||
|
|
@ -83,17 +82,122 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
{% for tag in domain_tags %}
|
||||
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }} <i>{{ domain_tags[tag] }}</i></span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
<br>
|
||||
<br>
|
||||
</div>
|
||||
|
||||
{% if 'pgp' in dict_domain%}
|
||||
<div id="accordionpgp" class="mt-3">
|
||||
<div class="card">
|
||||
<div class="card-header" id="headingPgp">
|
||||
<div class="row">
|
||||
<div class="col-11">
|
||||
<div class="mt-2">
|
||||
<i class="fas fa-key"></i> PGP Dumps
|
||||
<div class="badge badge-warning">{{l_64|length}}</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-1">
|
||||
<button class="btn btn-link collapsed rotate" data-toggle="collapse" data-target="#collapsePgp" aria-expanded="false" aria-controls="collapsePgp">
|
||||
<i class="fas fa-chevron-circle-down"></i>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div id="collapsePgp" class="collapse" aria-labelledby="headingPgp" data-parent="#accordionpgp">
|
||||
<div class="card-body">
|
||||
<table id="tablepgp" class="table table-striped">
|
||||
<thead class="thead-dark">
|
||||
<tr>
|
||||
<th>PGP Type</th>
|
||||
<th>Key ID</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for dict_key in dict_domain['pgp']%}
|
||||
{% if dict_key=="mail" %}
|
||||
{% set var_icon = "fas fa-at" %}
|
||||
{% elif dict_key=="name" %}
|
||||
{% set var_icon = "fas fa-user-tag" %}
|
||||
{% else %}
|
||||
{% set var_icon = "fas fa-key" %}
|
||||
{% endif %}
|
||||
{% for key_id in dict_domain['pgp'][dict_key]%}
|
||||
<tr>
|
||||
<td>
|
||||
<i class="{{ var_icon }}"></i>
|
||||
{{ dict_key }}
|
||||
</td>
|
||||
<td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ key_id }}">{{ key_id }}</a></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
|
||||
{% if 'cryptocurrency' in dict_domain%}
|
||||
<div id="accordioncurrency" class="mt-3">
|
||||
<div class="card">
|
||||
<div class="card-header" id="headingcurrency">
|
||||
<div class="row">
|
||||
<div class="col-11">
|
||||
<div class="mt-2">
|
||||
<i class="fas fa-coins"></i> Cryptocurrencies
|
||||
<div class="badge badge-warning">{{l_64|length}}</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-1">
|
||||
<button class="btn btn-link collapsed rotate" data-toggle="collapse" data-target="#collapsecurrency" aria-expanded="false" aria-controls="collapsecurrency">
|
||||
<i class="fas fa-chevron-circle-down"></i>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div id="collapsecurrency" class="collapse" aria-labelledby="headingcurrency" data-parent="#accordioncurrency">
|
||||
<div class="card-body">
|
||||
<table id="tablecurrency" class="table table-striped">
|
||||
<thead class="thead-dark">
|
||||
<tr>
|
||||
<th>Currency</th>
|
||||
<th>address</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for dict_key in dict_domain['cryptocurrency']%}
|
||||
{% if dict_key=="bitcoin" %}
|
||||
{% set var_icon = "fab fa-bitcoin" %}
|
||||
{% elif dict_key=="monero" %}
|
||||
{% set var_icon = "fab fa-monero" %}
|
||||
{% else %}
|
||||
{% set var_icon = "fas fa-coins" %}
|
||||
{% endif %}
|
||||
{% for key_id in dict_domain['cryptocurrency'][dict_key]%}
|
||||
<tr>
|
||||
<td>
|
||||
<i class="{{ var_icon }}"></i>
|
||||
{{ dict_key }}
|
||||
</td>
|
||||
<td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ key_id }}">{{ key_id }}</a></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
|
||||
<!-- TODO: ADD HASH DECODED -->
|
||||
|
||||
{% if l_pastes %}
|
||||
<hr>
|
||||
|
||||
<table class="table table-striped table-bordered table-hover" id="myTable_1">
|
||||
<thead class="thead-dark">
|
||||
<tr>
|
||||
|
|
@ -129,7 +233,7 @@
|
|||
</a>
|
||||
</div>
|
||||
|
||||
{% if domain_history %}
|
||||
{% if dict_domain["domain_history"] %}
|
||||
<table class="table table-hover mt-2" id="myTable_2">
|
||||
<thead>
|
||||
<tr>
|
||||
|
|
@ -142,7 +246,7 @@
|
|||
<td class="{% if epoch_item[0]==epoch %}text-dark table-info{% endif %}">
|
||||
<a href="{{ url_for('hiddenServices.show_domain') }}?domain={{domain}}&port={{port}}&epoch={{epoch_item[0]}}" class="text-secondary">
|
||||
<div class="d-flex justify-content-around" style="line-height:0.9;">
|
||||
<div>{{domain}}</div>
|
||||
<div>{{dict_domain["domain"]}}</div>
|
||||
{% if epoch_item[2] %}
|
||||
<div style="color:Green;"><i class="fas fa-check-circle"></i> UP</div>
|
||||
{% else %}
|
||||
|
|
@ -177,11 +281,13 @@
|
|||
</div>
|
||||
<canvas id="canvas" style="width:100%;"></canvas>
|
||||
<div class="text-center">
|
||||
<!--
|
||||
<small>
|
||||
<a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{screenshot['item']}}" class="text-info">
|
||||
<div style="line-height:0.9;">{{dict_links[screenshot['item']]}}</div>
|
||||
</a>
|
||||
<small>
|
||||
-->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
@ -196,13 +302,19 @@
|
|||
<script>
|
||||
var table;
|
||||
$(document).ready(function(){
|
||||
table = $('#myTable_1').DataTable(
|
||||
{
|
||||
//"aLengthMenu": [[5, 10, 15, 20, -1], [5, 10, 15, 20, "All"]],
|
||||
//"iDisplayLength": 5,
|
||||
//"order": [[ 0, "desc" ]]
|
||||
});
|
||||
});
|
||||
{% if 'pgp' in dict_domain%}
|
||||
$('#tablepgp').DataTable();
|
||||
{% endif %}
|
||||
{% if 'cryptocurrency' in dict_domain%}
|
||||
$('#tablecurrency').DataTable({});
|
||||
{% endif %}
|
||||
table = $('#myTable_1').DataTable(
|
||||
{
|
||||
//"aLengthMenu": [[5, 10, 15, 20, -1], [5, 10, 15, 20, "All"]],
|
||||
//"iDisplayLength": 5,
|
||||
//"order": [[ 0, "desc" ]]
|
||||
});
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
|
|
@ -230,7 +342,7 @@ img.onload = pixelate;
|
|||
img.addEventListener("error", img_error);
|
||||
var draw_img = false;
|
||||
|
||||
img.src = "{{ url_for('showsavedpastes.screenshot', filename=screenshot['screenshot']) }}";
|
||||
//img.src = "{{ url_for('showsavedpastes.screenshot', filename=screenshot['screenshot']) }}";
|
||||
|
||||
function pixelate() {
|
||||
/// use slider value
|
||||
Loading…
Reference in New Issue