chg: [Domain + UI Crawler] refractor show domain

pull/418/head
Terrtia 2019-10-30 17:12:04 +01:00
parent 44bb18a8be
commit a2d6874417
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
4 changed files with 327 additions and 29 deletions

View File

@ -56,6 +56,7 @@ def get_link_tree():
pass
def get_domain_tags(domain):
'''
Retun all tags of a given domain.
@ -103,11 +104,119 @@ def get_domain_all_correlation(domain, correlation_type=None):
domain_correl['pgp'] = res
return domain_correl
# TODO: handle port
def get_domain_history(domain, domain_type, port): # TODO: add date_range: from to + nb_elem
'''
Retun .
:param domain: crawled domain
:type domain: str
:return:
:rtype: list of tuple (item_core, epoch)
'''
return r_serv_onion.zrange('crawler_history_{}:{}:{}'.format(domain_type, domain, port), 0, -1, withscores=True)
def get_domain_history_with_status(domain, domain_type, port): # TODO: add date_range: from to + nb_elem
'''
Retun .
:param domain: crawled domain
:type domain: str
:return:
:rtype: list of dict (epoch, date: %Y/%m/%d - %H:%M.%S, boolean status)
'''
l_history = []
history = get_domain_history(domain, domain_type, port)
for root_item, epoch_val in history:
epoch_val = int(epoch_val) # force int
# domain down, root_item==epoch_val
try:
int(root_item)
status = False
# domain up, root_item=str
except ValueError:
status = True
l_history.append({"epoch": epoch_val, "date": time.strftime('%Y/%m/%d - %H:%M.%S', time.gmtime(epoch_val)), "status": status})
return l_history
class Domain(object):
"""docstring for Domain."""
def __init__(self, domain, port=80):
self.domain = str(domain)
## TODO: handle none port
self.type = get_domain_type(domain)
def get_domain_first_seen(self):
'''
Get domain first seen date
:return: domain first seen date
:rtype: str
'''
first_seen = r_serv_onion.hget('{}_metadata:{}'.format(self.type, self.domain), 'first_seen')
if first_seen is not None:
first_seen = '{}/{}/{}'.format(first_seen[0:4], first_seen[4:6], first_seen[6:8])
return first_seen
def get_domain_last_check(self):# # TODO: add epoch ???
'''
Get domain last check date
:return: domain last check date
:rtype: str
'''
last_check = r_serv_onion.hget('{}_metadata:{}'.format(self.type, self.domain), 'last_check')
if last_check is not None:
last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8])
return last_check
#def get_domain_all_ports(self):
# pass
def get_domain_metadata(self, first_seen=True, last_ckeck=True, ports=True):
'''
Get Domain basic metadata
:param first_seen: get domain first_seen
:type first_seen: boolean
:param last_ckeck: get domain last_check
:type last_ckeck: boolean
:param ports: get all domain ports
:type ports: boolean
:return: a dict of all metadata for a given domain
:rtype: dict
'''
dict_metadata = {}
if first_seen:
res = self.get_domain_first_seen()
if res is not None:
dict_metadata['first_seen'] = res
if last_ckeck:
res = self.get_domain_last_check()
if res is not None:
dict_metadata['last_check'] = res
return dict_metadata
def get_domain_tags(self):
'''
Retun all tags of a given domain.
:param domain: crawled domain
'''
return get_domain_tags(self.domain)
def get_domain_correlation(self):
'''
Retun all cryptocurrencies of a given domain.
'''
return get_domain_all_correlation(self.domain)
def get_domain_history_with_status(self):
'''
Retun the full history of a given domain and port.
'''
return get_domain_history_with_status(self.domain, self.type, 80)

View File

@ -108,6 +108,24 @@ class Correlation(object):
else:
return []
def _get_correlation_obj_domain(self, field_name, correlation_type):
'''
Return all domains that contain this correlation.
:param domain: field name
:type domain: str
:param correlation_type: correlation type
:type correlation_type: str
:return: a list of correlation
:rtype: list
'''
res = r_serv_metadata.smembers('set_domain_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
if res:
return list(res)
else:
return []
def get_domain_correlation_dict(self, domain, correlation_type=None):
'''
Return all correlation of a given domain.

View File

@ -0,0 +1,59 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
'''
import os
import sys
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
from flask_login import login_required, current_user, login_user, logout_user
sys.path.append('modules')
import Flask_config
# Import Role_Manager
from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
from Role_Manager import login_admin, login_analyst
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import Domain
r_cache = Flask_config.r_cache
r_serv_db = Flask_config.r_serv_db
r_serv_tags = Flask_config.r_serv_tags
bootstrap_label = Flask_config.bootstrap_label
# ============ BLUEPRINT ============
crawler_splash = Blueprint('crawler_splash', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/crawler/crawler_splash'))
# ============ VARIABLES ============
# ============ FUNCTIONS ============
# ============= ROUTES ==============
@crawler_splash.route('/crawlers/showDomain')
#@login_required
#@login_analyst
def showDomain():
domain_name = request.args.get('domain')
epoch = request.args.get('epoch')
port = request.args.get('port')
domain = Domain.Domain(domain_name)
dict_domain = domain.get_domain_metadata()
dict_domain = {**dict_domain, **domain.get_domain_correlation()}
dict_domain['domain'] = domain_name
dict_domain['tags'] = domain.get_domain_tags()
dict_domain['history'] = domain.get_domain_history_with_status()
print(dict_domain)
return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label, screenshot={'item': None, '':None}, dict_links={})

View File

@ -6,10 +6,10 @@
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
@ -45,7 +45,7 @@
</div>
{% endif %}
</span>
<h3 class="card-title text-white">{{ domain }} :</h3>
<h3 class="card-title text-white">{{ dict_domain['domain'] }} :</h3>
</div>
<div class="card-body">
<table class="table table-responsive table-condensed">
@ -58,23 +58,22 @@
</thead>
<tbody>
<tr>
<td class="panelText">{{ first_seen }}</td>
<td class="panelText">{{ last_check }}</td>
<td class="panelText">{%if "first_seen" in dict_domain%}{{ dict_domain['first_seen'] }}{%endif%}</td>
<td class="panelText">{%if "last_check" in dict_domain%}{{ dict_domain['last_check'] }}{%endif%}</td>
<td class="panelText">{{ ports }}</td>
</tr>
</tbody>
</table>
Origin Paste:
{% if origin_paste_name=='manual' or origin_paste_name=='auto' %}
<span class="badge badge-dark">{{ origin_paste_name }}</span>
{%else%}
<a class="badge badge-dark" target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste', paste=origin_paste) }}" />{{ origin_paste_name }}</a>
{%endif%}
<div>
{% for tag in origin_paste_tags %}
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag[1] }}">
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
{% for tag in dict_domain['tags'] %}
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
</a>
{% endfor %}
<br>
@ -83,17 +82,122 @@
</div>
</div>
<div>
{% for tag in domain_tags %}
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }} <i>{{ domain_tags[tag] }}</i></span>
</a>
{% endfor %}
<br>
<br>
</div>
{% if 'pgp' in dict_domain%}
<div id="accordionpgp" class="mt-3">
<div class="card">
<div class="card-header" id="headingPgp">
<div class="row">
<div class="col-11">
<div class="mt-2">
<i class="fas fa-key"></i> PGP Dumps&nbsp;&nbsp;
<div class="badge badge-warning">{{l_64|length}}</div>
</div>
</div>
<div class="col-1">
<button class="btn btn-link collapsed rotate" data-toggle="collapse" data-target="#collapsePgp" aria-expanded="false" aria-controls="collapsePgp">
<i class="fas fa-chevron-circle-down"></i>
</button>
</div>
</div>
</div>
<div id="collapsePgp" class="collapse" aria-labelledby="headingPgp" data-parent="#accordionpgp">
<div class="card-body">
<table id="tablepgp" class="table table-striped">
<thead class="thead-dark">
<tr>
<th>PGP Type</th>
<th>Key ID</th>
</tr>
</thead>
<tbody>
{% for dict_key in dict_domain['pgp']%}
{% if dict_key=="mail" %}
{% set var_icon = "fas fa-at" %}
{% elif dict_key=="name" %}
{% set var_icon = "fas fa-user-tag" %}
{% else %}
{% set var_icon = "fas fa-key" %}
{% endif %}
{% for key_id in dict_domain['pgp'][dict_key]%}
<tr>
<td>
<i class="{{ var_icon }}"></i>
&nbsp;&nbsp;{{ dict_key }}
</td>
<td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ key_id }}">{{ key_id }}</a></td>
</tr>
{% endfor %}
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
</div>
{% endif %}
{% if 'cryptocurrency' in dict_domain%}
<div id="accordioncurrency" class="mt-3">
<div class="card">
<div class="card-header" id="headingcurrency">
<div class="row">
<div class="col-11">
<div class="mt-2">
<i class="fas fa-coins"></i> Cryptocurrencies&nbsp;&nbsp;
<div class="badge badge-warning">{{l_64|length}}</div>
</div>
</div>
<div class="col-1">
<button class="btn btn-link collapsed rotate" data-toggle="collapse" data-target="#collapsecurrency" aria-expanded="false" aria-controls="collapsecurrency">
<i class="fas fa-chevron-circle-down"></i>
</button>
</div>
</div>
</div>
<div id="collapsecurrency" class="collapse" aria-labelledby="headingcurrency" data-parent="#accordioncurrency">
<div class="card-body">
<table id="tablecurrency" class="table table-striped">
<thead class="thead-dark">
<tr>
<th>Currency</th>
<th>address</th>
</tr>
</thead>
<tbody>
{% for dict_key in dict_domain['cryptocurrency']%}
{% if dict_key=="bitcoin" %}
{% set var_icon = "fab fa-bitcoin" %}
{% elif dict_key=="monero" %}
{% set var_icon = "fab fa-monero" %}
{% else %}
{% set var_icon = "fas fa-coins" %}
{% endif %}
{% for key_id in dict_domain['cryptocurrency'][dict_key]%}
<tr>
<td>
<i class="{{ var_icon }}"></i>
&nbsp;&nbsp;{{ dict_key }}
</td>
<td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ key_id }}">{{ key_id }}</a></td>
</tr>
{% endfor %}
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
</div>
{% endif %}
<!-- TODO: ADD HASH DECODED -->
{% if l_pastes %}
<hr>
<table class="table table-striped table-bordered table-hover" id="myTable_1">
<thead class="thead-dark">
<tr>
@ -129,7 +233,7 @@
</a>
</div>
{% if domain_history %}
{% if dict_domain["domain_history"] %}
<table class="table table-hover mt-2" id="myTable_2">
<thead>
<tr>
@ -142,7 +246,7 @@
<td class="{% if epoch_item[0]==epoch %}text-dark table-info{% endif %}">
<a href="{{ url_for('hiddenServices.show_domain') }}?domain={{domain}}&port={{port}}&epoch={{epoch_item[0]}}" class="text-secondary">
<div class="d-flex justify-content-around" style="line-height:0.9;">
<div>{{domain}}</div>
<div>{{dict_domain["domain"]}}</div>
{% if epoch_item[2] %}
<div style="color:Green;"><i class="fas fa-check-circle"></i> UP</div>
{% else %}
@ -177,11 +281,13 @@
</div>
<canvas id="canvas" style="width:100%;"></canvas>
<div class="text-center">
<!--
<small>
<a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{screenshot['item']}}" class="text-info">
<div style="line-height:0.9;">{{dict_links[screenshot['item']]}}</div>
</a>
<small>
-->
</div>
</div>
@ -196,13 +302,19 @@
<script>
var table;
$(document).ready(function(){
table = $('#myTable_1').DataTable(
{
//"aLengthMenu": [[5, 10, 15, 20, -1], [5, 10, 15, 20, "All"]],
//"iDisplayLength": 5,
//"order": [[ 0, "desc" ]]
});
});
{% if 'pgp' in dict_domain%}
$('#tablepgp').DataTable();
{% endif %}
{% if 'cryptocurrency' in dict_domain%}
$('#tablecurrency').DataTable({});
{% endif %}
table = $('#myTable_1').DataTable(
{
//"aLengthMenu": [[5, 10, 15, 20, -1], [5, 10, 15, 20, "All"]],
//"iDisplayLength": 5,
//"order": [[ 0, "desc" ]]
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
@ -230,7 +342,7 @@ img.onload = pixelate;
img.addEventListener("error", img_error);
var draw_img = false;
img.src = "{{ url_for('showsavedpastes.screenshot', filename=screenshot['screenshot']) }}";
//img.src = "{{ url_for('showsavedpastes.screenshot', filename=screenshot['screenshot']) }}";
function pixelate() {
/// use slider value