chg: [core + UI] refractor Domain + Show Domain, better handle of ports, select screenshot on the UI, ...

pull/418/head
Terrtia 2019-10-31 17:14:23 +01:00
parent a2d6874417
commit a3e5e44c9b
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
12 changed files with 280 additions and 180 deletions

View File

@ -11,6 +11,7 @@ import os
import sys
import time
import redis
import random
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Correlation
@ -32,6 +33,34 @@ def get_domain_type(domain):
else:
return 'regular'
def sanathyse_port(port, domain, domain_type, strict=False, current_port=None):
'''
Retun a port number, If the port number is invalid, a port of the provided domain is randomly selected
'''
try:
port = int(port)
except (TypeError, ValueError):
if strict:
port = current_port
else:
port = get_random_domain_port(domain, domain_type)
return port
def is_domain_up(domain, domain_type):
return r_serv_onion.hexists('{}_metadata:{}'.format(domain_type, domain), 'ports')
def get_domain_all_ports(domain, domain_type):
'''
Return a list of all crawled ports
'''
l_ports = r_serv_onion.hget('{}_metadata:{}'.format(domain_type, domain), 'ports')
if l_ports:
return l_ports.split(";")
return []
def get_random_domain_port(domain, domain_type):
return random.choice(get_domain_all_ports(domain, domain_type))
def get_all_domain_up_by_type(domain_type):
if domain_type in domains:
list_domain = list(r_serv_onion.smembers('full_{}_up'.format(domain_type)))
@ -52,6 +81,53 @@ def get_domain_item_children(domain, root_item_id):
all_items.extend(get_domain_item_children(domain, item_id))
return all_items
def get_domain_last_crawled_item_root(domain, domain_type, port):
'''
Retun last_crawled_item_core dict
'''
res = r_serv_onion.zrevrange('crawler_history_{}:{}:{}'.format(domain_type, domain, port), 0, 0, withscores=True)
if res:
return {"root_item": res[0][0], "epoch": int(res[0][1])}
else:
return {}
def get_domain_crawled_item_root(domain, domain_type, port, epoch=None):
'''
Retun the first item crawled for a given domain:port (and epoch)
'''
if epoch:
res = r_serv_onion.zrevrangebyscore('crawler_history_{}:{}:{}'.format(domain_type, domain, port), int(epoch), int(epoch))
if res:
return {"root_item": res[0], "epoch": int(epoch)}
# invalid epoch
epoch = None
if not epoch:
return get_domain_last_crawled_item_root(domain, domain_type, port)
def get_domain_items_crawled(domain, domain_type, port, epoch=None, items_link=False, item_screenshot=False, item_tag=False):
'''
'''
item_crawled = {}
item_root = get_domain_crawled_item_root(domain, domain_type, port, epoch=epoch)
if item_root:
item_crawled['port'] = port
item_crawled['epoch'] = item_root['epoch']
item_crawled['date'] = time.strftime('%Y/%m/%d - %H:%M.%S', time.gmtime(item_root['epoch']))
item_crawled['items'] = []
for item in get_domain_items(domain, item_root['root_item']):
dict_item = {"id": item}
if items_link:
dict_item['link'] = Item.get_item_link(item)
if item_screenshot:
dict_item['screenshot'] = Item.get_item_screenshot(item)
if item_tag:
dict_item['tags'] = Tag.get_item_tags_minimal(item)
item_crawled['items'].append(dict_item)
return item_crawled
def get_link_tree():
pass
@ -65,7 +141,7 @@ def get_domain_tags(domain):
'''
return Tag.get_item_tags(domain)
def get_domain_cryptocurrency(domain, currencies_type=None):
def get_domain_cryptocurrency(domain, currencies_type=None, get_nb=False):
'''
Retun all cryptocurrencies of a given domain.
@ -73,9 +149,9 @@ def get_domain_cryptocurrency(domain, currencies_type=None):
:param currencies_type: list of cryptocurrencies type
:type currencies_type: list, optional
'''
return cryptocurrency.get_domain_correlation_dict(domain, correlation_type=currencies_type)
return cryptocurrency.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb)
def get_domain_pgp(domain, currencies_type=None):
def get_domain_pgp(domain, currencies_type=None, get_nb=False):
'''
Retun all pgp of a given domain.
@ -83,9 +159,9 @@ def get_domain_pgp(domain, currencies_type=None):
:param currencies_type: list of pgp type
:type currencies_type: list, optional
'''
return pgp.get_domain_correlation_dict(domain, correlation_type=currencies_type)
return pgp.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb)
def get_domain_all_correlation(domain, correlation_type=None):
def get_domain_all_correlation(domain, correlation_type=None, get_nb=False):
'''
Retun all correlation of a given domain.
@ -96,10 +172,10 @@ def get_domain_all_correlation(domain, correlation_type=None):
:rtype: dict
'''
domain_correl = {}
res = get_domain_cryptocurrency(domain)
res = get_domain_cryptocurrency(domain, get_nb=get_nb)
if res:
domain_correl['cryptocurrency'] = res
res = get_domain_pgp(domain)
res = get_domain_pgp(domain, get_nb=get_nb)
if res:
domain_correl['pgp'] = res
return domain_correl
@ -141,13 +217,26 @@ def get_domain_history_with_status(domain, domain_type, port): # TODO: add date_
l_history.append({"epoch": epoch_val, "date": time.strftime('%Y/%m/%d - %H:%M.%S', time.gmtime(epoch_val)), "status": status})
return l_history
def verify_if_domain_exist(domain):
return r_serv_onion.exists('{}_metadata:{}'.format(get_domain_type(domain), domain))
def api_verify_if_domain_exist(domain):
if not verify_if_domain_exist(domain):
return ({'status': 'error', 'reason': 'Unknow Domain'}, 404)
else:
return None
class Domain(object):
"""docstring for Domain."""
def __init__(self, domain, port=80):
def __init__(self, domain, port=None):
self.domain = str(domain)
self.type = get_domain_type(domain)
if self.is_domain_up():
self.current_port = sanathyse_port(port, self.domain, self.type)
def get_current_port(self):
return self.current_port
def get_domain_first_seen(self):
'''
@ -161,7 +250,7 @@ class Domain(object):
first_seen = '{}/{}/{}'.format(first_seen[0:4], first_seen[4:6], first_seen[6:8])
return first_seen
def get_domain_last_check(self):# # TODO: add epoch ???
def get_domain_last_check(self):
'''
Get domain last check date
@ -173,10 +262,16 @@ class Domain(object):
last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8])
return last_check
#def get_domain_all_ports(self):
# pass
def is_domain_up(self): # # TODO: handle multiple ports
'''
Return True if this domain is UP
'''
return is_domain_up(self.domain, self.type)
def get_domain_metadata(self, first_seen=True, last_ckeck=True, ports=True):
def get_domain_all_ports(self):
return get_domain_all_ports(self.domain, self.type)
def get_domain_metadata(self, first_seen=True, last_ckeck=True, status=True, ports=True):
'''
Get Domain basic metadata
@ -199,6 +294,10 @@ class Domain(object):
res = self.get_domain_last_check()
if res is not None:
dict_metadata['last_check'] = res
if status:
dict_metadata['status'] = self.is_domain_up()
if ports:
dict_metadata['ports'] = self.get_domain_all_ports()
return dict_metadata
def get_domain_tags(self):
@ -213,10 +312,17 @@ class Domain(object):
'''
Retun all cryptocurrencies of a given domain.
'''
return get_domain_all_correlation(self.domain)
return get_domain_all_correlation(self.domain, get_nb=True)
def get_domain_history_with_status(self):
'''
Retun the full history of a given domain and port.
'''
return get_domain_history_with_status(self.domain, self.type, 80)
def get_domain_items_crawled(self, port=None, epoch=None, items_link=False, item_screenshot=False, item_tag=False):
'''
Return ........................
'''
port = sanathyse_port(port, self.domain, self.type, strict=True, current_port=self.current_port)
return get_domain_items_crawled(self.domain, self.type, port, epoch=epoch, items_link=items_link, item_screenshot=item_screenshot, item_tag=item_tag)

View File

@ -126,7 +126,7 @@ class Correlation(object):
else:
return []
def get_domain_correlation_dict(self, domain, correlation_type=None):
def get_domain_correlation_dict(self, domain, correlation_type=None, get_nb=False):
'''
Return all correlation of a given domain.
@ -143,6 +143,8 @@ class Correlation(object):
res = self._get_domain_correlation_obj(domain, correl)
if res:
dict_correlation[correl] = res
if get_nb:
dict_correlation['nb'] = dict_correlation.get('nb', 0) + len(dict_correlation[correl])
return dict_correlation

View File

@ -177,6 +177,14 @@ def is_item_in_domain(domain, item_id):
def get_item_domain(item_id):
return item_id[19:-36]
def get_item_children(item_id):
return list(r_serv_metadata.smembers('paste_children:{}'.format(item_id)))
def get_item_link(item_id):
return r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'real_link')
def get_item_screenshot(item_id):
screenshot = r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'screenshot')
if screenshot:
return os.path.join(screenshot[0:2], screenshot[2:4], screenshot[4:6], screenshot[6:8], screenshot[8:10], screenshot[10:12], screenshot[12:])
return ''

View File

@ -93,6 +93,22 @@ def get_item_tags(item_id):
else:
return []
def get_min_tag(tag):
tag = tag.split('=')
if len(tag) > 1:
if tag[1] != '':
tag = tag[1][1:-1]
# no value
else:
tag = tag[0][1:-1]
# custom tags
else:
tag = tag[0]
return tag
def get_item_tags_minimal(item_id):
return [ {"tag": tag, "min_tag": get_min_tag(tag)} for tag in get_item_tags(item_id) ]
# TEMPLATE + API QUERY
def add_items_tag(tags=[], galaxy_tags=[], item_id=None):
res_dict = {}

View File

@ -7,6 +7,8 @@
import os
import sys
import json
import random
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
from flask_login import login_required, current_user, login_user, logout_user
@ -34,7 +36,9 @@ crawler_splash = Blueprint('crawler_splash', __name__, template_folder=os.path.j
# ============ FUNCTIONS ============
def api_validator(api_response):
if api_response:
return Response(json.dumps(api_response[0], indent=2, sort_keys=True), mimetype='application/json'), api_response[1]
# ============= ROUTES ==============
@ -46,14 +50,19 @@ def showDomain():
epoch = request.args.get('epoch')
port = request.args.get('port')
domain = Domain.Domain(domain_name)
res = api_validator(Domain.api_verify_if_domain_exist(domain_name))
if res:
return res
domain = Domain.Domain(domain_name, port=port)
dict_domain = domain.get_domain_metadata()
dict_domain = {**dict_domain, **domain.get_domain_correlation()}
dict_domain['domain'] = domain_name
if domain.is_domain_up():
dict_domain = {**dict_domain, **domain.get_domain_correlation()}
dict_domain['tags'] = domain.get_domain_tags()
dict_domain['history'] = domain.get_domain_history_with_status()
dict_domain['crawler_history'] = domain.get_domain_items_crawled(items_link=True, epoch=epoch, item_screenshot=True, item_tag=True) # # TODO: handle multiple port
dict_domain['crawler_history']['random_item'] = random.choice(dict_domain['crawler_history']['items'])
print(dict_domain)
return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label, screenshot={'item': None, '':None}, dict_links={})
return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label)

View File

@ -730,91 +730,6 @@ def show_domains_by_daterange():
date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down,
domains_tags=domains_tags, type=service_type, bootstrap_label=bootstrap_label)
@hiddenServices.route("/crawlers/show_domain", methods=['GET'])
@login_required
@login_analyst
def show_domain():
domain = request.args.get('domain')
epoch = request.args.get('epoch')
try:
epoch = int(epoch)
except:
epoch = None
port = request.args.get('port')
faup.decode(domain)
unpack_url = faup.get()
## TODO: # FIXME: remove me
try:
domain = unpack_url['domain'].decode()
except:
domain = unpack_url['domain']
if not port:
if unpack_url['port']:
try:
port = unpack_url['port'].decode()
except:
port = unpack_url['port']
else:
port = 80
try:
port = int(port)
except:
port = 80
type = get_type_domain(domain)
if domain is None or not r_serv_onion.exists('{}_metadata:{}'.format(type, domain)):
return '404'
# # TODO: FIXME return 404
last_check = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'last_check')
if last_check is None:
last_check = '********'
last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8])
first_seen = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'first_seen')
if first_seen is None:
first_seen = '********'
first_seen = '{}/{}/{}'.format(first_seen[0:4], first_seen[4:6], first_seen[6:8])
ports = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'ports')
origin_paste = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'paste_parent')
h = HiddenServices(domain, type, port=port)
item_core = h.get_domain_crawled_core_item(epoch=epoch)
if item_core:
l_pastes = h.get_last_crawled_pastes(item_root=item_core['root_item'])
else:
l_pastes = []
dict_links = h.get_all_links(l_pastes)
if l_pastes:
status = True
else:
status = False
last_check = '{} - {}'.format(last_check, time.strftime('%H:%M.%S', time.gmtime(epoch)))
screenshot = h.get_domain_random_screenshot(l_pastes)
if screenshot:
screenshot = screenshot[0]
else:
screenshot = 'None'
domain_tags = h.get_domain_tags()
origin_paste_name = h.get_origin_paste_name()
origin_paste_tags = unpack_paste_tags(r_serv_metadata.smembers('tag:{}'.format(origin_paste)))
paste_tags = []
for path in l_pastes:
p_tags = r_serv_metadata.smembers('tag:'+path)
paste_tags.append(unpack_paste_tags(p_tags))
domain_history = h.extract_epoch_from_history(h.get_domain_crawled_history())
return render_template("showDomain.html", domain=domain, last_check=last_check, first_seen=first_seen,
l_pastes=l_pastes, paste_tags=paste_tags, bootstrap_label=bootstrap_label,
dict_links=dict_links, port=port, epoch=epoch,
ports=ports, domain_history=domain_history,
origin_paste_tags=origin_paste_tags, status=status,
origin_paste=origin_paste, origin_paste_name=origin_paste_name,
domain_tags=domain_tags, screenshot=screenshot)
@hiddenServices.route("/crawlers/download_domain", methods=['GET'])
@login_required
@login_analyst

View File

@ -67,7 +67,7 @@
title="<span class='badge badge-dark'>{{metadata_domain['domain']}}</span>"
data-content="port: <span class='badge badge-secondary'>{{metadata_domain['port']}}</span><br>
epoch: {{metadata_domain['epoch']}}">
<td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['domain_name'] }}</a></td>
<td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['domain_name'] }}</a></td>
<td>{{'{}/{}/{}'.format(metadata_domain['first_seen'][0:4], metadata_domain['first_seen'][4:6], metadata_domain['first_seen'][6:8])}}</td>
<td>{{'{}/{}/{}'.format(metadata_domain['last_check'][0:4], metadata_domain['last_check'][4:6], metadata_domain['last_check'][6:8])}}</td>
<td><div style="color:{{metadata_domain['status_color']}}; display:inline-block">
@ -205,7 +205,7 @@ function refresh_list_crawled(){
var newRow = tableRef.insertRow(tableRef.rows.length);
var newCell = newRow.insertCell(0);
newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('hiddenServices.show_domain') }}?onion_domain="+data_domain['domain']+"\">"+data_domain['domain']+"</a></td>";
newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('crawler_splash.showDomain') }}?onion_domain="+data_domain['domain']+"\">"+data_domain['domain']+"</a></td>";
newCell = newRow.insertCell(1);
newCell.innerHTML = "<td>"+data_domain['first_seen'].substr(0, 4)+"/"+data_domain['first_seen'].substr(4, 2)+"/"+data_domain['first_seen'].substr(6, 2)+"</td>"
@ -243,7 +243,7 @@ function refresh_list_crawled(){
newCell.innerHTML = "<td><i class=\"fa fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i>"+crawler['crawler_info']+"</td>";
newCell = newRow.insertCell(1);
newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('hiddenServices.show_domain') }}?onion_domain="+crawler['crawling_domain']+"\">"+crawler['crawling_domain']+"</a></td>";
newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('crawler_splash.showDomain') }}?onion_domain="+crawler['crawling_domain']+"\">"+crawler['crawling_domain']+"</a></td>";
newCell = newRow.insertCell(2);
newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";

View File

@ -45,7 +45,7 @@
<tbody id="tbody_last_crawled">
{% for metadata_domain in last_domains %}
<tr>
<td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['domain_name'] }}</a></td>
<td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['domain_name'] }}</a></td>
<td>{{'{}/{}/{}'.format(metadata_domain['first_seen'][0:4], metadata_domain['first_seen'][4:6], metadata_domain['first_seen'][6:8])}}</td>
<td>{{'{}/{}/{}'.format(metadata_domain['last_check'][0:4], metadata_domain['last_check'][4:6], metadata_domain['last_check'][6:8])}}</td>
<td><div style="color:{{metadata_domain['status_color']}}; display:inline-block">
@ -76,7 +76,7 @@
<tbody id="tbody_last_crawled">
{% for metadata_domain in auto_crawler_domain_onions_metadata %}
<tr>
<td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['url'] }}</a></td>
<td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['url'] }}</a></td>
<td><a class="btn btn-outline-danger px-1 py-0" href="{{ url_for('hiddenServices.remove_auto_crawler') }}?url={{ metadata_domain['url'] }}&page={{page}}">
<i class="fas fa-trash-alt"></i></a>
</td>
@ -110,7 +110,7 @@
<tbody id="tbody_last_crawled">
{% for metadata_domain in auto_crawler_domain_regular_metadata %}
<tr>
<td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['url'] }}</a></td>
<td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['url'] }}</a></td>
<td><a class="btn btn-outline-danger px-1 py-0" href="{{ url_for('hiddenServices.remove_auto_crawler') }}?url={{ metadata_domain['url'] }}&page={{page}}">
<i class="fas fa-trash-alt"></i></a>
</td>

View File

@ -64,7 +64,7 @@
{% for domain in domains_by_day[date] %}
<tr>
<td>
<a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ domain }}">{{ domain }}</a>
<a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ domain }}">{{ domain }}</a>
<div>
{% for tag in domain_metadata[domain]['tags'] %}
<a href="{{ url_for('Tags.Tags_page') }}?date_from={{date}}&date_to={{date}}&ltags={{ tag }}">

View File

@ -169,7 +169,7 @@
<tbody>
<tr>
<td>Domain</td>
<td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ crawler_metadata['domain'] }}" id='domain'>{{ crawler_metadata['domain'] }}</a></td>
<td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ crawler_metadata['domain'] }}" id='domain'>{{ crawler_metadata['domain'] }}</a></td>
</tr>
<tr>
<td>Father</td>

View File

@ -433,7 +433,7 @@
<tbody>
<tr>
<td>Domain</td>
<td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ crawler_metadata['domain'] }}" id='domain'>{{ crawler_metadata['domain'] }}</a></td>
<td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ crawler_metadata['domain'] }}" id='domain'>{{ crawler_metadata['domain'] }}</a></td>
</tr>
<tr>
<td>Father</td>

View File

@ -33,7 +33,7 @@
<div class="card mt-2">
<div class="card-header bg-dark">
<span class="badge badge-pill badge-light flex-row-reverse float-right">
{% if status %}
{% if dict_domain["status"] %}
<div style="color:Green;">
<i class="fas fa-check-circle fa-2x"></i>
UP
@ -60,7 +60,7 @@
<tr>
<td class="panelText">{%if "first_seen" in dict_domain%}{{ dict_domain['first_seen'] }}{%endif%}</td>
<td class="panelText">{%if "last_check" in dict_domain%}{{ dict_domain['last_check'] }}{%endif%}</td>
<td class="panelText">{{ ports }}</td>
<td class="panelText">{%if dict_domain["ports"]%}{{ dict_domain["ports"] }}{%endif%}</td>
</tr>
</tbody>
</table>
@ -91,7 +91,7 @@
<div class="col-11">
<div class="mt-2">
<i class="fas fa-key"></i> PGP Dumps&nbsp;&nbsp;
<div class="badge badge-warning">{{l_64|length}}</div>
<div class="badge badge-warning">{{dict_domain['pgp']['nb']}}</div>
</div>
</div>
<div class="col-1">
@ -112,6 +112,7 @@
</thead>
<tbody>
{% for dict_key in dict_domain['pgp']%}
{% if dict_key != "nb" %}
{% if dict_key=="mail" %}
{% set var_icon = "fas fa-at" %}
{% elif dict_key=="name" %}
@ -128,6 +129,7 @@
<td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ key_id }}">{{ key_id }}</a></td>
</tr>
{% endfor %}
{% endif %}
{% endfor %}
</tbody>
</table>
@ -146,7 +148,7 @@
<div class="col-11">
<div class="mt-2">
<i class="fas fa-coins"></i> Cryptocurrencies&nbsp;&nbsp;
<div class="badge badge-warning">{{l_64|length}}</div>
<div class="badge badge-warning">{{dict_domain['cryptocurrency']['nb']}}</div>
</div>
</div>
<div class="col-1">
@ -167,6 +169,7 @@
</thead>
<tbody>
{% for dict_key in dict_domain['cryptocurrency']%}
{% if dict_key != "nb" %}
{% if dict_key=="bitcoin" %}
{% set var_icon = "fab fa-bitcoin" %}
{% elif dict_key=="monero" %}
@ -183,6 +186,7 @@
<td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ key_id }}">{{ key_id }}</a></td>
</tr>
{% endfor %}
{% endif %}
{% endfor %}
</tbody>
</table>
@ -195,45 +199,70 @@
<!-- TODO: ADD HASH DECODED -->
{% if l_pastes %}
<hr>
{% if dict_domain["crawler_history"] %}
<hr class="my-4">
<div class="card mb-1">
<div class="card-header">
<div class="flex-row-reverse float-right text-secondary">
Date:
<span class="badge badge-pill badge-info mt-1">
{{dict_domain["crawler_history"]["date"]}}
</span>
&nbsp;&nbsp;PORT:
<span class="badge badge-pill badge-warning mt-1">
{{dict_domain["crawler_history"]["port"]}}
</span>
</div>
<h5>Crawled Items</h5>
</div>
</div>
<table class="table table-striped table-bordered table-hover" id="myTable_1">
<thead class="thead-dark">
<tr>
<th>Crawled Pastes</th>
<th class="text-center"><i class="fas fa-images"></i></th>
</tr>
</thead>
<tbody>
{% for path in l_pastes %}
{% for item in dict_domain["crawler_history"]["items"] %}
<tr>
<td>
<a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{path}}" class="text-secondary">
<div style="line-height:0.9;">{{ dict_links[path] }}</div>
<a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{item["id"]}}" class="text-secondary">
<div style="line-height:0.9;">{{ item["link"] }}</div>
</a>
<div>
{% for tag in paste_tags[loop.index0] %}
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag[1] }}">
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
{% for tag in item["tags"] %}
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag["tag"] }}">
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag["min_tag"] }}</span>
</a>
{% endfor %}
</div>
</td>
<td class="text-center">
{%if item["screenshot"]%}
<button class="btn" onclick="reload_image('{{ item["screenshot"] }}', '{{ item["link"] }}', '{{ item["id"] }}');">
<i class="far fa-image"></i>
</button>
{%endif%}
</td>
</tr>
{% endfor %}
</tbody>
</table>
{%endif%}
<div class="text-center my-3">
<div class="text-center mb-4">
<a href="{{ url_for('hiddenServices.download_domain') }}?domain={{domain}}&port={{port}}{% if epoch %}&epoch={{epoch}}{% endif %}" target="blank" style="font-size: 15px">
<button class="btn btn-info"><i class="fas fa-download"></i> Download Domain Content
</button>
</a>
</div>
{%endif%}
{% if dict_domain["domain_history"] %}
{% if dict_domain["history"] %}
<table class="table table-hover mt-2" id="myTable_2">
<thead>
<tr>
@ -241,18 +270,18 @@
</tr>
</thead>
<tbody>
{% for epoch_item in domain_history %}
{% for dom_history in dict_domain["history"] %}
<tr>
<td class="{% if epoch_item[0]==epoch %}text-dark table-info{% endif %}">
<a href="{{ url_for('hiddenServices.show_domain') }}?domain={{domain}}&port={{port}}&epoch={{epoch_item[0]}}" class="text-secondary">
<td class="{% if dom_history["epoch"]==dict_domain["crawler_history"]["epoch"] %}text-dark table-info{% endif %}">
<a href="{{ url_for('crawler_splash.showDomain') }}?domain={{dict_domain["domain"]}}&port={{dict_domain["crawler_history"]["port"]}}&epoch={{dom_history["epoch"]}}" class="text-secondary">
<div class="d-flex justify-content-around" style="line-height:0.9;">
<div>{{dict_domain["domain"]}}</div>
{% if epoch_item[2] %}
{% if dom_history["status"] %}
<div style="color:Green;"><i class="fas fa-check-circle"></i> UP</div>
{% else %}
<div style="color:Red;"><i class="fas fa-times-circle"></i> DOWN</div>
{% endif %}
<div>{{ epoch_item[1] }}</div>
<div>{{ dom_history["date"] }}</div>
</div>
</a>
</tr>
@ -281,13 +310,9 @@
</div>
<canvas id="canvas" style="width:100%;"></canvas>
<div class="text-center">
<!--
<small class="text-info" style="line-height:0.9;">
<a target="_blank" href="" id="screenshot_link"></a>
<small>
<a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{screenshot['item']}}" class="text-info">
<div style="line-height:0.9;">{{dict_links[screenshot['item']]}}</div>
</a>
<small>
-->
</div>
</div>
@ -333,6 +358,8 @@ function toggle_sidebar(){
<script>
var ctx = canvas.getContext('2d'), img = new Image();
var base_url = "{{ url_for('showsavedpastes.screenshot', filename="") }}"
var screenshot_href = "{{ url_for('showsavedpastes.showsavedpaste') }}?paste="
/// turn off image smoothing
ctx.webkitImageSmoothingEnabled = false;
@ -342,7 +369,15 @@ img.onload = pixelate;
img.addEventListener("error", img_error);
var draw_img = false;
//img.src = "{{ url_for('showsavedpastes.screenshot', filename=screenshot['screenshot']) }}";
{%if 'crawler_history' in dict_domain%}
var screenshot = "{{dict_domain['crawler_history']['random_item']['screenshot']}}"
$("#screenshot_link").attr("href", "screenshot_href + {{dict_domain['crawler_history']['random_item']['id']}}")
$("#screenshot_link").text("{{dict_domain['crawler_history']['random_item']['link']}}")
{%else%}
var screenshot = ""
{%endif%}
img.src = base_url + screenshot;
function pixelate() {
/// use slider value
@ -375,6 +410,15 @@ function img_error() {
blocks.value = 50;
pixelate;
}
function reload_image(screenshot, link, item_id) {
img.src=base_url + screenshot;
$("#screenshot_link").attr("href", screenshot_href + item_id)
$("#screenshot_link").text(link)
pixelate;
}
</script>
</html>