mirror of https://github.com/CIRCL/AIL-framework
chg: [core + UI] refractor Domain + Show Domain, better handle of ports, select screenshot on the UI, ...
Terrtia
parent
a2d6874417
commit
a3e5e44c9b
|
|
@ -11,6 +11,7 @@ import os
|
|||
import sys
|
||||
import time
|
||||
import redis
|
||||
import random
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||
import Correlation
|
||||
|
|
@ -32,6 +33,34 @@ def get_domain_type(domain):
|
|||
else:
|
||||
return 'regular'
|
||||
|
||||
def sanathyse_port(port, domain, domain_type, strict=False, current_port=None):
|
||||
'''
|
||||
Retun a port number, If the port number is invalid, a port of the provided domain is randomly selected
|
||||
'''
|
||||
try:
|
||||
port = int(port)
|
||||
except (TypeError, ValueError):
|
||||
if strict:
|
||||
port = current_port
|
||||
else:
|
||||
port = get_random_domain_port(domain, domain_type)
|
||||
return port
|
||||
|
||||
def is_domain_up(domain, domain_type):
|
||||
return r_serv_onion.hexists('{}_metadata:{}'.format(domain_type, domain), 'ports')
|
||||
|
||||
def get_domain_all_ports(domain, domain_type):
|
||||
'''
|
||||
Return a list of all crawled ports
|
||||
'''
|
||||
l_ports = r_serv_onion.hget('{}_metadata:{}'.format(domain_type, domain), 'ports')
|
||||
if l_ports:
|
||||
return l_ports.split(";")
|
||||
return []
|
||||
|
||||
def get_random_domain_port(domain, domain_type):
|
||||
return random.choice(get_domain_all_ports(domain, domain_type))
|
||||
|
||||
def get_all_domain_up_by_type(domain_type):
|
||||
if domain_type in domains:
|
||||
list_domain = list(r_serv_onion.smembers('full_{}_up'.format(domain_type)))
|
||||
|
|
@ -52,6 +81,53 @@ def get_domain_item_children(domain, root_item_id):
|
|||
all_items.extend(get_domain_item_children(domain, item_id))
|
||||
return all_items
|
||||
|
||||
def get_domain_last_crawled_item_root(domain, domain_type, port):
|
||||
'''
|
||||
Retun last_crawled_item_core dict
|
||||
'''
|
||||
res = r_serv_onion.zrevrange('crawler_history_{}:{}:{}'.format(domain_type, domain, port), 0, 0, withscores=True)
|
||||
if res:
|
||||
return {"root_item": res[0][0], "epoch": int(res[0][1])}
|
||||
else:
|
||||
return {}
|
||||
|
||||
def get_domain_crawled_item_root(domain, domain_type, port, epoch=None):
|
||||
'''
|
||||
Retun the first item crawled for a given domain:port (and epoch)
|
||||
'''
|
||||
if epoch:
|
||||
res = r_serv_onion.zrevrangebyscore('crawler_history_{}:{}:{}'.format(domain_type, domain, port), int(epoch), int(epoch))
|
||||
if res:
|
||||
return {"root_item": res[0], "epoch": int(epoch)}
|
||||
# invalid epoch
|
||||
epoch = None
|
||||
|
||||
if not epoch:
|
||||
return get_domain_last_crawled_item_root(domain, domain_type, port)
|
||||
|
||||
|
||||
def get_domain_items_crawled(domain, domain_type, port, epoch=None, items_link=False, item_screenshot=False, item_tag=False):
|
||||
'''
|
||||
|
||||
'''
|
||||
item_crawled = {}
|
||||
item_root = get_domain_crawled_item_root(domain, domain_type, port, epoch=epoch)
|
||||
if item_root:
|
||||
item_crawled['port'] = port
|
||||
item_crawled['epoch'] = item_root['epoch']
|
||||
item_crawled['date'] = time.strftime('%Y/%m/%d - %H:%M.%S', time.gmtime(item_root['epoch']))
|
||||
item_crawled['items'] = []
|
||||
for item in get_domain_items(domain, item_root['root_item']):
|
||||
dict_item = {"id": item}
|
||||
if items_link:
|
||||
dict_item['link'] = Item.get_item_link(item)
|
||||
if item_screenshot:
|
||||
dict_item['screenshot'] = Item.get_item_screenshot(item)
|
||||
if item_tag:
|
||||
dict_item['tags'] = Tag.get_item_tags_minimal(item)
|
||||
item_crawled['items'].append(dict_item)
|
||||
return item_crawled
|
||||
|
||||
def get_link_tree():
|
||||
pass
|
||||
|
||||
|
|
@ -65,7 +141,7 @@ def get_domain_tags(domain):
|
|||
'''
|
||||
return Tag.get_item_tags(domain)
|
||||
|
||||
def get_domain_cryptocurrency(domain, currencies_type=None):
|
||||
def get_domain_cryptocurrency(domain, currencies_type=None, get_nb=False):
|
||||
'''
|
||||
Retun all cryptocurrencies of a given domain.
|
||||
|
||||
|
|
@ -73,9 +149,9 @@ def get_domain_cryptocurrency(domain, currencies_type=None):
|
|||
:param currencies_type: list of cryptocurrencies type
|
||||
:type currencies_type: list, optional
|
||||
'''
|
||||
return cryptocurrency.get_domain_correlation_dict(domain, correlation_type=currencies_type)
|
||||
return cryptocurrency.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb)
|
||||
|
||||
def get_domain_pgp(domain, currencies_type=None):
|
||||
def get_domain_pgp(domain, currencies_type=None, get_nb=False):
|
||||
'''
|
||||
Retun all pgp of a given domain.
|
||||
|
||||
|
|
@ -83,9 +159,9 @@ def get_domain_pgp(domain, currencies_type=None):
|
|||
:param currencies_type: list of pgp type
|
||||
:type currencies_type: list, optional
|
||||
'''
|
||||
return pgp.get_domain_correlation_dict(domain, correlation_type=currencies_type)
|
||||
return pgp.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb)
|
||||
|
||||
def get_domain_all_correlation(domain, correlation_type=None):
|
||||
def get_domain_all_correlation(domain, correlation_type=None, get_nb=False):
|
||||
'''
|
||||
Retun all correlation of a given domain.
|
||||
|
||||
|
|
@ -96,10 +172,10 @@ def get_domain_all_correlation(domain, correlation_type=None):
|
|||
:rtype: dict
|
||||
'''
|
||||
domain_correl = {}
|
||||
res = get_domain_cryptocurrency(domain)
|
||||
res = get_domain_cryptocurrency(domain, get_nb=get_nb)
|
||||
if res:
|
||||
domain_correl['cryptocurrency'] = res
|
||||
res = get_domain_pgp(domain)
|
||||
res = get_domain_pgp(domain, get_nb=get_nb)
|
||||
if res:
|
||||
domain_correl['pgp'] = res
|
||||
return domain_correl
|
||||
|
|
@ -141,13 +217,26 @@ def get_domain_history_with_status(domain, domain_type, port): # TODO: add date_
|
|||
l_history.append({"epoch": epoch_val, "date": time.strftime('%Y/%m/%d - %H:%M.%S', time.gmtime(epoch_val)), "status": status})
|
||||
return l_history
|
||||
|
||||
def verify_if_domain_exist(domain):
|
||||
return r_serv_onion.exists('{}_metadata:{}'.format(get_domain_type(domain), domain))
|
||||
|
||||
def api_verify_if_domain_exist(domain):
|
||||
if not verify_if_domain_exist(domain):
|
||||
return ({'status': 'error', 'reason': 'Unknow Domain'}, 404)
|
||||
else:
|
||||
return None
|
||||
|
||||
class Domain(object):
|
||||
"""docstring for Domain."""
|
||||
|
||||
def __init__(self, domain, port=80):
|
||||
def __init__(self, domain, port=None):
|
||||
self.domain = str(domain)
|
||||
self.type = get_domain_type(domain)
|
||||
if self.is_domain_up():
|
||||
self.current_port = sanathyse_port(port, self.domain, self.type)
|
||||
|
||||
def get_current_port(self):
|
||||
return self.current_port
|
||||
|
||||
def get_domain_first_seen(self):
|
||||
'''
|
||||
|
|
@ -161,7 +250,7 @@ class Domain(object):
|
|||
first_seen = '{}/{}/{}'.format(first_seen[0:4], first_seen[4:6], first_seen[6:8])
|
||||
return first_seen
|
||||
|
||||
def get_domain_last_check(self):# # TODO: add epoch ???
|
||||
def get_domain_last_check(self):
|
||||
'''
|
||||
Get domain last check date
|
||||
|
||||
|
|
@ -173,10 +262,16 @@ class Domain(object):
|
|||
last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8])
|
||||
return last_check
|
||||
|
||||
#def get_domain_all_ports(self):
|
||||
# pass
|
||||
def is_domain_up(self): # # TODO: handle multiple ports
|
||||
'''
|
||||
Return True if this domain is UP
|
||||
'''
|
||||
return is_domain_up(self.domain, self.type)
|
||||
|
||||
def get_domain_metadata(self, first_seen=True, last_ckeck=True, ports=True):
|
||||
def get_domain_all_ports(self):
|
||||
return get_domain_all_ports(self.domain, self.type)
|
||||
|
||||
def get_domain_metadata(self, first_seen=True, last_ckeck=True, status=True, ports=True):
|
||||
'''
|
||||
Get Domain basic metadata
|
||||
|
||||
|
|
@ -199,6 +294,10 @@ class Domain(object):
|
|||
res = self.get_domain_last_check()
|
||||
if res is not None:
|
||||
dict_metadata['last_check'] = res
|
||||
if status:
|
||||
dict_metadata['status'] = self.is_domain_up()
|
||||
if ports:
|
||||
dict_metadata['ports'] = self.get_domain_all_ports()
|
||||
return dict_metadata
|
||||
|
||||
def get_domain_tags(self):
|
||||
|
|
@ -213,10 +312,17 @@ class Domain(object):
|
|||
'''
|
||||
Retun all cryptocurrencies of a given domain.
|
||||
'''
|
||||
return get_domain_all_correlation(self.domain)
|
||||
return get_domain_all_correlation(self.domain, get_nb=True)
|
||||
|
||||
def get_domain_history_with_status(self):
|
||||
'''
|
||||
Retun the full history of a given domain and port.
|
||||
'''
|
||||
return get_domain_history_with_status(self.domain, self.type, 80)
|
||||
|
||||
def get_domain_items_crawled(self, port=None, epoch=None, items_link=False, item_screenshot=False, item_tag=False):
|
||||
'''
|
||||
Return ........................
|
||||
'''
|
||||
port = sanathyse_port(port, self.domain, self.type, strict=True, current_port=self.current_port)
|
||||
return get_domain_items_crawled(self.domain, self.type, port, epoch=epoch, items_link=items_link, item_screenshot=item_screenshot, item_tag=item_tag)
|
||||
|
|
|
|||
|
|
@ -126,7 +126,7 @@ class Correlation(object):
|
|||
else:
|
||||
return []
|
||||
|
||||
def get_domain_correlation_dict(self, domain, correlation_type=None):
|
||||
def get_domain_correlation_dict(self, domain, correlation_type=None, get_nb=False):
|
||||
'''
|
||||
Return all correlation of a given domain.
|
||||
|
||||
|
|
@ -143,6 +143,8 @@ class Correlation(object):
|
|||
res = self._get_domain_correlation_obj(domain, correl)
|
||||
if res:
|
||||
dict_correlation[correl] = res
|
||||
if get_nb:
|
||||
dict_correlation['nb'] = dict_correlation.get('nb', 0) + len(dict_correlation[correl])
|
||||
return dict_correlation
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -177,6 +177,14 @@ def is_item_in_domain(domain, item_id):
|
|||
def get_item_domain(item_id):
|
||||
return item_id[19:-36]
|
||||
|
||||
|
||||
def get_item_children(item_id):
|
||||
return list(r_serv_metadata.smembers('paste_children:{}'.format(item_id)))
|
||||
|
||||
def get_item_link(item_id):
|
||||
return r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'real_link')
|
||||
|
||||
def get_item_screenshot(item_id):
|
||||
screenshot = r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'screenshot')
|
||||
if screenshot:
|
||||
return os.path.join(screenshot[0:2], screenshot[2:4], screenshot[4:6], screenshot[6:8], screenshot[8:10], screenshot[10:12], screenshot[12:])
|
||||
return ''
|
||||
|
|
|
|||
|
|
@ -93,6 +93,22 @@ def get_item_tags(item_id):
|
|||
else:
|
||||
return []
|
||||
|
||||
def get_min_tag(tag):
|
||||
tag = tag.split('=')
|
||||
if len(tag) > 1:
|
||||
if tag[1] != '':
|
||||
tag = tag[1][1:-1]
|
||||
# no value
|
||||
else:
|
||||
tag = tag[0][1:-1]
|
||||
# custom tags
|
||||
else:
|
||||
tag = tag[0]
|
||||
return tag
|
||||
|
||||
def get_item_tags_minimal(item_id):
|
||||
return [ {"tag": tag, "min_tag": get_min_tag(tag)} for tag in get_item_tags(item_id) ]
|
||||
|
||||
# TEMPLATE + API QUERY
|
||||
def add_items_tag(tags=[], galaxy_tags=[], item_id=None):
|
||||
res_dict = {}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@
|
|||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import random
|
||||
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
|
||||
from flask_login import login_required, current_user, login_user, logout_user
|
||||
|
|
@ -34,7 +36,9 @@ crawler_splash = Blueprint('crawler_splash', __name__, template_folder=os.path.j
|
|||
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
def api_validator(api_response):
|
||||
if api_response:
|
||||
return Response(json.dumps(api_response[0], indent=2, sort_keys=True), mimetype='application/json'), api_response[1]
|
||||
|
||||
|
||||
# ============= ROUTES ==============
|
||||
|
|
@ -46,14 +50,19 @@ def showDomain():
|
|||
epoch = request.args.get('epoch')
|
||||
port = request.args.get('port')
|
||||
|
||||
domain = Domain.Domain(domain_name)
|
||||
res = api_validator(Domain.api_verify_if_domain_exist(domain_name))
|
||||
if res:
|
||||
return res
|
||||
|
||||
domain = Domain.Domain(domain_name, port=port)
|
||||
|
||||
dict_domain = domain.get_domain_metadata()
|
||||
dict_domain = {**dict_domain, **domain.get_domain_correlation()}
|
||||
dict_domain['domain'] = domain_name
|
||||
if domain.is_domain_up():
|
||||
dict_domain = {**dict_domain, **domain.get_domain_correlation()}
|
||||
dict_domain['tags'] = domain.get_domain_tags()
|
||||
dict_domain['history'] = domain.get_domain_history_with_status()
|
||||
dict_domain['crawler_history'] = domain.get_domain_items_crawled(items_link=True, epoch=epoch, item_screenshot=True, item_tag=True) # # TODO: handle multiple port
|
||||
dict_domain['crawler_history']['random_item'] = random.choice(dict_domain['crawler_history']['items'])
|
||||
|
||||
print(dict_domain)
|
||||
|
||||
return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label, screenshot={'item': None, '':None}, dict_links={})
|
||||
return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label)
|
||||
|
|
|
|||
|
|
@ -730,91 +730,6 @@ def show_domains_by_daterange():
|
|||
date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down,
|
||||
domains_tags=domains_tags, type=service_type, bootstrap_label=bootstrap_label)
|
||||
|
||||
@hiddenServices.route("/crawlers/show_domain", methods=['GET'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def show_domain():
|
||||
domain = request.args.get('domain')
|
||||
epoch = request.args.get('epoch')
|
||||
try:
|
||||
epoch = int(epoch)
|
||||
except:
|
||||
epoch = None
|
||||
port = request.args.get('port')
|
||||
faup.decode(domain)
|
||||
unpack_url = faup.get()
|
||||
|
||||
## TODO: # FIXME: remove me
|
||||
try:
|
||||
domain = unpack_url['domain'].decode()
|
||||
except:
|
||||
domain = unpack_url['domain']
|
||||
|
||||
if not port:
|
||||
if unpack_url['port']:
|
||||
try:
|
||||
port = unpack_url['port'].decode()
|
||||
except:
|
||||
port = unpack_url['port']
|
||||
else:
|
||||
port = 80
|
||||
try:
|
||||
port = int(port)
|
||||
except:
|
||||
port = 80
|
||||
type = get_type_domain(domain)
|
||||
if domain is None or not r_serv_onion.exists('{}_metadata:{}'.format(type, domain)):
|
||||
return '404'
|
||||
# # TODO: FIXME return 404
|
||||
|
||||
last_check = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'last_check')
|
||||
if last_check is None:
|
||||
last_check = '********'
|
||||
last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8])
|
||||
first_seen = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'first_seen')
|
||||
if first_seen is None:
|
||||
first_seen = '********'
|
||||
first_seen = '{}/{}/{}'.format(first_seen[0:4], first_seen[4:6], first_seen[6:8])
|
||||
ports = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'ports')
|
||||
origin_paste = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'paste_parent')
|
||||
|
||||
h = HiddenServices(domain, type, port=port)
|
||||
item_core = h.get_domain_crawled_core_item(epoch=epoch)
|
||||
if item_core:
|
||||
l_pastes = h.get_last_crawled_pastes(item_root=item_core['root_item'])
|
||||
else:
|
||||
l_pastes = []
|
||||
dict_links = h.get_all_links(l_pastes)
|
||||
if l_pastes:
|
||||
status = True
|
||||
else:
|
||||
status = False
|
||||
last_check = '{} - {}'.format(last_check, time.strftime('%H:%M.%S', time.gmtime(epoch)))
|
||||
screenshot = h.get_domain_random_screenshot(l_pastes)
|
||||
if screenshot:
|
||||
screenshot = screenshot[0]
|
||||
else:
|
||||
screenshot = 'None'
|
||||
|
||||
domain_tags = h.get_domain_tags()
|
||||
|
||||
origin_paste_name = h.get_origin_paste_name()
|
||||
origin_paste_tags = unpack_paste_tags(r_serv_metadata.smembers('tag:{}'.format(origin_paste)))
|
||||
paste_tags = []
|
||||
for path in l_pastes:
|
||||
p_tags = r_serv_metadata.smembers('tag:'+path)
|
||||
paste_tags.append(unpack_paste_tags(p_tags))
|
||||
|
||||
domain_history = h.extract_epoch_from_history(h.get_domain_crawled_history())
|
||||
|
||||
return render_template("showDomain.html", domain=domain, last_check=last_check, first_seen=first_seen,
|
||||
l_pastes=l_pastes, paste_tags=paste_tags, bootstrap_label=bootstrap_label,
|
||||
dict_links=dict_links, port=port, epoch=epoch,
|
||||
ports=ports, domain_history=domain_history,
|
||||
origin_paste_tags=origin_paste_tags, status=status,
|
||||
origin_paste=origin_paste, origin_paste_name=origin_paste_name,
|
||||
domain_tags=domain_tags, screenshot=screenshot)
|
||||
|
||||
@hiddenServices.route("/crawlers/download_domain", methods=['GET'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@
|
|||
title="<span class='badge badge-dark'>{{metadata_domain['domain']}}</span>"
|
||||
data-content="port: <span class='badge badge-secondary'>{{metadata_domain['port']}}</span><br>
|
||||
epoch: {{metadata_domain['epoch']}}">
|
||||
<td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['domain_name'] }}</a></td>
|
||||
<td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['domain_name'] }}</a></td>
|
||||
<td>{{'{}/{}/{}'.format(metadata_domain['first_seen'][0:4], metadata_domain['first_seen'][4:6], metadata_domain['first_seen'][6:8])}}</td>
|
||||
<td>{{'{}/{}/{}'.format(metadata_domain['last_check'][0:4], metadata_domain['last_check'][4:6], metadata_domain['last_check'][6:8])}}</td>
|
||||
<td><div style="color:{{metadata_domain['status_color']}}; display:inline-block">
|
||||
|
|
@ -205,7 +205,7 @@ function refresh_list_crawled(){
|
|||
var newRow = tableRef.insertRow(tableRef.rows.length);
|
||||
|
||||
var newCell = newRow.insertCell(0);
|
||||
newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('hiddenServices.show_domain') }}?onion_domain="+data_domain['domain']+"\">"+data_domain['domain']+"</a></td>";
|
||||
newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('crawler_splash.showDomain') }}?onion_domain="+data_domain['domain']+"\">"+data_domain['domain']+"</a></td>";
|
||||
|
||||
newCell = newRow.insertCell(1);
|
||||
newCell.innerHTML = "<td>"+data_domain['first_seen'].substr(0, 4)+"/"+data_domain['first_seen'].substr(4, 2)+"/"+data_domain['first_seen'].substr(6, 2)+"</td>"
|
||||
|
|
@ -243,7 +243,7 @@ function refresh_list_crawled(){
|
|||
newCell.innerHTML = "<td><i class=\"fa fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i>"+crawler['crawler_info']+"</td>";
|
||||
|
||||
newCell = newRow.insertCell(1);
|
||||
newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('hiddenServices.show_domain') }}?onion_domain="+crawler['crawling_domain']+"\">"+crawler['crawling_domain']+"</a></td>";
|
||||
newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('crawler_splash.showDomain') }}?onion_domain="+crawler['crawling_domain']+"\">"+crawler['crawling_domain']+"</a></td>";
|
||||
|
||||
newCell = newRow.insertCell(2);
|
||||
newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@
|
|||
<tbody id="tbody_last_crawled">
|
||||
{% for metadata_domain in last_domains %}
|
||||
<tr>
|
||||
<td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['domain_name'] }}</a></td>
|
||||
<td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['domain_name'] }}</a></td>
|
||||
<td>{{'{}/{}/{}'.format(metadata_domain['first_seen'][0:4], metadata_domain['first_seen'][4:6], metadata_domain['first_seen'][6:8])}}</td>
|
||||
<td>{{'{}/{}/{}'.format(metadata_domain['last_check'][0:4], metadata_domain['last_check'][4:6], metadata_domain['last_check'][6:8])}}</td>
|
||||
<td><div style="color:{{metadata_domain['status_color']}}; display:inline-block">
|
||||
|
|
@ -76,7 +76,7 @@
|
|||
<tbody id="tbody_last_crawled">
|
||||
{% for metadata_domain in auto_crawler_domain_onions_metadata %}
|
||||
<tr>
|
||||
<td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['url'] }}</a></td>
|
||||
<td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['url'] }}</a></td>
|
||||
<td><a class="btn btn-outline-danger px-1 py-0" href="{{ url_for('hiddenServices.remove_auto_crawler') }}?url={{ metadata_domain['url'] }}&page={{page}}">
|
||||
<i class="fas fa-trash-alt"></i></a>
|
||||
</td>
|
||||
|
|
@ -110,7 +110,7 @@
|
|||
<tbody id="tbody_last_crawled">
|
||||
{% for metadata_domain in auto_crawler_domain_regular_metadata %}
|
||||
<tr>
|
||||
<td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['url'] }}</a></td>
|
||||
<td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['url'] }}</a></td>
|
||||
<td><a class="btn btn-outline-danger px-1 py-0" href="{{ url_for('hiddenServices.remove_auto_crawler') }}?url={{ metadata_domain['url'] }}&page={{page}}">
|
||||
<i class="fas fa-trash-alt"></i></a>
|
||||
</td>
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@
|
|||
{% for domain in domains_by_day[date] %}
|
||||
<tr>
|
||||
<td>
|
||||
<a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ domain }}">{{ domain }}</a>
|
||||
<a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ domain }}">{{ domain }}</a>
|
||||
<div>
|
||||
{% for tag in domain_metadata[domain]['tags'] %}
|
||||
<a href="{{ url_for('Tags.Tags_page') }}?date_from={{date}}&date_to={{date}}<ags={{ tag }}">
|
||||
|
|
|
|||
|
|
@ -169,7 +169,7 @@
|
|||
<tbody>
|
||||
<tr>
|
||||
<td>Domain</td>
|
||||
<td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ crawler_metadata['domain'] }}" id='domain'>{{ crawler_metadata['domain'] }}</a></td>
|
||||
<td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ crawler_metadata['domain'] }}" id='domain'>{{ crawler_metadata['domain'] }}</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Father</td>
|
||||
|
|
|
|||
|
|
@ -433,7 +433,7 @@
|
|||
<tbody>
|
||||
<tr>
|
||||
<td>Domain</td>
|
||||
<td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ crawler_metadata['domain'] }}" id='domain'>{{ crawler_metadata['domain'] }}</a></td>
|
||||
<td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ crawler_metadata['domain'] }}" id='domain'>{{ crawler_metadata['domain'] }}</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Father</td>
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@
|
|||
<div class="card mt-2">
|
||||
<div class="card-header bg-dark">
|
||||
<span class="badge badge-pill badge-light flex-row-reverse float-right">
|
||||
{% if status %}
|
||||
{% if dict_domain["status"] %}
|
||||
<div style="color:Green;">
|
||||
<i class="fas fa-check-circle fa-2x"></i>
|
||||
UP
|
||||
|
|
@ -60,7 +60,7 @@
|
|||
<tr>
|
||||
<td class="panelText">{%if "first_seen" in dict_domain%}{{ dict_domain['first_seen'] }}{%endif%}</td>
|
||||
<td class="panelText">{%if "last_check" in dict_domain%}{{ dict_domain['last_check'] }}{%endif%}</td>
|
||||
<td class="panelText">{{ ports }}</td>
|
||||
<td class="panelText">{%if dict_domain["ports"]%}{{ dict_domain["ports"] }}{%endif%}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
|
@ -91,7 +91,7 @@
|
|||
<div class="col-11">
|
||||
<div class="mt-2">
|
||||
<i class="fas fa-key"></i> PGP Dumps
|
||||
<div class="badge badge-warning">{{l_64|length}}</div>
|
||||
<div class="badge badge-warning">{{dict_domain['pgp']['nb']}}</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-1">
|
||||
|
|
@ -112,6 +112,7 @@
|
|||
</thead>
|
||||
<tbody>
|
||||
{% for dict_key in dict_domain['pgp']%}
|
||||
{% if dict_key != "nb" %}
|
||||
{% if dict_key=="mail" %}
|
||||
{% set var_icon = "fas fa-at" %}
|
||||
{% elif dict_key=="name" %}
|
||||
|
|
@ -128,6 +129,7 @@
|
|||
<td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ key_id }}">{{ key_id }}</a></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
|
|
@ -146,7 +148,7 @@
|
|||
<div class="col-11">
|
||||
<div class="mt-2">
|
||||
<i class="fas fa-coins"></i> Cryptocurrencies
|
||||
<div class="badge badge-warning">{{l_64|length}}</div>
|
||||
<div class="badge badge-warning">{{dict_domain['cryptocurrency']['nb']}}</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-1">
|
||||
|
|
@ -167,6 +169,7 @@
|
|||
</thead>
|
||||
<tbody>
|
||||
{% for dict_key in dict_domain['cryptocurrency']%}
|
||||
{% if dict_key != "nb" %}
|
||||
{% if dict_key=="bitcoin" %}
|
||||
{% set var_icon = "fab fa-bitcoin" %}
|
||||
{% elif dict_key=="monero" %}
|
||||
|
|
@ -183,6 +186,7 @@
|
|||
<td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ key_id }}">{{ key_id }}</a></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
|
|
@ -195,45 +199,70 @@
|
|||
|
||||
<!-- TODO: ADD HASH DECODED -->
|
||||
|
||||
{% if l_pastes %}
|
||||
<hr>
|
||||
{% if dict_domain["crawler_history"] %}
|
||||
<hr class="my-4">
|
||||
|
||||
<div class="card mb-1">
|
||||
<div class="card-header">
|
||||
<div class="flex-row-reverse float-right text-secondary">
|
||||
Date:
|
||||
<span class="badge badge-pill badge-info mt-1">
|
||||
{{dict_domain["crawler_history"]["date"]}}
|
||||
</span>
|
||||
PORT:
|
||||
<span class="badge badge-pill badge-warning mt-1">
|
||||
{{dict_domain["crawler_history"]["port"]}}
|
||||
</span>
|
||||
</div>
|
||||
<h5>Crawled Items</h5>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<table class="table table-striped table-bordered table-hover" id="myTable_1">
|
||||
<thead class="thead-dark">
|
||||
<tr>
|
||||
<th>Crawled Pastes</th>
|
||||
<th class="text-center"><i class="fas fa-images"></i></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for path in l_pastes %}
|
||||
{% for item in dict_domain["crawler_history"]["items"] %}
|
||||
<tr>
|
||||
<td>
|
||||
<a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{path}}" class="text-secondary">
|
||||
<div style="line-height:0.9;">{{ dict_links[path] }}</div>
|
||||
<a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{item["id"]}}" class="text-secondary">
|
||||
<div style="line-height:0.9;">{{ item["link"] }}</div>
|
||||
</a>
|
||||
<div>
|
||||
{% for tag in paste_tags[loop.index0] %}
|
||||
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag[1] }}">
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
|
||||
{% for tag in item["tags"] %}
|
||||
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag["tag"] }}">
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag["min_tag"] }}</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</td>
|
||||
<td class="text-center">
|
||||
{%if item["screenshot"]%}
|
||||
<button class="btn" onclick="reload_image('{{ item["screenshot"] }}', '{{ item["link"] }}', '{{ item["id"] }}');">
|
||||
<i class="far fa-image"></i>
|
||||
</button>
|
||||
{%endif%}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
|
||||
</tbody>
|
||||
</table>
|
||||
{%endif%}
|
||||
|
||||
|
||||
<div class="text-center my-3">
|
||||
<div class="text-center mb-4">
|
||||
<a href="{{ url_for('hiddenServices.download_domain') }}?domain={{domain}}&port={{port}}{% if epoch %}&epoch={{epoch}}{% endif %}" target="blank" style="font-size: 15px">
|
||||
<button class="btn btn-info"><i class="fas fa-download"></i> Download Domain Content
|
||||
</button>
|
||||
</a>
|
||||
</div>
|
||||
{%endif%}
|
||||
|
||||
{% if dict_domain["domain_history"] %}
|
||||
{% if dict_domain["history"] %}
|
||||
<table class="table table-hover mt-2" id="myTable_2">
|
||||
<thead>
|
||||
<tr>
|
||||
|
|
@ -241,18 +270,18 @@
|
|||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for epoch_item in domain_history %}
|
||||
{% for dom_history in dict_domain["history"] %}
|
||||
<tr>
|
||||
<td class="{% if epoch_item[0]==epoch %}text-dark table-info{% endif %}">
|
||||
<a href="{{ url_for('hiddenServices.show_domain') }}?domain={{domain}}&port={{port}}&epoch={{epoch_item[0]}}" class="text-secondary">
|
||||
<td class="{% if dom_history["epoch"]==dict_domain["crawler_history"]["epoch"] %}text-dark table-info{% endif %}">
|
||||
<a href="{{ url_for('crawler_splash.showDomain') }}?domain={{dict_domain["domain"]}}&port={{dict_domain["crawler_history"]["port"]}}&epoch={{dom_history["epoch"]}}" class="text-secondary">
|
||||
<div class="d-flex justify-content-around" style="line-height:0.9;">
|
||||
<div>{{dict_domain["domain"]}}</div>
|
||||
{% if epoch_item[2] %}
|
||||
{% if dom_history["status"] %}
|
||||
<div style="color:Green;"><i class="fas fa-check-circle"></i> UP</div>
|
||||
{% else %}
|
||||
<div style="color:Red;"><i class="fas fa-times-circle"></i> DOWN</div>
|
||||
{% endif %}
|
||||
<div>{{ epoch_item[1] }}</div>
|
||||
<div>{{ dom_history["date"] }}</div>
|
||||
</div>
|
||||
</a>
|
||||
</tr>
|
||||
|
|
@ -281,13 +310,9 @@
|
|||
</div>
|
||||
<canvas id="canvas" style="width:100%;"></canvas>
|
||||
<div class="text-center">
|
||||
<!--
|
||||
<small class="text-info" style="line-height:0.9;">
|
||||
<a target="_blank" href="" id="screenshot_link"></a>
|
||||
<small>
|
||||
<a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{screenshot['item']}}" class="text-info">
|
||||
<div style="line-height:0.9;">{{dict_links[screenshot['item']]}}</div>
|
||||
</a>
|
||||
<small>
|
||||
-->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
@ -333,6 +358,8 @@ function toggle_sidebar(){
|
|||
|
||||
<script>
|
||||
var ctx = canvas.getContext('2d'), img = new Image();
|
||||
var base_url = "{{ url_for('showsavedpastes.screenshot', filename="") }}"
|
||||
var screenshot_href = "{{ url_for('showsavedpastes.showsavedpaste') }}?paste="
|
||||
|
||||
/// turn off image smoothing
|
||||
ctx.webkitImageSmoothingEnabled = false;
|
||||
|
|
@ -342,7 +369,15 @@ img.onload = pixelate;
|
|||
img.addEventListener("error", img_error);
|
||||
var draw_img = false;
|
||||
|
||||
//img.src = "{{ url_for('showsavedpastes.screenshot', filename=screenshot['screenshot']) }}";
|
||||
{%if 'crawler_history' in dict_domain%}
|
||||
var screenshot = "{{dict_domain['crawler_history']['random_item']['screenshot']}}"
|
||||
$("#screenshot_link").attr("href", "screenshot_href + {{dict_domain['crawler_history']['random_item']['id']}}")
|
||||
$("#screenshot_link").text("{{dict_domain['crawler_history']['random_item']['link']}}")
|
||||
{%else%}
|
||||
var screenshot = ""
|
||||
{%endif%}
|
||||
|
||||
img.src = base_url + screenshot;
|
||||
|
||||
function pixelate() {
|
||||
/// use slider value
|
||||
|
|
@ -375,6 +410,15 @@ function img_error() {
|
|||
blocks.value = 50;
|
||||
pixelate;
|
||||
}
|
||||
|
||||
function reload_image(screenshot, link, item_id) {
|
||||
img.src=base_url + screenshot;
|
||||
$("#screenshot_link").attr("href", screenshot_href + item_id)
|
||||
$("#screenshot_link").text(link)
|
||||
pixelate;
|
||||
}
|
||||
|
||||
|
||||
</script>
|
||||
|
||||
</html>
|
||||
|
|
|
|||
Loading…
Reference in New Issue