chg: [core + UI] refractor Domain + Show Domain, better handle of ports, select screenshot on the UI, ...

bin/lib/Domain.py

@@ -11,6 +11,7 @@ import os
 import sys
 import time
 import redis
+import random
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
 import Correlation
@@ -32,6 +33,34 @@ def get_domain_type(domain):
     else:
         return 'regular'
 
+def sanathyse_port(port, domain, domain_type, strict=False, current_port=None):
+    '''
+    Retun a port number, If the port number is invalid, a port of the provided domain is randomly selected
+    '''
+    try:
+        port = int(port)
+    except (TypeError, ValueError):
+        if strict:
+            port = current_port
+        else:
+            port = get_random_domain_port(domain, domain_type)
+    return port
+
+def is_domain_up(domain, domain_type):
+    return r_serv_onion.hexists('{}_metadata:{}'.format(domain_type, domain), 'ports')
+
+def get_domain_all_ports(domain, domain_type):
+    '''
+    Return a list of all crawled ports
+    '''
+    l_ports = r_serv_onion.hget('{}_metadata:{}'.format(domain_type, domain), 'ports')
+    if l_ports:
+        return l_ports.split(";")
+    return []
+
+def get_random_domain_port(domain, domain_type):
+    return random.choice(get_domain_all_ports(domain, domain_type))
+
 def get_all_domain_up_by_type(domain_type):
     if domain_type in domains:
         list_domain = list(r_serv_onion.smembers('full_{}_up'.format(domain_type)))
@@ -52,6 +81,53 @@ def get_domain_item_children(domain, root_item_id):
             all_items.extend(get_domain_item_children(domain, item_id))
     return all_items
 
+def get_domain_last_crawled_item_root(domain, domain_type, port):
+    '''
+    Retun last_crawled_item_core dict
+    '''
+    res = r_serv_onion.zrevrange('crawler_history_{}:{}:{}'.format(domain_type, domain, port), 0, 0, withscores=True)
+    if res:
+        return {"root_item": res[0][0], "epoch": int(res[0][1])}
+    else:
+        return {}
+
+def get_domain_crawled_item_root(domain, domain_type, port, epoch=None):
+    '''
+    Retun the first item crawled for a given domain:port (and epoch)
+    '''
+    if epoch:
+        res = r_serv_onion.zrevrangebyscore('crawler_history_{}:{}:{}'.format(domain_type, domain, port), int(epoch), int(epoch))
+        if res:
+            return {"root_item": res[0], "epoch": int(epoch)}
+        # invalid epoch
+        epoch = None
+
+    if not epoch:
+        return get_domain_last_crawled_item_root(domain, domain_type, port)
+
+
+def get_domain_items_crawled(domain, domain_type, port, epoch=None, items_link=False, item_screenshot=False, item_tag=False):
+    '''
+
+    '''
+    item_crawled = {}
+    item_root = get_domain_crawled_item_root(domain, domain_type, port, epoch=epoch)
+    if item_root:
+        item_crawled['port'] = port
+        item_crawled['epoch'] = item_root['epoch']
+        item_crawled['date'] = time.strftime('%Y/%m/%d - %H:%M.%S', time.gmtime(item_root['epoch']))
+        item_crawled['items'] = []
+        for item in get_domain_items(domain, item_root['root_item']):
+            dict_item = {"id": item}
+            if items_link:
+                dict_item['link'] = Item.get_item_link(item)
+            if item_screenshot:
+                dict_item['screenshot'] = Item.get_item_screenshot(item)
+            if item_tag:
+                dict_item['tags'] = Tag.get_item_tags_minimal(item)
+            item_crawled['items'].append(dict_item)
+    return item_crawled
+
 def get_link_tree():
     pass
 
@@ -65,7 +141,7 @@ def get_domain_tags(domain):
     '''
     return Tag.get_item_tags(domain)
 
-def get_domain_cryptocurrency(domain, currencies_type=None):
+def get_domain_cryptocurrency(domain, currencies_type=None, get_nb=False):
     '''
     Retun all cryptocurrencies of a given domain.
 
@@ -73,9 +149,9 @@ def get_domain_cryptocurrency(domain, currencies_type=None):
     :param currencies_type: list of cryptocurrencies type
     :type currencies_type: list, optional
     '''
-    return cryptocurrency.get_domain_correlation_dict(domain, correlation_type=currencies_type)
+    return cryptocurrency.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb)
 
-def get_domain_pgp(domain, currencies_type=None):
+def get_domain_pgp(domain, currencies_type=None, get_nb=False):
     '''
     Retun all pgp of a given domain.
 
@@ -83,9 +159,9 @@ def get_domain_pgp(domain, currencies_type=None):
     :param currencies_type: list of pgp type
     :type currencies_type: list, optional
     '''
-    return pgp.get_domain_correlation_dict(domain, correlation_type=currencies_type)
+    return pgp.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb)
 
-def get_domain_all_correlation(domain, correlation_type=None):
+def get_domain_all_correlation(domain, correlation_type=None, get_nb=False):
     '''
     Retun all correlation of a given domain.
 
@@ -96,10 +172,10 @@ def get_domain_all_correlation(domain, correlation_type=None):
     :rtype: dict
     '''
     domain_correl = {}
-    res = get_domain_cryptocurrency(domain)
+    res = get_domain_cryptocurrency(domain, get_nb=get_nb)
     if res:
         domain_correl['cryptocurrency'] = res
-    res = get_domain_pgp(domain)
+    res = get_domain_pgp(domain, get_nb=get_nb)
     if res:
         domain_correl['pgp'] = res
     return domain_correl
@@ -141,13 +217,26 @@ def get_domain_history_with_status(domain, domain_type, port): # TODO: add date_
         l_history.append({"epoch": epoch_val, "date": time.strftime('%Y/%m/%d - %H:%M.%S', time.gmtime(epoch_val)), "status": status})
     return l_history
 
+def verify_if_domain_exist(domain):
+    return r_serv_onion.exists('{}_metadata:{}'.format(get_domain_type(domain), domain))
+
+def api_verify_if_domain_exist(domain):
+    if not verify_if_domain_exist(domain):
+        return ({'status': 'error', 'reason': 'Unknow Domain'}, 404)
+    else:
+        return None
 
 class Domain(object):
     """docstring for Domain."""
 
-    def __init__(self, domain, port=80):
+    def __init__(self, domain, port=None):
         self.domain = str(domain)
         self.type = get_domain_type(domain)
+        if self.is_domain_up():
+            self.current_port = sanathyse_port(port, self.domain, self.type)
+
+    def get_current_port(self):
+        return self.current_port
 
     def get_domain_first_seen(self):
         '''
@@ -161,7 +250,7 @@ class Domain(object):
             first_seen = '{}/{}/{}'.format(first_seen[0:4], first_seen[4:6], first_seen[6:8])
         return first_seen
 
-    def get_domain_last_check(self):# # TODO: add epoch ???
+    def get_domain_last_check(self):
         '''
         Get domain last check date
 
@@ -173,10 +262,16 @@ class Domain(object):
             last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8])
         return last_check
 
-    #def get_domain_all_ports(self):
+    def is_domain_up(self): # # TODO: handle multiple ports
-    #    pass
+        '''
+        Return True if this domain is UP
+        '''
+        return is_domain_up(self.domain, self.type)
 
-    def get_domain_metadata(self, first_seen=True, last_ckeck=True, ports=True):
+    def get_domain_all_ports(self):
+        return get_domain_all_ports(self.domain, self.type)
+
+    def get_domain_metadata(self, first_seen=True, last_ckeck=True, status=True, ports=True):
         '''
         Get Domain basic metadata
 
@@ -199,6 +294,10 @@ class Domain(object):
             res = self.get_domain_last_check()
             if res is not None:
                 dict_metadata['last_check'] = res
+        if status:
+            dict_metadata['status'] = self.is_domain_up()
+        if ports:
+            dict_metadata['ports'] = self.get_domain_all_ports()
         return dict_metadata
 
     def get_domain_tags(self):
@@ -213,10 +312,17 @@ class Domain(object):
         '''
         Retun all cryptocurrencies of a given domain.
         '''
-        return get_domain_all_correlation(self.domain)
+        return get_domain_all_correlation(self.domain, get_nb=True)
 
     def get_domain_history_with_status(self):
         '''
         Retun the full history of a given domain and port.
         '''
         return get_domain_history_with_status(self.domain, self.type, 80)
+
+    def get_domain_items_crawled(self, port=None, epoch=None, items_link=False, item_screenshot=False, item_tag=False):
+        '''
+        Return ........................
+        '''
+        port = sanathyse_port(port, self.domain, self.type, strict=True, current_port=self.current_port)
+        return get_domain_items_crawled(self.domain, self.type, port, epoch=epoch, items_link=items_link, item_screenshot=item_screenshot, item_tag=item_tag)

bin/packages/Correlation.py

@@ -126,7 +126,7 @@ class Correlation(object):
         else:
             return []
 
-    def get_domain_correlation_dict(self, domain, correlation_type=None):
+    def get_domain_correlation_dict(self, domain, correlation_type=None, get_nb=False):
         '''
         Return all correlation of a given domain.
 
@@ -143,6 +143,8 @@ class Correlation(object):
             res = self._get_domain_correlation_obj(domain, correl)
             if res:
                 dict_correlation[correl] = res
+                if get_nb:
+                    dict_correlation['nb'] = dict_correlation.get('nb', 0) + len(dict_correlation[correl])
         return dict_correlation
 
 

bin/packages/Item.py

@@ -177,6 +177,14 @@ def is_item_in_domain(domain, item_id):
 def get_item_domain(item_id):
     return item_id[19:-36]
 
-
 def get_item_children(item_id):
     return list(r_serv_metadata.smembers('paste_children:{}'.format(item_id)))
+
+def get_item_link(item_id):
+    return r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'real_link')
+
+def get_item_screenshot(item_id):
+    screenshot = r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'screenshot')
+    if screenshot:
+        return os.path.join(screenshot[0:2], screenshot[2:4], screenshot[4:6], screenshot[6:8], screenshot[8:10], screenshot[10:12], screenshot[12:])
+    return ''

bin/packages/Tag.py

@@ -93,6 +93,22 @@ def get_item_tags(item_id):
     else:
         return []
 
+def get_min_tag(tag):
+    tag = tag.split('=')
+    if len(tag) > 1:
+        if tag[1] != '':
+            tag = tag[1][1:-1]
+        # no value
+        else:
+            tag = tag[0][1:-1]
+    # custom tags
+    else:
+        tag = tag[0]
+    return tag
+
+def get_item_tags_minimal(item_id):
+    return [ {"tag": tag, "min_tag": get_min_tag(tag)} for tag in get_item_tags(item_id) ]
+
 # TEMPLATE + API QUERY
 def add_items_tag(tags=[], galaxy_tags=[], item_id=None):
     res_dict = {}

var/www/blueprints/crawler_splash.py

@@ -7,6 +7,8 @@
 
 import os
 import sys
+import json
+import random
 
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
 from flask_login import login_required, current_user, login_user, logout_user
@@ -34,7 +36,9 @@ crawler_splash = Blueprint('crawler_splash', __name__, template_folder=os.path.j
 
 
 # ============ FUNCTIONS ============
-
+def api_validator(api_response):
+    if api_response:
+        return Response(json.dumps(api_response[0], indent=2, sort_keys=True), mimetype='application/json'), api_response[1]
 
 
 # ============= ROUTES ==============
@@ -46,14 +50,19 @@ def showDomain():
     epoch = request.args.get('epoch')
     port = request.args.get('port')
 
-    domain = Domain.Domain(domain_name)
+    res = api_validator(Domain.api_verify_if_domain_exist(domain_name))
+    if res:
+        return res
+
+    domain = Domain.Domain(domain_name, port=port)
 
     dict_domain = domain.get_domain_metadata()
-    dict_domain = {**dict_domain, **domain.get_domain_correlation()}
     dict_domain['domain'] = domain_name
-    dict_domain['tags'] = domain.get_domain_tags()
+    if domain.is_domain_up():
-    dict_domain['history'] = domain.get_domain_history_with_status()
+        dict_domain = {**dict_domain, **domain.get_domain_correlation()}
+        dict_domain['tags'] = domain.get_domain_tags()
+        dict_domain['history'] = domain.get_domain_history_with_status()
+        dict_domain['crawler_history'] = domain.get_domain_items_crawled(items_link=True, epoch=epoch, item_screenshot=True, item_tag=True) # # TODO: handle multiple port
+        dict_domain['crawler_history']['random_item'] = random.choice(dict_domain['crawler_history']['items'])
 
-    print(dict_domain)
+    return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label)
-
-    return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label, screenshot={'item': None, '':None}, dict_links={})

var/www/modules/hiddenServices/Flask_hiddenServices.py

@@ -730,91 +730,6 @@ def show_domains_by_daterange():
                                 date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down,
                                 domains_tags=domains_tags, type=service_type, bootstrap_label=bootstrap_label)
 
-@hiddenServices.route("/crawlers/show_domain", methods=['GET'])
-@login_required
-@login_analyst
-def show_domain():
-    domain = request.args.get('domain')
-    epoch = request.args.get('epoch')
-    try:
-        epoch = int(epoch)
-    except:
-        epoch = None
-    port = request.args.get('port')
-    faup.decode(domain)
-    unpack_url = faup.get()
-
-    ## TODO: # FIXME: remove me
-    try:
-        domain = unpack_url['domain'].decode()
-    except:
-        domain = unpack_url['domain']
-
-    if not port:
-        if unpack_url['port']:
-            try:
-                port = unpack_url['port'].decode()
-            except:
-                port = unpack_url['port']
-        else:
-            port = 80
-    try:
-        port = int(port)
-    except:
-        port = 80
-    type = get_type_domain(domain)
-    if domain is None or not r_serv_onion.exists('{}_metadata:{}'.format(type, domain)):
-        return '404'
-        # # TODO: FIXME return 404
-
-    last_check = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'last_check')
-    if last_check is None:
-        last_check = '********'
-    last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8])
-    first_seen = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'first_seen')
-    if first_seen is None:
-        first_seen = '********'
-    first_seen = '{}/{}/{}'.format(first_seen[0:4], first_seen[4:6], first_seen[6:8])
-    ports = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'ports')
-    origin_paste = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'paste_parent')
-
-    h = HiddenServices(domain, type, port=port)
-    item_core = h.get_domain_crawled_core_item(epoch=epoch)
-    if item_core:
-        l_pastes = h.get_last_crawled_pastes(item_root=item_core['root_item'])
-    else:
-        l_pastes = []
-    dict_links = h.get_all_links(l_pastes)
-    if l_pastes:
-        status = True
-    else:
-        status = False
-    last_check = '{} - {}'.format(last_check, time.strftime('%H:%M.%S', time.gmtime(epoch)))
-    screenshot = h.get_domain_random_screenshot(l_pastes)
-    if screenshot:
-        screenshot = screenshot[0]
-    else:
-        screenshot = 'None'
-
-    domain_tags = h.get_domain_tags()
-
-    origin_paste_name = h.get_origin_paste_name()
-    origin_paste_tags = unpack_paste_tags(r_serv_metadata.smembers('tag:{}'.format(origin_paste)))
-    paste_tags = []
-    for path in l_pastes:
-        p_tags = r_serv_metadata.smembers('tag:'+path)
-        paste_tags.append(unpack_paste_tags(p_tags))
-
-    domain_history = h.extract_epoch_from_history(h.get_domain_crawled_history())
-
-    return render_template("showDomain.html", domain=domain, last_check=last_check, first_seen=first_seen,
-                            l_pastes=l_pastes, paste_tags=paste_tags, bootstrap_label=bootstrap_label,
-                            dict_links=dict_links, port=port, epoch=epoch,
-                            ports=ports, domain_history=domain_history,
-                            origin_paste_tags=origin_paste_tags, status=status,
-                            origin_paste=origin_paste, origin_paste_name=origin_paste_name,
-                            domain_tags=domain_tags, screenshot=screenshot)
-
 @hiddenServices.route("/crawlers/download_domain", methods=['GET'])
 @login_required
 @login_analyst

var/www/modules/hiddenServices/templates/Crawler_Splash_last_by_type.html

@@ -67,7 +67,7 @@
 												title="<span class='badge badge-dark'>{{metadata_domain['domain']}}</span>"
 												data-content="port: <span class='badge badge-secondary'>{{metadata_domain['port']}}</span><br>
 																			epoch: {{metadata_domain['epoch']}}">
-								        <td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['domain_name'] }}</a></td>
+								        <td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['domain_name'] }}</a></td>
 								        <td>{{'{}/{}/{}'.format(metadata_domain['first_seen'][0:4], metadata_domain['first_seen'][4:6], metadata_domain['first_seen'][6:8])}}</td>
 												<td>{{'{}/{}/{}'.format(metadata_domain['last_check'][0:4], metadata_domain['last_check'][4:6], metadata_domain['last_check'][6:8])}}</td>
 								        <td><div style="color:{{metadata_domain['status_color']}}; display:inline-block">
@@ -205,7 +205,7 @@ function refresh_list_crawled(){
 					var newRow   = tableRef.insertRow(tableRef.rows.length);
 
 			    var newCell  = newRow.insertCell(0);
-					newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('hiddenServices.show_domain') }}?onion_domain="+data_domain['domain']+"\">"+data_domain['domain']+"</a></td>";
+					newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('crawler_splash.showDomain') }}?onion_domain="+data_domain['domain']+"\">"+data_domain['domain']+"</a></td>";
 
 					newCell  = newRow.insertCell(1);
 					newCell.innerHTML = "<td>"+data_domain['first_seen'].substr(0, 4)+"/"+data_domain['first_seen'].substr(4, 2)+"/"+data_domain['first_seen'].substr(6, 2)+"</td>"
@@ -243,7 +243,7 @@ function refresh_list_crawled(){
 						newCell.innerHTML = "<td><i class=\"fa fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i>"+crawler['crawler_info']+"</td>";
 
 						newCell  = newRow.insertCell(1);
-						newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('hiddenServices.show_domain') }}?onion_domain="+crawler['crawling_domain']+"\">"+crawler['crawling_domain']+"</a></td>";
+						newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('crawler_splash.showDomain') }}?onion_domain="+crawler['crawling_domain']+"\">"+crawler['crawling_domain']+"</a></td>";
 
 						newCell  = newRow.insertCell(2);
 						newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";

var/www/modules/hiddenServices/templates/Crawler_auto.html

@@ -45,7 +45,7 @@
 							<tbody id="tbody_last_crawled">
 									{% for metadata_domain in last_domains %}
 										<tr>
-											<td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['domain_name'] }}</a></td>
+											<td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['domain_name'] }}</a></td>
 											<td>{{'{}/{}/{}'.format(metadata_domain['first_seen'][0:4], metadata_domain['first_seen'][4:6], metadata_domain['first_seen'][6:8])}}</td>
 											<td>{{'{}/{}/{}'.format(metadata_domain['last_check'][0:4], metadata_domain['last_check'][4:6], metadata_domain['last_check'][6:8])}}</td>
 											<td><div style="color:{{metadata_domain['status_color']}}; display:inline-block">
@@ -76,7 +76,7 @@
 								<tbody id="tbody_last_crawled">
 										{% for metadata_domain in auto_crawler_domain_onions_metadata %}
 											<tr>
-												<td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['url'] }}</a></td>
+												<td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['url'] }}</a></td>
 												<td><a class="btn btn-outline-danger px-1 py-0" href="{{ url_for('hiddenServices.remove_auto_crawler') }}?url={{ metadata_domain['url'] }}&page={{page}}">
 													<i class="fas fa-trash-alt"></i></a>
 												</td>
@@ -110,7 +110,7 @@
 								<tbody id="tbody_last_crawled">
 										{% for metadata_domain in auto_crawler_domain_regular_metadata %}
 											<tr>
-												<td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['url'] }}</a></td>
+												<td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ metadata_domain['domain'] }}&port={{metadata_domain['port']}}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['url'] }}</a></td>
 												<td><a class="btn btn-outline-danger px-1 py-0" href="{{ url_for('hiddenServices.remove_auto_crawler') }}?url={{ metadata_domain['url'] }}&page={{page}}">
 													<i class="fas fa-trash-alt"></i></a>
 												</td>

var/www/modules/hiddenServices/templates/domains.html

@@ -64,7 +64,7 @@
 											{% for domain in domains_by_day[date] %}
 									      <tr>
 									        <td>
-														<a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ domain }}">{{ domain }}</a>
+														<a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ domain }}">{{ domain }}</a>
 														<div>
 															{% for tag in domain_metadata[domain]['tags'] %}
 															<a href="{{ url_for('Tags.Tags_page') }}?date_from={{date}}&date_to={{date}}&ltags={{ tag }}">

var/www/modules/showpaste/templates/show_saved_item_min.html

@@ -169,7 +169,7 @@
             <tbody>
               <tr>
                 <td>Domain</td>
-                <td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ crawler_metadata['domain'] }}" id='domain'>{{ crawler_metadata['domain'] }}</a></td>
+                <td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ crawler_metadata['domain'] }}" id='domain'>{{ crawler_metadata['domain'] }}</a></td>
               </tr>
               <tr>
                 <td>Father</td>

var/www/modules/showpaste/templates/show_saved_paste.html

@@ -433,7 +433,7 @@
                         <tbody>
                           <tr>
                             <td>Domain</td>
-                            <td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ crawler_metadata['domain'] }}" id='domain'>{{ crawler_metadata['domain'] }}</a></td>
+                            <td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ crawler_metadata['domain'] }}" id='domain'>{{ crawler_metadata['domain'] }}</a></td>
                           </tr>
                           <tr>
                             <td>Father</td>

var/www/templates/crawler/crawler_splash/showDomain.html

@@ -33,7 +33,7 @@
 						<div class="card mt-2">
 							<div class="card-header bg-dark">
 								<span class="badge badge-pill badge-light flex-row-reverse float-right">
-									{% if status %}
+									{% if dict_domain["status"] %}
 										<div style="color:Green;">
 											<i class="fas fa-check-circle fa-2x"></i>
 											UP
@@ -60,7 +60,7 @@
 										<tr>
 											<td class="panelText">{%if "first_seen" in dict_domain%}{{ dict_domain['first_seen'] }}{%endif%}</td>
 											<td class="panelText">{%if "last_check" in dict_domain%}{{ dict_domain['last_check'] }}{%endif%}</td>
-                      <td class="panelText">{{ ports }}</td>
+                      <td class="panelText">{%if dict_domain["ports"]%}{{ dict_domain["ports"] }}{%endif%}</td>
 										</tr>
 									</tbody>
 								</table>
@@ -91,7 +91,7 @@
                       <div class="col-11">
                         <div class="mt-2">
                           <i class="fas fa-key"></i> PGP Dumps&nbsp;&nbsp;
-                          <div class="badge badge-warning">{{l_64|length}}</div>
+                          <div class="badge badge-warning">{{dict_domain['pgp']['nb']}}</div>
                         </div>
                       </div>
                       <div class="col-1">
@@ -112,22 +112,24 @@
                          </thead>
                         <tbody>
                           {% for dict_key in dict_domain['pgp']%}
-                            {% if dict_key=="mail" %}
+                            {% if dict_key != "nb" %}
-                              {% set var_icon = "fas fa-at" %}
+                              {% if dict_key=="mail" %}
-                            {% elif dict_key=="name" %}
+                                {% set var_icon = "fas fa-at" %}
-                              {% set var_icon = "fas fa-user-tag" %}
+                              {% elif dict_key=="name" %}
-                            {% else %}
+                                {% set var_icon = "fas fa-user-tag" %}
-                              {% set var_icon = "fas fa-key" %}
+                              {% else %}
+                                {% set var_icon = "fas fa-key" %}
+                              {% endif %}
+                              {% for key_id in dict_domain['pgp'][dict_key]%}
+                                <tr>
+                                  <td>
+                                    <i class="{{ var_icon }}"></i>
+                                    &nbsp;&nbsp;{{ dict_key }}
+                                  </td>
+                                  <td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ key_id }}">{{ key_id }}</a></td>
+                                </tr>
+                              {% endfor %}
                             {% endif %}
-                            {% for key_id in dict_domain['pgp'][dict_key]%}
-                              <tr>
-                                <td>
-                                  <i class="{{ var_icon }}"></i>
-                                  &nbsp;&nbsp;{{ dict_key }}
-                                </td>
-                                <td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ key_id }}">{{ key_id }}</a></td>
-                              </tr>
-                            {% endfor %}
                           {% endfor %}
                         </tbody>
                       </table>
@@ -146,7 +148,7 @@
                       <div class="col-11">
                         <div class="mt-2">
                           <i class="fas fa-coins"></i> Cryptocurrencies&nbsp;&nbsp;
-                          <div class="badge badge-warning">{{l_64|length}}</div>
+                          <div class="badge badge-warning">{{dict_domain['cryptocurrency']['nb']}}</div>
                         </div>
                       </div>
                       <div class="col-1">
@@ -167,22 +169,24 @@
                          </thead>
                         <tbody>
                           {% for dict_key in dict_domain['cryptocurrency']%}
-                            {% if dict_key=="bitcoin" %}
+                            {% if dict_key != "nb" %}
-                              {% set var_icon = "fab fa-bitcoin" %}
+                              {% if dict_key=="bitcoin" %}
-                            {% elif dict_key=="monero" %}
+                                {% set var_icon = "fab fa-bitcoin" %}
-                              {% set var_icon = "fab fa-monero" %}
+                              {% elif dict_key=="monero" %}
-                            {% else %}
+                                {% set var_icon = "fab fa-monero" %}
-                              {% set var_icon = "fas fa-coins" %}
+                              {% else %}
+                                {% set var_icon = "fas fa-coins" %}
+                              {% endif %}
+                              {% for key_id in dict_domain['cryptocurrency'][dict_key]%}
+                                <tr>
+                                  <td>
+                                    <i class="{{ var_icon }}"></i>
+                                    &nbsp;&nbsp;{{ dict_key }}
+                                  </td>
+                                  <td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ key_id }}">{{ key_id }}</a></td>
+                                </tr>
+                              {% endfor %}
                             {% endif %}
-                            {% for key_id in dict_domain['cryptocurrency'][dict_key]%}
-                              <tr>
-                                <td>
-                                  <i class="{{ var_icon }}"></i>
-                                  &nbsp;&nbsp;{{ dict_key }}
-                                </td>
-                                <td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ key_id }}">{{ key_id }}</a></td>
-                              </tr>
-                            {% endfor %}
                           {% endfor %}
                         </tbody>
                       </table>
@@ -195,45 +199,70 @@
 
             <!-- TODO: ADD HASH DECODED -->
 
-						{% if l_pastes %}
+						{% if dict_domain["crawler_history"] %}
-              <hr>
+              <hr class="my-4">
+
+              <div class="card mb-1">
+  							<div class="card-header">
+                  <div class="flex-row-reverse float-right text-secondary">
+                    Date:
+                    <span class="badge badge-pill badge-info mt-1">
+    									{{dict_domain["crawler_history"]["date"]}}
+    								</span>
+                    &nbsp;&nbsp;PORT:
+                    <span class="badge badge-pill badge-warning mt-1">
+    									{{dict_domain["crawler_history"]["port"]}}
+    								</span>
+                  </div>
+                  <h5>Crawled Items</h5>
+  							</div>
+
+              </div>
 
 							<table class="table table-striped table-bordered table-hover" id="myTable_1">
 						    <thead class="thead-dark">
 						      <tr>
 						        <th>Crawled Pastes</th>
+                    <th class="text-center"><i class="fas fa-images"></i></th>
 						      </tr>
 						    </thead>
 						    <tbody>
-					        {% for path in l_pastes %}
+					        {% for item in dict_domain["crawler_history"]["items"] %}
 					          <tr>
 				              <td>
-												<a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{path}}" class="text-secondary">
+												<a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{item["id"]}}" class="text-secondary">
-													<div style="line-height:0.9;">{{ dict_links[path] }}</div>
+													<div style="line-height:0.9;">{{ item["link"] }}</div>
 												</a>
 				                <div>
-				                  {% for tag in paste_tags[loop.index0] %}
+				                  {% for tag in item["tags"] %}
-					                  <a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag[1] }}">
+					                  <a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag["tag"] }}">
-					                    <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
+					                    <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag["min_tag"] }}</span>
 					                  </a>
 				                  {%  endfor %}
 				                </div>
 			              	</td>
+                    <td class="text-center">
+                      {%if item["screenshot"]%}
+                      <button class="btn" onclick="reload_image('{{ item["screenshot"] }}', '{{ item["link"] }}', '{{ item["id"] }}');">
+                        <i class="far fa-image"></i>
+                      </button>
+                      {%endif%}
+                    </td>
 						        </tr>
 						      {%  endfor %}
+
 						    </tbody>
 						  </table>
+
+              <div class="text-center mb-4">
+                <a href="{{ url_for('hiddenServices.download_domain') }}?domain={{domain}}&port={{port}}{% if epoch %}&epoch={{epoch}}{% endif %}" target="blank" style="font-size: 15px">
+                    <button class="btn btn-info"><i class="fas fa-download"></i> Download Domain Content
+                    </button>
+                </a>
+              </div>
 						{%endif%}
 
-
+            {% if dict_domain["history"] %}
-            <div class="text-center my-3">
-              <a href="{{ url_for('hiddenServices.download_domain') }}?domain={{domain}}&port={{port}}{% if epoch %}&epoch={{epoch}}{% endif %}" target="blank" style="font-size: 15px">
-  								<button class="btn btn-info"><i class="fas fa-download"></i> Download Domain Content
-  								</button>
-  						</a>
-            </div>
-
-            {% if dict_domain["domain_history"] %}
 							<table class="table table-hover mt-2" id="myTable_2">
 						    <thead>
 						      <tr>
@@ -241,18 +270,18 @@
 						      </tr>
 						    </thead>
 						    <tbody>
-					        {% for epoch_item in domain_history %}
+					        {% for dom_history in dict_domain["history"] %}
 					          <tr>
-				              <td class="{% if epoch_item[0]==epoch %}text-dark table-info{% endif %}">
+				              <td class="{% if dom_history["epoch"]==dict_domain["crawler_history"]["epoch"] %}text-dark table-info{% endif %}">
-												<a href="{{ url_for('hiddenServices.show_domain') }}?domain={{domain}}&port={{port}}&epoch={{epoch_item[0]}}" class="text-secondary">
+												<a href="{{ url_for('crawler_splash.showDomain') }}?domain={{dict_domain["domain"]}}&port={{dict_domain["crawler_history"]["port"]}}&epoch={{dom_history["epoch"]}}" class="text-secondary">
 													<div class="d-flex justify-content-around" style="line-height:0.9;">
                             <div>{{dict_domain["domain"]}}</div>
-                            {% if epoch_item[2] %}
+                            {% if dom_history["status"] %}
                               <div style="color:Green;"><i class="fas fa-check-circle"></i> UP</div>
                             {% else %}
                               <div style="color:Red;"><i class="fas fa-times-circle"></i>	DOWN</div>
                             {% endif %}
-                            <div>{{ epoch_item[1] }}</div>
+                            <div>{{ dom_history["date"] }}</div>
                           </div>
 												</a>
 						        </tr>
@@ -281,13 +310,9 @@
 						</div>
 						<canvas id="canvas" style="width:100%;"></canvas>
 						<div class="text-center">
-              <!--
+							<small class="text-info" style="line-height:0.9;">
+								<a target="_blank" href="" id="screenshot_link"></a>
 							<small>
-								<a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{screenshot['item']}}" class="text-info">
-									<div style="line-height:0.9;">{{dict_links[screenshot['item']]}}</div>
-								</a>
-							<small>
-              -->
 						</div>
 					</div>
 
@@ -333,6 +358,8 @@ function toggle_sidebar(){
 
 <script>
 var ctx = canvas.getContext('2d'), img = new Image();
+var base_url = "{{ url_for('showsavedpastes.screenshot', filename="") }}"
+var screenshot_href = "{{ url_for('showsavedpastes.showsavedpaste') }}?paste="
 
 /// turn off image smoothing
 ctx.webkitImageSmoothingEnabled = false;
@@ -342,7 +369,15 @@ img.onload = pixelate;
 img.addEventListener("error", img_error);
 var draw_img = false;
 
-//img.src = "{{ url_for('showsavedpastes.screenshot', filename=screenshot['screenshot']) }}";
+{%if 'crawler_history' in dict_domain%}
+  var screenshot = "{{dict_domain['crawler_history']['random_item']['screenshot']}}"
+  $("#screenshot_link").attr("href", "screenshot_href + {{dict_domain['crawler_history']['random_item']['id']}}")
+  $("#screenshot_link").text("{{dict_domain['crawler_history']['random_item']['link']}}")
+{%else%}
+var screenshot = ""
+{%endif%}
+
+img.src = base_url + screenshot;
 
 function pixelate() {
   /// use slider value
@@ -375,6 +410,15 @@ function img_error() {
 	blocks.value = 50;
 	pixelate;
 }
+
+function reload_image(screenshot, link, item_id) {
+  img.src=base_url + screenshot;
+  $("#screenshot_link").attr("href", screenshot_href + item_id)
+  $("#screenshot_link").text(link)
+  pixelate;
+}
+
+
 </script>
 
 </html>