chg: [UI Crawled Domains] show crawled domains by date range (options: domains tags, up domains, down domains)

todo: Shows domain history

bin/packages/HiddenServices.py

@@ -81,8 +81,12 @@ class HiddenServices(object):
             return ''
         return origin_paste.replace(self.paste_directory+'/', '')
 
-    def get_domain_tags(self):
-        return self.tags
+    def get_domain_tags(self, update=False):
+        if not update:
+            return self.tags
+        else:
+            self.get_last_crawled_pastes()
+            return self.tags
 
     def update_domain_tags(self, children):
         p_tags = self.r_serv_metadata.smembers('tag:'+children)

var/www/modules/hiddenServices/Flask_hiddenServices.py

@@ -8,7 +8,7 @@ import redis
 import datetime
 import sys
 import os
-from flask import Flask, render_template, jsonify, request, Blueprint
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
 
 from Date import Date
 from HiddenServices import HiddenServices
@@ -40,6 +40,16 @@ def get_date_range(num_day):
 
     return list(reversed(date_list))
 
+def substract_date(date_from, date_to):
+    date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
+    date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
+    delta = date_to - date_from # timedelta
+    l_date = []
+    for i in range(delta.days + 1):
+        date = date_from + datetime.timedelta(i)
+        l_date.append( date.strftime('%Y%m%d') )
+    return l_date
+
 def unpack_paste_tags(p_tags):
     l_tags = []
     for tag in p_tags:
@@ -94,6 +104,94 @@ def hiddenServices_page():
 
     return render_template("hiddenServices.html", last_onions=list_onion, statDomains=statDomains)
 
+@hiddenServices.route("/hiddenServices/get_onions_by_daterange", methods=['POST'])
+def get_onions_by_daterange():
+    date_from = request.form.get('date_from')
+    date_to = request.form.get('date_to')
+    domains_up = request.form.get('domains_up')
+    domains_down = request.form.get('domains_down')
+    domains_tags = request.form.get('domains_tags')
+
+    return redirect(url_for('hiddenServices.show_domains_by_daterange', date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down, domains_tags=domains_tags))
+
+@hiddenServices.route("/hiddenServices/show_domains_by_daterange", methods=['GET'])
+def show_domains_by_daterange():
+    date_from = request.args.get('date_from')
+    date_to = request.args.get('date_to')
+    domains_up = request.args.get('domains_up')
+    domains_down = request.args.get('domains_down')
+    domains_tags = request.args.get('domains_tags')
+
+    date_range = []
+    if date_from is not None and date_to is not None:
+        #change format
+        try:
+            if len(date_from) != 8:
+                date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
+                date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
+            date_range = substract_date(date_from, date_to)
+        except:
+            pass
+
+    if not date_range:
+        date_range.append(datetime.date.today().strftime("%Y%m%d"))
+        date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8]
+        date_to = date_from
+
+    else:
+        date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8]
+        date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8]
+
+    domains_by_day = {}
+    domain_metadata = {}
+    for date in date_range:
+        if domains_up:
+            domains_up = True
+            domains_by_day[date] = list(r_serv_onion.smembers('onion_up:{}'.format(date)))
+            for domain in domains_by_day[date]:
+                h = HiddenServices(domain, 'onion')
+                domain_metadata[domain] = {}
+                if domains_tags:
+                    domains_tags = True
+                    domain_metadata[domain]['tags'] = h.get_domain_tags(update=True)
+
+                domain_metadata[domain]['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'last_check')
+                if domain_metadata[domain]['last_check'] is None:
+                    domain_metadata[domain]['last_check'] = '********'
+                domain_metadata[domain]['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'first_seen')
+                if domain_metadata[domain]['first_seen'] is None:
+                    domain_metadata[domain]['first_seen'] = '********'
+                domain_metadata[domain]['status_text'] = 'UP'
+                domain_metadata[domain]['status_color'] = 'Green'
+                domain_metadata[domain]['status_icon'] = 'fa-check-circle'
+
+        if domains_down:
+            domains_down = True
+            domains_by_day_down = list(r_serv_onion.smembers('onion_down:{}'.format(date)))
+            if domains_up:
+                domains_by_day[date].extend(domains_by_day_down)
+            else:
+                domains_by_day[date] = domains_by_day_down
+            for domain in domains_by_day_down:
+                #h = HiddenServices(onion_domain, 'onion')
+                domain_metadata[domain] = {}
+                #domain_metadata[domain]['tags'] = h.get_domain_tags()
+
+                domain_metadata[domain]['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'last_check')
+                if domain_metadata[domain]['last_check'] is None:
+                    domain_metadata[domain]['last_check'] = '********'
+                domain_metadata[domain]['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'first_seen')
+                if domain_metadata[domain]['first_seen'] is None:
+                    domain_metadata[domain]['first_seen'] = '********'
+
+                domain_metadata[domain]['status_text'] = 'DOWN'
+                domain_metadata[domain]['status_color'] = 'Red'
+                domain_metadata[domain]['status_icon'] = 'fa-times-circle'
+
+    return render_template("domains.html", date_range=date_range, domains_by_day=domains_by_day, domain_metadata=domain_metadata,
+                                date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down,
+                                domains_tags=domains_tags, bootstrap_label=bootstrap_label)
+
 @hiddenServices.route("/hiddenServices/onion_domain", methods=['GET'])
 def onion_domain():
     onion_domain = request.args.get('onion_domain')

var/www/modules/hiddenServices/templates/domains.html

@@ -0,0 +1,291 @@
+<!DOCTYPE html>
+<html>
+
+	<head>
+	  <meta charset="utf-8">
+	  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+	  <title>Hidden Service - AIL</title>
+		<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+	  <!-- Core CSS -->
+	  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
+	  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
+	  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
+		<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet" type="text/css" />
+		<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
+	  <!-- JS -->
+	  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+		<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
+		<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
+		<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
+		<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
+
+		<style>
+		.red_table thead{
+			background: #d91f2d;
+			color: #fff;
+		}
+		.input-group .form-control {
+			position: unset;
+		}
+		</style>
+
+	</head>
+	<body>
+
+            {% include 'navbar.html' %}
+
+            <div id="page-wrapper">
+
+				<div class="row">
+					<div class="col-md-8">
+
+						{% for date in date_range %}
+							{% if domains_by_day[date]%}
+							<div class="panel panel-info" style="text-align:center;">
+								<div class="panel-heading">
+									<h3 style="text-align:center;">{{'{}/{}/{}'.format(date[0:4], date[4:6], date[6:8])}}</h3>
+								</div>
+							</div>
+
+								<table id="table_{{date}}" class="red_table table table-striped table-bordered">
+								  <thead>
+								    <tr>
+								      <th>Domain</th>
+								      <th>First Seen</th>
+											<th>Last Check</th>
+								      <th>Status</th>
+								    </tr>
+								  </thead>
+							    <tbody>
+										{% for domain in domains_by_day[date] %}
+								      <tr>
+								        <td>
+													<a target="_blank" href="{{ url_for('hiddenServices.onion_domain') }}?onion_domain={{ domain }}">{{ domain }}</a>
+													<div>
+														{% for tag in domain_metadata[domain]['tags'] %}
+														<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
+															<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }} <i>{{ domain_metadata[domain]['tags'][tag] }}</i></span>
+														</a>
+														{%  endfor %}
+													</div>
+												</td>
+								        <td>{{'{}/{}/{}'.format(domain_metadata[domain]['first_seen'][0:4], domain_metadata[domain]['first_seen'][4:6], domain_metadata[domain]['first_seen'][6:8])}}</td>
+												<td>{{'{}/{}/{}'.format(domain_metadata[domain]['last_check'][0:4], domain_metadata[domain]['last_check'][4:6], domain_metadata[domain]['last_check'][6:8])}}</td>
+								        <td><div style="color:{{domain_metadata[domain]['status_color']}}; display:inline-block">
+															<i class="fa {{domain_metadata[domain]['status_icon']}} fa-2x"></i>
+															{{domain_metadata[domain]['status_text']}}
+														</div>
+												</td>
+								      </tr>
+										{% endfor %}
+							    </tbody>
+							  </table>
+								<br>
+								<br>
+								<br>
+							{% endif %}
+						{% endfor %}
+
+
+					</div>
+
+					<div class="col-md-4">
+						<div class="panel panel-info" style="text-align:center;">
+							<div class="panel-heading">
+								Select domains by date range :
+								<br>
+								<br>
+								<form action="{{ url_for('hiddenServices.get_onions_by_daterange') }}" id="hash_selector_form" method='post'>
+									<div class="row">
+										<div class="col-md-6">
+											<div class="input-group">
+												<span class="input-group-addon"><i class="fa fa-calendar  fa" aria-hidden="true"></i></span>
+												<input class="form-control" id="date-range-from" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from">
+											</div>
+											<div class="input-group">
+												<span class="input-group-addon"><i class="fa fa-calendar  fa" aria-hidden="true"></i></span>
+												<input class="form-control" id="date-range-to" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to">
+											</div>
+											<div class="checkbox">
+												<label>
+													<input type="checkbox" name="domains_tags" value="True" {% if domains_tags %}checked{% endif %}>
+													<div style="color:#286090; display:inline-block">
+														Domains Tags <i class="fa fa-tags"></i>
+													</div>
+												</label>
+											</div>
+										</div>
+										<div class="col-md-6">
+											<div class="checkbox">
+											  <label>
+											    <input type="checkbox" name="domains_up" value="True" {% if domains_up %}checked{% endif %}>
+													<div style="color:Green; display:inline-block">
+														Domains UP <i class="fa fa-check-circle"></i>
+													</div>
+											  </label>
+											</div>
+											<div class="checkbox">
+											  <label>
+											    <input type="checkbox" name="domains_down" value="True" {% if domains_down %}checked{% endif %}>
+													<div style="color:Red; display:inline-block">
+														Domains DOWN 	<i class="fa fa-times-circle"></i>
+													</div>
+											  </label>
+											</div>
+										</div>
+									</div>
+									<button class="btn btn-primary" style="text-align:center;">
+										<i class="fa fa-eye-slash"></i> Show Onions
+									</button>
+								<form>
+							</div>
+						</div>
+
+					</div>
+
+				</div>
+
+            </div>
+            <!-- /#page-wrapper -->
+
+            <script>
+								var all_graph = {};
+                $(document).ready(function(){
+                    activePage = "page-hiddenServices"
+                    $("#"+activePage).addClass("active");
+
+										$('#date-range-from').dateRangePicker({
+											separator : ' to ',
+											getValue: function()
+											{
+												if ($('#date-range-from').val() && $('#date-range-to').val() )
+													return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
+												else
+													return '';
+											},
+											setValue: function(s,s1,s2)
+											{
+												$('#date-range-from').val(s1);
+												$('#date-range-to').val(s2);
+											}
+						  			});
+										$('#date-range-to').dateRangePicker({
+											separator : ' to ',
+											getValue: function()
+											{
+												if ($('#date-range-from').val() && $('#date-range-to').val() )
+													return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
+												else
+													return '';
+											},
+											setValue: function(s,s1,s2)
+											{
+												$('#date-range-from').val(s1);
+												$('#date-range-to').val(s2);
+											}
+						  			});
+
+										{% for date in date_range %}
+											$('#table_{{date}}').DataTable({
+							  				"aLengthMenu": [[5, 15, 30, -1], [5, 15, 30, "All"]],
+							  				"iDisplayLength": 15,
+							  				"order": [[ 0, "asc" ]]
+							  			});
+										{% endfor %}
+
+										//all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hiddenServices.domain_crawled_7days_json') }}?type=onion");
+                });
+								/*$(window).on("resize", function() {
+									all_graph.onResize();
+								});*/
+            </script>
+
+<script>
+/*
+all_graph.onResize = function () {
+	var aspect = 1000 / 500, all_graph = $("#graph_div");
+	var targetWidth = all_graph.parent().width();
+	all_graph.attr("width", targetWidth);
+	all_graph.attr("height", targetWidth / aspect);
+}
+window.all_graph = all_graph;
+
+function create_line_chart(id, url){
+	var width = 650;
+	var height = Math.round(width / 6);
+	var margin = {top: 20, right: 55, bottom: 50, left: 40};
+	var x = d3.scaleTime().range([0, width]);
+	var y = d3.scaleLinear().rangeRound([height, 0]);
+	var xAxis = d3.axisBottom(x);
+	var yAxis = d3.axisLeft(y);
+	var parseTime = d3.timeParse("%Y-%m-%d");
+	var line = d3.line()
+							.x(function(d) {
+          			return x(d.date);
+        			})
+							.y(function(d) {
+          			return y(d.value);
+        			});
+	var svg_line = d3.select('#'+id).append('svg')
+				.attr("id", "graph_div")
+				.attr("width", width + margin.left + margin.right)
+				.attr("height", height + margin.top + margin.bottom)
+				.append('g')
+						.attr('transform', "translate("+ margin.left +","+ margin.top +")");
+	var div = d3.select('body').append('div')
+				.attr('class', 'tooltip')
+				.style('opacity', 0);
+	//add div tooltip
+d3.json(url)
+	.then(function(data){
+		data.forEach(function(d) {
+			d.date_label = d.date;
+			d.date = parseTime(d.date);
+			d.value = +d.value;
+		});
+		// fit the data
+		x.domain(d3.extent(data, function(d) { return d.date; }));
+		//x.domain(data.map(function (d) { return d.date; })); //E
+		y.domain([0, d3.max(data, function(d){ return d.value ; })]);
+		//line
+		svg_line.append("path")
+				.data([data])
+				.attr("class", "line_graph")
+				.attr("d", line);
+		// add X axis
+		svg_line.append("g")
+		    .attr("transform", "translate(0," + height + ")")
+		    .call(d3.axisBottom(x))
+				.selectAll("text")
+					.style("text-anchor", "end")
+					.attr("transform", "rotate(-45)" );
+		// Add the Y Axis
+		svg_line.append("g")
+		    .call(d3.axisLeft(y));
+		//add a dot circle
+    svg_line.selectAll('dot')
+				.data(data).enter()
+				.append('circle')
+						.attr('r', 2)
+							.attr('cx', function(d) { return x(d.date); })
+							.attr('cy', function(d) { return y(d.value); })
+            	.on('mouseover', function(d) {
+	            	div.transition().style('opacity', .9);
+	            	div.html('' + d.date_label+ '<br/>' + d.value).style('left', (d3.event.pageX) + 'px')
+								.style("left", (d3.event.pageX) + "px")
+								.style("top", (d3.event.pageY - 28) + "px");
+          		})
+            	.on('mouseout', function(d)
+          			{
+            			div.transition().style('opacity', 0);
+          			});
+	});
+}
+*/
+</script>
+        </body>
+
+</html>

var/www/modules/hiddenServices/templates/hiddenServices.html

@@ -12,11 +12,17 @@
 	  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
 	  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
 	  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
+				<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet" type="text/css" />
 	  <!-- JS -->
 	  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+		<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
+		<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
 		<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
 
 		<style>
+		.input-group .form-control {
+			position: unset;
+		}
 		div.tooltip {
 			position: absolute;
 			text-align: center;
@@ -84,50 +90,103 @@
 					</div>
 
 					<div class="col-md-6">
-						<div class="panel panel-info">
-      						<div class="panel-heading">
-                    <i class="fa fa-eye-slash"></i> Domains Crawled Today
-      						</div>
+						<div class="panel panel-info" style="text-align:center;">
+							<div class="panel-heading">
+								Select domains by date range :
+								<br>
+								<br>
+								<form action="{{ url_for('hiddenServices.get_onions_by_daterange') }}" id="hash_selector_form" method='post'>
+									<div class="row">
+										<div class="col-md-6">
+											<div class="input-group">
+												<span class="input-group-addon"><i class="fa fa-calendar  fa" aria-hidden="true"></i></span>
+												<input class="form-control" id="date-range-from" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from">
+											</div>
+											<div class="input-group">
+												<span class="input-group-addon"><i class="fa fa-calendar  fa" aria-hidden="true"></i></span>
+												<input class="form-control" id="date-range-to" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to">
+											</div>
+											<div class="checkbox">
+												<label>
+													<input type="checkbox" name="domains_tags" value="True" checked>
+													<div style="color:#286090; display:inline-block">
+														Domains Tags <i class="fa fa-tags"></i>
+													</div>
+												</label>
+											</div>
+										</div>
+										<div class="col-md-6">
+											<div class="checkbox">
+											  <label>
+											    <input type="checkbox" name="domains_up" value="True" checked>
+													<div style="color:Green; display:inline-block">
+														Domains UP <i class="fa fa-check-circle"></i>
+													</div>
+											  </label>
+											</div>
+											<div class="checkbox">
+											  <label>
+											    <input type="checkbox" name="domains_down" value="True">
+													<div style="color:Red; display:inline-block">
+														Domains DOWN 	<i class="fa fa-times-circle"></i>
+													</div>
+											  </label>
+											</div>
 
-                      <table class="table table-hover table-striped">
-                        <tbody>
-                          <tr>
-                            <td>
-															<div style="color:Green; display:inline-block">
-																<i class="fa fa-check-circle fa-2x"></i>
-																Domains UP
-															</div>
-														</td>
-                            <td>
-															<div style="color:Green; display:inline-block">
-																{{ statDomains['domains_up'] }}
-															</div>
-														</td>
-                          </tr>
-                          <tr>
-                            <td>
-															<div style="color:Red; display:inline-block">
-																<i class="fa fa-times-circle fa-2x"></i>
-																Domains DOWN
-															</div>
-														</td>
-                            <td>
-															<div style="color:Red; display:inline-block">
-																{{ statDomains['domains_down'] }}
-															</div>
-														</td>
-                          </tr>
-                          <tr>
-                            <td>Crawled Domains</td>
-                            <td>{{ statDomains['total'] }}</td>
-                          </tr>
-													<tr>
-                            <td>Domains in Queue</td>
-                            <td>{{ statDomains['domains_queue'] }}</td>
-                          </tr>
-                        </tbody>
-                      </table>
-                </div>
+										</div>
+									</div>
+									<button class="btn btn-primary" style="text-align:center;">
+										<i class="fa fa-eye-slash"></i> Show Onions
+									</button>
+								<form>
+							</div>
+						</div>
+
+
+						<div class="panel panel-info">
+      				<div class="panel-heading">
+                <i class="fa fa-eye-slash"></i> Domains Crawled Today
+      				</div>
+
+              <table class="table table-hover table-striped">
+                <tbody>
+                  <tr>
+                    <td>
+											<div style="color:Green; display:inline-block">
+												<i class="fa fa-check-circle fa-2x"></i>
+												Domains UP
+											</div>
+										</td>
+                    <td>
+											<div style="color:Green; display:inline-block">
+												{{ statDomains['domains_up'] }}
+											</div>
+										</td>
+                  </tr>
+                  <tr>
+                    <td>
+											<div style="color:Red; display:inline-block">
+												<i class="fa fa-times-circle fa-2x"></i>
+												Domains DOWN
+											</div>
+										</td>
+                    <td>
+											<div style="color:Red; display:inline-block">
+												{{ statDomains['domains_down'] }}
+											</div>
+										</td>
+                  </tr>
+                  <tr>
+                    <td>Crawled Domains</td>
+                    <td>{{ statDomains['total'] }}</td>
+                  </tr>
+									<tr>
+                    <td>Domains in Queue</td>
+                    <td>{{ statDomains['domains_queue'] }}</td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
 					</div>
 
 				</div>
@@ -140,6 +199,38 @@
                 $(document).ready(function(){
                     activePage = "page-hiddenServices"
                     $("#"+activePage).addClass("active");
+
+										$('#date-range-from').dateRangePicker({
+											separator : ' to ',
+											getValue: function()
+											{
+												if ($('#date-range-from').val() && $('#date-range-to').val() )
+													return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
+												else
+													return '';
+											},
+											setValue: function(s,s1,s2)
+											{
+												$('#date-range-from').val(s1);
+												$('#date-range-to').val(s2);
+											}
+						  			});
+										$('#date-range-to').dateRangePicker({
+											separator : ' to ',
+											getValue: function()
+											{
+												if ($('#date-range-from').val() && $('#date-range-to').val() )
+													return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
+												else
+													return '';
+											},
+											setValue: function(s,s1,s2)
+											{
+												$('#date-range-from').val(s1);
+												$('#date-range-to').val(s2);
+											}
+						  			});
+
 										all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hiddenServices.domain_crawled_7days_json') }}?type=onion");
                 });
 								$(window).on("resize", function() {