chg: [UI tags] search domains by tags

bin/lib/Correlate_object.py

@@ -24,7 +24,7 @@ r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
 config_loader = None
 
 def is_valid_object_type(object_type):
-    if object_type in ['domain', 'item', 'screenshot']:
+    if object_type in ['domain', 'item', 'image']:
         return True
     else:
         return False

bin/packages/Tag.py

@@ -490,7 +490,8 @@ def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50
             l_tagged_obj.extend( date_day_obj )
 
         # handle pagination
-        nb_pages = len(l_tagged_obj) / nb_obj
+        nb_all_elem = len(l_tagged_obj)
+        nb_pages = nb_all_elem / nb_obj
         if not nb_pages.is_integer():
             nb_pages = int(nb_pages)+1
         else:
@@ -503,7 +504,7 @@ def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50
         stop = (nb_obj*page) -1
         l_tagged_obj = l_tagged_obj[start:stop]
 
-        return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages}
+        return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages, "nb_first_elem":start+1, "nb_last_elem":stop+1, "nb_all_elem":nb_all_elem}
 
     # without daterange
     else:
@@ -517,7 +518,8 @@ def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50
             return {"tagged_obj":l_tagged_obj, "page":0, "nb_pages":0}
 
         # handle pagination
-        nb_pages = len(l_tagged_obj) / nb_obj
+        nb_all_elem = len(l_tagged_obj)
+        nb_pages = nb_all_elem / nb_obj
         if not nb_pages.is_integer():
             nb_pages = int(nb_pages)+1
         else:
@@ -538,11 +540,16 @@ def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50
                     l_obj.append(elem)
                 current_index += 1
             l_tagged_obj = l_obj
+            stop += 1
+            if stop > nb_all_elem:
+                stop = nb_all_elem
         # only one page
         else:
+            start = 0
+            stop = nb_all_elem
             l_tagged_obj = list(l_tagged_obj)
 
-        return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages}
+        return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages, "nb_first_elem":start+1, "nb_last_elem":stop, "nb_all_elem":nb_all_elem}
 
 
 def get_obj_date(object_type, object_id): # # TODO: move me in another file

var/www/blueprints/tags_ui.py

@@ -94,6 +94,22 @@ def get_all_obj_tags():
         return jsonify(res)
     return jsonify(Tag.get_all_obj_tags(object_type))
 
+@tags_ui.route('/tag/search/domain')
+@login_required
+@login_read_only
+def tags_search_domains():
+    object_type = 'domain'
+    dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"}
+    return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
+
+@tags_ui.route('/tag/search/image')
+@login_required
+@login_read_only
+def tags_search_images():
+    object_type = 'image'
+    dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"}
+    return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
+
 @tags_ui.route('/tag/search/get_obj_by_tags')
 @login_required
 @login_read_only
@@ -102,7 +118,7 @@ def get_obj_by_tags():
     # # TODO: sanityze all
     object_type = request.args.get('object_type')
     ltags = request.args.get('ltags')
-    page = request.args.get('ltags')
+    page = request.args.get('page')
     date_from = request.args.get('ltags')
     date_to = request.args.get('ltags')
 
@@ -112,14 +128,23 @@ def get_obj_by_tags():
     for tag in list_tags:
         list_tag.append(tag.replace('"','\"'))
 
+    # object_type
     res = Correlate_object.sanitize_object_type(object_type)
     if res:
         return jsonify(res)
 
-    dict_obj = Tag.get_obj_by_tags(object_type, list_tag)
+    # page
+    try:
+        page = int(page)
+    except:
+        page = 1
+
+    dict_obj = Tag.get_obj_by_tags(object_type, list_tag, page=page)
 
     if dict_obj['tagged_obj']:
-        dict_tagged = {"object_type":object_type, "page":dict_obj['page'] ,"nb_pages":dict_obj['nb_pages'], "tagged_obj":[]}
+        dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s",
+                        "tagged_obj":[], "page":dict_obj['page'] ,"nb_pages":dict_obj['nb_pages'],
+                        "nb_first_elem":dict_obj['nb_first_elem'], "nb_last_elem":dict_obj['nb_last_elem'], "nb_all_elem":dict_obj['nb_all_elem']}
         for obj_id in dict_obj['tagged_obj']:
             obj_metadata = Correlate_object.get_object_metadata(object_type, obj_id)
             obj_metadata['id'] = obj_id
@@ -128,12 +153,16 @@ def get_obj_by_tags():
         dict_tagged['tab_keys'] = Correlate_object.get_obj_tag_table_keys(object_type)
 
         if len(list_tag) == 1:
-            dict_tagged['current_tags'] = ltags.replace('"', '').replace('=', '').replace(':', '')
+            dict_tagged['current_tags'] = [ltags.replace('"', '\"')]
         else:
             dict_tagged['current_tags'] = list_tag
+        dict_tagged['current_tags_str'] = ltags
 
         #return jsonify(dict_tagged)
-        return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
+    else:
+        dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"}
+
+    return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
 
 # # add route : /crawlers/show_domain
 # @tags_ui.route('/tags/search/domain')

var/www/modules/Tags/templates/Tags.html

@@ -217,8 +217,8 @@ var last_clicked_paste;
 var can_change_modal_content = true;
 
 $(document).ready(function(){
-	$("#nav_quick_search").removeClass("text-muted");
+	$("#nav_tags_search").removeClass("text-muted");
-	$("#nav_tag_{{tag_nav}}").addClass("active");
+	$("#nav_tags_search_item").addClass("active");
 	search_table = $('#myTable_').DataTable({ "order": [[ 0, "asc" ]] });
 
 	// Use to bind the button with the new displayed data

var/www/modules/hiddenServices/Flask_hiddenServices.py

@@ -253,7 +253,6 @@ def dashboard():
     statDomains_regular = get_stats_last_crawled_domains('regular', date)
 
     return render_template("Crawler_dashboard.html", crawler_metadata_onion = crawler_metadata_onion,
-                                object_type='domain',
                                 crawler_enabled=crawler_enabled, date=date,
                                 crawler_metadata_regular=crawler_metadata_regular,
                                 statDomains_onion=statDomains_onion, statDomains_regular=statDomains_regular)

var/www/modules/hiddenServices/templates/Crawler_dashboard.html

@@ -105,7 +105,9 @@
 					</div>
 				</div>
 
-				{% include 'tags/block_obj_tags_search.html' %}
+				{% with object_type='domain' %}
+					{% include 'tags/block_obj_tags_search.html' %}
+				{% endwith %}
 
 			</div>
 		</div>

var/www/templates/pagination.html

@@ -0,0 +1,50 @@
+<div class="my-2">
+	<div class="d-flex justify-content-center border-top">
+		<nav class="mt-4" aria-label="...">
+			<ul class="pagination">
+				<li class="page-item {%if page==1%}disabled{%endif%}">
+					<a class="page-link" href="{{ target_url }}&page={{page-1}}">Previous</a>
+				</li>
+
+				{%if page>3%}
+					<li class="page-item"><a class="page-link" href="{{ target_url }}&page=1">1</a></li>
+					<li class="page-item disabled"><a class="page-link" aria-disabled="true" href="#">...</a></li>
+					<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{page-1}}">{{page-1}}</a></li>
+					<li class="page-item active"><a class="page-link" href="{{ target_url }}&page={{page}}">{{page}}</a></li>
+				{%else%}
+				{%if page>2%}<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{page-2}}">{{page-2}}</a></li>{%endif%}
+				{%if page>1%}<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{page-1}}">{{page-1}}</a></li>{%endif%}
+					<li class="page-item active"><a class="page-link" href="{{ target_url }}&page={{page}}">{{page}}</a></li>
+				{%endif%}
+
+				{%if nb_page_max-page>3%}
+					<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{page+1}}">{{page+1}}</a></li>
+					<li class="page-item disabled"><a class="page-link" aria-disabled="true" href="#">...</a></li>
+					<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{nb_page_max}}">{{nb_page_max}}</a></li>
+				{%else%}
+				{%if nb_page_max-page>2%}<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{nb_page_max-2}}">{{nb_page_max-2}}</a></li>{%endif%}
+				{%if nb_page_max-page>1%}<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{nb_page_max-1}}">{{nb_page_max-1}}</a></li>{%endif%}
+				{%if nb_page_max-page>0%}<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{nb_page_max}}">{{nb_page_max}}</a></li>{%endif%}
+				{%endif%}
+
+				<li class="page-item {%if page==nb_page_max%}disabled{%endif%}">
+					<a class="page-link" href="{{ target_url }}&page={{page+1}}" aria-disabled="true">Next</a>
+				</li>
+			</ul>
+		</nav>
+	</div>
+
+	{%if nb_all_elem%}
+		<div class="d-flex justify-content-center">
+			<span class="badge badge-info text-white">
+				{{object_name}}:&nbsp;
+				<span class="badge badge-light">{{nb_first_elem}}-{{nb_last_elem}}</span>
+				<span> / </span>
+				<span class="badge badge-light">{{nb_all_elem}}</span>
+			 </span>
+		</div>
+		<br>
+		<br>
+		<br>
+	{%endif%}
+</div>

var/www/templates/tags/block_obj_tags_search.html

@@ -1,6 +1,6 @@
 <div class="card mb-3 mt-1">
 	<div class="card-header text-white bg-dark">
-		<h5 class="card-title">Search Domain by Tags :</h5>
+		<h5 class="card-title">Search {{object_name}} by Tags :</h5>
 	</div>
 		<div class="card-body">
 
@@ -29,21 +29,22 @@
 			</div>
 
 			<button class="btn btn-primary" type="button" id="button-search-tags" onclick="searchTags()">
-				<i class="fas fa-search"></i> Search Domains
+				<i class="fas fa-search"></i> Search {{object_name}}
 			</button>
 
 			</div>
 	</div>
 
+
 <link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
 <script src="{{ url_for('static', filename='js/tags.js') }}"></script>
 <script>
   var ltags;
-  $.getJSON("{{ url_for('tags_ui.get_all_obj_tags') }}?object_type={{dict_tagged['object_type']}}",
+  $.getJSON("{{ url_for('tags_ui.get_all_obj_tags') }}?object_type={{ object_type }}",
     function(data) {
       ltags = $('#ltags').tagSuggest({
         data: data,
-				value: [{%if "current_tags" in dict_tagged%}{% for tag in dict_tagged['current_tags'] %}'{{tag|safe}}',{%endfor%}{%endif%}],
+				value: [{%if "current_tags"%}{% for tag in current_tags %}'{{tag|safe}}',{%endfor%}{%endif%}],
         sortOrder: 'name',
         maxDropHeight: 200,
         name: 'ltags'
@@ -55,7 +56,8 @@
   	//var date_from = $('#date-range-from-input').val();
   	//var date_to =$('#date-range-to-input').val();
 		//parameter = parameter + "&date_from="+date_from+"&date_to="+date_to
-  	window.location.replace("{{ url_for('tags_ui.get_obj_by_tags') }}?ltags=" + data + "&object_type={{dict_tagged['object_type']}}&page={{dict_tagged['page']}}");
+		var parameter = "?ltags=" + data + "&object_type={{ object_type }}{%if page%}&page={{ page }}{%endif%}";
+  	window.location.replace("{{ url_for('tags_ui.get_obj_by_tags') }}" + parameter);
   }
   function emptyTags() {
   	ltags.clear();

var/www/templates/tags/menu_sidebar.html

@@ -6,6 +6,29 @@
       	</button>
 
       	<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
+          <h5 class="d-flex text-muted w-100" id="nav_tags_search">
+      		  <span>Tags Search</span>
+      		</h5>
+      	  <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100">
+      	    <li class="nav-item">
+      	      <a class="nav-link" href="{{ url_for('Tags.Tags_page') }}" id="nav_tags_search_item">
+      	        <i class="far fa-file"></i>
+      	        Search Items by Tags
+      	      </a>
+      	    </li>
+      	    <li class="nav-item">
+      	      <a class="nav-link" href="{{ url_for('tags_ui.tags_search_domains') }}" id="nav_tags_search_domain">
+      					<i class="fab fa-html5"></i>
+      	       	Search Domains by Tags
+      	      </a>
+      	    </li>
+            <li class="nav-item">
+      	      <a class="nav-link" href="{{ url_for('tags_ui.tags_search_images') }}" id="nav_tags_search_image">
+      					<i class="fas fa-image"></i>
+      	       	Search Images by Tags
+      	      </a>
+      	    </li>
+          </ul>
       		<h5 class="d-flex text-muted w-100">
       		  <span>Tags Management </span>
       		</h5>

var/www/templates/tags/search_obj_by_tags.html

@@ -25,24 +25,6 @@
 
   {% include 'nav_bar.html' %}
 
-	<!-- Modal -->
-  <div id="mymodal" class="modal fade" role="dialog">
-   <div class="modal-dialog modal-lg">
-
-     <!-- Modal content-->
-     <div id="mymodalcontent" class="modal-content">
-       <div id="mymodalbody" class="modal-body" max-width="850px">
-         <p>Loading paste information...</p>
-         <img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
-       </div>
-       <div class="modal-footer">
-         <a id="button_show_path" target="_blank" href=""><button type="button" class="btn btn-info">Show saved paste</button></a>
-         <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
-       </div>
-     </div>
-   </div>
-  </div>
-
 	<div class="container-fluid">
 		<div class="row">
 
@@ -50,10 +32,12 @@
 
 			<div class="col-12 col-lg-10" id="core_content">
 
-				{% include 'tags/block_obj_tags_search.html' %}
+        {% with object_type=dict_tagged['object_type'], current_tags=dict_tagged['current_tags'], object_name=dict_tagged['object_name'] %}
+					{% include 'tags/block_obj_tags_search.html' %}
+				{% endwith %}
 
 				<div>
-					{%if dict_tagged%}
+					{%if 'tagged_obj' in dict_tagged%}
 						<table class="table table-bordered table-hover" id="myTable_">
 				      <thead class="thead-dark">
 				        <tr>
@@ -80,7 +64,7 @@
                         <div class="mb-2">
   				                {% for tag in dict_obj['tags'] %}
     				                <a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
-    				                  <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
+    				                  <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
     				                </a>
   				                {%  endfor %}
   				              </div>
@@ -99,15 +83,22 @@
       									{% endif %}
   										</td>
   				          </tr>
-  				        {%  endfor %}
+  				        {% endfor %}
 
-                {%  endif %}
+                {% endif %}
 
 				      </tbody>
 				    </table>
 
 					{%endif%}
 				</div>
+        {%if 'tagged_obj' in dict_tagged%}
+          {% with page=dict_tagged['page'], nb_page_max=dict_tagged['nb_pages'],  nb_first_elem=dict_tagged['nb_first_elem'], nb_last_elem=dict_tagged['nb_last_elem'], nb_all_elem=dict_tagged['nb_all_elem'] %}
+            {% set object_name= dict_tagged['object_name'] %}
+            {% set target_url=url_for('tags_ui.get_obj_by_tags') + "?object_type=" + dict_tagged['object_type'] + "&ltags=" + dict_tagged['current_tags_str'] %}
+            {% include 'pagination.html' %}
+  				{% endwith %}
+        {%endif%}
 
     	</div>
 		</div>
@@ -120,7 +111,8 @@ var ltags;
 var search_table;
 
 $(document).ready(function(){
-	//search_table = $('#myTable_').DataTable({ "order": [[ 0, "asc" ]] });
+  $("#nav_tags_search").removeClass("text-muted");
+	$("#nav_tags_search_{{dict_tagged['object_type']}}").addClass("active");
   $('#myTable_').DataTable({ "order": [[ 0, "asc" ]] });
 
 });