diff --git a/OVERVIEW.md b/OVERVIEW.md index 305a8958..f30f763d 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -14,7 +14,6 @@ Redis and ARDB overview * ARDB on TCP port 6382 - DB 1 - Curve DB 2 - TermFreq DB 3 - Trending @@ -24,7 +23,7 @@ Redis and ARDB overview DB 7 - Metadata DB 8 - Statistics DB 9 - Crawler - + * ARDB on TCP port - DB 0 - Lines duplicate - DB 1 - Hashes @@ -36,32 +35,58 @@ Redis and ARDB overview ##### Hset: | Key | Field | Value | | ------ | ------ | ------ | -| daily_tags:**** | **** | **** | +| daily_tags:**date** | **tag** | **nb tagged this day** | | | | -| tag_metadata:**** | first_seen | **** | -| tag_metadata:**** | last_seen | **** | +| tag_metadata:**tag** | first_seen | **date** | +| tag_metadata:**tag** | last_seen | **date** | ##### Set: | Key | Value | | ------ | ------ | -| list_tags | **** | -| active_taxonomies | **** | -| active_galaxies | **** | -| active_tag_**** | **** | -| synonym_tag_misp-galaxy:**** | **** | -| list_export_tags | **** | -| ****:**** | **** | +| list_tags | **tag** | +| active_taxonomies | **taxonomie** | +| active_galaxies | **galaxie** | +| active_tag_**taxonomie or galaxy** | **tag** | +| synonym_tag_misp-galaxy:**galaxy** | **tag synonym** | +| list_export_tags | **user_tag** | +| **tag**:**date** | **paste** | ##### old: | Key | Value | | ------ | ------ | -| **** | **** | +| *tag* | *paste* | +## DB9 - Crawler: +##### Hset: +| Key | Field | Value | +| ------ | ------ | ------ | +| **service type**:**domain** | first_seen | **date** | +| | last_check | **date** | +| | paste_parent | **parent last crawling (can be auto or manual)** | +##### Zset: +| Key | Field | Value | +| ------ | ------ | ------ | +| crawler_history_**service type**:**domain** | **item root (first crawled item)** | **epoch (seconds)** | +##### Regular key: +| Key | Value | +| ------ | ------ | +| crawler_history_**service type**:**domain** | **json config** | + +##### exemple json config: +```json +{ + "closespider_pagecount": 1, + "time": 3600, + "depth_limit": 0, + "har": 0, + "png": 0 +} +``` ARDB overview --------------------------- @@ -130,5 +155,3 @@ ARDB_DB GET - 'base64_decoded:'+date nd_decoded GET - 'binary_decoded:'+date nd_decoded - - diff --git a/bin/Tags.py b/bin/Tags.py index c0ea4bdc..cf2f06e6 100755 --- a/bin/Tags.py +++ b/bin/Tags.py @@ -75,12 +75,12 @@ if __name__ == '__main__': if res == 1: print("new tags added : {}".format(tag)) # add the path to the tag set - date = get_paste_date(path) - res = server.sadd('{}:{}'.format(tag, date), path) + curr_date = datetime.date.today().strftime("%Y%m%d") + res = server.sadd('{}:{}'.format(tag, curr_date), path) if res == 1: print("new paste: {}".format(path)) print(" tagged: {}".format(tag)) - set_tag_metadata(tag, date) + set_tag_metadata(tag, curr_date) server_metadata.sadd('tag:{}'.format(path), tag) curr_date = datetime.date.today().strftime("%Y%m%d") diff --git a/var/www/modules/Tags/Flask_Tags.py b/var/www/modules/Tags/Flask_Tags.py index bbc918ed..57094b83 100644 --- a/var/www/modules/Tags/Flask_Tags.py +++ b/var/www/modules/Tags/Flask_Tags.py @@ -8,7 +8,7 @@ import redis from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for import json -from datetime import datetime +import datetime import ssdeep import Paste @@ -71,9 +71,98 @@ def get_tags_with_synonyms(tag): # ============= ROUTES ============== -@Tags.route("/Tags/", methods=['GET']) +@Tags.route("/tags/", methods=['GET']) def Tags_page(): - return render_template("Tags.html") + current_date = datetime.date.today().strftime("%Y-%m-%d") + + tags = request.args.get('ltags') + if tags is None: + return render_template("Tags.html", date_from=current_date, date_to=current_date) + + else: + + + tags = request.args.get('ltags') + + list_tags = tags.split(',') + list_tag = [] + for tag in list_tags: + list_tag.append(tag.replace('"','\"')) + + # TODO verify input + + if(type(list_tags) is list): + # no tag + if list_tags is False: + print('empty') + # 1 tag + elif len(list_tags) < 2: + tagged_pastes = r_serv_tags.smembers(list_tags[0]) + + # 2 tags or more + else: + tagged_pastes = r_serv_tags.sinter(list_tags[0], *list_tags[1:]) + + else : + return 'INCORRECT INPUT' + + all_content = [] + paste_date = [] + paste_linenum = [] + all_path = [] + allPastes = list(tagged_pastes) + paste_tags = [] + + for path in allPastes[0:50]: ######################moduleName + all_path.append(path) + paste = Paste.Paste(path) + content = paste.get_p_content() + content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 + all_content.append(content[0:content_range].replace("\"", "\'").replace("\r", " ").replace("\n", " ")) + curr_date = str(paste._get_p_date()) + curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] + paste_date.append(curr_date) + paste_linenum.append(paste.get_lines_info()[0]) + p_tags = r_serv_metadata.smembers('tag:'+path) + complete_tags = [] + l_tags = [] + for tag in p_tags: + complete_tag = tag + + tag = tag.split('=') + if len(tag) > 1: + if tag[1] != '': + tag = tag[1][1:-1] + # no value + else: + tag = tag[0][1:-1] + # use for custom tags + else: + tag = tag[0] + + l_tags.append( (tag,complete_tag) ) + + paste_tags.append(l_tags) + + if len(allPastes) > 10: + finished = False + else: + finished = True + + return render_template("Tags.html", + all_path=all_path, + tags=tags, + list_tag = list_tag, + date_from=current_date, + date_to=current_date, + paste_tags=paste_tags, + bootstrap_label=bootstrap_label, + content=all_content, + paste_date=paste_date, + paste_linenum=paste_linenum, + char_to_display=max_preview_modal, + finished=finished) + @Tags.route("/Tags/get_all_tags") def get_all_tags(): @@ -173,94 +262,6 @@ def get_tags_galaxy(): else: return 'this galaxy is disable' - -@Tags.route("/Tags/get_tagged_paste") -def get_tagged_paste(): - - tags = request.args.get('ltags') - - list_tags = tags.split(',') - list_tag = [] - for tag in list_tags: - list_tag.append(tag.replace('"','\"')) - - # TODO verify input - - if(type(list_tags) is list): - # no tag - if list_tags is False: - print('empty') - # 1 tag - elif len(list_tags) < 2: - tagged_pastes = r_serv_tags.smembers(list_tags[0]) - - # 2 tags or more - else: - tagged_pastes = r_serv_tags.sinter(list_tags[0], *list_tags[1:]) - - else : - return 'INCORRECT INPUT' - - #TODO FIXME - currentSelectYear = int(datetime.now().year) - - all_content = [] - paste_date = [] - paste_linenum = [] - all_path = [] - allPastes = list(tagged_pastes) - paste_tags = [] - - for path in allPastes[0:50]: ######################moduleName - all_path.append(path) - paste = Paste.Paste(path) - content = paste.get_p_content() - content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 - all_content.append(content[0:content_range].replace("\"", "\'").replace("\r", " ").replace("\n", " ")) - curr_date = str(paste._get_p_date()) - curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] - paste_date.append(curr_date) - paste_linenum.append(paste.get_lines_info()[0]) - p_tags = r_serv_metadata.smembers('tag:'+path) - complete_tags = [] - l_tags = [] - for tag in p_tags: - complete_tag = tag - - tag = tag.split('=') - if len(tag) > 1: - if tag[1] != '': - tag = tag[1][1:-1] - # no value - else: - tag = tag[0][1:-1] - # use for custom tags - else: - tag = tag[0] - - l_tags.append( (tag,complete_tag) ) - - paste_tags.append(l_tags) - - if len(allPastes) > 10: - finished = False - else: - finished = True - - return render_template("tagged.html", - year=currentSelectYear, - all_path=all_path, - tags=tags, - list_tag = list_tag, - paste_tags=paste_tags, - bootstrap_label=bootstrap_label, - content=all_content, - paste_date=paste_date, - paste_linenum=paste_linenum, - char_to_display=max_preview_modal, - finished=finished) - - @Tags.route("/Tags/remove_tag") def remove_tag(): diff --git a/var/www/modules/Tags/templates/Tags.html b/var/www/modules/Tags/templates/Tags.html index 43221deb..ead15331 100644 --- a/var/www/modules/Tags/templates/Tags.html +++ b/var/www/modules/Tags/templates/Tags.html @@ -9,12 +9,15 @@ + + + @@ -68,6 +71,48 @@ +
+ {%if all_path%} + + + + + + + + + + + + {% for path in all_path %} + + + + + + + {% endfor %} + + +
DatePath# of linesAction
{{ paste_date[loop.index0] }} +
{{ path }}
+ +
+ {% for tag in paste_tags[loop.index0] %} + + {{ tag[0] }} + + {% endfor %} +
+ 		{{ paste_linenum[loop.index0] }}

+ +

+
+ {%endif%} +
+


@@ -98,14 +143,38 @@ + + + + diff --git a/var/www/modules/hiddenServices/templates/header_hiddenServices.html b/var/www/modules/hiddenServices/templates/header_hiddenServices.html index 5c77963c..34e20319 100644 --- a/var/www/modules/hiddenServices/templates/header_hiddenServices.html +++ b/var/www/modules/hiddenServices/templates/header_hiddenServices.html @@ -1 +1 @@ -
  • hidden Services
    • +
  • hidden Services
    • diff --git a/var/www/modules/hiddenServices/templates/showDomain.html b/var/www/modules/hiddenServices/templates/showDomain.html index e797f6b2..22224fd0 100644 --- a/var/www/modules/hiddenServices/templates/showDomain.html +++ b/var/www/modules/hiddenServices/templates/showDomain.html @@ -7,6 +7,7 @@ + @@ -70,7 +71,7 @@ {%endif%}
    {% for tag in origin_paste_tags %} - + {{ tag[0] }} {% endfor %} @@ -82,7 +83,7 @@
    {% for tag in domain_tags %} - + {{ tag }} {{ domain_tags[tag] }} {% endfor %} @@ -106,7 +107,7 @@
    {% for tag in paste_tags[loop.index0] %} - + {{ tag[0] }} {% endfor %} diff --git a/var/www/modules/search/templates/search.html b/var/www/modules/search/templates/search.html index 5754a0ee..46b73091 100644 --- a/var/www/modules/search/templates/search.html +++ b/var/www/modules/search/templates/search.html @@ -101,7 +101,7 @@ {{ path }}
    {% for tag in paste_tags[loop.index0] %} - + {{ tag[0] }} {% endfor %} @@ -201,7 +201,7 @@ var curr_preview = data.preview_array[i].replace(/\"/g, "\'"); var tag = "" for(j=0; j" + tag = tag + "" + "" + data.list_tags[i][j][0] + "" + "" } diff --git a/var/www/templates/nav_bar.html b/var/www/templates/nav_bar.html index 83f174bf..57d05ab5 100644 --- a/var/www/templates/nav_bar.html +++ b/var/www/templates/nav_bar.html @@ -22,7 +22,7 @@ Leaks Hunter
  • - Crawlers + Crawlers
  • Decoded