From da5579875db909b6503452742bbda78d2d225c5f Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 20 May 2019 14:32:07 +0200 Subject: [PATCH] chg: [pgpdump] reprocess tagged items + fix pgpdump --- bin/LAUNCH.sh | 2 + bin/PgpDump.py | 1 + bin/helper/reprocess_pgp_message.py | 50 +++++++++++++++++++ .../modules/hashDecoded/Flask_hashDecoded.py | 16 +++++- .../hashDecoded/templates/PgpDecoded.html | 2 +- 5 files changed, 68 insertions(+), 3 deletions(-) create mode 100755 bin/helper/reprocess_pgp_message.py diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 6d7c6bf7..e4175b90 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -187,6 +187,8 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x" sleep 0.1 + screen -S "Script_AIL" -X screen -t "PgpDump" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./PgpDump.py; read x" + sleep 0.1 screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Decoder.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Bitcoin.py; read x" diff --git a/bin/PgpDump.py b/bin/PgpDump.py index 20009235..f46cbce9 100755 --- a/bin/PgpDump.py +++ b/bin/PgpDump.py @@ -8,6 +8,7 @@ import os import re +import time import redis import signal import datetime diff --git a/bin/helper/reprocess_pgp_message.py b/bin/helper/reprocess_pgp_message.py new file mode 100755 index 00000000..bf8d78ee --- /dev/null +++ b/bin/helper/reprocess_pgp_message.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import gzip +import base64 +import uuid +import datetime +import base64 +import redis +import json +import time + +sys.path.append(os.environ['AIL_BIN']) +from Helper import Process + +def substract_date(date_from, date_to): + date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8])) + date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8])) + delta = date_to - date_from # timedelta + l_date = [] + for i in range(delta.days + 1): + date = date_from + datetime.timedelta(i) + l_date.append( date.strftime('%Y%m%d') ) + return l_date + +config_section = 'Keys' +p = Process(config_section) + +r_tags = redis.StrictRedis( + host=p.config.get("ARDB_Tags", "host"), + port=p.config.getint("ARDB_Tags", "port"), + db=p.config.getint("ARDB_Tags", "db"), + decode_responses=True) + +tag = 'infoleak:automatic-detection="pgp-message"' + +# get tag first/last seen +first_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'first_seen') +last_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'last_seen') + +l_dates = substract_date(first_seen, last_seen) + +# get all tagged items +for date in l_dates: + daily_tagged_items = r_tags.smembers('{}:{}'.format(tag, date)) + + for item in daily_tagged_items: + p.populate_set_out(item, 'PgpDump') diff --git a/var/www/modules/hashDecoded/Flask_hashDecoded.py b/var/www/modules/hashDecoded/Flask_hashDecoded.py index c5e34d0f..29a48726 100644 --- a/var/www/modules/hashDecoded/Flask_hashDecoded.py +++ b/var/www/modules/hashDecoded/Flask_hashDecoded.py @@ -102,7 +102,7 @@ def get_file_icon_text(estimated_type): return file_icon_text def get_pgp_id_icon_text(type_id): - # set file icon + # set type_id icon if type_id == 'key': file_icon_text = '\uf084' elif type_id == 'name': @@ -113,6 +113,18 @@ def get_pgp_id_icon_text(type_id): file_icon_text = '\uf249' return file_icon_text +def get_pgp_icon(type_id): + # set type_id icon + if type_id == 'key': + pgp_icon_text = 'key' + elif type_id == 'name': + pgp_icon_text = 'user-tag' + elif type_id == 'mail': + pgp_icon_text = 'at' + else: + pgp_icon_text = 'times' + return pgp_icon_text + def verify_pgp_type_id(type_id): if type_id in ['key', 'name', 'mail']: return True @@ -846,7 +858,7 @@ def pgpdump_page(): if pgp_metadata[dump_id]: pgp_metadata[dump_id]['type_id'] = typ_id - #file_icon = get_file_icon(estimated_type) + pgp_metadata[dump_id]['type_icon'] = get_pgp_icon(typ_id) pgp_metadata[dump_id]['sparklines_data'] = list_sparkline_pgp_values(date_range_sparkline, typ_id, dump_id) pgp_metadata[dump_id]['sparklines_id'] = sparkline_id diff --git a/var/www/modules/hashDecoded/templates/PgpDecoded.html b/var/www/modules/hashDecoded/templates/PgpDecoded.html index e7e21feb..88da78e0 100644 --- a/var/www/modules/hashDecoded/templates/PgpDecoded.html +++ b/var/www/modules/hashDecoded/templates/PgpDecoded.html @@ -143,7 +143,7 @@ {% for pgp_dump in l_pgpdump %} -   {{ l_pgpdump[pgp_dump]['type_id'] }} +   {{ l_pgpdump[pgp_dump]['type_id'] }} {{ pgp_dump }} {{ l_pgpdump[pgp_dump]['first_seen'] }} {{ l_pgpdump[pgp_dump]['last_seen'] }}