mirror of https://github.com/CIRCL/AIL-framework
chg: [Tags alertHandler] remove alertHandler + add quick tag search sidebar + use tag last_seen to display tagged items
Terrtia
parent
ce6475e96e
commit
c2bc72c72e
|
|
@ -57,8 +57,6 @@ def search_api_key(message):
|
|||
msg = 'infoleak:automatic-detection="api-key";{}'.format(filename)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
||||
msg = 'apikey;{}'.format(filename)
|
||||
p.populate_set_out(msg, 'alertHandler')
|
||||
#Send to duplicate
|
||||
p.populate_set_out(filename, 'Duplicate')
|
||||
|
||||
|
|
@ -86,7 +84,7 @@ if __name__ == "__main__":
|
|||
|
||||
if message is not None:
|
||||
|
||||
search_api_key(message)
|
||||
search_api_key(message)
|
||||
|
||||
else:
|
||||
publisher.debug("Script ApiKey is Idling 10s")
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ def decode_base58(bc, length):
|
|||
for char in bc:
|
||||
n = n * 58 + digits58.index(char)
|
||||
return n.to_bytes(length, 'big')
|
||||
|
||||
|
||||
def check_bc(bc):
|
||||
try:
|
||||
bcbytes = decode_base58(bc, 25)
|
||||
|
|
@ -62,8 +62,6 @@ def search_key(content, message, paste):
|
|||
to_print = 'Bitcoin found: {} address and {} private Keys'.format(len(bitcoin_address), len(bitcoin_private_key))
|
||||
print(to_print)
|
||||
publisher.warning(to_print)
|
||||
msg = ('bitcoin;{}'.format(message))
|
||||
p.populate_set_out( msg, 'alertHandler')
|
||||
|
||||
msg = 'infoleak:automatic-detection="bitcoin-address";{}'.format(message)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
|
|
|||
|
|
@ -107,9 +107,6 @@ if __name__ == "__main__":
|
|||
publisher.warning(to_print)
|
||||
#Send to duplicate
|
||||
p.populate_set_out(filepath, 'Duplicate')
|
||||
#Send to alertHandler
|
||||
msg = 'credential;{}'.format(filepath)
|
||||
p.populate_set_out(msg, 'alertHandler')
|
||||
|
||||
msg = 'infoleak:automatic-detection="credential";{}'.format(filepath)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
|
|
|||
|
|
@ -82,9 +82,6 @@ if __name__ == "__main__":
|
|||
to_print, len(creditcard_set), paste.p_path))
|
||||
#Send to duplicate
|
||||
p.populate_set_out(filename, 'Duplicate')
|
||||
#send to Browse_warning_paste
|
||||
msg = 'creditcard;{}'.format(filename)
|
||||
p.populate_set_out(msg, 'alertHandler')
|
||||
|
||||
msg = 'infoleak:automatic-detection="credit-card";{}'.format(filename)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
|
|
|||
|
|
@ -31,10 +31,6 @@ def search_cve(message):
|
|||
print('{} contains CVEs'.format(paste.p_name))
|
||||
publisher.warning('{} contains CVEs'.format(paste.p_name))
|
||||
|
||||
#send to Browse_warning_paste
|
||||
msg = 'cve;{}'.format(filepath)
|
||||
p.populate_set_out(msg, 'alertHandler')
|
||||
|
||||
msg = 'infoleak:automatic-detection="cve";{}'.format(filepath)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
#Send to duplicate
|
||||
|
|
|
|||
|
|
@ -147,9 +147,6 @@ def set_out_paste(decoder_name, message):
|
|||
publisher.warning(decoder_name+' decoded')
|
||||
#Send to duplicate
|
||||
p.populate_set_out(message, 'Duplicate')
|
||||
#send to Browse_warning_paste
|
||||
msg = (decoder_name+';{}'.format(message))
|
||||
p.populate_set_out( msg, 'alertHandler')
|
||||
|
||||
msg = 'infoleak:automatic-detection="'+decoder_name+'";{}'.format(message)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
|
|
|||
|
|
@ -112,10 +112,7 @@ def search_key(paste):
|
|||
|
||||
#Send to duplicate
|
||||
p.populate_set_out(message, 'Duplicate')
|
||||
#send to Browse_warning_paste
|
||||
msg = ('keys;{}'.format(message))
|
||||
print(message)
|
||||
p.populate_set_out( msg, 'alertHandler')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -206,8 +206,6 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "LibInjection" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./LibInjection.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "alertHandler" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./alertHandler.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tags.py; read x"
|
||||
|
|
|
|||
|
|
@ -51,8 +51,7 @@ def analyse(url, path):
|
|||
publisher.warning(to_print)
|
||||
#Send to duplicate
|
||||
p.populate_set_out(path, 'Duplicate')
|
||||
#send to Browse_warning_paste
|
||||
p.populate_set_out('sqlinjection;{}'.format(path), 'alertHandler')
|
||||
|
||||
msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
||||
|
|
|
|||
|
|
@ -83,7 +83,6 @@ if __name__ == "__main__":
|
|||
publisher.warning(to_print)
|
||||
#Send to duplicate
|
||||
p.populate_set_out(filename, 'Duplicate')
|
||||
p.populate_set_out('mail;{}'.format(filename), 'alertHandler')
|
||||
|
||||
msg = 'infoleak:automatic-detection="mail";{}'.format(filename)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
|
|
|||
|
|
@ -235,7 +235,6 @@ if __name__ == "__main__":
|
|||
else:
|
||||
for url in fetch(p, r_cache, urls, domains_list, path):
|
||||
publisher.info('{}Checked {};{}'.format(to_print, url, PST.p_path))
|
||||
#p.populate_set_out('onion;{}'.format(PST.p_path), 'alertHandler')
|
||||
|
||||
# TAG Item
|
||||
msg = 'infoleak:automatic-detection="onion";{}'.format(PST.p_path)
|
||||
|
|
|
|||
|
|
@ -32,14 +32,11 @@ def search_phone(message):
|
|||
if len(results) > 4:
|
||||
print(results)
|
||||
publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name))
|
||||
#send to Browse_warning_paste
|
||||
msg = 'phone;{}'.format(message)
|
||||
p.populate_set_out(msg, 'alertHandler')
|
||||
#Send to duplicate
|
||||
|
||||
msg = 'infoleak:automatic-detection="phone-number";{}'.format(message)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
||||
#Send to duplicate
|
||||
p.populate_set_out(message, 'Duplicate')
|
||||
stats = {}
|
||||
for phone_number in results:
|
||||
|
|
|
|||
|
|
@ -82,8 +82,6 @@ def analyse(url, path):
|
|||
publisher.warning(to_print)
|
||||
#Send to duplicate
|
||||
p.populate_set_out(path, 'Duplicate')
|
||||
#send to Browse_warning_paste
|
||||
p.populate_set_out('sqlinjection;{}'.format(path), 'alertHandler')
|
||||
|
||||
msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
|
|
|||
|
|
@ -1,63 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
The Browse_warning_paste module
|
||||
====================
|
||||
|
||||
This module saved signaled paste (logged as 'warning') in redis for further usage
|
||||
like browsing by category
|
||||
|
||||
Its input comes from other modules, namely:
|
||||
Credential, CreditCard, SQLinjection, CVE, Keys, Mail and Phone
|
||||
|
||||
"""
|
||||
|
||||
import redis
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from packages import Paste
|
||||
from pubsublogger import publisher
|
||||
from Helper import Process
|
||||
|
||||
import sys
|
||||
sys.path.append('../')
|
||||
|
||||
flag_misp = False
|
||||
|
||||
if __name__ == "__main__":
|
||||
publisher.port = 6380
|
||||
publisher.channel = "Script"
|
||||
|
||||
config_section = 'alertHandler'
|
||||
|
||||
p = Process(config_section)
|
||||
|
||||
# port generated automatically depending on the date
|
||||
curYear = datetime.now().year
|
||||
server = redis.StrictRedis(
|
||||
host=p.config.get("ARDB_DB", "host"),
|
||||
port=p.config.get("ARDB_DB", "port"),
|
||||
db=curYear,
|
||||
decode_responses=True)
|
||||
|
||||
# FUNCTIONS #
|
||||
publisher.info("Script duplicate started")
|
||||
|
||||
while True:
|
||||
message = p.get_from_set()
|
||||
if message is not None:
|
||||
module_name, p_path = message.split(';')
|
||||
print("new alert : {}".format(module_name))
|
||||
#PST = Paste.Paste(p_path)
|
||||
else:
|
||||
publisher.debug("Script Attribute is idling 10s")
|
||||
time.sleep(10)
|
||||
continue
|
||||
|
||||
# Add in redis for browseWarningPaste
|
||||
# Format in set: WARNING_moduleName -> p_path
|
||||
key = "WARNING_" + module_name
|
||||
server.sadd(key, p_path)
|
||||
|
||||
publisher.info('Saved warning paste {}'.format(p_path))
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
echo -e "\t* Checking configuration"
|
||||
bash -c "./Update-conf.py"
|
||||
exitStatus=$?
|
||||
if [ $exitStatus -ge 1 ]; then
|
||||
echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
|
||||
exit
|
||||
fi
|
||||
echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
|
||||
|
||||
screen -dmS "Script"
|
||||
sleep 0.1
|
||||
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
|
||||
|
||||
screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Mixer" bash -c './Mixer.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Attributes" bash -c './Attributes.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Lines" bash -c './Lines.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "CreditCards" bash -c './CreditCards.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Web" bash -c './Web.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "CurveManageTopSets" bash -c './CurveManageTopSets.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "RegexForTermsFrequency" bash -c './RegexForTermsFrequency.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "SetForTermsFrequency" bash -c './SetForTermsFrequency.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "alertHandler" bash -c './alertHandler.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
|
||||
|
|
@ -51,7 +51,7 @@ publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Re
|
|||
|
||||
[CreditCards]
|
||||
subscribe = Redis_CreditCards
|
||||
publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler,Redis_Tags
|
||||
publish = Redis_Duplicate,Redis_ModuleStats,Redis_Tags
|
||||
|
||||
[BankAccount]
|
||||
subscribe = Redis_Global
|
||||
|
|
@ -59,12 +59,12 @@ publish = Redis_Duplicate,Redis_Tags
|
|||
|
||||
[Mail]
|
||||
subscribe = Redis_Mail
|
||||
publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler,Redis_Tags
|
||||
publish = Redis_Duplicate,Redis_ModuleStats,Redis_Tags
|
||||
|
||||
[Onion]
|
||||
subscribe = Redis_Onion
|
||||
publish = Redis_ValidOnion,ZMQ_FetchedOnion,Redis_alertHandler,Redis_Tags,Redis_Crawler
|
||||
#publish = Redis_Global,Redis_ValidOnion,ZMQ_FetchedOnion,Redis_alertHandler
|
||||
publish = Redis_ValidOnion,ZMQ_FetchedOnion,Redis_Tags,Redis_Crawler
|
||||
#publish = Redis_Global,Redis_ValidOnion,ZMQ_FetchedOnion
|
||||
|
||||
[DumpValidOnion]
|
||||
subscribe = Redis_ValidOnion
|
||||
|
|
@ -78,18 +78,15 @@ subscribe = Redis_Url
|
|||
|
||||
[LibInjection]
|
||||
subscribe = Redis_Url
|
||||
publish = Redis_alertHandler,Redis_Duplicate,Redis_Tags
|
||||
publish = Redis_Duplicate,Redis_Tags
|
||||
|
||||
[SQLInjectionDetection]
|
||||
subscribe = Redis_Url
|
||||
publish = Redis_alertHandler,Redis_Duplicate,Redis_Tags
|
||||
publish = Redis_Duplicate,Redis_Tags
|
||||
|
||||
[ModuleStats]
|
||||
subscribe = Redis_ModuleStats
|
||||
|
||||
[alertHandler]
|
||||
subscribe = Redis_alertHandler
|
||||
|
||||
[Tags]
|
||||
subscribe = Redis_Tags
|
||||
publish = Redis_Tags_feed
|
||||
|
|
@ -99,7 +96,7 @@ subscribe = Redis_Tags_feed
|
|||
|
||||
#[send_to_queue]
|
||||
#subscribe = Redis_Cve
|
||||
#publish = Redis_alertHandler,Redis_Tags
|
||||
#publish = Redis_Tags
|
||||
|
||||
[SentimentAnalysis]
|
||||
subscribe = Redis_Global
|
||||
|
|
@ -109,31 +106,31 @@ subscribe = Redis_Global
|
|||
|
||||
[Credential]
|
||||
subscribe = Redis_Credential
|
||||
publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler,Redis_Tags
|
||||
publish = Redis_Duplicate,Redis_ModuleStats,Redis_Tags
|
||||
|
||||
[Cve]
|
||||
subscribe = Redis_Cve
|
||||
publish = Redis_alertHandler,Redis_Duplicate,Redis_Tags
|
||||
publish = Redis_Duplicate,Redis_Tags
|
||||
|
||||
[Phone]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags
|
||||
publish = Redis_Duplicate,Redis_Tags
|
||||
|
||||
[Keys]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags
|
||||
publish = Redis_Duplicate,Redis_Tags
|
||||
|
||||
[ApiKey]
|
||||
subscribe = Redis_ApiKey
|
||||
publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags
|
||||
publish = Redis_Duplicate,Redis_Tags
|
||||
|
||||
[Decoder]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags
|
||||
publish = Redis_Duplicate,Redis_Tags
|
||||
|
||||
[Bitcoin]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags
|
||||
publish = Redis_Duplicate,Redis_Tags
|
||||
|
||||
[submit_paste]
|
||||
subscribe = Redis
|
||||
|
|
@ -142,4 +139,3 @@ publish = Redis_Mixer
|
|||
[Crawler]
|
||||
subscribe = Redis_Crawler
|
||||
publish = Redis_Mixer,Redis_Tags
|
||||
|
||||
|
|
|
|||
|
|
@ -118,6 +118,16 @@ def get_all_dates_range(date_from, date_to):
|
|||
all_dates['date_range'] = date_range
|
||||
return all_dates
|
||||
|
||||
def get_last_seen_from_tags_list(list_tags):
|
||||
min_last_seen = 99999999
|
||||
for tag in list_tags:
|
||||
tag_last_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
|
||||
if tag_last_seen:
|
||||
tag_last_seen = int(tag_last_seen)
|
||||
if tag_last_seen < min_last_seen:
|
||||
min_last_seen = tag_last_seen
|
||||
return str(min_last_seen)
|
||||
|
||||
def add_item_tag(tag, item_path):
|
||||
item_date = int(get_item_date(item_path))
|
||||
|
||||
|
|
@ -211,123 +221,133 @@ def update_tag_last_seen(tag, tag_first_seen, tag_last_seen):
|
|||
def Tags_page():
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
tags = request.args.get('ltags')
|
||||
|
||||
if tags is None:
|
||||
dates = get_all_dates_range(date_from, date_to)
|
||||
return render_template("Tags.html", date_from=dates['date_from'], date_to=dates['date_to'])
|
||||
|
||||
# unpack tags
|
||||
list_tags = tags.split(',')
|
||||
list_tag = []
|
||||
for tag in list_tags:
|
||||
list_tag.append(tag.replace('"','\"'))
|
||||
|
||||
#no search by date, use last_seen for date_from/date_to
|
||||
if date_from is None and date_to is None and tags is not None:
|
||||
date_from = get_last_seen_from_tags_list(list_tags)
|
||||
date_to = date_from
|
||||
|
||||
# TODO verify input
|
||||
|
||||
dates = get_all_dates_range(date_from, date_to)
|
||||
|
||||
tags = request.args.get('ltags')
|
||||
if tags is None:
|
||||
return render_template("Tags.html", date_from=dates['date_from'], date_to=dates['date_to'])
|
||||
if(type(list_tags) is list):
|
||||
# no tag
|
||||
if list_tags is False:
|
||||
print('empty')
|
||||
# 1 tag
|
||||
elif len(list_tags) < 2:
|
||||
tagged_pastes = []
|
||||
for date in dates['date_range']:
|
||||
tagged_pastes.extend(r_serv_tags.smembers('{}:{}'.format(list_tags[0], date)))
|
||||
|
||||
else:
|
||||
tags = request.args.get('ltags')
|
||||
|
||||
list_tags = tags.split(',')
|
||||
list_tag = []
|
||||
for tag in list_tags:
|
||||
list_tag.append(tag.replace('"','\"'))
|
||||
|
||||
# TODO verify input
|
||||
|
||||
if(type(list_tags) is list):
|
||||
# no tag
|
||||
if list_tags is False:
|
||||
print('empty')
|
||||
# 1 tag
|
||||
elif len(list_tags) < 2:
|
||||
tagged_pastes = []
|
||||
for date in dates['date_range']:
|
||||
tagged_pastes.extend(r_serv_tags.smembers('{}:{}'.format(list_tags[0], date)))
|
||||
|
||||
# 2 tags or more
|
||||
else:
|
||||
tagged_pastes = []
|
||||
for date in dates['date_range']:
|
||||
tag_keys = []
|
||||
for tag in list_tags:
|
||||
tag_keys.append('{}:{}'.format(tag, date))
|
||||
|
||||
if len(tag_keys) > 1:
|
||||
daily_items = r_serv_tags.sinter(tag_keys[0], *tag_keys[1:])
|
||||
else:
|
||||
daily_items = r_serv_tags.sinter(tag_keys[0])
|
||||
tagged_pastes.extend(daily_items)
|
||||
|
||||
else :
|
||||
return 'INCORRECT INPUT'
|
||||
|
||||
all_content = []
|
||||
paste_date = []
|
||||
paste_linenum = []
|
||||
all_path = []
|
||||
allPastes = list(tagged_pastes)
|
||||
paste_tags = []
|
||||
|
||||
try:
|
||||
page = int(request.args.get('page'))
|
||||
except:
|
||||
page = 1
|
||||
if page <= 0:
|
||||
page = 1
|
||||
nb_page_max = len(tagged_pastes)/(max_tags_result)
|
||||
if not nb_page_max.is_integer():
|
||||
nb_page_max = int(nb_page_max)+1
|
||||
# 2 tags or more
|
||||
else:
|
||||
nb_page_max = int(nb_page_max)
|
||||
if page > nb_page_max:
|
||||
page = nb_page_max
|
||||
start = max_tags_result*(page -1)
|
||||
stop = max_tags_result*page
|
||||
tagged_pastes = []
|
||||
for date in dates['date_range']:
|
||||
tag_keys = []
|
||||
for tag in list_tags:
|
||||
tag_keys.append('{}:{}'.format(tag, date))
|
||||
|
||||
for path in allPastes[start:stop]: ######################moduleName
|
||||
all_path.append(path)
|
||||
paste = Paste.Paste(path)
|
||||
content = paste.get_p_content()
|
||||
content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
|
||||
all_content.append(content[0:content_range].replace("\"", "\'").replace("\r", " ").replace("\n", " "))
|
||||
curr_date = str(paste._get_p_date())
|
||||
curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
|
||||
paste_date.append(curr_date)
|
||||
paste_linenum.append(paste.get_lines_info()[0])
|
||||
p_tags = r_serv_metadata.smembers('tag:'+path)
|
||||
complete_tags = []
|
||||
l_tags = []
|
||||
for tag in p_tags:
|
||||
complete_tag = tag
|
||||
|
||||
tag = tag.split('=')
|
||||
if len(tag) > 1:
|
||||
if tag[1] != '':
|
||||
tag = tag[1][1:-1]
|
||||
# no value
|
||||
else:
|
||||
tag = tag[0][1:-1]
|
||||
# use for custom tags
|
||||
if len(tag_keys) > 1:
|
||||
daily_items = r_serv_tags.sinter(tag_keys[0], *tag_keys[1:])
|
||||
else:
|
||||
tag = tag[0]
|
||||
daily_items = r_serv_tags.sinter(tag_keys[0])
|
||||
tagged_pastes.extend(daily_items)
|
||||
|
||||
l_tags.append( (tag,complete_tag) )
|
||||
else :
|
||||
return 'INCORRECT INPUT'
|
||||
|
||||
paste_tags.append(l_tags)
|
||||
all_content = []
|
||||
paste_date = []
|
||||
paste_linenum = []
|
||||
all_path = []
|
||||
allPastes = list(tagged_pastes)
|
||||
paste_tags = []
|
||||
|
||||
if len(allPastes) > 10:
|
||||
finished = False
|
||||
else:
|
||||
finished = True
|
||||
try:
|
||||
page = int(request.args.get('page'))
|
||||
except:
|
||||
page = 1
|
||||
if page <= 0:
|
||||
page = 1
|
||||
nb_page_max = len(tagged_pastes)/(max_tags_result)
|
||||
if not nb_page_max.is_integer():
|
||||
nb_page_max = int(nb_page_max)+1
|
||||
else:
|
||||
nb_page_max = int(nb_page_max)
|
||||
if page > nb_page_max:
|
||||
page = nb_page_max
|
||||
start = max_tags_result*(page -1)
|
||||
stop = max_tags_result*page
|
||||
|
||||
return render_template("Tags.html",
|
||||
all_path=all_path,
|
||||
tags=tags,
|
||||
list_tag = list_tag,
|
||||
date_from=dates['date_from'],
|
||||
date_to=dates['date_to'],
|
||||
page=page, nb_page_max=nb_page_max,
|
||||
paste_tags=paste_tags,
|
||||
bootstrap_label=bootstrap_label,
|
||||
content=all_content,
|
||||
paste_date=paste_date,
|
||||
paste_linenum=paste_linenum,
|
||||
char_to_display=max_preview_modal,
|
||||
finished=finished)
|
||||
for path in allPastes[start:stop]:
|
||||
all_path.append(path)
|
||||
paste = Paste.Paste(path)
|
||||
content = paste.get_p_content()
|
||||
content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
|
||||
all_content.append(content[0:content_range].replace("\"", "\'").replace("\r", " ").replace("\n", " "))
|
||||
curr_date = str(paste._get_p_date())
|
||||
curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
|
||||
paste_date.append(curr_date)
|
||||
paste_linenum.append(paste.get_lines_info()[0])
|
||||
p_tags = r_serv_metadata.smembers('tag:'+path)
|
||||
complete_tags = []
|
||||
l_tags = []
|
||||
for tag in p_tags:
|
||||
complete_tag = tag
|
||||
|
||||
tag = tag.split('=')
|
||||
if len(tag) > 1:
|
||||
if tag[1] != '':
|
||||
tag = tag[1][1:-1]
|
||||
# no value
|
||||
else:
|
||||
tag = tag[0][1:-1]
|
||||
# use for custom tags
|
||||
else:
|
||||
tag = tag[0]
|
||||
|
||||
l_tags.append( (tag,complete_tag) )
|
||||
|
||||
paste_tags.append(l_tags)
|
||||
|
||||
if len(allPastes) > 10:
|
||||
finished = False
|
||||
else:
|
||||
finished = True
|
||||
|
||||
if len(list_tag) == 1:
|
||||
tag_nav=tags.replace('"', '').replace('=', '').replace(':', '')
|
||||
else:
|
||||
tag_nav='empty'
|
||||
|
||||
return render_template("Tags.html",
|
||||
all_path=all_path,
|
||||
tags=tags,
|
||||
tag_nav=tag_nav,
|
||||
list_tag = list_tag,
|
||||
date_from=dates['date_from'],
|
||||
date_to=dates['date_to'],
|
||||
page=page, nb_page_max=nb_page_max,
|
||||
paste_tags=paste_tags,
|
||||
bootstrap_label=bootstrap_label,
|
||||
content=all_content,
|
||||
paste_date=paste_date,
|
||||
paste_linenum=paste_linenum,
|
||||
char_to_display=max_preview_modal,
|
||||
finished=finished)
|
||||
|
||||
|
||||
@Tags.route("/Tags/get_all_tags")
|
||||
|
|
|
|||
|
|
@ -217,7 +217,8 @@ var last_clicked_paste;
|
|||
var can_change_modal_content = true;
|
||||
|
||||
$(document).ready(function(){
|
||||
$("#page-Browse-Items").addClass("active");
|
||||
$("#nav_quick_search").removeClass("text-muted");
|
||||
$("#nav_tag_{{tag_nav}}").addClass("active");
|
||||
search_table = $('#myTable_').DataTable({ "order": [[ 0, "asc" ]] });
|
||||
|
||||
// Use to bind the button with the new displayed data
|
||||
|
|
|
|||
|
|
@ -34,5 +34,58 @@
|
|||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
<h5 class="d-flex text-muted w-100" id="nav_quick_search">
|
||||
<span>Quick Search </span>
|
||||
</h5>
|
||||
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100">
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href='{{url_for('Tags.Tags_page')}}?ltags=infoleak:automatic-detection="credential"' id='nav_tag_infoleakautomatic-detectioncredential'>
|
||||
<i class="fas fa-unlock-alt"></i>
|
||||
Credentials
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href='{{url_for('Tags.Tags_page')}}?ltags=infoleak:automatic-detection="credit-card"' id='nav_tag_infoleakautomatic-detectioncredit-card'>
|
||||
<i class="far fa-credit-card"></i>
|
||||
Credit cards
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href='{{url_for('Tags.Tags_page')}}?ltags=infoleak:automatic-detection="mail"' id='nav_tag_infoleakautomatic-detectionmail'>
|
||||
<i class="fas fa-envelope"></i>
|
||||
Mails
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href='{{url_for('Tags.Tags_page')}}?ltags=infoleak:automatic-detection="cve"' id='nav_tag_infoleakautomatic-detectioncve'>
|
||||
<i class="fas fa-bug"></i>
|
||||
CVEs
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href='{{url_for('Tags.Tags_page')}}?ltags=infoleak:automatic-detection="onion"' id='nav_tag_infoleakautomatic-detectiononion'>
|
||||
<i class="fas fa-user-secret"></i>
|
||||
Onions
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href='{{url_for('Tags.Tags_page')}}?ltags=infoleak:automatic-detection="bitcoin-address"' id='nav_tag_infoleakautomatic-detectionbitcoin-address'>
|
||||
<i class="fab fa-bitcoin"></i>
|
||||
Bitcoin
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href='{{url_for('Tags.Tags_page')}}?ltags=infoleak:automatic-detection="base64"' id='nav_tag_infoleakautomatic-detectionbase64'>
|
||||
<i class="fas fa-lock-open"></i>
|
||||
Base64
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href='{{url_for('Tags.Tags_page')}}?ltags=infoleak:automatic-detection="phone-number"' id='nav_tag_infoleakautomatic-detectionphone-number'>
|
||||
<i class="fas fa-phone"></i>
|
||||
Phones
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
|
|||
Loading…
Reference in New Issue