CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [cleanup] remove ARDB + fix hive case

pull/594/head
Terrtia 2023-01-18 16:28:08 +01:00
parent 0e41c95b5c
commit 3365a054a8
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
42 changed files with 431 additions and 3083 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/DB_KVROCKS_MIGRATION.py
View File

@ -252,6 +252,8 @@ def trackers_migration():
for id in old_Tracker.get_retro_hunt_items_by_daterange(task_uuid, meta['date_from'], meta['date_to']):
Tracker.save_retro_hunt_match(task_uuid, id)
Tracker._fix_db_custom_tags()
def investigations_migration():
print('INVESTIGATION MIGRATION...')

14
bin/Helper.py
View File

@ -140,12 +140,6 @@ class Process(object):
db=self.config.get('RedisPubSub', 'db'),
decode_responses=True)
self.serv_statistics = redis.StrictRedis(
host=self.config.get('ARDB_Statistics', 'host'),
port=self.config.get('ARDB_Statistics', 'port'),
db=self.config.get('ARDB_Statistics', 'db'),
decode_responses=True)
self.moduleNum = os.getpid()
def populate_set_in(self):
@ -181,9 +175,9 @@ class Process(object):
try:
if '.gz' in message:
path = message.split(".")[-2].split("/")[-1]
#find start of path with AIL_HOME
# find start of path with AIL_HOME
index_s = message.find(os.environ['AIL_HOME'])
#Stop when .gz
# Stop when .gz
index_e = message.find(".gz")+3
if(index_s == -1):
complete_path = message[0:index_e]
@ -200,7 +194,6 @@ class Process(object):
self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
curr_date = datetime.date.today()
self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'paste_by_modules_in:'+self.subscriber_name, 1)
return message
except:
@ -238,6 +231,3 @@ class Process(object):
continue
self.pubsub.publish(message)
def incr_module_timeout_statistic(self):
curr_date = datetime.date.today()
self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'paste_by_modules_timeout:'+self.subscriber_name, 1)

4
bin/LAUNCH.sh
View File

@ -237,6 +237,8 @@ function launching_scripts {
sleep 0.1
screen -S "Script_AIL" -X screen -t "Telegram" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Telegram.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Tools" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Tools.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Hosts" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Hosts.py; read x"
sleep 0.1
@ -283,8 +285,6 @@ function launching_scripts {
sleep 0.1
screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Tools" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tools.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "IPAddress" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./IPAddress.py; read x"

69
bin/Release.py
View File

@ -1,69 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import time
from lib.objects.Items import Item
from pubsublogger import publisher
from Helper import Process
import re
import signal
class TimeoutException(Exception):
pass
def timeout_handler(signum, frame):
raise TimeoutException
signal.signal(signal.SIGALRM, timeout_handler)
'''
This module takes its input from the global module.
It applies some regex and publish matched content
'''
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
config_section = "Release"
p = Process(config_section)
max_execution_time = p.config.getint("Curve", "max_execution_time")
publisher.info("Release scripts to find release names")
movie = "[a-zA-Z0-9.]+\.[0-9]{4}.[a-zA-Z0-9.]+\-[a-zA-Z]+"
tv = "[a-zA-Z0-9.]+\.S[0-9]{2}E[0-9]{2}.[a-zA-Z0-9.]+\.[a-zA-Z0-9.]+\-[a-zA-Z0-9]+"
xxx = "[a-zA-Z0-9._]+.XXX.[a-zA-Z0-9.]+\-[a-zA-Z0-9]+"
regexs = [movie, tv, xxx]
regex = '|'.join(regexs)
while True:
signal.alarm(max_execution_time)
filepath = p.get_from_set()
if filepath is None:
publisher.debug("Script Release is Idling 10s")
print('Sleeping')
time.sleep(10)
continue
item = Item(filepath)
content = item.get_content()
#signal.alarm(max_execution_time)
try:
releases = set(re.findall(regex, content))
if len(releases) == 0:
continue
to_print = f'Release;{item.get_source()};{item.get_date()};{item.get_basename()};{len(releases)} releases;{item.get_id()}'
print(to_print)
if len(releases) > 30:
publisher.warning(to_print)
else:
publisher.info(to_print)
except TimeoutException:
p.incr_module_timeout_statistic()
print(f"{item.get_id()} processing timeout")
continue
else:
signal.alarm(0)

55
bin/SourceCode.py
View File

@ -1,55 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import time
from lib.objects.Items import Item
from pubsublogger import publisher
from Helper import Process
import re
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
config_section = "SourceCode"
p = Process(config_section)
publisher.info("Finding Source Code")
critical = 0 # AS TO BE IMPORTANT, MIGHT BE REMOVED
# RELEVANT LANGUAGES
shell = r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+\:\~\$"
c = r"\#include\ \<[a-z\/]+.h\>"
php = r"\<\?php"
python = r"import\ [\w]+"
bash = r"#!\/[\w]*\/bash"
javascript = r"function\(\)"
ruby = r"require \ [\w]+"
adr = r"0x[a-f0-9]{2}"
# asm = r"\"((?s).{1}x[0-9a-f]{2}){3,}" ISSUES WITH FINDALL, pattern like \x54\xaf\x23\..
languages = [shell, c, php, bash, python, javascript, bash, ruby, adr]
regex = '|'.join(languages)
print(regex)
while True:
message = p.get_from_set()
if message is None:
publisher.debug("Script Source Code is Idling 10s")
print('Sleeping')
time.sleep(10)
continue
filepath, count = message.split()
item = Item(filepath)
content = item.get_content()
match_set = set(re.findall(regex, content))
if len(match_set) == 0:
continue
to_print = f'SourceCode;{item.get_source()};{item.get_date()};{item.get_basename()};{item.get_id()}'
if len(match_set) > critical:
publisher.warning(to_print)
else:
publisher.info(to_print)

19
bin/export/Export.py
View File

@ -72,6 +72,25 @@ def is_hive_connected():
HIVE_CLIENT = get_hive_client()
def sanitize_threat_level_hive(threat_level):
try:
int(threat_level)
if 1 <= threat_level <= 3:
return threat_level
else:
return 2
except:
return 2
def sanitize_tlp_hive(tlp):
try:
int(tlp)
if 0 <= tlp <= 3:
return tlp
else:
return 2
except:
return 2
def create_thehive_alert(item_id, tag_trigger):
item = Item(item_id)

76
bin/helper/dump_crawler_history.py
View File

@ -1,76 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import gzip
import datetime
import redis
import json
import time
import shutil
sys.path.append(os.environ['AIL_BIN'])
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
from HiddenServices import HiddenServices
from Helper import Process
def substract_date(date_from, date_to):
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
delta = date_to - date_from # timedelta
l_date = []
for i in range(delta.days + 1):
date = date_from + datetime.timedelta(i)
l_date.append( date.strftime('%Y%m%d') )
return l_date
config_section = 'Keys'
p = Process(config_section)
r_serv_onion = redis.StrictRedis(
host=p.config.get("ARDB_Onion", "host"),
port=p.config.getint("ARDB_Onion", "port"),
db=p.config.getint("ARDB_Onion", "db"),
decode_responses=True)
date_from = '20190614'
date_to = '20190615'
service_type = 'onion'
date_range = substract_date(date_from, date_to)
dir_path = os.path.join(os.environ['AIL_HOME'], 'temp')
domain_skipped = []
for date in date_range:
domains_up = list(r_serv_onion.smembers('{}_up:{}'.format(service_type, date)))
if domains_up:
save_path = os.path.join(dir_path, date[0:4], date[4:6], date[6:8])
try:
os.makedirs(save_path)
except FileExistsError:
pass
for domain in domains_up:
print(domain)
h = HiddenServices(domain, 'onion')
item_core = h.get_domain_crawled_core_item()
if 'root_item' in item_core:
l_pastes = h.get_last_crawled_pastes(item_root=item_core['root_item'])
try:
res = h.create_domain_basic_archive(l_pastes)
filename = os.path.join(save_path, '{}'.format(domain))
with open(filename, 'wb') as f:
shutil.copyfileobj(res, f)
print('done')
except Exception as e:
print('skipped')
domain_skipped.append(domain)
pass
print()
print()
print('DOMAINS SKIPPED: ')
for domain in domain_skipped:
print(domain)

50
bin/helper/reprocess_bitcoin.py
View File

@ -1,50 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import gzip
import base64
import uuid
import datetime
import base64
import redis
import json
import time
sys.path.append(os.environ['AIL_BIN'])
from Helper import Process
def substract_date(date_from, date_to):
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
delta = date_to - date_from # timedelta
l_date = []
for i in range(delta.days + 1):
date = date_from + datetime.timedelta(i)
l_date.append( date.strftime('%Y%m%d') )
return l_date
config_section = 'Global'
p = Process(config_section)
r_tags = redis.StrictRedis(
host=p.config.get("ARDB_Tags", "host"),
port=p.config.getint("ARDB_Tags", "port"),
db=p.config.getint("ARDB_Tags", "db"),
decode_responses=True)
tag = 'infoleak:automatic-detection="bitcoin-address"'
# get tag first/last seen
first_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'first_seen')
last_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
l_dates = substract_date(first_seen, last_seen)
# get all tagged items
for date in l_dates:
daily_tagged_items = r_tags.smembers('{}:{}'.format(tag, date))
for item in daily_tagged_items:
p.populate_set_out(item)

50
bin/helper/reprocess_pgp_message.py
View File

@ -1,50 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import gzip
import base64
import uuid
import datetime
import base64
import redis
import json
import time
sys.path.append(os.environ['AIL_BIN'])
from Helper import Process
def substract_date(date_from, date_to):
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
delta = date_to - date_from # timedelta
l_date = []
for i in range(delta.days + 1):
date = date_from + datetime.timedelta(i)
l_date.append( date.strftime('%Y%m%d') )
return l_date
config_section = 'Keys'
p = Process(config_section)
r_tags = redis.StrictRedis(
host=p.config.get("ARDB_Tags", "host"),
port=p.config.getint("ARDB_Tags", "port"),
db=p.config.getint("ARDB_Tags", "db"),
decode_responses=True)
tag = 'infoleak:automatic-detection="pgp-message"'
# get tag first/last seen
first_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'first_seen')
last_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
l_dates = substract_date(first_seen, last_seen)
# get all tagged items
for date in l_dates:
daily_tagged_items = r_tags.smembers('{}:{}'.format(tag, date))
for item in daily_tagged_items:
p.populate_set_out(item, 'PgpDump')

1
bin/import/importer.py
View File

@ -10,7 +10,6 @@ Recieve Json Items (example: Twitter feeder)
import os
import importlib
import json
import redis
import sys
import time

2
bin/lib/Config_DB.py
View File

@ -15,7 +15,7 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
from lib import ConfigLoader
config_loader = ConfigLoader.ConfigLoader()
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
r_serv_db = config_loader.get_redis_conn("_DB")
config_loader = None
#### TO PUT IN CONFIG

4
bin/lib/MispModules.py
View File

@ -3,6 +3,7 @@
import os
import json
import requests
import sys
import configparser
misp_module_url = 'http://localhost:6666'
@ -16,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN'])
from lib import ConfigLoader
config_loader = ConfigLoader.ConfigLoader()
r_serv = config_loader.get_redis_conn("ARDB_DB")
r_serv = config_loader.get_redis_conn("_DB")
config_loader = None
def init_config(config_path=default_config_path):
@ -114,6 +115,7 @@ def parse_module_enrichment_response(misp_module_response):
#print(response_types)
return response_values
if __name__ == "__main__":
load_modules_list()

7
bin/lib/Statistics.py
View File

@ -12,8 +12,7 @@ sys.path.append(os.environ['AIL_BIN'])
from lib import ConfigLoader
config_loader = ConfigLoader.ConfigLoader()
r_statistics = config_loader.get_redis_conn("ARDB_Statistics")
# r_serv_trend = ConfigLoader().get_redis_conn("ARDB_Trending")
r_statistics = config_loader.get_db_conn("Kvrocks_Stats")
config_loader = None
PIE_CHART_MAX_CARDINALITY = 8
@ -113,11 +112,11 @@ def update_module_stats(module_name, num, keyword, date):
if r_statistics.zcard(f'top_{module_name}_set_{date}') < PIE_CHART_MAX_CARDINALITY:
r_statistics.zadd(f'top_{module_name}_set_{date}', {keyword: float(keyword_total_sum)})
else: # zset at full capacity
else: # zset at full capacity
member_set = r_statistics.zrangebyscore(f'top_{module_name}_set_{date}', '-inf', '+inf', withscores=True, start=0, num=1)
# Member set is a list of (value, score) pairs
if int(member_set[0][1]) < keyword_total_sum:
#remove min from set and add the new one
# remove min from set and add the new one
r_statistics.zrem(f'top_{module_name}_set_{date}', member_set[0][0])
r_statistics.zadd(f'top_{module_name}_set_{date}', {keyword: float(keyword_total_sum)})

70
bin/lib/Tag.py
View File

@ -620,7 +620,7 @@ def update_tag_metadata(tag, date, delete=False): # # TODO: delete Tags
# old
# r_tags.smembers(f'{tag}:{date}')
# r_tags.smembers(f'{obj_type}:{tag}')
def get_tag_objects(obj_type, subtype='', date=''):
def get_tag_objects(tag, obj_type, subtype='', date=''):
if obj_type == 'item':
return r_tags.smembers(f'{obj_type}:{subtype}:{tag}:{date}')
else:
@ -1079,6 +1079,10 @@ def get_modal_add_tags(object_id, object_type='item', object_subtype=''):
return {"active_taxonomies": get_active_taxonomies(), "active_galaxies": get_active_galaxies(),
"object_id": object_id, "object_type": object_type, "object_subtype": object_subtype}
#####################################################################################
#####################################################################################
#####################################################################################
######## NEW VERSION ########
def create_custom_tag(tag):
r_tags.sadd('tags:custom', tag)
@ -1137,28 +1141,45 @@ def get_enabled_tags_with_synonyms_ui():
## Objects tags ##
###################################################################################
###################################################################################
###################################################################################
###################################################################################
###################################################################################
###################################################################################
###################################################################################
###################################################################################
###################################################################################
def add_global_tag(tag, object_type=None):
'''
Create a set of all tags used in AIL (all + by object)
# TYPE -> taxonomy/galaxy/custom
:param tag: tag
:type tag: str
:param object_type: object type
:type object_type: str
'''
r_tags.sadd('list_tags', tag)
if object_type:
r_tags.sadd('list_tags:{}'.format(object_type), tag)
class Tag:
def __int__(self, t_type, t_id, obj='item'):
self.type = t_type
self.id = t_id
self.obj = obj
def get_first_seen(self):
pass
def get_last_seen(self):
pass
def get_color(self):
pass
def is_enabled(self):
pass
def get_meta(self):
meta = {'first_seen': self.get_first_seen(),
'last_seen': self.get_last_seen(),
'obj': self.obj,
'tag': self.id,
'type': self.type}
###################################################################################
###################################################################################
###################################################################################
###################################################################################
def add_obj_tags(object_id, object_subtype, object_type, tags=[], galaxy_tags=[]):
for tag in tags:
@ -1201,23 +1222,6 @@ def api_add_obj_tags(tags=[], galaxy_tags=[], object_id=None, object_type="item"
res_dict['type'] = object_type
return res_dict, 200
# def add_tag(object_type, tag, object_id, obj_date=None):
# # new tag
# if not is_obj_tagged(object_id, tag):
# # # TODO: # FIXME: sanitize object_type
# if obj_date:
# try:
# obj_date = int(obj_date)
# except:
# obj_date = None
# if not obj_date:
# obj_date = get_obj_date(object_type, object_id)
# add_global_tag(tag, object_type=object_type)
# add_obj_tag(object_type, object_id, tag, obj_date=obj_date)
# update_tag_metadata(tag, obj_date, object_type=object_type)
#
# # create tags stats # # TODO: put me in cache
# r_tags.hincrby('daily_tags:{}'.format(datetime.date.today().strftime("%Y%m%d")), tag, 1)
# def delete_obj_tag(object_type, object_id, tag, obj_date):
# if object_type=="item": # # TODO: # FIXME: # REVIEW: !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

15
bin/lib/Tracker.py
View File

@ -551,9 +551,11 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti
# create tracker tags list
for tag in tags:
r_serv_tracker.sadd(f'tracker:tags:{tracker_uuid}', escape(tag))
tag = escape(tag)
r_serv_tracker.sadd(f'tracker:tags:{tracker_uuid}', tag)
Tag.create_custom_tag(tag)
# create tracker tags mail notification list
# create tracker mail notification list
for mail in mails:
r_serv_tracker.sadd(f'tracker:mail:{tracker_uuid}', escape(mail))
@ -1030,7 +1032,9 @@ def create_retro_hunt_task(name, rule, date_from, date_to, creator, sources=[],
for source in sources:
r_serv_tracker.sadd(f'tracker:retro_hunt:task:sources:{task_uuid}', escape(source))
for tag in tags:
r_serv_tracker.sadd(f'tracker:retro_hunt:task:tags:{task_uuid}', escape(tag))
tag = escape(tag)
r_serv_tracker.sadd(f'tracker:retro_hunt:task:tags:{task_uuid}', tag)
Tag.create_custom_tag(tag)
for mail in mails:
r_serv_tracker.sadd(f'tracker:retro_hunt:task:mails:{task_uuid}', escape(mail))
@ -1364,14 +1368,13 @@ def get_trackers_tags():
def _fix_db_custom_tags():
for tag in get_trackers_tags():
if not Tag.is_taxonomie_tag(tag) and not Tag.is_galaxy_tag(tag):
print(tag)
Tag.create_custom_tag(tag)
#### -- ####
#if __name__ == '__main__':
if __name__ == '__main__':
#_fix_db_custom_tags()
_fix_db_custom_tags()
# fix_all_tracker_uuid_list()
# res = get_all_tracker_uuid()
# print(len(res))

1
bin/lib/Users.py
View File

@ -17,7 +17,6 @@ from lib.ConfigLoader import ConfigLoader
# Config
config_loader = ConfigLoader()
# r_serv_db = config_loader.get_redis_conn("ARDB_DB")
r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
config_loader = None

150
bin/lib/ail_updates.py
View File

@ -10,42 +10,42 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from lib.ConfigLoader import ConfigLoader
from lib import Users
config_loader = ConfigLoader()
#r_serv_db = config_loader.get_redis_conn("Kvrocks_DB")
r_serv = config_loader.get_redis_conn("ARDB_DB")
r_db = config_loader.get_redis_conn("Kvrocks_DB")
config_loader = None
BACKGROUND_UPDATES = {
'v1.5':{
'nb_updates': 5,
'message': 'Tags and Screenshots'
},
'v2.4':{
'nb_updates': 1,
'message': ' Domains Tags and Correlations'
},
'v2.6':{
'nb_updates': 1,
'message': 'Domains Tags and Correlations'
},
'v2.7':{
'nb_updates': 1,
'message': 'Domains Tags'
},
'v3.4':{
'nb_updates': 1,
'message': 'Domains Languages'
},
'v3.7':{
'nb_updates': 1,
'message': 'Trackers first_seen/last_seen'
}
'v1.5': {
'nb_updates': 5,
'message': 'Tags and Screenshots'
},
'v2.4': {
'nb_updates': 1,
'message': ' Domains Tags and Correlations'
},
'v2.6': {
'nb_updates': 1,
'message': 'Domains Tags and Correlations'
},
'v2.7': {
'nb_updates': 1,
'message': 'Domains Tags'
},
'v3.4': {
'nb_updates': 1,
'message': 'Domains Languages'
},
'v3.7': {
'nb_updates': 1,
'message': 'Trackers first_seen/last_seen'
}
}
def get_ail_version():
return r_serv.get('ail:version')
return r_db.get('ail:version')
def get_ail_float_version():
version = get_ail_version()
@ -55,20 +55,23 @@ def get_ail_float_version():
version = 0
return version
def get_ail_all_updates(date_separator='-'):
dict_update = r_serv.hgetall('ail:update_date')
dict_update = r_db.hgetall('ail:update_date')
if date_separator:
for version in dict_update:
u_date = dict_update[version]
dict_update[version] = f'{u_date[0:4]}{date_separator}{u_date[4:6]}{date_separator}{u_date[6:8]}'
return dict_update
def add_ail_update(version):
#Add new AIL version
r_serv.hset('ail:update_date', version, datetime.datetime.now().strftime("%Y%m%d"))
#Set current ail version
# Add new AIL version
r_db.hset('ail:update_date', version, datetime.datetime.now().strftime("%Y%m%d"))
# Set current ail version
if float(version[1:]) > get_ail_float_version():
r_serv.set('ail:version', version)
r_db.set('ail:version', version)
def check_version(version):
if version[0] != 'v':
@ -83,75 +86,94 @@ def check_version(version):
return False
return True
#### UPDATE BACKGROUND ####
def exits_background_update_to_launch():
return r_serv.scard('ail:update:to_update') != 0
return r_db.scard('ail:update:to_update') != 0
def is_version_in_background_update(version):
return r_serv.sismember('ail:update:to_update', version)
return r_db.sismember('ail:update:to_update', version)
def get_all_background_updates_to_launch():
return r_serv.smembers('ail:update:to_update')
return r_db.smembers('ail:update:to_update')
def get_current_background_update():
return r_serv.get('ail:update:update_in_progress')
return r_db.get('ail:update:update_in_progress')
def get_current_background_update_script():
return r_serv.get('ail:update:current_background_script')
return r_db.get('ail:update:current_background_script')
def get_current_background_update_script_path(version, script_name):
return os.path.join(os.environ['AIL_HOME'], 'update', version, script_name)
def get_current_background_nb_update_completed():
return r_serv.scard('ail:update:update_in_progress:completed')
return r_db.scard('ail:update:update_in_progress:completed')
def get_current_background_update_progress():
progress = r_serv.get('ail:update:current_background_script_stat')
progress = r_db.get('ail:update:current_background_script_stat')
if not progress:
progress = 0
return int(progress)
def get_background_update_error():
return r_serv.get('ail:update:error')
return r_db.get('ail:update:error')
def add_background_updates_to_launch(version):
return r_serv.sadd('ail:update:to_update', version)
return r_db.sadd('ail:update:to_update', version)
def start_background_update(version):
r_serv.delete('ail:update:error')
r_serv.set('ail:update:update_in_progress', version)
r_db.delete('ail:update:error')
r_db.set('ail:update:update_in_progress', version)
def set_current_background_update_script(script_name):
r_serv.set('ail:update:current_background_script', script_name)
r_serv.set('ail:update:current_background_script_stat', 0)
r_db.set('ail:update:current_background_script', script_name)
r_db.set('ail:update:current_background_script_stat', 0)
def set_current_background_update_progress(progress):
r_serv.set('ail:update:current_background_script_stat', progress)
r_db.set('ail:update:current_background_script_stat', progress)
def set_background_update_error(error):
r_serv.set('ail:update:error', error)
r_db.set('ail:update:error', error)
def end_background_update_script():
r_serv.sadd('ail:update:update_in_progress:completed')
r_db.sadd('ail:update:update_in_progress:completed')
def end_background_update(version):
r_serv.delete('ail:update:update_in_progress')
r_serv.delete('ail:update:current_background_script')
r_serv.delete('ail:update:current_background_script_stat')
r_serv.delete('ail:update:update_in_progress:completed')
r_serv.srem('ail:update:to_update', version)
r_db.delete('ail:update:update_in_progress')
r_db.delete('ail:update:current_background_script')
r_db.delete('ail:update:current_background_script_stat')
r_db.delete('ail:update:update_in_progress:completed')
r_db.srem('ail:update:to_update', version)
def clear_background_update():
r_serv.delete('ail:update:error')
r_serv.delete('ail:update:update_in_progress')
r_serv.delete('ail:update:current_background_script')
r_serv.delete('ail:update:current_background_script_stat')
r_serv.delete('ail:update:update_in_progress:completed')
r_db.delete('ail:update:error')
r_db.delete('ail:update:update_in_progress')
r_db.delete('ail:update:current_background_script')
r_db.delete('ail:update:current_background_script_stat')
r_db.delete('ail:update:update_in_progress:completed')
def get_update_background_message(version):
return BACKGROUND_UPDATES[version]['message']
# TODO: Detect error in subprocess
def get_update_background_metadata():
dict_update = {}
@ -162,7 +184,7 @@ def get_update_background_metadata():
dict_update['script_progress'] = get_current_background_update_progress()
dict_update['nb_update'] = BACKGROUND_UPDATES[dict_update['version']]['nb_updates']
dict_update['nb_completed'] = get_current_background_nb_update_completed()
dict_update['progress'] = int(dict_update['nb_completed']*100/dict_update['nb_update'])
dict_update['progress'] = int(dict_update['nb_completed'] * 100 / dict_update['nb_update'])
dict_update['error'] = get_background_update_error()
return dict_update
@ -170,14 +192,6 @@ def get_update_background_metadata():
##-- UPDATE BACKGROUND --##
##########################
if __name__ == '__main__':
res = check_version('v3.1..1')
print(res)

1
bin/lib/objects/Decodeds.py
View File

@ -36,7 +36,6 @@ except:
config_loader = ConfigLoader()
r_objects = config_loader.get_db_conn("Kvrocks_Objects")
r_metadata = config_loader.get_redis_conn("ARDB_Metadata")
HASH_DIR = config_loader.get_config_str('Directories', 'hash')
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
config_loader = None

28
bin/lib/objects/abstract_subtype_object.py
View File

@ -8,7 +8,6 @@ Base Class for AIL Objects
##################################
import os
import sys
from abc import abstractmethod
# from flask import url_for
@ -25,7 +24,6 @@ from packages import Date
# LOAD CONFIG
config_loader = ConfigLoader()
r_metadata = config_loader.get_redis_conn("ARDB_Metadata")
r_object = config_loader.get_db_conn("Kvrocks_Objects")
config_loader = None
@ -51,13 +49,6 @@ class AbstractSubtypeObject(AbstractObject):
def exists(self):
return r_object.exists(f'meta:{self.type}:{self.subtype}:{self.id}')
# def exists(self):
# res = r_metadata.zscore(f'{self.type}_all:{self.subtype}', self.id)
# if res is not None:
# return True
# else:
# return False
def get_first_seen(self, r_int=False):
first_seen = r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'first_seen')
if r_int:
@ -79,11 +70,11 @@ class AbstractSubtypeObject(AbstractObject):
return last_seen
def get_nb_seen(self):
return int(r_metadata.zscore(f'{self.type}_all:{self.subtype}', self.id))
return int(r_object.zscore(f'{self.type}_all:{self.subtype}', self.id))
# # TODO: CHECK RESULT
def get_nb_seen_by_date(self, date_day):
nb = r_metadata.hget(f'{self.type}:{self.subtype}:{date_day}', self.id)
nb = r_object.hget(f'{self.type}:{self.subtype}:{date_day}', self.id)
if nb is None:
return 0
else:
@ -134,9 +125,9 @@ class AbstractSubtypeObject(AbstractObject):
self.update_daterange(date)
update_obj_date(date, self.type, self.subtype)
# daily
r_metadata.hincrby(f'{self.type}:{self.subtype}:{date}', self.id, 1)
r_object.hincrby(f'{self.type}:{self.subtype}:{date}', self.id, 1)
# all subtypes
r_metadata.zincrby(f'{self.type}_all:{self.subtype}', 1, self.id)
r_object.zincrby(f'{self.type}_all:{self.subtype}', 1, self.id)
#######################################################################
#######################################################################
@ -158,14 +149,5 @@ class AbstractSubtypeObject(AbstractObject):
def _delete(self):
pass
####################################
#
# _get_items
# get_metadata
#
#
def get_all_id(obj_type, subtype):
return r_metadata.zrange(f'{obj_type}_all:{subtype}', 0, -1)
return r_object.zrange(f'{obj_type}_all:{subtype}', 0, -1)

3
bin/modules/Credential.py
View File

@ -69,8 +69,7 @@ class Credential(AbstractModule):
# Database
config_loader = ConfigLoader.ConfigLoader()
# self.server_cred = config_loader.get_redis_conn("ARDB_TermCred")
self.server_statistics = config_loader.get_redis_conn("ARDB_Statistics")
# self.server_cred = config_loader.get_redis_conn("_TermCred")
# Config values
self.minimumLengthThreshold = config_loader.get_config_int("Credential", "minimumLengthThreshold")

8
bin/modules/Global.py
View File

@ -43,6 +43,7 @@ from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader
from lib.data_retention_engine import update_obj_date
from lib import item_basic
# from lib import Statistics
class Global(AbstractModule):
"""
@ -52,8 +53,6 @@ class Global(AbstractModule):
def __init__(self):
super(Global, self).__init__()
self.r_stats = ConfigLoader().get_redis_conn("ARDB_Statistics")
self.processed_item = 0
self.time_last_stats = time.time()
@ -197,12 +196,13 @@ class Global(AbstractModule):
self.redis_logger.warning(f'Global; Incomplete file: {filename}')
print(f'Global; Incomplete file: {filename}')
# save daily stats
self.r_stats.zincrby('module:Global:incomplete_file', 1, datetime.datetime.now().strftime('%Y%m%d'))
# self.r_stats.zincrby('module:Global:incomplete_file', 1, datetime.datetime.now().strftime('%Y%m%d'))
# Statistics.
except OSError:
self.redis_logger.warning(f'Global; Not a gzipped file: {filename}')
print(f'Global; Not a gzipped file: {filename}')
# save daily stats
self.r_stats.zincrby('module:Global:invalid_file', 1, datetime.datetime.now().strftime('%Y%m%d'))
# self.r_stats.zincrby('module:Global:invalid_file', 1, datetime.datetime.now().strftime('%Y%m%d'))
return curr_file_content

7
bin/modules/LibInjection.py
View File

@ -13,7 +13,6 @@ It tries to identify SQL Injections with libinjection.
import os
import sys
import urllib.request
import pylibinjection
from datetime import datetime
@ -28,6 +27,7 @@ sys.path.append(os.environ['AIL_BIN'])
from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader
from lib.objects.Items import Item
from lib import Statistics
class LibInjection(AbstractModule):
"""docstring for LibInjection module."""
@ -37,9 +37,6 @@ class LibInjection(AbstractModule):
self.faup = Faup()
config_loader = ConfigLoader()
self.server_statistics = config_loader.get_redis_conn("ARDB_Statistics")
self.redis_logger.info(f"Module: {self.module_name} Launched")
def compute(self, message):
@ -94,7 +91,7 @@ class LibInjection(AbstractModule):
tld = url_parsed['tld']
if tld is not None:
date = datetime.now().strftime("%Y%m")
self.server_statistics.hincrby(f'SQLInjection_by_tld:{date}', tld, 1)
Statistics.add_module_tld_stats_by_date(self.module_name, date, tld, 1)
if __name__ == "__main__":

6
bin/modules/Onion.py
View File

@ -34,7 +34,6 @@ class Onion(AbstractModule):
config_loader = ConfigLoader()
self.r_cache = config_loader.get_redis_conn("Redis_Cache")
self.r_onion = config_loader.get_redis_conn("ARDB_Onion")
self.pending_seconds = config_loader.get_config_int("Onion", "max_execution_time")
# regex timeout
@ -91,7 +90,7 @@ class Onion(AbstractModule):
if onion_urls:
if crawlers.is_crawler_activated():
for domain in domains:# TODO LOAD DEFAULT SCREENSHOT + HAR
for domain in domains: # TODO LOAD DEFAULT SCREENSHOT + HAR
task_uuid = crawlers.add_crawler_task(domain, parent=item.get_id())
if task_uuid:
print(f'{domain} added to crawler queue: {task_uuid}')
@ -109,6 +108,3 @@ if __name__ == "__main__":
module = Onion()
# module.compute('submitted/2022/10/10/submitted_705d1d92-7e9a-4a44-8c21-ccd167bfb7db.gz 9')
module.run()
# 5ajw6aqf3ep7sijnscdzw77t7xq4xjpsy335yb2wiwgouo7yfxtjlmid.onion to debian.org

6
bin/modules/SQLInjectionDetection.py
View File

@ -27,6 +27,7 @@ sys.path.append(os.environ['AIL_BIN'])
from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader
from lib.objects.Items import Item
from lib import Statistics
class SQLInjectionDetection(AbstractModule):
"""docstring for SQLInjectionDetection module."""
@ -40,9 +41,6 @@ class SQLInjectionDetection(AbstractModule):
self.faup = Faup()
config_loader = ConfigLoader()
self.server_statistics = config_loader.get_redis_conn("ARDB_Statistics")
self.redis_logger.info(f"Module: {self.module_name} Launched")
def compute(self, message):
@ -75,7 +73,7 @@ class SQLInjectionDetection(AbstractModule):
except:
pass
date = datetime.now().strftime("%Y%m")
self.server_statistics.hincrby(f'SQLInjection_by_tld:{date}', tld, 1)
Statistics.add_module_tld_stats_by_date(self.module_name, date, tld, 1)
# Try to detect if the url passed might be an sql injection by applying the regex
# defined above on it.

2
bin/modules/SentimentAnalysis.py
View File

@ -62,7 +62,7 @@ class SentimentAnalysis(AbstractModule):
self.sentiment_lexicon_file = ConfigLoader.ConfigLoader().get_config_str("Directories", "sentiment_lexicon_file")
# REDIS_LEVEL_DB #
self.db = ConfigLoader.ConfigLoader().get_redis_conn("ARDB_Sentiment")
self.db = ConfigLoader.ConfigLoader().get_redis_conn("_Sentiment")
self.time1 = time.time()

253
bin/packages/config.cfg.docker-compose-sample
View File

@ -1,253 +0,0 @@
[Directories]
bloomfilters = Blooms
dicofilters = Dicos
pastes = PASTES
hash = HASHS
crawled = crawled
crawled_screenshot = CRAWLED_SCREENSHOT
wordtrending_csv = var/www/static/csv/wordstrendingdata
wordsfile = files/wordfile
protocolstrending_csv = var/www/static/csv/protocolstrendingdata
protocolsfile = files/protocolsfile
tldstrending_csv = var/www/static/csv/tldstrendingdata
tldsfile = faup/src/data/mozilla.tlds
domainstrending_csv = var/www/static/csv/domainstrendingdata
pystemonpath = /opt/pystemon/
sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt
##### Notifications ######
[Notifications]
ail_domain = http://localhost:7000
sender = sender@example.com
sender_host = smtp.example.com
sender_port = 1337
sender_pw = None
# optional for using with authenticated SMTP over SSL
# sender_pw = securepassword
##### Flask #####
[Flask]
#Proxying requests to the app
baseUrl = /
#Number of logs to display in the dashboard
max_dashboard_logs = 15
#Maximum number of character to display in the toolip
max_preview_char = 250
#Maximum number of character to display in the modal
max_preview_modal = 800
#Default number of header to display in trending graphs
default_display = 10
#Number of minutes displayed for the number of processed pastes.
minute_processed_paste = 10
#Maximum line length authorized to make a diff between duplicates
DiffMaxLineLength = 10000
#### Modules ####
[BankAccount]
max_execution_time = 60
[Categ]
#Minimum number of match between the paste and the category file
matchingThreshold=1
[Credential]
#Minimum length that a credential must have to be considered as such
minimumLengthThreshold=3
#Will be pushed as alert if the number of credentials is greater to that number
criticalNumberToAlert=8
#Will be considered as false positive if less that X matches from the top password list
minTopPassList=5
[Curve]
max_execution_time = 90
[Onion]
max_execution_time = 180
[Base64]
path = Base64/
max_execution_time = 60
[Binary]
path = Base64/
max_execution_time = 60
[Hex]
path = Base64/
max_execution_time = 60
[Modules_Duplicates]
#Number of month to look back
maximum_month_range = 3
#The value where two pastes are considerate duplicate for ssdeep.
threshold_duplicate_ssdeep = 50
#The value where two pastes are considerate duplicate for tlsh.
threshold_duplicate_tlsh = 52
#Minimum size of the paste considered
min_paste_size = 0.3
[Module_ModuleInformation]
#Threshold to deduce if a module is stuck or not, in seconds.
threshold_stucked_module=600
[Module_Mixer]
#Define the configuration of the mixer, possible value: 1, 2 or 3
operation_mode = 3
#Define the time that a paste will be considerate duplicate. in seconds (1day = 86400)
ttl_duplicate = 86400
default_unnamed_feed_name = unnamed_feeder
[RegexForTermsFrequency]
max_execution_time = 60
##### Redis #####
[Redis_Cache]
host = localhost
port = 6379
db = 0
[Redis_Log]
host = localhost
port = 6380
db = 0
[Redis_Log_submit]
host = localhost
port = 6380
db = 1
[Redis_Queues]
host = localhost
port = 6381
db = 0
[Redis_Data_Merging]
host = localhost
port = 6379
db = 1
[Redis_Paste_Name]
host = localhost
port = 6379
db = 2
[Redis_Mixer_Cache]
host = localhost
port = 6381
db = 1
##### ARDB #####
[ARDB_Curve]
host = localhost
port = 6382
db = 1
[ARDB_Sentiment]
host = localhost
port = 6382
db = 4
[ARDB_TermFreq]
host = localhost
port = 6382
db = 2
[ARDB_TermCred]
host = localhost
port = 6382
db = 5
[ARDB_DB]
host = localhost
port = 6382
db = 0
[ARDB_Trending]
host = localhost
port = 6382
db = 3
[ARDB_Hashs]
host = localhost
db = 1
[ARDB_Tags]
host = localhost
port = 6382
db = 6
[ARDB_Metadata]
host = localhost
port = 6382
db = 7
[ARDB_Statistics]
host = localhost
port = 6382
db = 8
[ARDB_Onion]
host = localhost
port = 6382
db = 9
[Url]
cc_critical = DE
[DomClassifier]
cc = DE
cc_tld = r'\.de$'
dns = 8.8.8.8
[Mail]
dns = 8.8.8.8
[Web]
dns = 149.13.33.69
# Indexer configuration
[Indexer]
type = whoosh
path = indexdir
register = indexdir/all_index.txt
#size in Mb
index_max_size = 2000
[ailleakObject]
maxDuplicateToPushToMISP=10
###############################################################################
# For multiple feed, add them with "," without space
# e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557
[ZMQ_Global]
#address = tcp://crf.circl.lu:5556
address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
channel = 102
bind = tcp://127.0.0.1:5556
[ZMQ_Url]
address = tcp://127.0.0.1:5004
channel = urls
[ZMQ_FetchedOnion]
address = tcp://127.0.0.1:5005
channel = FetchedOnion
[RedisPubSub]
host = localhost
port = 6381
db = 0
[Crawler]
activate_crawler = False
crawler_depth_limit = 1
splash_url_onion = http://172.17.0.1
splash_onion_port = 8050

39
bin/packages/modules.cfg
View File

@ -12,10 +12,6 @@ publish = Redis_Mixer,Redis_Tags
subscribe = Redis_Mixer
publish = Redis_Global,Redis_ModuleStats
#[PreProcessFeed]
#subscribe = Redis_preProcess1
#publish = Redis_Mixer
[Duplicates]
subscribe = Redis_Duplicate
@ -62,7 +58,7 @@ subscribe = Redis_Global
[Categ]
subscribe = Redis_Global
publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Urls,Redis_Credential,Redis_SourceCode,Redis_Cve,Redis_ApiKey,Redis_SyncModule
publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Urls,Redis_Credential,Redis_Cve,Redis_ApiKey,Redis_SyncModule
[CreditCards]
subscribe = Redis_CreditCards
@ -78,13 +74,9 @@ publish = Redis_ModuleStats,Redis_Tags
[Onion]
subscribe = Redis_Onion
publish = Redis_ValidOnion,Redis_Tags,Redis_Crawler
publish = Redis_Tags,Redis_Crawler
#publish = Redis_ValidOnion,ZMQ_FetchedOnion,Redis_Tags,Redis_Crawler
# TODO remove me
[DumpValidOnion]
subscribe = Redis_ValidOnion
[Urls]
subscribe = Redis_Urls
publish = Redis_Url
@ -112,15 +104,8 @@ subscribe = Redis_SyncModule
[MISP_The_hive_feeder]
subscribe = Redis_Tags_feed
#[send_to_queue]
#subscribe = Redis_Cve
#publish = Redis_Tags
[SentimentAnalysis]
subscribe = Redis_Global
[Release]
subscribe = Redis_Global
#[SentimentAnalysis]
#subscribe = Redis_Global
[Credential]
subscribe = Redis_Credential
@ -130,9 +115,10 @@ publish = Redis_Duplicate,Redis_ModuleStats,Redis_Tags
subscribe = Redis_Cve
publish = Redis_Tags
[Phone]
subscribe = Redis_Global
publish = Redis_Tags
# Disabled
#[Phone]
#subscribe = Redis_Global
#publish = Redis_Tags
[Keys]
subscribe = Redis_Global
@ -159,7 +145,6 @@ subscribe = Redis
publish = Redis_Mixer
[Crawler]
subscribe = Redis_Crawler
publish = Redis_Mixer,Redis_Tags
[IP]
@ -169,3 +154,11 @@ publish = Redis_Tags
[Zerobins]
subscribe = Redis_Url
#[PreProcessFeed]
#subscribe = Redis_preProcess1
#publish = Redis_Mixer
# [My_Module]
# subscribe = Redis_Global
# publish = Redis_Tags

10
bin/template.py
View File

@ -36,11 +36,11 @@ class Template(AbstractModule):
# Send module state to logs
self.redis_logger.info(f'Module {self.module_name} initialized')
def computeNone(self):
"""
Do something when there is no message in the queue
"""
self.redis_logger.debug("No message in queue")
# def computeNone(self):
# """
# Do something when there is no message in the queue
# """
# self.redis_logger.debug("No message in queue")
def compute(self, message):
"""

31
var/www/blueprints/import_export.py
View File

@ -81,7 +81,7 @@ def import_object_file():
return render_template("import_object.html", all_imported_obj=all_imported_obj, error=error)
@import_export.route("/objects/misp/export", methods=['GET'])
@import_export.route("/misp/objects/export", methods=['GET'])
@login_required
@login_analyst
def objects_misp_export():
@ -91,7 +91,7 @@ def objects_misp_export():
return render_template("export_object.html", object_types=object_types, to_export=to_export)
@import_export.route("/objects/misp/export/post", methods=['POST'])
@import_export.route("/misp/objects/export/post", methods=['POST'])
@login_required
@login_analyst
def objects_misp_export_post():
@ -159,7 +159,7 @@ def objects_misp_export_post():
misp_url=event['url'])
@import_export.route("/objects/misp/export/add", methods=['GET'])
@import_export.route("/misp/objects/export/add", methods=['GET'])
@login_required
@login_analyst
def add_object_id_to_export():
@ -181,7 +181,7 @@ def add_object_id_to_export():
return redirect(url_for('import_export.objects_misp_export'))
@import_export.route("/objects/misp/export/delete", methods=['GET'])
@import_export.route("/misp/objects/export/delete", methods=['GET'])
@login_required
@login_analyst
def delete_object_id_to_export():
@ -194,7 +194,7 @@ def delete_object_id_to_export():
return jsonify(success=True)
@import_export.route("/import_export/investigation", methods=['GET'])
@import_export.route("/investigation/misp/export", methods=['GET'])
@login_required
@login_analyst
def export_investigation():
@ -206,3 +206,24 @@ def export_investigation():
return Response(json.dumps({"error": "Can't reach MISP Instance"}, indent=2, sort_keys=True),
mimetype='application/json'), 400
return redirect(url_for('investigations_b.show_investigation', uuid=investigation_uuid))
@import_export.route("/thehive/objects/case/export", methods=['POST'])
@login_required
@login_analyst
def create_thehive_case():
description = request.form['hive_description']
title = request.form['hive_case_title']
threat_level = Export.sanitize_threat_level_hive(request.form['threat_level_hive'])
tlp = Export.sanitize_tlp_hive(request.form['hive_tlp'])
item_id = request.form['obj_id']
item = Item(item_id)
if not item.exists():
abort(404)
case_id = Export.create_thehive_case(item_id, title=title, tlp=tlp, threat_level=threat_level, description=description)
if case_id:
return redirect(Export.get_case_url(case_id))
else:
return 'error'

93
var/www/modules/Flask_config.py
View File

@ -30,15 +30,9 @@ r_cache = config_loader.get_redis_conn("Redis_Cache")
r_serv_log = config_loader.get_redis_conn("Redis_Log")
r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit")
r_serv_charts = config_loader.get_redis_conn("ARDB_Trending")
r_serv_sentiment = config_loader.get_redis_conn("ARDB_Sentiment")
r_serv_term = config_loader.get_redis_conn("ARDB_Tracker")
r_serv_cred = config_loader.get_redis_conn("ARDB_TermCred")
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
r_serv_statistics = config_loader.get_redis_conn("ARDB_Statistics")
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
r_serv_charts = config_loader.get_redis_conn("ARDB_Trending") # -> TODO MIGRATE Stats Graphs
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") # -> TODO MIGRATE /correlation/ subtypes objects
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") # -> TODO MIGRATE AUTO CRAWLER
# # # # # # #
r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
@ -51,72 +45,45 @@ redis_logger.port = 6380
# Channel name to publish logs
redis_logger.channel = 'Flask'
sys.path.append('../../configs/keys')
# MISP #
try:
from pymisp import PyMISP
from mispKEYS import misp_url, misp_key, misp_verifycert
pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
misp_event_url = misp_url + '/events/view/'
print('Misp connected')
except:
print('Misp not connected')
pymisp = False
misp_event_url = '#'
# The Hive #
try:
from thehive4py.api import TheHiveApi
import thehive4py.exceptions
from theHiveKEYS import the_hive_url, the_hive_key, the_hive_verifycert
if the_hive_url == '':
HiveApi = False
hive_case_url = '#'
print('The HIVE not connected')
else:
HiveApi = TheHiveApi(the_hive_url, the_hive_key, cert=the_hive_verifycert)
hive_case_url = the_hive_url+'/index.html#/case/id_here/details'
except:
print('The HIVE not connected')
HiveApi = False
hive_case_url = '#'
if HiveApi != False:
try:
HiveApi.get_alert(0)
print('The Hive connected')
except thehive4py.exceptions.AlertException:
HiveApi = False
print('The Hive not connected')
#### VARIABLES ####
baseUrl = config_loader.get_config_str("Flask", "baseurl")
baseUrl = baseUrl.replace('/', '')
if baseUrl != '':
baseUrl = '/'+baseUrl
baseUrl = '/' + baseUrl
max_preview_char = int(config_loader.get_config_str("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
max_preview_modal = int(config_loader.get_config_str("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
max_preview_char = int(
config_loader.get_config_str("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
max_preview_modal = int(
config_loader.get_config_str("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
max_tags_result = 50
DiffMaxLineLength = int(config_loader.get_config_str("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value
DiffMaxLineLength = int(config_loader.get_config_str("Flask",
"DiffMaxLineLength")) # Use to display the estimated percentage instead of a raw value
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
dict_update_description = {'v1.5':{'nb_background_update': 5, 'update_warning_message': 'An Update is running on the background. Some informations like Tags, screenshot can be',
'update_warning_message_notice_me': 'missing from the UI.'},
'v2.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
'update_warning_message_notice_me': 'missing from the UI.'},
'v2.6':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
'update_warning_message_notice_me': 'missing from the UI.'},
'v2.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags can be',
'update_warning_message_notice_me': 'missing from the UI.'},
'v3.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Languages can be',
'update_warning_message_notice_me': 'missing from the UI.'},
'v3.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Tracker first_seen/last_seen can be',
'update_warning_message_notice_me': 'missing from the UI.'}
}
dict_update_description = {'v1.5': {'nb_background_update': 5,
'update_warning_message': 'An Update is running on the background. Some informations like Tags, screenshot can be',
'update_warning_message_notice_me': 'missing from the UI.'},
'v2.4': {'nb_background_update': 1,
'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
'update_warning_message_notice_me': 'missing from the UI.'},
'v2.6': {'nb_background_update': 1,
'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
'update_warning_message_notice_me': 'missing from the UI.'},
'v2.7': {'nb_background_update': 1,
'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags can be',
'update_warning_message_notice_me': 'missing from the UI.'},
'v3.4': {'nb_background_update': 1,
'update_warning_message': 'An Update is running on the background. Some informations like Domain Languages can be',
'update_warning_message_notice_me': 'missing from the UI.'},
'v3.7': {'nb_background_update': 1,
'update_warning_message': 'An Update is running on the background. Some informations like Tracker first_seen/last_seen can be',
'update_warning_message_notice_me': 'missing from the UI.'}
}
UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')
@ -137,10 +104,10 @@ SUBMIT_PASTE_TEXT_MAX_SIZE = int(config_loader.get_config_str("SubmitPaste", "TE
SUBMIT_PASTE_FILE_MAX_SIZE = int(config_loader.get_config_str("SubmitPaste", "FILE_MAX_SIZE"))
SUBMIT_PASTE_FILE_ALLOWED_EXTENSIONS = [item.strip() for item in config_loader.get_config_str("SubmitPaste", "FILE_ALLOWED_EXTENSIONS").split(',')]
# VT
try:
from virusTotalKEYS import vt_key
if vt_key != '':
vt_auth = vt_key
vt_enabled = True

40
var/www/modules/PasteSubmit/Flask_PasteSubmit.py
View File

@ -11,10 +11,8 @@ import re
import os
import sys
import string
import datetime
import unicodedata
import uuid
from io import BytesIO
from functools import wraps
@ -42,9 +40,6 @@ import Flask_config
app = Flask_config.app
baseUrl = Flask_config.baseUrl
r_serv_metadata = Flask_config.r_serv_metadata # TODO REMOVE ME
r_serv_db = Flask_config.r_serv_db # TODO REMOVE ME
r_serv_log_submit = Flask_config.r_serv_log_submit # TODO REMOVE ME
@ -317,35 +312,16 @@ def submit_status():
else:
return 'INVALID UUID'
# TODO MOVE ME IN import_export blueprint
@PasteSubmit.route("/PasteSubmit/create_hive_case", methods=['POST'])
@login_required
@login_analyst
def create_hive_case():
tlp = request.form['hive_tlp']
if tlp:
tlp = int(tlp)
else:
tlp = 2
threat_level = request.form['threat_level_hive']
if threat_level:
threat_level = int(threat_level)
else:
threat_level = 2
description = request.form['hive_description']
title = request.form['hive_case_title']
item_id = request.form['obj_id']
######################################################################################################
######################################################################################################
######################################################################################################
######################################################################################################
if (0 <= tlp <= 3) and (1 <= threat_level <= 4):
# TODO MIGRATE TAGS PUSH
case_id = Export.create_thehive_case(item_id, title=title, tlp=tlp, threat_level=threat_level, description=description)
if case_id:
return redirect(get_case_url(case_id))
else:
return 'error'
# TEMPORARY: 2 SET OF CUSTOM + infoleak tags ?????????
return 'error'
@PasteSubmit.route("/PasteSubmit/edit_tag_export")
@login_required
@ -410,8 +386,8 @@ def edit_tag_export():
status_hive=status_hive,
nb_tags_whitelist_misp=nb_tags_whitelist_misp,
nb_tags_whitelist_hive=nb_tags_whitelist_hive,
flag_misp=flag_misp,
flag_hive=flag_hive)
flag_misp=True,
flag_hive=True)
@PasteSubmit.route("/PasteSubmit/tag_export_edited", methods=['POST'])
@login_required

1
var/www/modules/hiddenServices/Flask_hiddenServices.py
View File

@ -27,7 +27,6 @@ app = Flask_config.app
baseUrl = Flask_config.baseUrl
r_cache = Flask_config.r_cache
r_serv_onion = Flask_config.r_serv_onion
r_serv_metadata = Flask_config.r_serv_metadata
bootstrap_label = Flask_config.bootstrap_label
hiddenServices = Blueprint('hiddenServices', __name__, template_folder='templates')

5
var/www/modules/restApi/Flask_restApi.py
View File

@ -41,8 +41,6 @@ app = Flask_config.app
baseUrl = Flask_config.baseUrl
r_cache = Flask_config.r_cache
r_serv_db = Flask_config.r_serv_db
r_serv_onion = Flask_config.r_serv_onion
r_serv_metadata = Flask_config.r_serv_metadata
restApi = Blueprint('restApi', __name__, template_folder='templates')
@ -258,7 +256,8 @@ def add_item_tags():
tags = data.get('tags', [])
galaxy = data.get('galaxy', [])
res = Tag.api_add_obj_tags(tags=tags, galaxy_tags=galaxy, object_id=object_id, object_type="item")
# res = Tag.api_add_obj_tags(tags=tags, galaxy_tags=galaxy, object_id=object_id, object_type="item")
res = {'error': 'disabled endpoint'}, 500
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

314
var/www/modules/sentiment/Flask_sentiment.py
View File

@ -1,160 +1,160 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
Flask functions and routes for the trending modules page
'''
import os
import sys
import datetime
import calendar
import flask
from flask import Flask, render_template, jsonify, request, Blueprint
from Role_Manager import login_admin, login_analyst, login_read_only
from flask_login import login_required
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from packages.Date import Date
# ============ VARIABLES ============
import Flask_config
app = Flask_config.app
baseUrl = Flask_config.baseUrl
r_serv_charts = Flask_config.r_serv_charts
r_serv_sentiment = Flask_config.r_serv_sentiment
sentiments = Blueprint('sentiments', __name__, template_folder='templates')
# ============ FUNCTIONS ============
def get_date_range(num_day):
curr_date = datetime.date.today()
date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
date_list = []
for i in range(0, num_day+1):
date_list.append(date.substract_day(i))
return date_list
# ============ ROUTES ============
@sentiments.route("/sentiment_analysis_trending/")
@login_required
@login_read_only
def sentiment_analysis_trending():
return render_template("sentiment_analysis_trending.html")
@sentiments.route("/sentiment_analysis_getplotdata/", methods=['GET'])
@login_required
@login_read_only
def sentiment_analysis_getplotdata():
# Get the top providers based on number of pastes
oneHour = 60*60
sevenDays = oneHour*24*7
dateStart = datetime.datetime.now()
dateStart = dateStart.replace(minute=0, second=0, microsecond=0)
dateStart_timestamp = calendar.timegm(dateStart.timetuple())
getAllProviders = request.args.get('getProviders')
provider = request.args.get('provider')
allProvider = request.args.get('all')
if getAllProviders == 'True':
if allProvider == "True":
range_providers = r_serv_charts.smembers('all_provider_set')
return jsonify(list(range_providers))
else:
range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8)
# if empty, get yesterday top providers
range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers
# if still empty, takes from all providers
if range_providers == []:
print('today provider empty')
range_providers = r_serv_charts.smembers('all_provider_set')
return jsonify(list(range_providers))
elif provider is not None:
to_return = {}
cur_provider_name = provider + '_'
list_date = {}
for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour):
cur_set_name = cur_provider_name + str(cur_timestamp)
list_value = []
for cur_id in r_serv_sentiment.smembers(cur_set_name):
cur_value = (r_serv_sentiment.get(cur_id))
list_value.append(cur_value)
list_date[cur_timestamp] = list_value
to_return[provider] = list_date
return jsonify(to_return)
return "Bad request"
@sentiments.route("/sentiment_analysis_plot_tool/")
@login_required
@login_read_only
def sentiment_analysis_plot_tool():
return render_template("sentiment_analysis_plot_tool.html")
@sentiments.route("/sentiment_analysis_plot_tool_getdata/", methods=['GET'])
@login_required
@login_read_only
def sentiment_analysis_plot_tool_getdata():
getProviders = request.args.get('getProviders')
if getProviders == 'True':
providers = []
for cur_provider in r_serv_charts.smembers('all_provider_set'):
providers.append(cur_provider)
return jsonify(providers)
else:
query = request.args.get('query')
query = query.split(',')
Qdate = request.args.get('Qdate')
date1 = (Qdate.split('-')[0]).split('/')
date1 = datetime.date(int(date1[2]), int(date1[0]), int(date1[1]))
date2 = (Qdate.split('-')[1]).split('/')
date2 = datetime.date(int(date2[2]), int(date2[0]), int(date2[1]))
timestamp1 = calendar.timegm(date1.timetuple())
timestamp2 = calendar.timegm(date2.timetuple())
oneHour = 60*60
oneDay = oneHour*24
to_return = {}
for cur_provider in query:
list_date = {}
cur_provider_name = cur_provider + '_'
for cur_timestamp in range(int(timestamp1), int(timestamp2)+oneDay, oneHour):
cur_set_name = cur_provider_name + str(cur_timestamp)
list_value = []
for cur_id in r_serv_sentiment.smembers(cur_set_name):
cur_value = (r_serv_sentiment.get(cur_id))
list_value.append(cur_value)
list_date[cur_timestamp] = list_value
to_return[cur_provider] = list_date
return jsonify(to_return)
# ========= REGISTRATION =========
app.register_blueprint(sentiments, url_prefix=baseUrl)
# '''
# Flask functions and routes for the trending modules page
# '''
# import os
# import sys
# import datetime
# import calendar
# import flask
# from flask import Flask, render_template, jsonify, request, Blueprint
#
# from Role_Manager import login_admin, login_analyst, login_read_only
# from flask_login import login_required
#
# sys.path.append(os.environ['AIL_BIN'])
# ##################################
# # Import Project packages
# ##################################
# from packages.Date import Date
#
# # ============ VARIABLES ============
# import Flask_config
#
# app = Flask_config.app
# baseUrl = Flask_config.baseUrl
# r_serv_charts = Flask_config.r_serv_charts
# r_serv_sentiment = Flask_config.r_serv_sentiment
#
# sentiments = Blueprint('sentiments', __name__, template_folder='templates')
#
# # ============ FUNCTIONS ============
#
# def get_date_range(num_day):
# curr_date = datetime.date.today()
# date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
# date_list = []
#
# for i in range(0, num_day+1):
# date_list.append(date.substract_day(i))
# return date_list
#
#
# # ============ ROUTES ============
#
# @sentiments.route("/sentiment_analysis_trending/")
# @login_required
# @login_read_only
# def sentiment_analysis_trending():
# return render_template("sentiment_analysis_trending.html")
#
#
# @sentiments.route("/sentiment_analysis_getplotdata/", methods=['GET'])
# @login_required
# @login_read_only
# def sentiment_analysis_getplotdata():
# # Get the top providers based on number of pastes
# oneHour = 60*60
# sevenDays = oneHour*24*7
# dateStart = datetime.datetime.now()
# dateStart = dateStart.replace(minute=0, second=0, microsecond=0)
# dateStart_timestamp = calendar.timegm(dateStart.timetuple())
#
# getAllProviders = request.args.get('getProviders')
# provider = request.args.get('provider')
# allProvider = request.args.get('all')
# if getAllProviders == 'True':
# if allProvider == "True":
# range_providers = r_serv_charts.smembers('all_provider_set')
#
# return jsonify(list(range_providers))
# else:
# range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8)
# # if empty, get yesterday top providers
# range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers
#
#
# # if still empty, takes from all providers
# if range_providers == []:
# print('today provider empty')
# range_providers = r_serv_charts.smembers('all_provider_set')
#
# return jsonify(list(range_providers))
#
# elif provider is not None:
# to_return = {}
#
# cur_provider_name = provider + '_'
# list_date = {}
# for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour):
# cur_set_name = cur_provider_name + str(cur_timestamp)
#
# list_value = []
# for cur_id in r_serv_sentiment.smembers(cur_set_name):
# cur_value = (r_serv_sentiment.get(cur_id))
# list_value.append(cur_value)
# list_date[cur_timestamp] = list_value
# to_return[provider] = list_date
#
# return jsonify(to_return)
# return "Bad request"
#
#
#
# @sentiments.route("/sentiment_analysis_plot_tool/")
# @login_required
# @login_read_only
# def sentiment_analysis_plot_tool():
# return render_template("sentiment_analysis_plot_tool.html")
#
#
#
# @sentiments.route("/sentiment_analysis_plot_tool_getdata/", methods=['GET'])
# @login_required
# @login_read_only
# def sentiment_analysis_plot_tool_getdata():
# getProviders = request.args.get('getProviders')
#
# if getProviders == 'True':
# providers = []
# for cur_provider in r_serv_charts.smembers('all_provider_set'):
# providers.append(cur_provider)
# return jsonify(providers)
#
# else:
# query = request.args.get('query')
# query = query.split(',')
# Qdate = request.args.get('Qdate')
#
# date1 = (Qdate.split('-')[0]).split('/')
# date1 = datetime.date(int(date1[2]), int(date1[0]), int(date1[1]))
#
# date2 = (Qdate.split('-')[1]).split('/')
# date2 = datetime.date(int(date2[2]), int(date2[0]), int(date2[1]))
#
# timestamp1 = calendar.timegm(date1.timetuple())
# timestamp2 = calendar.timegm(date2.timetuple())
#
# oneHour = 60*60
# oneDay = oneHour*24
#
# to_return = {}
# for cur_provider in query:
# list_date = {}
# cur_provider_name = cur_provider + '_'
# for cur_timestamp in range(int(timestamp1), int(timestamp2)+oneDay, oneHour):
# cur_set_name = cur_provider_name + str(cur_timestamp)
#
# list_value = []
# for cur_id in r_serv_sentiment.smembers(cur_set_name):
# cur_value = (r_serv_sentiment.get(cur_id))
# list_value.append(cur_value)
# list_date[cur_timestamp] = list_value
# to_return[cur_provider] = list_date
#
# return jsonify(to_return)
#
# # ========= REGISTRATION =========
# app.register_blueprint(sentiments, url_prefix=baseUrl)

14
var/www/modules/sentiment/templates/header_sentiment.html
View File

@ -1,7 +1,7 @@
<li id='page-sentiment'><a class="dropdown-toggle" data-toggle="dropdown" href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-heart"></i> Sentiment Analysis
<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-bar-chart-o"> </i> Sentiment trending</a></li>
<li><a href="{{ url_for('sentiments.sentiment_analysis_plot_tool') }}"><i class="fa fa-wrench"> </i> Sentiment plot Tool</a></li>
</ul>
</li>
{#<li id='page-sentiment'><a class="dropdown-toggle" data-toggle="dropdown" href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-heart"></i> Sentiment Analysis#}
{# <span class="caret"></span></a>#}
{# <ul class="dropdown-menu">#}
{# <li><a href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-bar-chart-o"> </i> Sentiment trending</a></li>#}
{# <li><a href="{{ url_for('sentiments.sentiment_analysis_plot_tool') }}"><i class="fa fa-wrench"> </i> Sentiment plot Tool</a></li>#}
{# </ul>#}
{#</li>#}

356
var/www/modules/terms/Flask_terms.py
View File

@ -1,356 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
Flask functions and routes for the trending modules page
note: The matching of credential against supplied credential is done using Levenshtein distance
'''
import os
import sys
import flask
from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, Response
from Role_Manager import login_admin, login_analyst, login_user_no_api, login_read_only
from flask_login import login_required, current_user
import Levenshtein
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib.objects.Items import Item
# ============ VARIABLES ============
import Flask_config
app = Flask_config.app
baseUrl = Flask_config.baseUrl
r_serv_term = Flask_config.r_serv_term
r_serv_cred = Flask_config.r_serv_cred
r_serv_db = Flask_config.r_serv_db
bootstrap_label = Flask_config.bootstrap_label
terms = Blueprint('terms', __name__, template_folder='templates')
'''TERM'''
DEFAULT_MATCH_PERCENT = 50
#tracked
TrackedTermsSet_Name = "TrackedSetTermSet"
TrackedTermsDate_Name = "TrackedTermDate"
#black
BlackListTermsDate_Name = "BlackListTermDate"
BlackListTermsSet_Name = "BlackListSetTermSet"
#regex
TrackedRegexSet_Name = "TrackedRegexSet"
TrackedRegexDate_Name = "TrackedRegexDate"
#set
TrackedSetSet_Name = "TrackedSetSet"
TrackedSetDate_Name = "TrackedSetDate"
# notifications enabled/disabled
# same value as in `bin/NotificationHelper.py`
TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
# associated notification email addresses for a specific term`
# same value as in `bin/NotificationHelper.py`
# Keys will be e.g. TrackedNotificationEmails_<TERMNAME>
TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
'''CRED'''
REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+'
REDIS_KEY_NUM_USERNAME = 'uniqNumForUsername'
REDIS_KEY_NUM_PATH = 'uniqNumForUsername'
REDIS_KEY_ALL_CRED_SET = 'AllCredentials'
REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev'
REDIS_KEY_ALL_PATH_SET = 'AllPath'
REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev'
REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
# ============ FUNCTIONS ============
def Term_getValueOverRange(word, startDate, num_day, per_paste=""):
passed_days = 0
oneDay = 60*60*24
to_return = []
curr_to_return = 0
for timestamp in range(startDate, startDate - max(num_day)*oneDay, -oneDay):
value = r_serv_term.hget(per_paste+str(timestamp), word)
curr_to_return += int(value) if value is not None else 0
for i in num_day:
if passed_days == i-1:
to_return.append(curr_to_return)
passed_days += 1
return to_return
#Mix suplied username, if extensive is set, slice username(s) with different windows
def mixUserName(supplied, extensive=False):
#e.g.: John Smith
terms = supplied.split()[:2]
usernames = []
if len(terms) == 1:
terms.append(' ')
#john, smith, John, Smith, JOHN, SMITH
usernames += [terms[0].lower()]
usernames += [terms[1].lower()]
usernames += [terms[0][0].upper() + terms[0][1:].lower()]
usernames += [terms[1][0].upper() + terms[1][1:].lower()]
usernames += [terms[0].upper()]
usernames += [terms[1].upper()]
#johnsmith, smithjohn, JOHNsmith, johnSMITH, SMITHjohn, smithJOHN
usernames += [(terms[0].lower() + terms[1].lower()).strip()]
usernames += [(terms[1].lower() + terms[0].lower()).strip()]
usernames += [(terms[0].upper() + terms[1].lower()).strip()]
usernames += [(terms[0].lower() + terms[1].upper()).strip()]
usernames += [(terms[1].upper() + terms[0].lower()).strip()]
usernames += [(terms[1].lower() + terms[0].upper()).strip()]
#Jsmith, JSmith, jsmith, jSmith, johnS, Js, JohnSmith, Johnsmith, johnSmith
usernames += [(terms[0][0].upper() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].upper() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
usernames += [(terms[0].lower() + terms[1][0].upper()).strip()]
usernames += [(terms[0].upper() + terms[1][0].lower()).strip()]
usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
if not extensive:
return usernames
#Slice the supplied username(s)
mixedSupplied = supplied.replace(' ','')
minWindow = 3 if len(mixedSupplied)/2 < 4 else len(mixedSupplied)/2
for winSize in range(3,len(mixedSupplied)):
for startIndex in range(0, len(mixedSupplied)-winSize):
usernames += [mixedSupplied[startIndex:startIndex+winSize]]
filtered_usernames = []
for usr in usernames:
if len(usr) > 2:
filtered_usernames.append(usr)
return filtered_usernames
def save_tag_to_auto_push(list_tag):
for tag in set(list_tag):
#limit tag length
if len(tag) > 49:
tag = tag[0:48]
r_serv_db.sadd('list_export_tags', tag)
# ============ ROUTES ============
# TODO: remove + clean
# @terms.route("/terms_plot_tool/")
# @login_required
# @login_read_only
# def terms_plot_tool():
# term = request.args.get('term')
# if term is not None:
# return render_template("terms_plot_tool.html", term=term)
# else:
# return render_template("terms_plot_tool.html", term="")
#
#
# @terms.route("/terms_plot_tool_data/")
# @login_required
# @login_read_only
# def terms_plot_tool_data():
# oneDay = 60*60*24
# range_start = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0;
# range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0)
# range_start = calendar.timegm(range_start.timetuple())
# range_end = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0;
# range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0)
# range_end = calendar.timegm(range_end.timetuple())
# term = request.args.get('term')
#
# per_paste = request.args.get('per_paste')
# if per_paste == "1" or per_paste is None:
# per_paste = "per_paste_"
# else:
# per_paste = ""
#
# if term is None:
# return "None"
#
# else:
# value_range = []
# for timestamp in range(range_start, range_end+oneDay, oneDay):
# value = r_serv_term.hget(per_paste+str(timestamp), term)
# curr_value_range = int(value) if value is not None else 0
# value_range.append([timestamp, curr_value_range])
# value_range.insert(0,term)
# return jsonify(value_range)
#
# @terms.route("/terms_plot_top/"
# @login_required
# @login_read_only
# def terms_plot_top():
# per_paste = request.args.get('per_paste')
# per_paste = per_paste if per_paste is not None else 1
# return render_template("terms_plot_top.html", per_paste=per_paste)
#
#
# @terms.route("/terms_plot_top_data/")
# @login_required
# @login_read_only
# def terms_plot_top_data():
# oneDay = 60*60*24
# today = datetime.datetime.now()
# today = today.replace(hour=0, minute=0, second=0, microsecond=0)
# today_timestamp = calendar.timegm(today.timetuple())
#
# per_paste = request.args.get('per_paste')
# if per_paste == "1" or per_paste is None:
# per_paste = "per_paste_"
# else:
# per_paste = ""
#
# set_day = per_paste + "TopTermFreq_set_day_" + str(today_timestamp)
# set_week = per_paste + "TopTermFreq_set_week";
# set_month = per_paste + "TopTermFreq_set_month";
#
# the_set = per_paste + request.args.get('set')
# num_day = int(request.args.get('num_day'))
#
# if the_set is None:
# return "None"
# else:
# to_return = []
# if "TopTermFreq_set_day" in the_set:
# the_set += "_" + str(today_timestamp)
#
# for term, tot_value in r_serv_term.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20):
# position = {}
# position['day'] = r_serv_term.zrevrank(set_day, term)
# position['day'] = position['day']+1 if position['day'] is not None else "<20"
# position['week'] = r_serv_term.zrevrank(set_week, term)
# position['week'] = position['week']+1 if position['week'] is not None else "<20"
# position['month'] = r_serv_term.zrevrank(set_month, term)
# position['month'] = position['month']+1 if position['month'] is not None else "<20"
# value_range = []
# for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay):
# value = r_serv_term.hget(per_paste+str(timestamp), term)
# curr_value_range = int(value) if value is not None else 0
# value_range.append([timestamp, curr_value_range])
#
# to_return.append([term, value_range, tot_value, position])
#
# return jsonify(to_return)
@terms.route("/credentials_tracker/")
@login_required
@login_read_only
def credentials_tracker():
return render_template("credentials_tracker.html")
@terms.route("/credentials_management_query_paste/", methods=['GET', 'POST'])
@login_required
@login_user_no_api
def credentials_management_query_paste():
cred = request.args.get('cred')
allPath = request.json['allPath']
paste_info = []
for pathNum in allPath:
path = r_serv_cred.hget(REDIS_KEY_ALL_PATH_SET_REV, pathNum)
item = Item(path)
p_date = item.get_date(separator=True)
p_source = item.get_source()
p_content = item.get_content()
p_encoding = item.get_mimetype()
p_size = item.get_size()
p_mime = p_encoding
lineinfo = item.get_meta_lines(content=p_content)
p_lineinfo = lineinfo['nb'], lineinfo['max_length']
if p_content != 0:
p_content = p_content[0:400]
paste_info.append({"path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content})
return jsonify(paste_info)
@terms.route("/credentials_management_action/", methods=['GET'])
@login_required
@login_user_no_api
def cred_management_action():
supplied = request.args.get('term')
action = request.args.get('action')
section = request.args.get('section')
extensive = request.args.get('extensive')
extensive = True if extensive == "true" else False
if extensive:
#collectDico
AllUsernameInRedis = r_serv_cred.hgetall(REDIS_KEY_ALL_CRED_SET).keys()
uniq_num_set = set()
if action == "seek":
possibilities = mixUserName(supplied, extensive)
for poss in possibilities:
num = r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET, poss)
if num is not None:
uniq_num_set.add(num)
for num in r_serv_cred.smembers(poss):
uniq_num_set.add(num)
#Extensive /!\
if extensive:
iter_num = 0
tot_iter = len(AllUsernameInRedis)*len(possibilities)
for tempUsername in AllUsernameInRedis:
for poss in possibilities:
#FIXME print progress
if(iter_num % int(tot_iter/20) == 0):
#print("searching: {}% done".format(int(iter_num/tot_iter*100)), sep=' ', end='\r', flush=True)
print("searching: {}% done".format(float(iter_num)/float(tot_iter)*100))
iter_num += 1
if poss in tempUsername:
num = (r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET, tempUsername))
if num is not None:
uniq_num_set.add(num)
for num in r_serv_cred.smembers(tempUsername):
uniq_num_set.add(num)
data = {'usr': [], 'path': [], 'numPaste': [], 'simil': []}
for Unum in uniq_num_set:
levenRatio = 2.0
username = (r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum))
# Calculate Levenshtein distance, ignore negative ratio
supp_splitted = supplied.split()
supp_mixed = supplied.replace(' ','')
supp_splitted.append(supp_mixed)
for indiv_supplied in supp_splitted:
levenRatio = float(Levenshtein.ratio(indiv_supplied, username))
levenRatioStr = "{:.1%}".format(levenRatio)
data['usr'].append(username)
allPathNum = list(r_serv_cred.smembers(REDIS_KEY_MAP_CRED_TO_PATH+'_'+Unum))
data['path'].append(allPathNum)
data['numPaste'].append(len(allPathNum))
data['simil'].append(levenRatioStr)
to_return = {}
to_return["section"] = section
to_return["action"] = action
to_return["term"] = supplied
to_return["data"] = data
return jsonify(to_return)
# ========= REGISTRATION =========
app.register_blueprint(terms, url_prefix=baseUrl)

253
var/www/modules/terms/templates/credentials_tracker.html
View File

@ -1,253 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Credentials Tracker - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.pie.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
<style>
.btn-link {
color: #000000
}
</style>
</head>
<body>
<!-- Modal -->
<div id="mymodal" class="modal fade" role="dialog">
<div class="modal-dialog modal-lg">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div id="mymodalbody" class="modal-body" max-width="8500px">
<p>Loading paste information...</p>
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
</div>
<div class="modal-footer">
<a id="button_show_plot" target="_blank" href=""><button type="button" class="btn btn-info">Plot term</button></a>
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-termsfrequency" >Credential seeker</h1>
</div>
<!-- /.col-lg-12 -->
<!-- Panel OPTIONS -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-success">
<div class="panel-heading">
<strong>Credential seeker</strong>
</div>
<div class="panel-body">
<label><input type="checkbox" id="extensive" style="margin-bottom: 10px;"><strong>Extensive search (takes time)</strong></input></label>
<div class="form-group input-group" style="margin-bottom: 30px;">
<span class="input-group-addon"><span class="glyphicon glyphicon-screenshot"></span></span>
<input id="seekInput" class="form-control" placeholder="Credential to seek." type="text" style="max-width: 400px;">
<button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="seek" data-action="seek"> Seek</button>
<span id="nodata" class="alert alert-info" style="margin-left: 10px; display: none; padding: 6px;">No data</span>
</div>
<table class="table table-striped table-bordered table-hover" id="myTable">
<thead>
<tr>
<th style="max-width: 800px;">Username</th>
<th>Similarity</th>
<th># concerned paste(s)</th>
<th>Action</th>
</tr>
</thead>
<tbody>
</tbody>
</table>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
<script>
$(document).ready(function(){
activePage = $('h1.page-header').attr('data-page');
$("#"+activePage).addClass("active");
$('[data-toggle="tooltip"]').tooltip();
table_track = $('#myTable').DataTable({ "order": [[ 1, "dec" ]] });
table_track.on( 'draw.dt', function () {
perform_binding();
});
$("#seekInput").on("keyup", function(event){
if(event.keyCode == 13){
$("#followTermBtn").click();
$("#seekInput").val("");
}
});
perform_binding();
$("#mymodal").on('hidden.bs.modal', function () {
$("#mymodalbody").html("<p>Loading paste information...</p>");
var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
$("#mymodalbody").append(loading_gif); // Show the loading GIF
});
});
</script>
<script>
function perform_binding() {
$(".btn-interaction").off("click.interaction");
$(".btn-interaction").on("click.interaction", perform_operation);
}
function perform_modal_binding() {
// On click, get html content from url and update the corresponding modal
$("[data-toggle='modal']").on("click.openmodal", function (event) {
event.preventDefault();
var the_modal=$(this);
var url = "{{ url_for('terms.credentials_management_query_paste') }}?cred=" + encodeURIComponent($(this).attr('data-term'));
$.ajax({
type: 'POST',
url: url,
dataType: "json",
data: JSON.stringify({ 'allPath': JSON.parse($(this).attr('data-path')) }),
contentType : "application/json"
}).done(function (data) {
if (data.length != 0) {
var html_to_add = "";
html_to_add += "<table id=\"modal-table\" class=\"table table-striped\">";
html_to_add += "<thead>";
html_to_add += "<tr>";
html_to_add += "<th>Source</th>";
html_to_add += "<th>Date</th>";
html_to_add += "<th>Encoding</th>";
html_to_add += "<th>Size (Kb)</th>";
html_to_add += "<th># lines</th>";
html_to_add += "<th>Max length</th>";
html_to_add += "<th>Preview</th>";
html_to_add += "</tr>";
html_to_add += "</thead>";
html_to_add += "<tbody>";
for (i=0; i<data.length; i++) {
curr_data = data[i];
html_to_add += "<tr>";
html_to_add += "<td>"+curr_data.source+"</td>";
html_to_add += "<td>"+curr_data.date+"</td>";
html_to_add += "<td>"+curr_data.encoding+"</td>";
html_to_add += "<td>"+curr_data.size+"</td>";
html_to_add += "<td>"+curr_data.lineinfo[0]+"</td>";
html_to_add += "<td>"+curr_data.lineinfo[1]+"</td>";
html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content.replace(/\"/g, "\'")+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('objects_item.showItem') }}?id="+curr_data.path+"&num=0\"> <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
html_to_add += "</tr>";
}
html_to_add += "</tbody>";
html_to_add += "</table>";
$("#mymodalbody").html(html_to_add);
$("[data-toggle=popover]").popover();
$('#modal-table').DataTable();
} else {
$("#mymodalbody").html("No paste containing this term has been received yet.");
}
}
)});
}
function perform_operation(){
var curr_section = $(this).attr('data-section');
var curr_action = $(this).attr('data-action');
if (curr_action == "add") {
var curr_term = $('#'+curr_section+'Input').val();
} else if (curr_action == "seek") {
var curr_term = $('#'+curr_section+'Input').val();
} else {
var curr_term = $(this).attr('data-content');
}
var data_to_send = { section: curr_section, action:curr_action, term: curr_term, extensive: $("#extensive").is(":checked")};
if (curr_term != "") {
//console.log(data_to_send);
$.get("{{ url_for('terms.cred_management_action') }}", data_to_send, function(data, status){
if(status == "success") {
var json = data;
if(json.action == "add") {
//not used for the moment
} else if (json.action == "seek") {
table_track.clear().draw();
var rep = json.data;
//var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Track this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"add\">";
var action_button = "";
if (rep.usr.length == 0) {
$( "#nodata" ).removeClass('alert-info').addClass('alert-danger');
$( "#nodata" ).text("No data");
$( "#nodata" ).fadeIn( "fast", function() {
setTimeout(function() {
$( "#nodata" ).fadeOut("fast");
}, 2000);
});
} else {
for(i=0; i < rep.usr.length; i++) {
$( "#nodata" ).removeClass('alert-danger').addClass('alert-info');
$( "#nodata" ).text(curr_term);
$( "#nodata" ).fadeIn( "fast");
toAdd = "</button><span data-toggle=\"modal\" data-target=\"#mymodal\" data-term=\""+rep.usr[i]+"\" data-path=\"["+rep.path[i]+"]\" ><button class=\"btn-link\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Show concerned paste(s)\"><span class=\"glyphicon glyphicon-info-sign\"></span></button></span>";
table_track.row.add( [
rep.usr[i],
rep.simil[i],
rep.numPaste[i],
toAdd+action_button ] ).draw( false );
}
perform_binding();
perform_modal_binding();
}
}
}
});
}
}
</script>
</body>
</html>

6
var/www/modules/terms/templates/header_terms.html
View File

@ -1,6 +0,0 @@
<li id='page-termsfrequency'><a class="dropdown-toggle" data-toggle="dropdown" href="{{ url_for('terms.credentials_tracker') }}"><i class="fa fa-eye"></i> Terms frequency
<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="{{ url_for('terms.credentials_tracker') }}"><i class="glyphicon glyphicon-screenshot"> </i> Credentials seeker</a></li>
</ul>
</li>

591
var/www/modules/terms/templates/terms_management.html
View File

@ -1,591 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Terms Management</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/switch_checkbox.css') }}" rel="stylesheet" type="text/css" />
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
<style>
.sparkLineStats ul {
padding-left:0;
list-style:none
}
.btn-link {
color: #000000
}
.popover-content {
white-space:pre-wrap;
word-wrap:break-word;
}
.mouse_pointer{
cursor: pointer;
}
.lb-md {
font-size: 16px;
}
</style>
</head>
<body>
<!-- Modal -->
<div id="mymodal" class="modal fade" role="dialog">
<div class="modal-dialog modal-lg">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div id="mymodalbody" class="modal-body" max-width="8500px">
<p>Loading paste information...</p>
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
</div>
<div class="modal-footer">
<a id="button_show_plot" target="_blank" href=""><button type="button" class="btn btn-info">Plot term</button></a>
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-termsfrequency" >Terms frequency: Management interface</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
<div class="row">
<!-- Panel OPTIONS -->
<div class="row">
<div class="col-lg-12">
<div class="row">
{% set uniq_id = namespace(modal_id=0)%}
<div class="col-lg-12">
<label class="switch">
<input id="per_paste" class="switch-input" value="per_paste" type="checkbox" onclick="reload_per_paste()">
<span class="switch-label" data-on="On" data-off="Off"></span>
<span class="switch-handle"></span>
</label>
<strong style="top: 3px; position: relative;">1 term per paste</strong>
<div id="panel-today" class="panel panel-success">
<div class="panel-heading">
<strong>Manage tracked terms</strong>
</div>
<div class="panel-body">
<div style="margin-bottom: 10px;">
<table>
<tr><td><b>Regex</b>: surround the term by '<b>/</b>'. </td> <td><b style="margin-left: 20px;">/([a-z])\w+([a-z])\n/</b></td></tr>
<tr><td><b>Set of terms</b>: surround the list by '<b>\</b>'. </td> <td><b style="margin-left: 20px;">\[term1, term2, ...]\</b></td></tr>
<tr><td> - To set a custom matching <b>threshold</b> (defaut=50), append it at the end as a inner list '<b>[thresh]</b>'. </td> <td><b style="margin-left: 20px;">\[term1, term2, ..., [75]]\</b></td></tr>
</table>
</div>
<div class="form-group input-group" style="margin-bottom: 30px;">
<span class="input-group-addon"><span class="fa fa-eye"></span></span>
<input id="followTermInput" class="form-control" placeholder="Term to track." type="text" style="max-width: 400px;">
<input id="followTermEMailNotificationReceiversInput" class="form-control" placeholder="Notification E-Mails (optional, space separated)" type="text" style="max-width: 400px;">
<input id="followTermTag" class="form-control" placeholder="Tags (optional, space separated)" type="text" style="max-width: 400px;">
<button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Add term</button>
</div>
<table class="table table-striped table-bordered table-hover" id="myTable">
<thead>
<tr>
<th style="max-width: 800px;">Term</th>
<th>Added date</th>
<th>Day occurence</th>
<th>Week occurence</th>
<th>Month occurence</th>
<th># tracked paste</th>
<th>Action</th>
<th>Notification E-Mails</th>
</tr>
</thead>
<tbody>
<!-- SET -->
{% for set in trackSet_list %}
<tr style="background-color: #cdffca;">
<td>
<span class="term_name">{{ set }}</span>
<div>
{% for tag in notificationTagsTermMapping[set] %}
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
</a>
{% endfor %}
{% if notificationTagsTermMapping[set] %}
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
<div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
<div class="modal-dialog">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
<h2 class="text-center">Remove Custom Tag</h2>
</div>
<div class="modal-body">
<form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
{% for tag in notificationTagsTermMapping[set] %}
<div class="form-check">
<input type="hidden" class="form-control" name="term" value="{{ set }}">
<input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
<label class="form-check-label">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
</label>
<br>
</div>
{% endfor %}
</form>
</div>
<div class="modal-footer">
<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
<span class="glyphicon glyphicon-trash"></span>
<span class="label-icon">Remove Tags</span>
</button>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
{% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
{% endif %}
</div>
</td>
<td>{{ trackSet_list_values[loop.index0][3] }}</td>
<td>{{ trackSet_list_values[loop.index0][0] }}</td>
<td>{{ trackSet_list_values[loop.index0][1] }}</td>
<td>{{ trackSet_list_values[loop.index0][2] }}</td>
<td>{{ trackSet_list_num_of_paste[loop.index0] }}</td>
<td><p style="margin: 0px; white-space: nowrap;">
<span data-toggle="modal" data-target="#mymodal" data-term="{{ set }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ set }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
&nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ set }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[set] %} checked {% endif %}>
</p></td>
<td>
{% for email in notificationEMailTermMapping[set] %}
<a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{set}}">
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
<span class="glyphicon glyphicon-trash" style="color:Red;" ></span>
</div>
</a>
{{ email }}
<br>
{% endfor %}
</td>
</tr>
{% endfor %}
<!-- REGEX -->
{% for regex in trackReg_list %}
<tr style="background-color: #fffdca;">
<td>
<span class="term_name">{{ regex }}</span>
<div>
{% for tag in notificationTagsTermMapping[regex] %}
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
</a>
{% endfor %}
{% if notificationTagsTermMapping[regex] %}
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
<div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
<div class="modal-dialog">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
<h2 class="text-center">Remove Custom Tag</h2>
</div>
<div class="modal-body">
<form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
{% for tag in notificationTagsTermMapping[regex] %}
<div class="form-check">
<input type="hidden" class="form-control" name="term" value="{{ regex }}">
<input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
<label class="form-check-label">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
</label>
<br>
</div>
{% endfor %}
</form>
</div>
<div class="modal-footer">
<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
<span class="glyphicon glyphicon-trash"></span>
<span class="label-icon">Remove Tags</span>
</button>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
{% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
{% endif %}
</div>
</td>
<td>{{ trackReg_list_values[loop.index0][3] }}</td>
<td>{{ trackReg_list_values[loop.index0][0] }}</td>
<td>{{ trackReg_list_values[loop.index0][1] }}</td>
<td>{{ trackReg_list_values[loop.index0][2] }}</td>
<td>{{ trackReg_list_num_of_paste[loop.index0] }}</td>
<td><p style="margin: 0px; white-space: nowrap;">
<span data-toggle="modal" data-target="#mymodal" data-term="{{ regex }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ regex }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
&nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ regex }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[regex] %} checked {% endif %}>
</p></td>
<td>
{% for email in notificationEMailTermMapping[regex] %}
<a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{regex}}">
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
<span class="glyphicon glyphicon-trash" style="color:Red;"></span>
</div>
</a>
{{ email }}
<br>
{% endfor %}
</td>
</tr>
{% endfor %}
<!-- Normal term -->
{% for term in track_list %}
<tr>
<td>
<span class="term_name">{{ term }}</span>
<div>
{% for tag in notificationTagsTermMapping[term] %}
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
</a>
{% endfor %}
{% if notificationTagsTermMapping[term] %}
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
<div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
<div class="modal-dialog">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
<h2 class="text-center">Remove Custom Tag</h2>
</div>
<div class="modal-body">
<form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
{% for tag in notificationTagsTermMapping[term] %}
<div class="form-check">
<input type="hidden" class="form-control" name="term" value="{{ term }}">
<input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
<label class="form-check-label">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
</label>
<br>
</div>
{% endfor %}
</form>
</div>
<div class="modal-footer">
<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
<span class="glyphicon glyphicon-trash"></span>
<span class="label-icon">Remove Tags</span>
</button>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
{% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
{% endif %}
</div>
</td>
<td>{{ track_list_values[loop.index0][3] }}</td>
<td>{{ track_list_values[loop.index0][0] }}</td>
<td>{{ track_list_values[loop.index0][1] }}</td>
<td>{{ track_list_values[loop.index0][2] }}</td>
<td>{{ track_list_num_of_paste[loop.index0] }}</td>
<td><p style="margin: 0px; white-space: nowrap;">
<span data-toggle="modal" data-target="#mymodal" data-term="{{ term }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ term }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
&nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ term }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[term] %} checked {% endif %}>
</p></td>
<td>
{% for email in notificationEMailTermMapping[term] %}
<a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{term}}">
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
<span class="glyphicon glyphicon-trash" style="color:Red;"></span>
</div>
</a>
{{ email }}
<br>
{% endfor %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- Panel OPTIONS -->
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-danger">
<div class="panel-heading">
<strong>Manage blacklisted terms</strong>
</div>
<div class="panel-body">
<div class="form-group input-group" style="margin-bottom: 30px;">
<span class="input-group-addon"><span class="fa fa-eye-slash "></span></span>
<input id="blacklistTermInput" class="form-control" placeholder="Term to track" type="text" style="max-width: 400px;">
<button id="blacklistTermBtn" class="btn btn-danger btn-interaction" style="margin-left: 10px;" data-section="blacklistTerm" data-action="add"> Black list a term</button>
</div>
<table class="table table-striped table-bordered table-hover" id="myTable2">
<thead>
<tr>
<th style="max-width: 800px;">Term</th>
<th>Added date</th>
<th>Action</th>
</tr>
</thead>
<tbody>
{% for term, date in black_list %}
<tr>
<td>{{ term }}</td>
<td>{{ date }}</td>
<td><p style="margin: 0px;">
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="right" title="Remove this term" data-content="{{ term }}" data-section="blacklistTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
</p></td>
</tr>
{% endfor %}
</tbody>
</table>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
</div>
<!-- import graph function -->
<script>
function reload_per_paste() {
var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
window.location.href = {{ url_for('terms.terms_management') }}+"?per_paste="+checked;
}
var table_track;
var table_black;
function bindEventsForCurrentPage() {
// On click, get html content from url and update the corresponding modal
$("[data-toggle='modal']").unbind().on("click.openmodal", function (event) {
//console.log(data);
event.preventDefault();
var the_modal=$(this);
var url = "{{ url_for('terms.terms_management_query_paste') }}?term=" + encodeURIComponent($(this).attr('data-term'));
$.getJSON(url, function (data) {
if (data.length != 0) {
var html_to_add = "";
html_to_add += "<table id=\"modal-table\" class=\"table table-striped\">";
html_to_add += "<thead>";
html_to_add += "<tr>";
html_to_add += "<th>Source</th>";
html_to_add += "<th>Date</th>";
html_to_add += "<th>Mime</th>";
html_to_add += "<th>Size (Kb)</th>";
html_to_add += "<th># lines</th>";
html_to_add += "<th>Max length</th>";
html_to_add += "<th>Preview</th>";
html_to_add += "</tr>";
html_to_add += "</thead>";
html_to_add += "<tbody>";
for (i=0; i<data.length; i++) {
curr_data = data[i];
html_to_add += "<tr>";
html_to_add += "<td>"+curr_data.source+"</td>";
html_to_add += "<td>"+curr_data.date+"</td>";
html_to_add += "<td>"+curr_data.mime+"</td>";
html_to_add += "<td>"+curr_data.size+"</td>";
html_to_add += "<td>"+curr_data.lineinfo[0]+"</td>";
html_to_add += "<td>"+curr_data.lineinfo[1]+"</td>";
html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content.replace(/\"/g, "\'")+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('objects_item.showItem') }}?id="+curr_data.path+"&num=0\"> <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
html_to_add += "</tr>";
}
html_to_add += "</tbody>";
html_to_add += "</table>";
$("#mymodalbody").html(html_to_add);
$("[data-toggle=popover]").popover();
$("#button_show_plot").attr("href", "{{ url_for('terms.terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
$('#modal-table').DataTable();
} else {
$("#mymodalbody").html("No paste containing this term has been received yet.");
$("#button_show_plot").attr("href", "{{ url_for('terms.terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
}
});
});
}
$(document).ready(function(){
bindEventsForCurrentPage();
activePage = $('h1.page-header').attr('data-page');
$("#"+activePage).addClass("active");
if({{ per_paste }} == 1) {
$("#per_paste").attr('checked', true)
}
$('[data-toggle="tooltip"]').tooltip();
table_track = $('#myTable').DataTable();
table_black = $('#myTable2').DataTable();
table_track.on( 'draw.dt', function () {
perform_binding();
});
table_black.on( 'draw.dt', function () {
perform_binding();
});
$("#followTermInput").keyup(function(event){
if(event.keyCode == 13){
$("#followTermBtn").click();
$("#followTermInput").val("");
}
});
$("#blacklistTermInput").keyup(function(event){
if(event.keyCode == 13){
$("#blacklistTermBtn").click();
$("#blacklistTermInput").val("");
}
});
perform_binding();
$("#mymodal").on('hidden.bs.modal', function () {
$("#mymodalbody").html("<p>Loading paste information...</p>");
var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
$("#mymodalbody").append(loading_gif); // Show the loading GIF
});
});
</script>
<script>
function perform_binding() {
$(".btn-interaction").unbind("click.interaction");
$(".btn-interaction").bind("click.interaction", perform_operation);
}
function perform_operation(){
var curr_section = $(this).attr('data-section');
var curr_action = $(this).attr('data-action');
var row_tr = $(this).closest("tr");
if (curr_action == "add") {
var curr_term = $('#'+curr_section+'Input').val();
var email_addresses = $('#followTermEMailNotificationReceiversInput').val();
var tags = $('#followTermTag').val();
} else {
var curr_term = $(this).attr('data-content');
var email_addresses = "";
}
var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses, tags: tags};
if (curr_term != "") {
//console.log(data_to_send);
$.get("{{ url_for('terms.terms_management_action') }}", data_to_send, function(data, status){
if(status == "success") {
var json = data;
if(json.section == "followTerm") {
if(json.action == "add") {
// query data
$.get("{{ url_for('terms.terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
reload_per_paste();
});
} else if (json.action == "delete") {
row_tr.remove()
}
} else if(json.section == "blacklistTerm"){
if(json.action == "add") {
$.get("{{ url_for('terms.terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
console.log(data2);
var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"blacklistTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
table_black.row.add( [ json.term, data2[3], action_button ] ).draw( false );
perform_binding();
});
} else if (json.action == "delete") {
// Find indexes of row which have the term in the first column
var index = table_black.rows().eq( 0 ).filter( function (rowIdx) {
return table_black.cell( rowIdx, 0 ).data() === json.term;
} );
table_black.rows(index).remove().draw( false );
}
}
}
});
}
}
</script>

275
var/www/modules/terms/templates/terms_plot_tool.html
View File

@ -1,275 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Terms Plot Tool - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/switch_checkbox.css') }}" rel="stylesheet" type="text/css" />
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery-ui.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
<style>
.sparkLineStats ul {
padding-left:0;
list-style:none
}
</style>
</head>
<body>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-termsfrequency" >Terms plot tool</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
<div class="row">
<!-- Panel OPTIONS -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-default">
<div class="panel-heading">
<strong>Select options</strong>
</div>
<div class="panel-body">
<div aria-disabled="false" class="slider sliderRange sliderBlue ui-slider ui-slider-horizontal ui-widget ui-widget-content ui-corner-all" style="margin-bottom: 5px;"></div>
<strong>Date:</strong> <input type="text" id="amount" readonly style="border:0; color:#f6931f; font-weight:bold;"> <button id="plot-month" onclick="replot();" class="btn btn-info" style="float: right;">Replot</button>
<div class="form-group input-group" style="margin-top: 30px;">
<span class="input-group-addon"><span class="glyphicon glyphicon-stats"></span></span>
<input id="TermInput" class="form-control" placeholder="Term to plot" type="text" style="max-width: 400px;" data-init-plot="{{ term }}">
<button id="plot-btn" class="btn btn-info" style="margin-left: 10px;"><span class="fa fa-caret-down"> Plot a term</button>
<button id="plot-btn-add" class="btn btn-success" style="margin-left: 6px;"><span class="fa fa-plus"></span> Add the term to the chart</button>
</div>
<div class='pull_right' style="float: right;">
<label class="switch">
<input id="per_paste" class="switch-input" value="per_paste" type="checkbox" checked>
<span class="switch-label" data-on="On" data-off="Off"></span>
<span class="switch-handle"></span>
</label>
<strong style="top: 3px; position: relative;">1 term per paste</strong>
</div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- Panel PLOT -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-default">
<div class="panel-heading">
<strong>Graph</strong>
</div>
<div class="panel-body">
<div id="graph" style="height: 300px;"></div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
</div>
<div style="position: absolute; border: 1px solid rgb(255, 221, 221); padding: 2px; background-color: #333; color:white; opacity: 0.8; top: 423px; left: 616px; display: none;" id="tooltip"></div>
<!-- import graph function -->
<script>
$(document).ready(function(){
activePage = $('h1.page-header').attr('data-page');
$("#"+activePage).addClass("active");
/* Create the slider and button*/
$('#plot-btn-add').hide();
var today = Date.now();
var old_day = today - (12*31*24*60*60)*1000;
$( ".sliderRange" ).slider({
range: true,
min: old_day,
max: today,
values: [ today - (7*24*60*60)*1000, today ],
step: 24*60*60*1000,
slide: function( event, ui ) {
$( "#amount" ).val( new Date(ui.values[ 0 ]).toLocaleDateString() + " - " + new Date(ui.values[ 1 ]).toLocaleDateString() );
}
});
$( "#amount" ).val( new Date($( ".sliderRange" ).slider( "values", 0 )).toLocaleDateString() +
" - " + new Date($( ".sliderRange" ).slider( "values", 1 )).toLocaleDateString() );
$('#plot-btn').click(plotData);
$('#plot-btn-add').click(addData);
$("#TermInput").val($("#TermInput").attr("data-init-plot"));
if($("#TermInput").attr("data-init-plot") != "") {
$("#plot-btn").click();
}
$("#TermInput").keyup(function(event){
if(event.keyCode == 13){
$("#plot-btn").click();
$("#TermInput").val("");
}
});
});
</script>
<script>
var plot;
var graph_data = [];
var plotted_terms = [];
var graph_options = {
series: {
lines: {
show: true,
lineWidth: 2
},
bars: {show: false, barWidth: 60*60*1000},
shadowSize: 0
},
grid: {
hoverable: true,
clickable: true,
tickColor: "#f9f9f9",
borderWidth: 0
},
xaxis: {
mode: "time",
timeformat: "%m/%d",
minTickSize: [1, "day"]
},
yaxis: {
autoscaleMargin: 0.1,
},
}
function plotData() {
$('#plot-btn-add').show("fast");
var curthis = $(this);
var term = $('#TermInput').val();
plotted_terms = [term]
var range_start = new Date($( ".sliderRange" ).slider( "values", 0 )).getTime() / 1000;
var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
$.getJSON("{{ url_for('terms.terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term, per_paste: checked }, function(data, status){
graph_data = [];
var to_plot = [];
var curr_data = [];
for(i=1; i<data.length; i++) {
curr_data.push([data[i][0]*1000, data[i][1]]);
}
to_plot.push({ data: curr_data, label: term});
graph_data.push({ data: curr_data, label: term});
plot = $.plot($("#graph"), to_plot, graph_options);
$("#graph").bind("plothover", function (event, pos, item) {
if (item) {
var date = new Date(item.datapoint[0]);
var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
var y = item.datapoint[1];
$("#tooltip").html(item.series.label + " for "+x + " = " + y)
.css({top: item.pageY-15, left: item.pageX+5})
.fadeIn(200);
} else {
$("#tooltip").hide();
}
});
$("#TermInput").val("");
})
}
function addData() {
var curthis = $(this);
var term = $('#TermInput').val();
plotted_terms.push(term)
var range_start = new Date($( ".sliderRange" ).slider( "values", 0 )).getTime() / 1000;
var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
$.getJSON("{{ url_for('terms.terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term, per_paste: checked }, function(data, status){
var curr_data = [];
for(i=1; i<data.length; i++) {
curr_data.push([data[i][0]*1000, data[i][1]]);
}
graph_data.push({ data: curr_data, label: term});
plot = $.plot($("#graph"), graph_data, graph_options);
$("#TermInput").val("");
})
}
function replot() {
graph_data = [];
promises = [];
for(i=0; i<plotted_terms.length; i++) {
var term = plotted_terms[i];
var range_start = new Date($( ".sliderRange" ).slider( "values", 0 )).getTime() / 1000;
var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
promises.push($.getJSON("{{ url_for('terms.terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term, per_paste: checked }, function(data, status){
var curr_data = [];
for(i=1; i<data.length; i++) {
curr_data.push([data[i][0]*1000, data[i][1]]);
}
graph_data.push({ data: curr_data, label: data[0]});
$("#TermInput").val("");
}))
}
$.when.apply($, promises).done( function () {
plot = $.plot($("#graph"), graph_data, graph_options);
});
}
</script>

576
var/www/modules/terms/templates/terms_plot_top.html
View File

@ -1,576 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Terms Plot Top - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/switch_checkbox.css') }}" rel="stylesheet" type="text/css" />
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
<style>
.sparkLineStats ul {
padding-left:0;
list-style:none
}
.table > tbody > tr > td {
padding: 5px;
}
</style>
</head>
<body>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-termsfrequency" >Terms frequency: Top set information</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
<div class="row">
<!-- Panel OPTIONS -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div class='pull_right'>
<label class="switch">
<input id="per_paste" class="switch-input" value="per_paste" type="checkbox" onclick="reload_per_paste()">
<span class="switch-label" data-on="On" data-off="Off"></span>
<span class="switch-handle"></span>
</label>
<strong style="top: 3px; position: relative;">1 term per paste</strong>
<div>
<div id="panel-today" class="panel panel-info">
<div class="panel-heading">
<strong>Today</strong>
</div>
<div class="panel-body">
<div id="panel-today" class="panel panel-default">
<div class="panel-heading">
<strong>Today top word</strong>
</div>
<div class="panel-body">
<div class="col-lg-6">
<table class="table table-striped">
<thead>
<tr>
<th>Term</th>
<th>Value</th>
<th>Action</th>
<th>Show</th>
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the week and month set">Position</th>
</tr>
</thead>
<tbody id="table-today">
</tbody>
</table>
</div>
<div class="col-lg-6">
<table class="table table-striped">
<thead>
<tr>
<th>Term</th>
<th>Value</th>
<th>Action</th>
<th>Show</th>
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the week and month set">Position</th>
</tr>
</thead>
<tbody id="table-today2">
</tbody>
</table>
</div>
</div>
<!-- /.panel-body -->
</div>
<div id="panel-today" class="panel panel-default">
<div class="panel-heading">
<strong>Graph</strong>
</div>
<div class="panel-body">
<div id="graph-today" style="height: 400px;"></div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- Panel OPTIONS -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-info">
<div class="panel-heading">
<strong>Week</strong>
</div>
<div class="panel-body">
<div id="panel-week" class="panel panel-default">
<div class="panel-heading">
<strong>Week top word</strong>
</div>
<div class="panel-body">
<div class="col-lg-6">
<table class="table table-striped">
<thead>
<tr>
<th>Term</th>
<th>Value</th>
<th>Action</th>
<th>Show</th>
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and month set">Position</th>
</tr>
</thead>
<tbody id="table-week">
</tbody>
</table>
</div>
<div class="col-lg-6">
<table class="table table-striped">
<thead>
<tr>
<th>Term</th>
<th>Value</th>
<th>Action</th>
<th>Show</th>
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and month set">Position</th>
</tr>
</thead>
<tbody id="table-week2">
</tbody>
</table>
</div>
</div>
<!-- /.panel-body -->
</div>
<div id="panel-week" class="panel panel-default">
<div class="panel-heading">
<strong>Graph</strong>
</div>
<div class="panel-body">
<div id="graph-week" style="height: 400px;"></div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- Panel OPTIONS -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-info">
<div class="panel-heading">
<strong>Month</strong>
</div>
<div class="panel-body">
<div id="panel-month" class="panel panel-default">
<div class="panel-heading">
<strong>Month top word</strong>
</div>
<div class="panel-body">
<div class="col-lg-6">
<table class="table table-striped">
<thead>
<tr>
<th>Term</th>
<th>Value</th>
<th>Action</th>
<th>Show</th>
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and week set">Position</th>
</tr>
</thead>
<tbody id="table-month">
</tbody>
</table>
</div>
<div class="col-lg-6">
<table class="table table-striped">
<thead>
<tr>
<th>Term</th>
<th>Value</th>
<th>Action</th>
<th>Show</th>
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and week set">Position</th>
</tr>
</thead>
<tbody id="table-month2">
</tbody>
</table>
</div>
</div>
<!-- /.panel-body -->
</div>
<div id="panel-month" class="panel panel-default">
<div class="panel-heading">
<strong>Graph</strong>
</div>
<div class="panel-body">
<div id="graph-month" style="height: 400px;"></div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
</div>
<div style="position: absolute; border: 1px solid rgb(255, 221, 221); padding: 2px; background-color: #333; color:white; opacity: 0.8; top: 423px; left: 616px; display: none;" id="tooltip"></div>
<!-- import graph function -->
<script>
$(document).ready(function(){
activePage = $('h1.page-header').attr('data-page');
$("#"+activePage).addClass("active");
if({{ per_paste }} == 1) {
$("#per_paste").attr('checked', true)
}
});
function reload_per_paste() {
var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
window.location.href = {{ url_for('terms.terms_plot_top') }}+"?per_paste="+checked;
}
</script>
<script>
var graph_options = {
series: {
lines: {
show: true,
lineWidth: 2
},
bars: {show: false, barWidth: 60*60*1000},
shadowSize: 0
},
grid: {
hoverable: true,
clickable: true,
tickColor: "#f9f9f9",
borderWidth: 0
},
legend: { show: true,
noColumns: 1,
position: "nw",
labelFormatter: function(label, series) {
return "<a href=\"#\" onClick=\"hide_or_show2("+series.idx+", "+series.graphNum+"); return false;\" >" + label + "</a>";
}
},
xaxis: {
mode: "time",
timeformat: "%m/%d",
minTickSize: [1, "day"]
},
yaxis: {
//transform: function (v) { return v < 1 ? v : Math.log(v); }
autoscaleMargin: 0.1,
},
tooltip: true,
tooltipOpts: {
content: " %s (%x.1 is %y.4) ",
shifts: {
x: -60,
y: 25
}
}
}
set_today = "TopTermFreq_set_day";
set_week = "TopTermFreq_set_week";
set_month = "TopTermFreq_set_month";
default_num_curves = 8;
per_paste = {{ per_paste }}
var plot_today;
var plot_week;
var plot_month;
var promises = []; // Used to know when everything has been received
promises.push($.getJSON("{{ url_for('terms.terms_plot_top_data') }}", { set: set_today, num_day: 5, per_paste: per_paste }, function(data, status){
data.sort(function(a, b){return b[2]-a[2];});
// Sort data
var table_today = $("#table-today")
var table_today2 = $("#table-today2")
var to_plot = [];
var unchecked_label = [];
for(i=0; i<data.length; i++) {
var highlight = data[i][3].week == "<20"? " style=\"background-color: lightgreen;\" " : "";
var curr_data = [];
for(j=0; j<data[i][1].length; j++) {
curr_data.push([data[i][1][j][0]*1000, data[i][1][j][1]]);
}
if (i>=default_num_curves) {
unchecked_label.push(data[i][0]);
}
to_plot.push({ data: curr_data, label: data[i][0], idx: i});
if ( i < (data.length/2))
table_today.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("today", data[i][0], i<default_num_curves)+"</td><td>"+data[i][3].week+", "+data[i][3].month+"</td></tr>");
else
table_today2.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("today", data[i][0], i<default_num_curves)+"</td><td>"+data[i][3].week+", "+data[i][3].month+"</td></tr>");
}
graph_options.series.graphNum=1;
plot_today = $.plot($("#graph-today"), to_plot, graph_options);
hide_unchecked_curves(plot_today, unchecked_label);
$("#graph-today").bind("plothover", function (event, pos, item) {
if (item) {
var date = new Date(item.datapoint[0]);
var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
var y = item.datapoint[1];
$("#tooltip").html(item.series.label + " for "+x + " = " + y)
.css({top: item.pageY+5, left: item.pageX+5})
.fadeIn(200);
} else {
$("#tooltip").hide();
}
});
}));
promises.push($.getJSON("{{ url_for('terms.terms_plot_top_data') }}", { set: set_week, num_day: 7, per_paste: per_paste }, function(data, status){
data.sort(function(a, b){return b[2]-a[2];});
// Sort data
var table = $("#table-week")
var table2 = $("#table-week2")
var to_plot = [];
var unchecked_label = [];
for(i=0; i<data.length; i++) {
var highlight = parseInt(data[i][3].day) > 20? " style=\"background-color: orange;\" " : "";
var curr_data = [];
for(j=0; j<data[i][1].length; j++) {
curr_data.push([data[i][1][j][0]*1000, data[i][1][j][1]]);
}
if (i>=default_num_curves) {
unchecked_label.push(data[i][0]);
}
to_plot.push({ data: curr_data, label: data[i][0], idx: i});
if ( i < (data.length/2))
table.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("week", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].month+"</td></tr>");
else
table2.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("week", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].month+"</td></tr>");
}
graph_options.series.graphNum=2;
plot_week = $.plot($("#graph-week"), to_plot, graph_options);
hide_unchecked_curves(plot_week, unchecked_label);
$("#graph-week").bind("plothover", function (event, pos, item) {
if (item) {
var date = new Date(item.datapoint[0]);
var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
var y = item.datapoint[1];
$("#tooltip").html(item.series.label + " for "+x + " = " + y)
.css({top: item.pageY+5, left: item.pageX+5})
.fadeIn(200);
} else {
$("#tooltip").hide();
}
});
}));
promises.push($.getJSON("{{ url_for('terms.terms_plot_top_data') }}", { set: set_month, num_day: 31, per_paste: per_paste }, function(data, status){
data.sort(function(a, b){return b[2]-a[2];});
// Sort data
var table = $("#table-month")
var table2 = $("#table-month2")
var to_plot = [];
var unchecked_label = [];
for(i=0; i<data.length; i++) {
var highlight = parseInt(data[i][3].day) > 20? " style=\"background-color: orange;\" " : "";
var curr_data = [];
for(j=0; j<data[i][1].length; j++) {
curr_data.push([data[i][1][j][0]*1000, data[i][1][j][1]]);
}
if (i>=default_num_curves) {
unchecked_label.push(data[i][0]);
}
to_plot.push({ data: curr_data, label: data[i][0], idx: i});
if ( i < (data.length/2))
table.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("month", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].week+"</td></tr>");
else
table2.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("month", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].week+"</td></tr>");
}
graph_options.series.graphNum=3;
plot_month = $.plot($("#graph-month"), to_plot, graph_options);
hide_unchecked_curves(plot_month, unchecked_label);
$("#graph-month").bind("plothover", function (event, pos, item) {
if (item) {
var date = new Date(item.datapoint[0]);
var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
var y = item.datapoint[1];
$("#tooltip").html(item.series.label + " for "+x + " = " + y)
.css({top: item.pageY+5, left: item.pageX+5})
.fadeIn(200);
} else {
$("#tooltip").hide();
}
});
}));
/* When everything has been received, start adding tooltip */
$.when.apply($, promises).done( function (arg) {
$('[data-toggle="tooltip"]').tooltip();
$(".btn-interaction").unbind("click.interaction");
$(".btn-interaction").bind("click.interaction", perform_operation);
$(".check-interaction").unbind("click.interaction");
$(".check-interaction").bind("click.interaction", hide_or_show);
});
function addbuttons(term) {
return "<button class=\"fa fa-eye btn-interaction btn btn-success\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Add to tracked list\" style=\"margin-right: 5px;\""+
"data-section=\"followTerm\" data-term=\""+term+"\"></button>"+
"<button class=\"fa fa-eye-slash btn-interaction btn btn-danger\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Add to black-list\""+
"data-section=\"blacklistTerm\" data-term=\""+term+"\"></button>";
}
function addcheckbox(graph, term, checked) {
var checked_text = checked ? "checked" : "";
return "<input type=checkbox "+checked_text+" class=\"check-interaction\" data-term=\""+term+"\" data-graph=\""+graph+"\"></input>";
}
function perform_operation(){
var curr_section = $(this).attr('data-section');
var curr_term = $(this).attr('data-term');
var data_to_send = { section: curr_section, action:"add", term: curr_term};
$.get("{{ url_for('terms.terms_management_action') }}", data_to_send, function(data, status){
if(status == "success") {
location.reload();
}
});
}
function hide_unchecked_curves(plot, unchecked_label) {
var graphData = plot.getData();
var index;
for(i=0; i<graphData.length; i++) {
if($.inArray( graphData[i].label, unchecked_label ) != -1){
graphData[i].lines.show = false;
}
}
plot.setData(graphData);
plot.draw();
}
function hide_or_show() {
var curr_term = $(this).attr('data-term');
var graph = $(this).attr('data-graph');
var checked = $(this).prop('checked')
if(graph == "today") {
var graphData = plot_today.getData();
var index;
for(i=0; i<graphData.length; i++){
if(graphData[i].label == curr_term){
index = i;
break;
}
}
graphData[index].lines.show = checked;
plot_today.setData(graphData);
plot_today.draw();
} else if (graph == "week") {
var graphData = plot_week.getData();
var index;
for(i=0; i<graphData.length; i++){
if(graphData[i].label == curr_term){
index = i;
break;
}
}
graphData[index].lines.show = checked;
plot_week.setData(graphData);
plot_week.draw();
} else if (graph == "month") {
var graphData = plot_month.getData();
var index;
for(i=0; i<graphData.length; i++){
if(graphData[i].label == curr_term){
index = i;
break;
}
}
graphData[index].lines.show = checked;
plot_month.setData(graphData);
plot_month.draw();
}
// graph, hide curve
}
function hide_or_show2(index, graphNum) {
if (graphNum == 1)
var plot = plot_today;
else if (graphNum == 2)
var plot = plot_week;
else if (graphNum == 3)
var plot = plot_month;
var graphData = plot.getData();
graphData[index].lines.show = !graphData[index].lines.show;
plot.setData(graphData);
plot.draw();
}
</script>
</body>
</html>

7
var/www/templates/modals/create_hive_case.html
View File

@ -6,9 +6,9 @@
<img id="hive-logo" src="{{url_for('static', filename='image/thehive-logo.png') }}" width="500" >
</div>
<form method="post" action="{{ url_for('PasteSubmit.create_hive_case') }}" target="_blank">
<form method="post" action="{{ url_for('import_export.create_thehive_case') }}" target="_blank">
<div class="modal-body">
<div class="modal-body">
<div class="input clear required">
<label for="EventThreatLevelId">Threat Level</label>
@ -35,8 +35,9 @@
<label for="hive_description">Description</label>
<input name="hive_description" class="form-control span6" placeholder="Quick Case Description" type="text" id="hive_description"/>
</div>
<input type="hidden" id="obj_type" name="obj_type" value="item">
<input type="hidden" id="obj_id" name="obj_id" value="{{ dict_item['id'] }}">
</div>
</div>
<div class="modal-footer">
<button class="btn btn-primary" type="submit">