mirror of https://github.com/CIRCL/AIL-framework
chg: [tracker + update] add update v3.7 + add map item_id:tracker_uuid (data retention) + fix tracker first_seen/last_seen
Terrtia
parent
92cfe300f8
commit
564280935b
|
|
@ -67,6 +67,24 @@ def verify_mail_list(mail_list):
|
|||
##-- UTILS --##
|
||||
###############
|
||||
|
||||
def get_all_tracker_type():
|
||||
return ['word', 'set', 'regex', 'yara']
|
||||
|
||||
def get_all_tracker_uuid():
|
||||
return r_serv_tracker.smembers(f'trackers:all')
|
||||
|
||||
def get_all_tracker_by_type(tracker_type):
|
||||
r_serv_tracker.smembers(f'trackers:all:{tracker_type}')
|
||||
|
||||
# def get_all_tracker():
|
||||
# l_keys_name = []
|
||||
# for tracker_type in get_all_tracker_type():
|
||||
# l_keys_name.append(f'all:tracker:{tracker_type}')
|
||||
# return r_serv_tracker.sunion(l_keys_name[0], *l_keys_name[1:])
|
||||
|
||||
def get_all_tracker_by_type(tracker_type):
|
||||
return r_serv_tracker.smembers(f'all:tracker:{tracker_type}')
|
||||
|
||||
def get_tracker_by_uuid(tracker_uuid):
|
||||
return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'tracked')
|
||||
|
||||
|
|
@ -94,17 +112,20 @@ def get_tracker_uuid_sources(tracker_uuid):
|
|||
def get_tracker_description(tracker_uuid):
|
||||
return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'description')
|
||||
|
||||
def get_tracker_date(tracker_uuid):
|
||||
return r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'date')
|
||||
|
||||
def get_tracker_first_seen(tracker_uuid):
|
||||
res = r_serv_tracker.zrange('tracker:stat:{}'.format(tracker_uuid), 0, 0)
|
||||
res = r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'first_seen')
|
||||
if res:
|
||||
return res[0]
|
||||
return res
|
||||
else:
|
||||
return None
|
||||
|
||||
def get_tracker_last_seen(tracker_uuid):
|
||||
res = r_serv_tracker.zrevrange('tracker:stat:{}'.format(tracker_uuid), 0, 0)
|
||||
res = r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'last_seen')
|
||||
if res:
|
||||
return res[0]
|
||||
return res
|
||||
else:
|
||||
return None
|
||||
|
||||
|
|
@ -112,7 +133,7 @@ def get_tracker_metedata(tracker_uuid, user_id=False, description=False, level=F
|
|||
dict_uuid = {}
|
||||
dict_uuid['tracker'] = get_tracker_by_uuid(tracker_uuid)
|
||||
dict_uuid['type'] = get_tracker_type(tracker_uuid)
|
||||
dict_uuid['date'] = r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'date')
|
||||
dict_uuid['date'] = get_tracker_date(tracker_uuid)
|
||||
dict_uuid['description'] = get_tracker_description(tracker_uuid)
|
||||
dict_uuid['first_seen'] = get_tracker_first_seen(tracker_uuid)
|
||||
dict_uuid['last_seen'] = get_tracker_last_seen(tracker_uuid)
|
||||
|
|
@ -131,15 +152,12 @@ def get_tracker_metedata(tracker_uuid, user_id=False, description=False, level=F
|
|||
dict_uuid['uuid'] = tracker_uuid
|
||||
return dict_uuid
|
||||
|
||||
################################################################################
|
||||
################################################################################
|
||||
################################################################################
|
||||
# # TODO: FIXME
|
||||
# tracker sparkline
|
||||
def get_tracker_sparkline(tracker_uuid, num_day=6):
|
||||
date_range_sparkline = Date.get_date_range(num_day)
|
||||
sparklines_value = []
|
||||
for date_day in date_range_sparkline:
|
||||
nb_seen_this_day = r_serv_tracker.scard('tracker:item:{}:{}'.format(tracker_uuid, date_day))
|
||||
nb_seen_this_day = r_serv_tracker.zscore(f'tracker:stat:{tracker_uuid}', int(date_day))
|
||||
if nb_seen_this_day is None:
|
||||
nb_seen_this_day = 0
|
||||
sparklines_value.append(int(nb_seen_this_day))
|
||||
|
|
@ -159,11 +177,40 @@ def get_tracker_items_by_daterange(tracker_uuid, date_from, date_to):
|
|||
def add_tracked_item(tracker_uuid, item_id):
|
||||
item_date = item_basic.get_item_date(item_id)
|
||||
# track item
|
||||
# r_serv_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid)
|
||||
r_serv_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid)
|
||||
res = r_serv_tracker.sadd(f'tracker:item:{tracker_uuid}:{item_date}', item_id)
|
||||
# track nb item by date
|
||||
if res == 1:
|
||||
r_serv_tracker.zincrby('tracker:stat:{}'.format(tracker_uuid), int(item_date), 1)
|
||||
nb_items = r_serv_tracker.zincrby('tracker:stat:{}'.format(tracker_uuid), int(item_date), 1)
|
||||
if nb_items == 1:
|
||||
update_tracker_daterange(tracker_uuid, item_date)
|
||||
|
||||
def set_tracker_first_seen(tracker_uuid, date):
|
||||
r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'first_seen', int(date))
|
||||
|
||||
def set_tracker_last_seen(tracker_uuid, date):
|
||||
r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'last_seen', int(date))
|
||||
|
||||
# # TODO: ADD CACHE ???
|
||||
def update_tracker_daterange(tracker_uuid, date, op='add'):
|
||||
date = int(date)
|
||||
first_seen = get_tracker_first_seen(tracker_uuid)
|
||||
|
||||
if op == 'add':
|
||||
if not first_seen:
|
||||
set_tracker_first_seen(tracker_uuid, date)
|
||||
set_tracker_last_seen(tracker_uuid, date)
|
||||
else:
|
||||
first_seen = int(first_seen)
|
||||
last_seen = int(get_tracker_last_seen(tracker_uuid))
|
||||
if date < first_seen:
|
||||
set_tracker_first_seen(tracker_uuid, date)
|
||||
if date > last_seen:
|
||||
set_tracker_last_seen(tracker_uuid, date)
|
||||
|
||||
if op == 'del':
|
||||
pass
|
||||
|
||||
|
||||
def remove_tracked_item(item_id):
|
||||
item_date = item_basic.get_item_date(item_id)
|
||||
|
|
@ -237,13 +284,40 @@ def api_is_allowed_to_edit_tracker(tracker_uuid, user_id):
|
|||
|
||||
#### FIX DB ####
|
||||
def fix_tracker_stats_per_day(tracker_uuid):
|
||||
date_from = get_tracker_first_seen(tracker_uuid)
|
||||
date_to = get_tracker_last_seen(tracker_uuid)
|
||||
date_from = get_tracker_date(tracker_uuid)
|
||||
date_to = Date.get_today_date_str()
|
||||
# delete stats
|
||||
r_serv_tracker.delete(f'tracker:stat:{tracker_uuid}')
|
||||
# create new stats
|
||||
for date_day in Date.substract_date(date_from, date_to):
|
||||
pass
|
||||
date_day = int(date_day)
|
||||
|
||||
nb_items = r_serv_tracker.scard(f'tracker:item:{tracker_uuid}:{date_day}')
|
||||
if nb_items:
|
||||
r_serv_tracker.zincrby('tracker:stat:{}'.format(tracker_uuid), int(date_day), nb_items)
|
||||
|
||||
# update first_seen/last_seen
|
||||
update_tracker_daterange(tracker_uuid, date_day)
|
||||
|
||||
def fix_tracker_item_link(tracker_uuid):
|
||||
date_from = get_tracker_first_seen(tracker_uuid)
|
||||
date_to = get_tracker_last_seen(tracker_uuid)
|
||||
|
||||
for date_day in Date.substract_date(date_from, date_to):
|
||||
l_items = r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}')
|
||||
for item_id in l_items:
|
||||
r_serv_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid)
|
||||
|
||||
def fix_all_tracker_uuid_list():
|
||||
r_serv_tracker.delete(f'trackers:all')
|
||||
r_serv_tracker.delete(f'trackers:all:{tracker_type}')
|
||||
for tracker_type in get_all_tracker_type():
|
||||
l_tracker = get_all_tracker_by_type(tracker_type)
|
||||
for tracker in l_tracker:
|
||||
l_tracker_uuid = get_tracker_uuid_list(tracker, tracker_type)
|
||||
for tracker_uuid in l_tracker_uuid:
|
||||
r_serv_tracker.sadd(f'trackers:all', tracker_uuid)
|
||||
r_serv_tracker.sadd(f'trackers:all:{tracker_type}', tracker_uuid)
|
||||
|
||||
##-- FIX DB --##
|
||||
|
||||
|
|
@ -359,6 +433,9 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti
|
|||
# create tracker - uuid map
|
||||
r_serv_tracker.sadd('all:tracker_uuid:{}:{}'.format(tracker_type, tracker), tracker_uuid)
|
||||
|
||||
r_serv_tracker.sadd(f'trackers:all', tracker_uuid)
|
||||
r_serv_tracker.sadd(f'trackers:all:{tracker_type}', tracker_uuid)
|
||||
|
||||
# add display level set
|
||||
if level == 0: # user only
|
||||
r_serv_tracker.sadd('user:tracker:{}'.format(user_id), tracker_uuid)
|
||||
|
|
@ -1122,6 +1199,20 @@ def api_delete_retro_hunt_task(task_uuid):
|
|||
return (delete_retro_hunt_task(task_uuid), 200)
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# fix_all_tracker_uuid_list()
|
||||
# res = get_all_tracker_uuid()
|
||||
# print(len(res))
|
||||
|
||||
# import Term
|
||||
# Term.delete_term('5262ab6c-8784-4a55-b0ff-a471018414b4')
|
||||
|
||||
#fix_tracker_stats_per_day('5262ab6c-8784-4a55-b0ff-a471018414b4')
|
||||
|
||||
# tracker_uuid = '5262ab6c-8784-4a55-b0ff-a471018414b4'
|
||||
# fix_tracker_item_link(tracker_uuid)
|
||||
# res = get_item_all_trackers_uuid('archive/')
|
||||
# print(res)
|
||||
|
||||
#res = is_valid_yara_rule('rule dummy { }')
|
||||
|
||||
# res = create_tracker('test', 'word', 'admin@admin.test', 1, [], [], None, sources=['crawled', 'pastebin.com', 'rt/pastebin.com'])
|
||||
|
|
|
|||
|
|
@ -290,11 +290,16 @@ def parse_tracked_term_to_delete(dict_input, user_id):
|
|||
delete_term(term_uuid)
|
||||
return ({"uuid": term_uuid}, 200)
|
||||
|
||||
# # TODO: MOVE IN TRACKER
|
||||
def delete_term(term_uuid):
|
||||
term = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked')
|
||||
term_type = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type')
|
||||
level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level')
|
||||
r_serv_term.srem('all:tracker_uuid:{}:{}'.format(term_type, term), term_uuid)
|
||||
|
||||
r_serv_term.srem(f'trackers:all', term_uuid)
|
||||
r_serv_term.srem(f'trackers:all:{term_type}', term_uuid)
|
||||
|
||||
# Term not tracked by other users
|
||||
if not r_serv_term.exists('all:tracker_uuid:{}:{}'.format(term_type, term)):
|
||||
r_serv_term.srem('all:tracker:{}'.format(term_type), term)
|
||||
|
|
@ -323,9 +328,14 @@ def delete_term(term_uuid):
|
|||
r_serv_term.delete('tracker:sources:{}'.format(term_uuid))
|
||||
|
||||
# remove item set
|
||||
all_item_date = r_serv_term.zrange('tracker:stat:{}'.format(term_uuid), 0, -1)
|
||||
#########################3
|
||||
all_item_date = r_serv_term.zrange(f'tracker:stat:{term_uuid}', 0, -1, withscores=True)
|
||||
if all_item_date:
|
||||
all_item_date = dict(all_item_date)
|
||||
for date in all_item_date:
|
||||
r_serv_term.delete('tracker:item:{}:{}'.format(term_uuid, date))
|
||||
for item_id in r_serv_term.smembers(f'tracker:item:{term_uuid}:{date}'):
|
||||
r_serv_term.srem(f'obj:trackers:item:{item_id}', term_uuid)
|
||||
r_serv_term.delete(f'tracker:item:{term_uuid}:{date}')
|
||||
r_serv_term.delete('tracker:stat:{}'.format(term_uuid))
|
||||
|
||||
if term_type == 'yara':
|
||||
|
|
|
|||
|
|
@ -65,3 +65,4 @@ if __name__ == "__main__":
|
|||
launch_background_upgrade('v2.6', ['Update_screenshots.py'])
|
||||
launch_background_upgrade('v2.7', ['Update_domain_tags.py'])
|
||||
launch_background_upgrade('v3.4', ['Update_domain.py'])
|
||||
launch_background_upgrade('v3.7', ['Update_trackers.py'])
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
import Tracker
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
|
||||
from ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
||||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
def update(self):
|
||||
"""
|
||||
Update Domain Languages
|
||||
"""
|
||||
print('Fixing Tracker_uuid list ...')
|
||||
Tracker.fix_all_tracker_uuid_list()
|
||||
nb = 0
|
||||
for tracker_uuid in get_all_tracker_uuid:
|
||||
self.r_serv.sadd('trackers_update_v3.7', tracker_uuid)
|
||||
nb += 1
|
||||
|
||||
self.r_serv.set('update:nb_elem_to_convert', nb)
|
||||
self.r_serv.set('update:nb_elem_converted',0)
|
||||
|
||||
# Add background update
|
||||
self.r_serv.sadd('ail:to_update', self.version)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
updater = Updater('v3.7')
|
||||
updater.run_update()
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
# SUBMODULES #
|
||||
git submodule update
|
||||
|
||||
|
||||
echo -e $GREEN"Updating thirdparty ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ut
|
||||
wait
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v3.7/Update.py
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
exit 0
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
import Domain
|
||||
|
||||
def update_update_stats():
|
||||
nb_updated = int(r_serv_db.get('update:nb_elem_converted'))
|
||||
progress = int((nb_updated * 100) / nb_elem_to_update)
|
||||
print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress))
|
||||
r_serv_db.set('ail:current_background_script_stat', progress)
|
||||
|
||||
def update_domain_language(domain_obj, item_id):
|
||||
domain_name = domain_obj.get_domain_name()
|
||||
Domain.add_domain_languages_by_item_id(domain_name, item_id)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||
config_loader = None
|
||||
|
||||
r_serv_db.set('ail:current_background_script', 'trackers update')
|
||||
|
||||
nb_elem_to_update = r_serv_db.get('update:nb_elem_to_convert')
|
||||
if not nb_elem_to_update:
|
||||
nb_elem_to_update = 1
|
||||
else:
|
||||
nb_elem_to_update = int(nb_elem_to_update)
|
||||
|
||||
|
||||
|
||||
while True:
|
||||
tracker_uuid = r_serv_onion.spop('trackers_update_v3.7')
|
||||
if tracker_uuid is not None:
|
||||
date_from =
|
||||
date_to =
|
||||
|
||||
# FIX STATS
|
||||
|
||||
print(tracker_uuid)
|
||||
# get all dates
|
||||
|
||||
# get items id
|
||||
|
||||
# convert
|
||||
domain = Domain.Domain(domain)
|
||||
for domain_history in domain.get_domain_history():
|
||||
domain_item = domain.get_domain_items_crawled(epoch=domain_history[1]) # item_tag
|
||||
if "items" in domain_item:
|
||||
for item_dict in domain_item['items']:
|
||||
update_domain_language(domain, item_dict['id'])
|
||||
|
||||
r_serv_db.incr('update:nb_elem_converted')
|
||||
update_update_stats()
|
||||
|
||||
else:
|
||||
r_serv_db.set('ail:current_background_script_stat', 100)
|
||||
sys.exit(0)
|
||||
|
|
@ -109,6 +109,8 @@ dict_update_description = {'v1.5':{'nb_background_update': 5, 'update_warning_me
|
|||
'v2.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags can be',
|
||||
'update_warning_message_notice_me': 'missing from the UI.'},
|
||||
'v3.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Languages can be',
|
||||
'update_warning_message_notice_me': 'missing from the UI.'},
|
||||
'v3.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Tracker first_seen/last_seen can be',
|
||||
'update_warning_message_notice_me': 'missing from the UI.'}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -209,13 +209,15 @@
|
|||
<div class="col-md-6">
|
||||
<div class="input-group" id="date-range-from">
|
||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ tracker_metadata['date_from'] }}" name="date_from" autocomplete="off">
|
||||
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" name="date_from" autocomplete="off"
|
||||
{%if tracker_metadata['date_from']%}value="{{ tracker_metadata['date_from'] }}"{%else%}value="{{tracker_metadata['first_seen']}}"{%endif%}>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="input-group" id="date-range-to">
|
||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ tracker_metadata['date_to'] }}" name="date_to" autocomplete="off">
|
||||
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" name="date_to" autocomplete="off"
|
||||
{%if tracker_metadata['date_to']%}value="{{ tracker_metadata['date_to'] }}"{%else%}value="{{tracker_metadata['last_seen']}}"{%endif%}>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -270,7 +272,7 @@
|
|||
<script>
|
||||
$(document).ready(function(){
|
||||
$('#div_edit_mails').hide();
|
||||
$('#div_edit_tags').hide();edit_description
|
||||
$('#div_edit_tags').hide();
|
||||
$('#div_edit_description').hide();
|
||||
$("#page-Decoded").addClass("active");
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue