CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [updates] remove v2.4 update + cleanup

pull/594/head
Terrtia 2022-11-23 11:02:33 +01:00
parent 4c2746a2d6
commit e67252c65b
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
33 changed files with 314 additions and 503 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
update/bin/old_ail_updater.py
View File

@ -2,14 +2,15 @@
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib import ConfigLoader
class AIL_Updater(object):
"""docstring for AIL_Updater."""
@ -38,9 +39,9 @@ class AIL_Updater(object):
"""
Update DB version
"""
#Set current ail version
# Set current ail version
self.r_serv.hset('ail:update_date', self.version, datetime.datetime.now().strftime("%Y%m%d"))
#Set current ail version
# Set current ail version
if self.f_version > self.current_f_version:
self.r_serv.set('ail:version', self.version)

7
update/default_update/Update.py
View File

@ -4,10 +4,7 @@
import os
import sys
import time
import redis
import argparse
import datetime
import configparser
sys.path.append(os.environ['AIL_BIN'])
##################################
@ -17,7 +14,7 @@ from lib import ail_updates
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='AIL default update')
parser.add_argument('-t', help='version tag' , type=str, dest='tag', required=True)
parser.add_argument('-t', help='version tag', type=str, dest='tag', required=True)
args = parser.parse_args()
if not args.tag:
@ -28,7 +25,7 @@ if __name__ == '__main__':
update_tag = args.tag.replace(' ', '')
if not ail_updates.check_version(update_tag):
parser.print_help()
print(f'Error: Invalid update tag {update_tag})
print(f'Error: Invalid update tag {update_tag}')
sys.exit(0)
start_deb = time.time()

3
update/v1.5/Update-ARDB_Metadata.py
View File

@ -4,10 +4,9 @@
import os
import sys
import time
import redis
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
from lib import ConfigLoader
def update_tracked_terms(main_key, tracked_container_key):
for tracked_item in r_serv_term.smembers(main_key):

5
update/v1.5/Update-ARDB_Onions.py
View File

@ -4,11 +4,10 @@
import os
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.environ['AIL_BIN'])
from lib import ConfigLoader
def substract_date(date_from, date_to):
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))

6
update/v1.5/Update-ARDB_Onions_screenshots.py
View File

@ -4,13 +4,12 @@
import os
import sys
import time
import redis
import datetime
from hashlib import sha256
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.environ['AIL_BIN'])
from lib import ConfigLoader
def rreplace(s, old, new, occurrence):
li = s.rsplit(old, occurrence)
@ -26,6 +25,7 @@ def substract_date(date_from, date_to):
l_date.append( date.strftime('%Y%m%d') )
return l_date
if __name__ == '__main__':
start_deb = time.time()

2
update/v1.5/Update-ARDB_Tags.py
View File

@ -7,7 +7,7 @@ import time
import redis
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
from lib import ConfigLoader
if __name__ == '__main__':

12
update/v1.5/Update-ARDB_Tags_background.py
View File

@ -4,10 +4,9 @@
import os
import sys
import time
import redis
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.environ['AIL_BIN'])
from lib import ConfigLoader
def tags_key_fusion(old_item_path_key, new_item_path_key):
print('fusion:')
@ -17,10 +16,10 @@ def tags_key_fusion(old_item_path_key, new_item_path_key):
r_serv_metadata.sadd(new_item_path_key, tag)
r_serv_metadata.srem(old_item_path_key, tag)
if __name__ == '__main__':
start_deb = time.time()
config_loader = ConfigLoader.ConfigLoader()
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
@ -38,7 +37,7 @@ if __name__ == '__main__':
print('Updating ARDB_Tags ...')
start = time.time()
#update item metadata tags
# update item metadata tags
tag_not_updated = True
total_to_update = r_serv_tag.scard('maj:v1.5:absolute_path_to_rename')
nb_updated = 0
@ -57,7 +56,7 @@ if __name__ == '__main__':
if r_serv_tag.scard('maj:v1.5:absolute_path_to_rename') == 0:
tag_not_updated = False
else:
progress = int((nb_updated * 100) /total_to_update)
progress = int((nb_updated * 100) / total_to_update)
print('{}/{} Tags updated {}%'.format(nb_updated, total_to_update, progress))
# update progress stats
if progress != last_progress:
@ -66,7 +65,6 @@ if __name__ == '__main__':
end = time.time()
print('Updating ARDB_Tags Done: {} s'.format(end - start))
r_serv.sadd('ail:update_v1.5', 'tags_background')

5
update/v1.5/Update.py
View File

@ -4,11 +4,10 @@
import os
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.environ['AIL_BIN'])
from lib import ConfigLoader
if __name__ == '__main__':

9
update/v1.7/Update.py
View File

@ -4,11 +4,10 @@
import os
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.environ['AIL_BIN'])
from lib import ConfigLoader
if __name__ == '__main__':
@ -19,8 +18,8 @@ if __name__ == '__main__':
r_serv = config_loader.get_redis_conn("ARDB_DB")
config_loader = None
#Set current ail version
# Set current ail version
r_serv.set('ail:version', 'v1.7')
#Set current ail version
# Set current ail version
r_serv.set('ail:update_date_v1.7', datetime.datetime.now().strftime("%Y%m%d"))

9
update/v2.0/Update.py
View File

@ -4,11 +4,10 @@
import os
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.environ['AIL_BIN'])
from lib import ConfigLoader
if __name__ == '__main__':
@ -19,7 +18,7 @@ if __name__ == '__main__':
r_serv = config_loader.get_redis_conn("ARDB_DB")
config_loader = None
#Set current ail version
# Set current ail version
r_serv.set('ail:version', 'v2.0')
# use new update_date format
@ -30,5 +29,5 @@ if __name__ == '__main__':
r_serv.hset('ail:update_date', tag, date_tag)
r_serv.delete('ail:update_date_{}'.format(tag))
#Set current ail version
# Set current ail version
r_serv.hset('ail:update_date', 'v2.0', datetime.datetime.now().strftime("%Y%m%d"))

53
update/v2.2/Update.py
View File

@ -2,51 +2,58 @@
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
import Item
import Term
sys.path.append(os.environ['AIL_BIN'])
from packages import Term
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
from lib import ConfigLoader
def rreplace(s, old, new, occurrence):
li = s.rsplit(old, occurrence)
return new.join(li)
def get_item_id(full_path):
return full_path.replace(PASTES_FOLDER, '', 1)
def get_item_date(id_item):
l_dir = id_item.split('/')
return f'{l_dir[-4]}{l_dir[-3]}{l_dir[-2]}'
if __name__ == '__main__':
start_deb = time.time()
config_loader = ConfigLoader.ConfigLoader()
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
PASTES_FOLDER = os.path.join(os.path.realpath(PASTES_FOLDER), '')
r_serv = config_loader.get_redis_conn("ARDB_DB")
r_serv_term_stats = config_loader.get_redis_conn("ARDB_Trending")
r_serv_termfreq = config_loader.get_redis_conn("ARDB_TermFreq")
config_loader = None
r_serv_term_stats.flushdb()
#convert all regex:
# convert all regex:
all_regex = r_serv_termfreq.smembers('TrackedRegexSet')
for regex in all_regex:
tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex)) )
mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex)) )
tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex)))
mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex)))
new_term = regex[1:-1]
res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1},
'admin@admin.test')
if res[1] == 200:
term_uuid = res[0]['uuid']
list_items = r_serv_termfreq.smembers('regex_{}'.format(regex))
for paste_item in list_items:
item_id = Item.get_item_id(paste_item)
item_date = Item.get_item_date(item_id)
item_id = get_item_id(paste_item)
item_date = get_item_date(item_id)
Term.add_tracked_item(term_uuid, item_id, item_date)
# Invalid Tracker => remove it
@ -58,16 +65,16 @@ if __name__ == '__main__':
all_tokens = r_serv_termfreq.smembers('TrackedSetTermSet')
for token in all_tokens:
tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token)) )
mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token)) )
tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token)))
mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token)))
res = Term.parse_json_term_to_add({"term": token, "type": 'word', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
if res[1] == 200:
term_uuid = res[0]['uuid']
list_items = r_serv_termfreq.smembers('tracked_{}'.format(token))
for paste_item in list_items:
item_id = Item.get_item_id(paste_item)
item_date = Item.get_item_date(item_id)
item_id = get_item_id(paste_item)
item_date = get_item_date(item_id)
Term.add_tracked_item(term_uuid, item_id, item_date)
# Invalid Tracker => remove it
else:
@ -78,8 +85,8 @@ if __name__ == '__main__':
all_set = r_serv_termfreq.smembers('TrackedSetSet')
for curr_set in all_set:
tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set)) )
mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set)) )
tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set)))
mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set)))
to_remove = ',{}'.format(curr_set.split(',')[-1])
new_set = rreplace(curr_set, to_remove, '', 1)
@ -91,8 +98,8 @@ if __name__ == '__main__':
term_uuid = res[0]['uuid']
list_items = r_serv_termfreq.smembers('tracked_{}'.format(curr_set))
for paste_item in list_items:
item_id = Item.get_item_id(paste_item)
item_date = Item.get_item_date(item_id)
item_id = get_item_id(paste_item)
item_date = get_item_date(item_id)
Term.add_tracked_item(term_uuid, item_id, item_date)
# Invalid Tracker => remove it
else:
@ -103,8 +110,8 @@ if __name__ == '__main__':
r_serv_termfreq.flushdb()
#Set current ail version
# Set current ail version
r_serv.set('ail:version', 'v2.2')
#Set current ail version
# Set current ail version
r_serv.hset('ail:update_date', 'v2.2', datetime.datetime.now().strftime("%Y%m%d"))

36
update/v2.4/Update.py
View File

@ -1,36 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
new_version = 'v2.4'
if __name__ == '__main__':
start_deb = time.time()
config_loader = ConfigLoader.ConfigLoader()
r_serv = config_loader.get_redis_conn("ARDB_DB")
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
config_loader = None
r_serv_onion.sunionstore('domain_update_v2.4', 'full_onion_up', 'full_regular_up')
r_serv.set('update:nb_elem_to_convert', r_serv_onion.scard('domain_update_v2.4'))
r_serv.set('update:nb_elem_converted',0)
# Add background update
r_serv.sadd('ail:to_update', new_version)
#Set current ail version
r_serv.set('ail:version', new_version)
#Set current ail version
r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))

42
update/v2.4/Update.sh
View File

@ -1,42 +0,0 @@
#!/bin/bash
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
export PATH=$AIL_HOME:$PATH
export PATH=$AIL_REDIS:$PATH
export PATH=$AIL_ARDB:$PATH
export PATH=$AIL_BIN:$PATH
export PATH=$AIL_FLASK:$PATH
GREEN="\\033[1;32m"
DEFAULT="\\033[0;39m"
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
bash ${AIL_BIN}/LAUNCH.sh -ks
wait
bash ${AIL_BIN}/LAUNCH.sh -lav &
wait
echo ""
cp ${AIL_BIN}/packages/config.cfg ${AIL_HOME}/configs/core.cfg
rm ${AIL_BIN}/packages/config.cfg
echo ""
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
echo ""
python ${AIL_HOME}/update/v2.4/Update.py
wait
echo ""
echo ""
echo ""
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
bash ${AIL_BIN}/LAUNCH.sh -ks
wait
exit 0

86
update/v2.4/Update_domain.py
View File

@ -1,86 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Item
import Tag
from Cryptocurrency import cryptocurrency
from Pgp import pgp
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
import Decoded
import Domain
def update_update_stats():
nb_updated = int(r_serv_db.get('update:nb_elem_converted'))
progress = int((nb_updated * 100) / nb_elem_to_update)
print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress))
r_serv_db.set('ail:current_background_script_stat', progress)
def update_domain_by_item(domain_obj, item_id):
domain_name = domain_obj.get_domain_name()
# update domain tags
for tag in Tag.get_obj_tag(item_id):
if tag != 'infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"':
Tag.add_tag("domain", tag, domain_name, obj_date=Item.get_item_date(item_id))
# update domain correlation
item_correlation = Item.get_item_all_correlation(item_id)
for correlation_name in item_correlation:
for correlation_type in item_correlation[correlation_name]:
if correlation_name in ('pgp', 'cryptocurrency'):
for correl_value in item_correlation[correlation_name][correlation_type]:
if correlation_name=='pgp':
pgp.save_domain_correlation(domain_name, correlation_type, correl_value)
if correlation_name=='cryptocurrency':
cryptocurrency.save_domain_correlation(domain_name, correlation_type, correl_value)
if correlation_name=='decoded':
for decoded_item in item_correlation['decoded']:
Decoded.save_domain_decoded(domain_name, decoded_item)
if __name__ == '__main__':
start_deb = time.time()
config_loader = ConfigLoader.ConfigLoader()
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
config_loader = None
r_serv.set('ail:current_background_script', 'domain update')
nb_elem_to_update = r_serv_db.get('update:nb_elem_to_convert')
if not nb_elem_to_update:
nb_elem_to_update = 0
else:
nb_elem_to_update = int(nb_elem_to_update)
while True:
domain = r_serv_onion.spop('domain_update_v2.4')
if domain is not None:
print(domain)
domain = Domain.Domain(domain)
for domain_history in domain.get_domain_history():
domain_item = domain.get_domain_items_crawled(epoch=domain_history[1]) # item_tag
if "items" in domain_item:
for item_dict in domain_item['items']:
update_domain_by_item(domain, item_dict['id'])
r_serv_db.incr('update:nb_elem_converted')
update_update_stats()
else:
r_serv.delete('update:nb_elem_to_convert')
r_serv.delete('update:nb_elem_converted')
r_serv_db.set('ail:current_background_script_stat', 100)
sys.exit(0)

10
update/v2.5/Update.py
View File

@ -2,14 +2,12 @@
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.environ['AIL_BIN'])
from lib import ConfigLoader
new_version = 'v2.5'
@ -30,8 +28,8 @@ if __name__ == '__main__':
r_serv.sadd('user_role:user_no_api', user)
r_serv.sadd('user_role:read_only', user)
#Set current ail version
# Set current ail version
r_serv.set('ail:version', new_version)
#Set current ail version
# Set current ail version
r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))

11
update/v2.6/Update.py
View File

@ -2,19 +2,16 @@
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.environ['AIL_BIN'])
from lib import ConfigLoader
new_version = 'v2.6'
if __name__ == '__main__':
start_deb = time.time()
config_loader = ConfigLoader.ConfigLoader()
@ -23,8 +20,8 @@ if __name__ == '__main__':
r_serv.sadd('ail:to_update', new_version)
#Set current ail version
# Set current ail version
r_serv.set('ail:version', new_version)
#Set current ail version
# Set current ail version
r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))

41
update/v2.6/Update_screenshots.py
View File

@ -4,21 +4,19 @@
import os
import sys
import time
import redis
import datetime
from hashlib import sha256
from pyfaup.faup import Faup
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Item
sys.path.append(os.environ['AIL_BIN'])
from lib import ConfigLoader
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
def get_domain(item_id):
item_id = item_id.split('/')
item_id = item_id[-1]
return item_id[:-36]
def get_all_item(screenshot_sha256):
return r_serv_onion.smembers('screenshot:{}'.format(screenshot_sha256))
def get_all_item(s_sha256):
return r_serv_onion.smembers(f'screenshot:{s_sha256}')
def sanitize_domain(domain):
faup.decode(domain)
@ -30,27 +28,24 @@ def sanitize_domain(domain):
pass
return domain_sanitized.lower()
def update_db(screenshot_sha256):
screenshot_items = get_all_item(screenshot_sha256)
def update_db(s_sha256):
screenshot_items = get_all_item(s_sha256)
if screenshot_items:
for item_id in screenshot_items:
item_id = item_id.replace(PASTES_FOLDER+'/', '', 1) # remove root path
domain = Item.get_domain(item_id)
item_id = item_id.replace(PASTES_FOLDER+'/', '', 1) # remove root path
domain = get_domain(item_id)
domain_sanitized = sanitize_domain(domain)
if domain != domain_sanitized:
r_serv_onion.sadd('incorrect_domain', domain)
domain = domain_sanitized
#print(item_id)
#print(domain)
r_serv_onion.sadd('domain_screenshot:{}'.format(domain), screenshot_sha256)
r_serv_onion.sadd('screenshot_domain:{}'.format(screenshot_sha256), domain)
r_serv_onion.sadd('domain_screenshot:{}'.format(domain), s_sha256)
r_serv_onion.sadd('screenshot_domain:{}'.format(s_sha256), domain)
else:
pass
# broken screenshot
r_serv_onion.sadd('broken_screenshot', screenshot_sha256)
r_serv_onion.sadd('broken_screenshot', s_sha256)
if __name__ == '__main__':
@ -77,15 +72,15 @@ if __name__ == '__main__':
if os.path.isdir(SCREENSHOT_FOLDER):
for root, dirs, files in os.walk(SCREENSHOT_FOLDER, topdown=False):
#print(dirs)
# print(dirs)
for name in files:
nb = nb + 1
screenshot_sha256 = os.path.join(root, name)
screenshot_sha256 = screenshot_sha256[:-4] # remove .png
screenshot_sha256 = screenshot_sha256[:-4] # remove .png
screenshot_sha256 = screenshot_sha256.replace(SCREENSHOT_FOLDER, '', 1)
screenshot_sha256 = screenshot_sha256.replace('/', '')
update_db(screenshot_sha256)
#print('Screenshot updated: {}'.format(nb))
# print('Screenshot updated: {}'.format(nb))
if nb % 1000 == 0:
r_serv_db.set('ail:current_background_script', 'screenshot updated: {}'.format(nb))

10
update/v2.7/Update.py
View File

@ -2,14 +2,12 @@
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.environ['AIL_BIN'])
from lib import ConfigLoader
new_version = 'v2.7'
@ -32,8 +30,8 @@ if __name__ == '__main__':
r_serv.set('ail:update_v2.7:deletetagrange', 1)
#### ####
#Set current ail version
# Set current ail version
r_serv.set('ail:version', new_version)
#Set current ail version
# Set current ail version
r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))

35
update/v2.7/Update_domain_tags.py
View File

@ -4,19 +4,13 @@
import os
import sys
import time
import redis
import datetime
from hashlib import sha256
from pyfaup.faup import Faup
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Date
sys.path.append(os.environ['AIL_BIN'])
from packages import Date
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
import Tag
from lib import ConfigLoader
def sanitize_domain(domain):
faup.decode(domain)
@ -28,8 +22,23 @@ def sanitize_domain(domain):
pass
return domain_sanitized.lower()
def get_all_obj_tags(obj_type):
return list(r_serv_tags.smembers(f'list_tags:{obj_type}'))
def add_global_tag(tag, object_type=None):
r_serv_tags.sadd('list_tags', tag)
if object_type:
r_serv_tags.sadd('list_tags:{}'.format(object_type), tag)
def get_obj_tag(object_id):
res = r_serv_metadata.smembers('tag:{}'.format(object_id))
if res:
return list(res)
else:
return []
def delete_domain_tag_daterange():
all_domains_tags = Tag.get_all_obj_tags('domain')
all_domains_tags = get_all_obj_tags('domain')
nb_updated = 0
nb_to_update = len(all_domains_tags)
if nb_to_update == 0:
@ -49,13 +58,13 @@ def update_domain_tags(domain):
r_serv_onion.sadd('incorrect_domain', domain)
domain = domain_sanitized
domain_tags = Tag.get_obj_tag(domain)
domain_tags = get_obj_tag(domain)
for tag in domain_tags:
# delete incorrect tags
if tag == 'infoleak:submission="crawler"' or tag == 'infoleak:submission="manual"':
r_serv_metadata.srem('tag:{}'.format(domain), tag)
else:
Tag.add_global_tag(tag, object_type='domain')
add_global_tag(tag, object_type='domain')
r_serv_tags.sadd('{}:{}'.format('domain', tag), domain)
def update_progress(refresh_time, nb_updated, nb_elem_to_update):
@ -88,6 +97,7 @@ def update_db():
r_serv_onion.sort('full_onion_up', alpha=True)
r_serv_onion.sort('full_regular_up', alpha=True)
if __name__ == '__main__':
start_deb = time.time()
@ -113,6 +123,5 @@ if __name__ == '__main__':
r_serv_db.set('ail:current_background_script_stat', 100)
end = time.time()
print('ALL domains tags updated in {} s'.format(end - start_deb))

33
update/v3.0/Update.py
View File

@ -3,27 +3,20 @@
import os
import sys
import time
import redis
import argparse
import datetime
import configparser
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.environ['AIL_HOME'])
##################################
# Import Project packages
##################################
from update.bin.old_ail_updater import AIL_Updater
class Updater(AIL_Updater):
"""default Updater."""
def __init__(self, version):
super(Updater, self).__init__(version)
new_version = 'v3.0'
if __name__ == '__main__':
start_deb = time.time()
config_loader = ConfigLoader.ConfigLoader()
r_serv = config_loader.get_redis_conn("ARDB_DB")
config_loader = None
#Set current ail version
r_serv.set('ail:version', new_version)
#Set current ail version
r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
updater = Updater('v3.0')
updater.run_update()

43
update/v3.1.1/Update.py
View File

@ -3,37 +3,20 @@
import os
import sys
import time
import redis
import argparse
import datetime
import configparser
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.environ['AIL_HOME'])
##################################
# Import Project packages
##################################
from update.bin.old_ail_updater import AIL_Updater
class Updater(AIL_Updater):
"""default Updater."""
def __init__(self, version):
super(Updater, self).__init__(version)
new_version = 'v3.1.1'
if __name__ == '__main__':
start_deb = time.time()
config_loader = ConfigLoader.ConfigLoader()
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
config_loader = None
#### NEW EXPORTER
# remove old tags errors
#r_serv_db.delete('mess_not_saved_export')
# move solo tags to export in tags_db
#all_misp_tags = r_serv_db.smembers('whitelist_misp')
#all_hive_tags = r_serv_db.smembers('whitelist_hive')
# # TODO: save them in tags db
#### NEW EXPORTER
#Set current ail version
r_serv_db.set('ail:version', new_version)
#Set current ail version
r_serv_db.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
updater = Updater('v3.1.1')
updater.run_update()

32
update/v3.1/Update.py
View File

@ -3,27 +3,21 @@
import os
import sys
import time
import redis
import argparse
import datetime
import configparser
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.environ['AIL_HOME'])
##################################
# Import Project packages
##################################
from update.bin.old_ail_updater import AIL_Updater
class Updater(AIL_Updater):
"""default Updater."""
def __init__(self, version):
super(Updater, self).__init__(version)
new_version = 'v3.1'
if __name__ == '__main__':
updater = Updater('v3.1')
updater.run_update()
start_deb = time.time()
config_loader = ConfigLoader.ConfigLoader()
r_serv = config_loader.get_redis_conn("ARDB_DB")
config_loader = None
#Set current ail version
r_serv.set('ail:version', new_version)
#Set current ail version
r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))

43
update/v3.2/Update.py
View File

@ -3,37 +3,20 @@
import os
import sys
import time
import redis
import argparse
import datetime
import configparser
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.environ['AIL_HOME'])
##################################
# Import Project packages
##################################
from update.bin.old_ail_updater import AIL_Updater
class Updater(AIL_Updater):
"""default Updater."""
def __init__(self, version):
super(Updater, self).__init__(version)
new_version = 'v3.2'
if __name__ == '__main__':
start_deb = time.time()
config_loader = ConfigLoader.ConfigLoader()
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
config_loader = None
#### NEW EXPORTER
# remove old tags errors
#r_serv_db.delete('mess_not_saved_export')
# move solo tags to export in tags_db
#all_misp_tags = r_serv_db.smembers('whitelist_misp')
#all_hive_tags = r_serv_db.smembers('whitelist_hive')
# # TODO: save them in tags db
#### NEW EXPORTER
#Set current ail version
r_serv_db.set('ail:version', new_version)
#Set current ail version
r_serv_db.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
updater = Updater('v3.2')
updater.run_update()

33
update/v3.3/Update.py
View File

@ -3,27 +3,20 @@
import os
import sys
import time
import redis
import argparse
import datetime
import configparser
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.environ['AIL_HOME'])
##################################
# Import Project packages
##################################
from update.bin.old_ail_updater import AIL_Updater
class Updater(AIL_Updater):
"""default Updater."""
def __init__(self, version):
super(Updater, self).__init__(version)
new_version = 'v3.3'
if __name__ == '__main__':
start_deb = time.time()
config_loader = ConfigLoader.ConfigLoader()
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
config_loader = None
#Set current ail version
r_serv_db.set('ail:version', new_version)
#Set current ail version
r_serv_db.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
updater = Updater('v3.3')
updater.run_update()

18
update/v3.4/Update.py
View File

@ -2,17 +2,13 @@
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
from old_ail_updater import AIL_Updater
sys.path.append(os.environ['AIL_HOME'])
##################################
# Import Project packages
##################################
from update.bin.old_ail_updater import AIL_Updater
class Updater(AIL_Updater):
"""default Updater."""
@ -27,12 +23,12 @@ class Updater(AIL_Updater):
"""
self.r_serv_onion.sunionstore('domain_update_v3.4', 'full_onion_up', 'full_regular_up')
self.r_serv.set('update:nb_elem_to_convert', self.r_serv_onion.scard('domain_update_v3.4'))
self.r_serv.set('update:nb_elem_converted',0)
self.r_serv.set('update:nb_elem_converted', 0)
# Add background update
self.r_serv.sadd('ail:to_update', self.version)
if __name__ == '__main__':
if __name__ == '__main__':
updater = Updater('v3.4')
updater.run_update()

93
update/v3.4/Update_domain.py
View File

@ -2,33 +2,95 @@
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
import Domain
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib import ConfigLoader
from lib.objects.Items import Item
from lib import Domain
def get_domain_type(domain_name):
if str(domain_name).endswith('.onion'):
return 'onion'
else:
return 'regular'
def add_domain_language(domain_name, language):
language = language.split('-')[0]
domain_type = get_domain_type(domain_name)
r_serv_onion.sadd('all_domains_languages', language)
r_serv_onion.sadd(f'all_domains_languages:{domain_type}', language)
r_serv_onion.sadd(f'language:domains:{domain_type}:{language}', domain_name)
r_serv_onion.sadd(f'domain:language:{domain_name}', language)
def add_domain_languages_by_item_id(domain_name, item_id):
item = Item(item_id)
for lang in item.get_languages():
add_domain_language(domain_name, lang.language)
def update_update_stats():
nb_updated = int(r_serv_db.get('update:nb_elem_converted'))
progress = int((nb_updated * 100) / nb_elem_to_update)
print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress))
print(f'{nb_updated}/{nb_elem_to_update} updated {progress}%')
r_serv_db.set('ail:current_background_script_stat', progress)
def update_domain_language(domain_obj, item_id):
domain_name = domain_obj.get_domain_name()
Domain.add_domain_languages_by_item_id(domain_name, item_id)
add_domain_languages_by_item_id(domain_name, item_id)
def get_domain_history(domain_type, domain_name):
return r_serv_onion.zrange(f'crawler_history_{domain_type}:{domain_name}:80', 0, -1, withscores=True)
def get_item_children(item_id):
return r_serv_metadata.smembers(f'paste_children:{item_id}')
def get_domain_items(domain_name, root_item_id):
dom_item = get_domain_item_children(domain_name, root_item_id)
dom_item.append(root_item_id)
return dom_item
def is_item_in_domain(domain_name, item_id):
is_in_domain = False
domain_length = len(domain_name)
if len(item_id) > (domain_length+48):
if item_id[-36-domain_length:-36] == domain_name:
is_in_domain = True
return is_in_domain
def get_domain_item_children(domain_name, root_item_id):
all_items = []
for item_id in get_item_children(root_item_id):
if is_item_in_domain(domain_name, item_id):
all_items.append(item_id)
all_items.extend(get_domain_item_children(domain_name, item_id))
return all_items
def get_domain_crawled_item_root(domain_name, domain_type, epoch):
res = r_serv_onion.zrevrangebyscore(f'crawler_history_{domain_type}:{domain_name}:80', int(epoch), int(epoch))
return {"root_item": res[0], "epoch": int(epoch)}
def get_domain_items_crawled(domain_name, domain_type, epoch):
item_crawled = []
item_root = get_domain_crawled_item_root(domain_name, domain_type, epoch)
if item_root:
if item_root['root_item'] != str(item_root['epoch']) and item_root['root_item']:
for item_id in get_domain_items(domain_name, item_root['root_item']):
item_crawled.append(item_id)
return item_crawled
if __name__ == '__main__':
start_deb = time.time()
config_loader = ConfigLoader.ConfigLoader()
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
config_loader = None
r_serv_db.set('ail:current_background_script', 'domain languages update')
@ -39,18 +101,19 @@ if __name__ == '__main__':
else:
nb_elem_to_update = int(nb_elem_to_update)
#Domain._delete_all_domains_languages()
# _delete_all_domains_languages()
while True:
domain = r_serv_onion.spop('domain_update_v3.4')
if domain is not None:
print(domain)
domain = str(domain)
domain_t = get_domain_type(domain)
domain = Domain.Domain(domain)
for domain_history in domain.get_domain_history():
domain_item = domain.get_domain_items_crawled(epoch=domain_history[1]) # item_tag
if "items" in domain_item:
for item_dict in domain_item['items']:
update_domain_language(domain, item_dict['id'])
for domain_history in get_domain_history(domain_t, domain):
domain_items = get_domain_items_crawled(domain, domain_t, domain_history[1])
for id_item in domain_items:
update_domain_language(domain, id_item)
r_serv_db.incr('update:nb_elem_converted')
update_update_stats()

16
update/v3.5/Update.py
View File

@ -2,17 +2,13 @@
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
from old_ail_updater import AIL_Updater
sys.path.append(os.environ['AIL_HOME'])
##################################
# Import Project packages
##################################
from update.bin.old_ail_updater import AIL_Updater
class Updater(AIL_Updater):
"""default Updater."""
@ -20,7 +16,7 @@ class Updater(AIL_Updater):
def __init__(self, version):
super(Updater, self).__init__(version)
if __name__ == '__main__':
if __name__ == '__main__':
updater = Updater('v3.5')
updater.run_update()

16
update/v3.6/Update.py
View File

@ -2,17 +2,13 @@
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
from old_ail_updater import AIL_Updater
sys.path.append(os.environ['AIL_HOME'])
##################################
# Import Project packages
##################################
from update.bin.old_ail_updater import AIL_Updater
class Updater(AIL_Updater):
"""default Updater."""
@ -20,7 +16,7 @@ class Updater(AIL_Updater):
def __init__(self, version):
super(Updater, self).__init__(version)
if __name__ == '__main__':
if __name__ == '__main__':
updater = Updater('v3.6')
updater.run_update()

18
update/v3.7/Update.py
View File

@ -2,18 +2,14 @@
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
import Tracker
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
from old_ail_updater import AIL_Updater
sys.path.append(os.environ['AIL_HOME'])
##################################
# Import Project packages
##################################
from update.bin.old_ail_updater import AIL_Updater
from lib import Tracker
class Updater(AIL_Updater):
"""default Updater."""
@ -38,7 +34,7 @@ class Updater(AIL_Updater):
# Add background update
self.r_serv.sadd('ail:to_update', self.version)
if __name__ == '__main__':
if __name__ == '__main__':
updater = Updater('v3.7')
updater.run_update()

19
update/v3.7/Update_trackers.py
View File

@ -2,29 +2,31 @@
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
import Tracker
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib import ConfigLoader
from lib import Tracker
def update_update_stats():
nb_updated = int(r_serv_db.get('update:nb_elem_converted'))
progress = int((nb_updated * 100) / nb_elem_to_update)
print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress))
print(f'{nb_updated}/{nb_elem_to_update} updated {progress}%')
r_serv_db.set('ail:current_background_script_stat', progress)
if __name__ == '__main__':
if __name__ == '__main__':
start_deb = time.time()
config_loader = ConfigLoader.ConfigLoader()
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
r_serv_tracker = config_loader.get_db_conn("Kvrocks_DB")
config_loader = None
Tracker.r_serv_tracker = r_serv_tracker
r_serv_db.set('ail:current_background_script', 'trackers update')
@ -43,7 +45,6 @@ if __name__ == '__main__':
# MAP TRACKER - ITEM_ID
Tracker.fix_tracker_item_link(tracker_uuid)
r_serv_db.incr('update:nb_elem_converted')
update_update_stats()

16
update/v4.0/Update.py
View File

@ -2,17 +2,13 @@
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
from old_ail_updater import AIL_Updater
sys.path.append(os.environ['AIL_HOME'])
##################################
# Import Project packages
##################################
from update.bin.old_ail_updater import AIL_Updater
class Updater(AIL_Updater):
"""default Updater."""
@ -20,7 +16,7 @@ class Updater(AIL_Updater):
def __init__(self, version):
super(Updater, self).__init__(version)
if __name__ == '__main__':
if __name__ == '__main__':
updater = Updater('v4.0')
updater.run_update()

26
update/v4.1/Update.py
View File

@ -2,17 +2,14 @@
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
from old_ail_updater import AIL_Updater
sys.path.append(os.environ['AIL_HOME'])
##################################
# Import Project packages
##################################
from update.bin.old_ail_updater import AIL_Updater
class Updater(AIL_Updater):
"""default Updater."""
@ -21,17 +18,14 @@ class Updater(AIL_Updater):
super(Updater, self).__init__(version)
def update(self):
r_tracking = redis.StrictRedis( host='localhost',
port=6382,
db=2,
decode_responses=True )
r_tracking = redis.StrictRedis(host='localhost',
port=6382,
db=2,
decode_responses=True)
# FLUSH OLD DB
r_tracking.flushdb()
if __name__ == '__main__':
updater = Updater('v4.1')
updater.run_update()

16
update/v4.2/Update.py
View File

@ -2,17 +2,13 @@
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
from old_ail_updater import AIL_Updater
sys.path.append(os.environ['AIL_HOME'])
##################################
# Import Project packages
##################################
from update.bin.old_ail_updater import AIL_Updater
class Updater(AIL_Updater):
"""default Updater."""
@ -20,7 +16,7 @@ class Updater(AIL_Updater):
def __init__(self, version):
super(Updater, self).__init__(version)
if __name__ == '__main__':
if __name__ == '__main__':
updater = Updater('v4.2')
updater.run_update()