CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [v1.5 background update] create update stats

pull/342/head
Terrtia 2019-04-17 17:07:09 +02:00
parent 4bff603d4f
commit ddaad5db93
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
8 changed files with 110 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
bin/update-background.py
View File

@ -47,12 +47,13 @@ if __name__ == "__main__":
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags.py')
process = subprocess.run(['python' ,update_file])
if if not r_serv.sismember('ail:update_v1.5', 'tags_background'):
if not r_serv.sismember('ail:update_v1.5', 'tags_background'):
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags_background.py')
process = subprocess.run(['python' ,update_file])
if r_serv.scard('ail:update_v1.5') != 4:
r_serv.set('ail:update_error', 'Update v1.5 Failed, please relaunch the bin/update-background.py script')
else:
r_serv.delete('ail:update_in_progress')
r_serv_db.delete('ail:current_background_script')
r_serv.delete('ail:current_background_update', 'v1.5')
r_serv.delete('ail:current_background_script')
r_serv.delete('ail:current_background_script_stat')
r_serv.delete('ail:current_background_update')

38
update/v1.4/Update-ARDB_Metadata.py
View File

@ -97,13 +97,29 @@ if __name__ == '__main__':
index = 0
start = time.time()
#update stats
r_serv.set('ail:current_background_script_stat', 0)
# Update base64
update_hash_item('base64')
#update stats
r_serv.set('ail:current_background_script_stat', 20)
# Update binary
update_hash_item('binary')
#update stats
r_serv.set('ail:current_background_script_stat', 40)
# Update binary
update_hash_item('hexadecimal')
#update stats
r_serv.set('ail:current_background_script_stat', 60)
total_onion = r_serv_tag.scard('infoleak:submission=\"crawler\"')
nb_updated = 0
last_progress = 0
# Update onion metadata
all_crawled_items = r_serv_tag.smembers('infoleak:submission=\"crawler\"')
for item_path in all_crawled_items:
@ -131,17 +147,37 @@ if __name__ == '__main__':
if PASTES_FOLDER in father:
r_serv_metadata.hset(new_item_metadata, 'father', father.replace(PASTES_FOLDER, '', 1))
end = time.time()
nb_updated += 1
progress = int((nb_updated * 30) /total_onion)
print('{}/{} updated {}%'.format(nb_updated, total_onion, progress + 60))
# update progress stats
if progress != last_progress:
r_serv.set('ail:current_background_script_stat', progress + 60)
last_progress = progress
#update stats
r_serv.set('ail:current_background_script_stat', 90)
## update tracked term/set/regex
# update tracked term
update_tracked_terms('TrackedSetTermSet', 'tracked_{}')
#update stats
r_serv.set('ail:current_background_script_stat', 93)
# update tracked set
update_tracked_terms('TrackedSetSet', 'set_{}')
#update stats
r_serv.set('ail:current_background_script_stat', 96)
# update tracked regex
update_tracked_terms('TrackedRegexSet', 'regex_{}')
#update stats
r_serv.set('ail:current_background_script_stat', 100)
##
end = time.time()
print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start))
print()

15
update/v1.4/Update-ARDB_Onions.py
View File

@ -74,6 +74,7 @@ if __name__ == '__main__':
decode_responses=True)
r_serv.set('ail:current_background_script', 'onions')
r_serv.set('ail:current_background_script_stat', 0)
## Update Onion ##
print('Updating ARDB_Onion ...')
@ -93,11 +94,15 @@ if __name__ == '__main__':
all_onion_history = r_serv_onion.lrange('onion_history:{}'.format(onion_domain), 0 ,-1)
if all_onion_history:
for date_history in all_onion_history:
pass
#print('onion_history:{}:{}'.format(onion_domain, date_history))
r_serv_onion.delete('onion_history:{}:{}'.format(onion_domain, date_history))
r_serv_onion.delete('onion_history:{}'.format(onion_domain))
#stats
total_domain = r_serv_onion.scard('full_onion_up')
nb_updated = 0
last_progress = 0
# clean up domain
all_domain_up = r_serv_onion.smembers('full_onion_up')
for onion_domain in all_domain_up:
@ -130,6 +135,14 @@ if __name__ == '__main__':
r_serv_onion.hset('onion_metadata:{}'.format(onion_domain), 'ports', '80')
r_serv_onion.hdel('onion_metadata:{}'.format(onion_domain), 'last_seen')
nb_updated += 1
progress = int((nb_updated * 100) /total_domain)
print('{}/{} updated {}%'.format(nb_updated, total_domain, progress))
# update progress stats
if progress != last_progress:
r_serv.set('ail:current_background_script_stat', progress)
last_progress = progress
end = time.time()
print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start))

15
update/v1.4/Update-ARDB_Tags.py
View File

@ -58,11 +58,15 @@ if __name__ == '__main__':
decode_responses=True)
r_serv.set('ail:current_background_script', 'tags')
r_serv.set('ail:current_background_script_stat', 0)
if r_serv.exists('v1.5:onions') and r_serv.exists('v1.5:metadata'):
if r_serv.sismember('ail:update_v1.5', 'onions') and r_serv.sismember('ail:update_v1.5', 'metadata'):
print('Updating ARDB_Tags ...')
index = 0
nb_tags_to_update = 0
nb_updated = 0
last_progress = 0
start = time.time()
tags_list = r_serv_tag.smembers('list_tags')
@ -81,6 +85,7 @@ if __name__ == '__main__':
tag_metadata[tag]['last_seen'] = 0
else:
tag_metadata[tag]['last_seen'] = int(tag_metadata[tag]['last_seen'])
nb_tags_to_update += r_serv_tag.scard(tag)
for tag in tags_list:
@ -123,6 +128,14 @@ if __name__ == '__main__':
r_serv_tag.srem(tag, item_path)
index = index + 1
nb_updated += 1
progress = int((nb_updated * 100) /nb_tags_to_update)
print('{}/{} updated {}%'.format(nb_updated, nb_tags_to_update, progress))
# update progress stats
if progress != last_progress:
r_serv.set('ail:current_background_script_stat', progress)
last_progress = progress
#flush browse importante pastes db
r_important_paste_2018.flushdb()
r_important_paste_2019.flushdb()

60
update/v1.4/Update-ARDB_Tags_background.py
View File

@ -47,35 +47,43 @@ if __name__ == '__main__':
db=cfg.getint("ARDB_Tags", "db"),
decode_responses=True)
r_serv.set('ail:current_background_script', 'tags_background')
if r_serv.sismember('ail:update_v1.5', 'tags'):
print('Updating ARDB_Tags ...')
start = time.time()
r_serv.set('ail:current_background_script', 'tags_background')
r_serv.set('ail:current_background_script_stat', 0)
#update item metadata tags
tag_not_updated = True
total_to_update = r_serv_tag.scard('maj:v1.5:absolute_path_to_rename')
nb_updated = 0
if total_to_update > 0:
while tag_not_updated:
item_path = r_serv_tag.srandmember('maj:v1.5:absolute_path_to_rename')
old_tag_item_key = 'tag:{}'.format(item_path)
new_item_path = item_path.replace(PASTES_FOLDER, '', 1)
new_tag_item_key = 'tag:{}'.format(new_item_path)
res = r_serv_metadata.renamenx(old_tag_item_key, new_tag_item_key)
if res == 0:
tags_key_fusion(old_tag_item_key, new_tag_item_key)
nb_updated += 1
if r_serv_tag.scard('maj:v1.5:absolute_path_to_rename') == 0:
tag_not_updated = False
else:
progress = int((nb_updated * 100) /total_to_update)
print('{}/{} Tags updated {}%'.format(nb_updated, total_to_update, progress))
r_serv_tag.srem('maj:v1.5:absolute_path_to_rename', item_path)
print('Updating ARDB_Tags ...')
start = time.time()
end = time.time()
#update item metadata tags
tag_not_updated = True
total_to_update = r_serv_tag.scard('maj:v1.5:absolute_path_to_rename')
nb_updated = 0
last_progress = 0
if total_to_update > 0:
while tag_not_updated:
item_path = r_serv_tag.srandmember('maj:v1.5:absolute_path_to_rename')
old_tag_item_key = 'tag:{}'.format(item_path)
new_item_path = item_path.replace(PASTES_FOLDER, '', 1)
new_tag_item_key = 'tag:{}'.format(new_item_path)
res = r_serv_metadata.renamenx(old_tag_item_key, new_tag_item_key)
if res == 0:
tags_key_fusion(old_tag_item_key, new_tag_item_key)
nb_updated += 1
r_serv_tag.srem('maj:v1.5:absolute_path_to_rename', item_path)
if r_serv_tag.scard('maj:v1.5:absolute_path_to_rename') == 0:
tag_not_updated = False
else:
progress = int((nb_updated * 100) /total_to_update)
print('{}/{} Tags updated {}%'.format(nb_updated, total_to_update, progress))
# update progress stats
if progress != last_progress:
r_serv.set('ail:current_background_script_stat', progress)
last_progress = progress
end = time.time()
print('Updating ARDB_Tags Done: {} s'.format(end - start))
print('Updating ARDB_Tags Done: {} s'.format(end - start))
r_serv.sadd('ail:update_v1.5', 'tags_background')
r_serv.sadd('ail:update_v1.5', 'tags_background')

9
var/www/modules/settings/Flask_settings.py
View File

@ -106,9 +106,6 @@ def get_background_update_stats_json():
else:
update_stats['background_stats'] = int(update_stats['background_stats'])
## DEBUG:
update_stats['background_stats'] =12
update_progression = r_serv_db.scard('ail:update_{}'.format(current_update))
update_nb_scripts = dict_update_stat[current_update]['nb_background_update']
update_stats['update_stat'] = int(update_progression*100/update_nb_scripts)
@ -122,7 +119,11 @@ def get_background_update_stats_json():
else:
update_stats['error_message'] = 'Please relaunch the bin/update-background.py script'
else:
update_stats['error'] = False
if update_stats['background_name'] is None:
update_stats['error'] = True
update_stats['error_message'] = 'Please launch the bin/update-background.py script'
else:
update_stats['error'] = False
return jsonify(update_stats)

1
var/www/modules/settings/templates/header_settings.html Normal file
View File

@ -0,0 +1 @@
<li id='page-hiddenServices'><a href="{{ url_for('settings.settings_page') }}"><i class="fa fa-cog"></i> Server Management </a></li>

8
var/www/modules/settings/templates/settings_index.html
View File

@ -58,7 +58,6 @@
{{git_metadata['current_branch']}}
</td>
</tr>
{%if git_metadata['is_clone']%}
<tr
{%if git_metadata['new_git_update_available']%}
class="table-warning"
@ -85,11 +84,6 @@
{{git_metadata['last_local_tag']}}
</td>
</tr>
{%else%}
<tr>
<td>This working directory is a fork</td>
</tr>
{%endif%}
</tbody>
</table>
</div>
@ -196,7 +190,7 @@ update_progress();
//Interval
var progress_interval = setInterval(function(){
update_progress()
}, 10000);
}, 4000);
</script>
</html>