chg: [v1.5 background update] create update stats

pull/342/head
Terrtia 2019-04-17 17:07:09 +02:00
parent 4bff603d4f
commit ddaad5db93
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
8 changed files with 110 additions and 43 deletions

View File

@ -47,12 +47,13 @@ if __name__ == "__main__":
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags.py') update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags.py')
process = subprocess.run(['python' ,update_file]) process = subprocess.run(['python' ,update_file])
if if not r_serv.sismember('ail:update_v1.5', 'tags_background'): if not r_serv.sismember('ail:update_v1.5', 'tags_background'):
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags_background.py') update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags_background.py')
process = subprocess.run(['python' ,update_file]) process = subprocess.run(['python' ,update_file])
if r_serv.scard('ail:update_v1.5') != 4: if r_serv.scard('ail:update_v1.5') != 4:
r_serv.set('ail:update_error', 'Update v1.5 Failed, please relaunch the bin/update-background.py script') r_serv.set('ail:update_error', 'Update v1.5 Failed, please relaunch the bin/update-background.py script')
else: else:
r_serv.delete('ail:update_in_progress') r_serv.delete('ail:update_in_progress')
r_serv_db.delete('ail:current_background_script') r_serv.delete('ail:current_background_script')
r_serv.delete('ail:current_background_update', 'v1.5') r_serv.delete('ail:current_background_script_stat')
r_serv.delete('ail:current_background_update')

View File

@ -97,13 +97,29 @@ if __name__ == '__main__':
index = 0 index = 0
start = time.time() start = time.time()
#update stats
r_serv.set('ail:current_background_script_stat', 0)
# Update base64 # Update base64
update_hash_item('base64') update_hash_item('base64')
#update stats
r_serv.set('ail:current_background_script_stat', 20)
# Update binary # Update binary
update_hash_item('binary') update_hash_item('binary')
#update stats
r_serv.set('ail:current_background_script_stat', 40)
# Update binary # Update binary
update_hash_item('hexadecimal') update_hash_item('hexadecimal')
#update stats
r_serv.set('ail:current_background_script_stat', 60)
total_onion = r_serv_tag.scard('infoleak:submission=\"crawler\"')
nb_updated = 0
last_progress = 0
# Update onion metadata # Update onion metadata
all_crawled_items = r_serv_tag.smembers('infoleak:submission=\"crawler\"') all_crawled_items = r_serv_tag.smembers('infoleak:submission=\"crawler\"')
for item_path in all_crawled_items: for item_path in all_crawled_items:
@ -131,17 +147,37 @@ if __name__ == '__main__':
if PASTES_FOLDER in father: if PASTES_FOLDER in father:
r_serv_metadata.hset(new_item_metadata, 'father', father.replace(PASTES_FOLDER, '', 1)) r_serv_metadata.hset(new_item_metadata, 'father', father.replace(PASTES_FOLDER, '', 1))
end = time.time() nb_updated += 1
progress = int((nb_updated * 30) /total_onion)
print('{}/{} updated {}%'.format(nb_updated, total_onion, progress + 60))
# update progress stats
if progress != last_progress:
r_serv.set('ail:current_background_script_stat', progress + 60)
last_progress = progress
#update stats
r_serv.set('ail:current_background_script_stat', 90)
## update tracked term/set/regex ## update tracked term/set/regex
# update tracked term # update tracked term
update_tracked_terms('TrackedSetTermSet', 'tracked_{}') update_tracked_terms('TrackedSetTermSet', 'tracked_{}')
#update stats
r_serv.set('ail:current_background_script_stat', 93)
# update tracked set # update tracked set
update_tracked_terms('TrackedSetSet', 'set_{}') update_tracked_terms('TrackedSetSet', 'set_{}')
#update stats
r_serv.set('ail:current_background_script_stat', 96)
# update tracked regex # update tracked regex
update_tracked_terms('TrackedRegexSet', 'regex_{}') update_tracked_terms('TrackedRegexSet', 'regex_{}')
#update stats
r_serv.set('ail:current_background_script_stat', 100)
## ##
end = time.time()
print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start)) print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start))
print() print()

View File

@ -74,6 +74,7 @@ if __name__ == '__main__':
decode_responses=True) decode_responses=True)
r_serv.set('ail:current_background_script', 'onions') r_serv.set('ail:current_background_script', 'onions')
r_serv.set('ail:current_background_script_stat', 0)
## Update Onion ## ## Update Onion ##
print('Updating ARDB_Onion ...') print('Updating ARDB_Onion ...')
@ -93,11 +94,15 @@ if __name__ == '__main__':
all_onion_history = r_serv_onion.lrange('onion_history:{}'.format(onion_domain), 0 ,-1) all_onion_history = r_serv_onion.lrange('onion_history:{}'.format(onion_domain), 0 ,-1)
if all_onion_history: if all_onion_history:
for date_history in all_onion_history: for date_history in all_onion_history:
pass
#print('onion_history:{}:{}'.format(onion_domain, date_history)) #print('onion_history:{}:{}'.format(onion_domain, date_history))
r_serv_onion.delete('onion_history:{}:{}'.format(onion_domain, date_history)) r_serv_onion.delete('onion_history:{}:{}'.format(onion_domain, date_history))
r_serv_onion.delete('onion_history:{}'.format(onion_domain)) r_serv_onion.delete('onion_history:{}'.format(onion_domain))
#stats
total_domain = r_serv_onion.scard('full_onion_up')
nb_updated = 0
last_progress = 0
# clean up domain # clean up domain
all_domain_up = r_serv_onion.smembers('full_onion_up') all_domain_up = r_serv_onion.smembers('full_onion_up')
for onion_domain in all_domain_up: for onion_domain in all_domain_up:
@ -130,6 +135,14 @@ if __name__ == '__main__':
r_serv_onion.hset('onion_metadata:{}'.format(onion_domain), 'ports', '80') r_serv_onion.hset('onion_metadata:{}'.format(onion_domain), 'ports', '80')
r_serv_onion.hdel('onion_metadata:{}'.format(onion_domain), 'last_seen') r_serv_onion.hdel('onion_metadata:{}'.format(onion_domain), 'last_seen')
nb_updated += 1
progress = int((nb_updated * 100) /total_domain)
print('{}/{} updated {}%'.format(nb_updated, total_domain, progress))
# update progress stats
if progress != last_progress:
r_serv.set('ail:current_background_script_stat', progress)
last_progress = progress
end = time.time() end = time.time()
print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start)) print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start))

View File

@ -58,11 +58,15 @@ if __name__ == '__main__':
decode_responses=True) decode_responses=True)
r_serv.set('ail:current_background_script', 'tags') r_serv.set('ail:current_background_script', 'tags')
r_serv.set('ail:current_background_script_stat', 0)
if r_serv.exists('v1.5:onions') and r_serv.exists('v1.5:metadata'): if r_serv.sismember('ail:update_v1.5', 'onions') and r_serv.sismember('ail:update_v1.5', 'metadata'):
print('Updating ARDB_Tags ...') print('Updating ARDB_Tags ...')
index = 0 index = 0
nb_tags_to_update = 0
nb_updated = 0
last_progress = 0
start = time.time() start = time.time()
tags_list = r_serv_tag.smembers('list_tags') tags_list = r_serv_tag.smembers('list_tags')
@ -81,6 +85,7 @@ if __name__ == '__main__':
tag_metadata[tag]['last_seen'] = 0 tag_metadata[tag]['last_seen'] = 0
else: else:
tag_metadata[tag]['last_seen'] = int(tag_metadata[tag]['last_seen']) tag_metadata[tag]['last_seen'] = int(tag_metadata[tag]['last_seen'])
nb_tags_to_update += r_serv_tag.scard(tag)
for tag in tags_list: for tag in tags_list:
@ -123,6 +128,14 @@ if __name__ == '__main__':
r_serv_tag.srem(tag, item_path) r_serv_tag.srem(tag, item_path)
index = index + 1 index = index + 1
nb_updated += 1
progress = int((nb_updated * 100) /nb_tags_to_update)
print('{}/{} updated {}%'.format(nb_updated, nb_tags_to_update, progress))
# update progress stats
if progress != last_progress:
r_serv.set('ail:current_background_script_stat', progress)
last_progress = progress
#flush browse importante pastes db #flush browse importante pastes db
r_important_paste_2018.flushdb() r_important_paste_2018.flushdb()
r_important_paste_2019.flushdb() r_important_paste_2019.flushdb()

View File

@ -47,35 +47,43 @@ if __name__ == '__main__':
db=cfg.getint("ARDB_Tags", "db"), db=cfg.getint("ARDB_Tags", "db"),
decode_responses=True) decode_responses=True)
r_serv.set('ail:current_background_script', 'tags_background') if r_serv.sismember('ail:update_v1.5', 'tags'):
print('Updating ARDB_Tags ...') r_serv.set('ail:current_background_script', 'tags_background')
start = time.time() r_serv.set('ail:current_background_script_stat', 0)
#update item metadata tags print('Updating ARDB_Tags ...')
tag_not_updated = True start = time.time()
total_to_update = r_serv_tag.scard('maj:v1.5:absolute_path_to_rename')
nb_updated = 0
if total_to_update > 0:
while tag_not_updated:
item_path = r_serv_tag.srandmember('maj:v1.5:absolute_path_to_rename')
old_tag_item_key = 'tag:{}'.format(item_path)
new_item_path = item_path.replace(PASTES_FOLDER, '', 1)
new_tag_item_key = 'tag:{}'.format(new_item_path)
res = r_serv_metadata.renamenx(old_tag_item_key, new_tag_item_key)
if res == 0:
tags_key_fusion(old_tag_item_key, new_tag_item_key)
nb_updated += 1
if r_serv_tag.scard('maj:v1.5:absolute_path_to_rename') == 0:
tag_not_updated = False
else:
progress = int((nb_updated * 100) /total_to_update)
print('{}/{} Tags updated {}%'.format(nb_updated, total_to_update, progress))
r_serv_tag.srem('maj:v1.5:absolute_path_to_rename', item_path)
end = time.time() #update item metadata tags
tag_not_updated = True
total_to_update = r_serv_tag.scard('maj:v1.5:absolute_path_to_rename')
nb_updated = 0
last_progress = 0
if total_to_update > 0:
while tag_not_updated:
item_path = r_serv_tag.srandmember('maj:v1.5:absolute_path_to_rename')
old_tag_item_key = 'tag:{}'.format(item_path)
new_item_path = item_path.replace(PASTES_FOLDER, '', 1)
new_tag_item_key = 'tag:{}'.format(new_item_path)
res = r_serv_metadata.renamenx(old_tag_item_key, new_tag_item_key)
if res == 0:
tags_key_fusion(old_tag_item_key, new_tag_item_key)
nb_updated += 1
r_serv_tag.srem('maj:v1.5:absolute_path_to_rename', item_path)
if r_serv_tag.scard('maj:v1.5:absolute_path_to_rename') == 0:
tag_not_updated = False
else:
progress = int((nb_updated * 100) /total_to_update)
print('{}/{} Tags updated {}%'.format(nb_updated, total_to_update, progress))
# update progress stats
if progress != last_progress:
r_serv.set('ail:current_background_script_stat', progress)
last_progress = progress
end = time.time()
print('Updating ARDB_Tags Done: {} s'.format(end - start)) print('Updating ARDB_Tags Done: {} s'.format(end - start))
r_serv.sadd('ail:update_v1.5', 'tags_background') r_serv.sadd('ail:update_v1.5', 'tags_background')

View File

@ -106,9 +106,6 @@ def get_background_update_stats_json():
else: else:
update_stats['background_stats'] = int(update_stats['background_stats']) update_stats['background_stats'] = int(update_stats['background_stats'])
## DEBUG:
update_stats['background_stats'] =12
update_progression = r_serv_db.scard('ail:update_{}'.format(current_update)) update_progression = r_serv_db.scard('ail:update_{}'.format(current_update))
update_nb_scripts = dict_update_stat[current_update]['nb_background_update'] update_nb_scripts = dict_update_stat[current_update]['nb_background_update']
update_stats['update_stat'] = int(update_progression*100/update_nb_scripts) update_stats['update_stat'] = int(update_progression*100/update_nb_scripts)
@ -122,7 +119,11 @@ def get_background_update_stats_json():
else: else:
update_stats['error_message'] = 'Please relaunch the bin/update-background.py script' update_stats['error_message'] = 'Please relaunch the bin/update-background.py script'
else: else:
update_stats['error'] = False if update_stats['background_name'] is None:
update_stats['error'] = True
update_stats['error_message'] = 'Please launch the bin/update-background.py script'
else:
update_stats['error'] = False
return jsonify(update_stats) return jsonify(update_stats)

View File

@ -0,0 +1 @@
<li id='page-hiddenServices'><a href="{{ url_for('settings.settings_page') }}"><i class="fa fa-cog"></i> Server Management </a></li>

View File

@ -58,7 +58,6 @@
{{git_metadata['current_branch']}} {{git_metadata['current_branch']}}
</td> </td>
</tr> </tr>
{%if git_metadata['is_clone']%}
<tr <tr
{%if git_metadata['new_git_update_available']%} {%if git_metadata['new_git_update_available']%}
class="table-warning" class="table-warning"
@ -85,11 +84,6 @@
{{git_metadata['last_local_tag']}} {{git_metadata['last_local_tag']}}
</td> </td>
</tr> </tr>
{%else%}
<tr>
<td>This working directory is a fork</td>
</tr>
{%endif%}
</tbody> </tbody>
</table> </table>
</div> </div>
@ -196,7 +190,7 @@ update_progress();
//Interval //Interval
var progress_interval = setInterval(function(){ var progress_interval = setInterval(function(){
update_progress() update_progress()
}, 10000); }, 4000);
</script> </script>
</html> </html>