chg: [v1.5 background update] create update stats

bin/update-background.py

@@ -47,12 +47,13 @@ if __name__ == "__main__":
             update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags.py')
             process = subprocess.run(['python' ,update_file])
 
-        if if not r_serv.sismember('ail:update_v1.5', 'tags_background'):
+        if not r_serv.sismember('ail:update_v1.5', 'tags_background'):
             update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags_background.py')
             process = subprocess.run(['python' ,update_file])
         if r_serv.scard('ail:update_v1.5') != 4:
             r_serv.set('ail:update_error', 'Update v1.5 Failed, please relaunch the bin/update-background.py script')
         else:
             r_serv.delete('ail:update_in_progress')
-            r_serv_db.delete('ail:current_background_script')
+            r_serv.delete('ail:current_background_script')
-            r_serv.delete('ail:current_background_update', 'v1.5')
+            r_serv.delete('ail:current_background_script_stat')
+            r_serv.delete('ail:current_background_update')

update/v1.4/Update-ARDB_Metadata.py

@@ -97,13 +97,29 @@ if __name__ == '__main__':
     index = 0
     start = time.time()
 
+    #update stats
+    r_serv.set('ail:current_background_script_stat', 0)
+
     # Update base64
     update_hash_item('base64')
+
+    #update stats
+    r_serv.set('ail:current_background_script_stat', 20)
     # Update binary
     update_hash_item('binary')
+
+    #update stats
+    r_serv.set('ail:current_background_script_stat', 40)
     # Update binary
     update_hash_item('hexadecimal')
 
+    #update stats
+    r_serv.set('ail:current_background_script_stat', 60)
+
+    total_onion = r_serv_tag.scard('infoleak:submission=\"crawler\"')
+    nb_updated = 0
+    last_progress = 0
+
     # Update onion metadata
     all_crawled_items = r_serv_tag.smembers('infoleak:submission=\"crawler\"')
     for item_path in all_crawled_items:
@@ -131,17 +147,37 @@ if __name__ == '__main__':
             if PASTES_FOLDER in father:
                 r_serv_metadata.hset(new_item_metadata, 'father', father.replace(PASTES_FOLDER, '', 1))
 
-    end = time.time()
+        nb_updated += 1
+        progress = int((nb_updated * 30) /total_onion)
+        print('{}/{}    updated    {}%'.format(nb_updated, total_onion, progress + 60))
+        # update progress stats
+        if progress != last_progress:
+            r_serv.set('ail:current_background_script_stat', progress + 60)
+            last_progress = progress
+
+    #update stats
+    r_serv.set('ail:current_background_script_stat', 90)
 
     ## update tracked term/set/regex
     # update tracked term
     update_tracked_terms('TrackedSetTermSet', 'tracked_{}')
+
+    #update stats
+    r_serv.set('ail:current_background_script_stat', 93)
     # update tracked set
     update_tracked_terms('TrackedSetSet', 'set_{}')
+
+    #update stats
+    r_serv.set('ail:current_background_script_stat', 96)
     # update tracked regex
     update_tracked_terms('TrackedRegexSet', 'regex_{}')
+
+    #update stats
+    r_serv.set('ail:current_background_script_stat', 100)
     ##
 
+    end = time.time()
+
     print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start))
     print()
 

update/v1.4/Update-ARDB_Onions.py

@@ -74,6 +74,7 @@ if __name__ == '__main__':
         decode_responses=True)
 
     r_serv.set('ail:current_background_script', 'onions')
+    r_serv.set('ail:current_background_script_stat', 0)
 
     ## Update Onion ##
     print('Updating ARDB_Onion ...')
@@ -93,11 +94,15 @@ if __name__ == '__main__':
                 all_onion_history = r_serv_onion.lrange('onion_history:{}'.format(onion_domain), 0 ,-1)
                 if all_onion_history:
                     for date_history in all_onion_history:
-                        pass
                         #print('onion_history:{}:{}'.format(onion_domain, date_history))
                         r_serv_onion.delete('onion_history:{}:{}'.format(onion_domain, date_history))
                     r_serv_onion.delete('onion_history:{}'.format(onion_domain))
 
+    #stats
+    total_domain = r_serv_onion.scard('full_onion_up')
+    nb_updated = 0
+    last_progress = 0
+
     # clean up domain
     all_domain_up = r_serv_onion.smembers('full_onion_up')
     for onion_domain in all_domain_up:
@@ -130,6 +135,14 @@ if __name__ == '__main__':
         r_serv_onion.hset('onion_metadata:{}'.format(onion_domain), 'ports', '80')
         r_serv_onion.hdel('onion_metadata:{}'.format(onion_domain), 'last_seen')
 
+        nb_updated += 1
+        progress = int((nb_updated * 100) /total_domain)
+        print('{}/{}    updated    {}%'.format(nb_updated, total_domain, progress))
+        # update progress stats
+        if progress != last_progress:
+            r_serv.set('ail:current_background_script_stat', progress)
+            last_progress = progress
+
 
     end = time.time()
     print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start))

update/v1.4/Update-ARDB_Tags.py

@@ -58,11 +58,15 @@ if __name__ == '__main__':
         decode_responses=True)
 
     r_serv.set('ail:current_background_script', 'tags')
+    r_serv.set('ail:current_background_script_stat', 0)
 
-    if r_serv.exists('v1.5:onions') and r_serv.exists('v1.5:metadata'):
+    if r_serv.sismember('ail:update_v1.5', 'onions') and r_serv.sismember('ail:update_v1.5', 'metadata'):
 
         print('Updating ARDB_Tags ...')
         index = 0
+        nb_tags_to_update = 0
+        nb_updated = 0
+        last_progress = 0
         start = time.time()
 
         tags_list = r_serv_tag.smembers('list_tags')
@@ -81,6 +85,7 @@ if __name__ == '__main__':
                 tag_metadata[tag]['last_seen'] = 0
             else:
                 tag_metadata[tag]['last_seen'] = int(tag_metadata[tag]['last_seen'])
+            nb_tags_to_update += r_serv_tag.scard(tag)
 
         for tag in tags_list:
 
@@ -123,6 +128,14 @@ if __name__ == '__main__':
                 r_serv_tag.srem(tag, item_path)
                 index = index + 1
 
+            nb_updated += 1
+            progress = int((nb_updated * 100) /nb_tags_to_update)
+            print('{}/{}    updated    {}%'.format(nb_updated, nb_tags_to_update, progress))
+            # update progress stats
+            if progress != last_progress:
+                r_serv.set('ail:current_background_script_stat', progress)
+                last_progress = progress
+
         #flush browse importante pastes db
         r_important_paste_2018.flushdb()
         r_important_paste_2019.flushdb()

update/v1.4/Update-ARDB_Tags_background.py

@@ -47,35 +47,43 @@ if __name__ == '__main__':
         db=cfg.getint("ARDB_Tags", "db"),
         decode_responses=True)
 
-    r_serv.set('ail:current_background_script', 'tags_background')
+    if r_serv.sismember('ail:update_v1.5', 'tags'):
 
-    print('Updating ARDB_Tags ...')
+        r_serv.set('ail:current_background_script', 'tags_background')
-    start = time.time()
+        r_serv.set('ail:current_background_script_stat', 0)
 
-    #update item metadata tags
+        print('Updating ARDB_Tags ...')
-    tag_not_updated = True
+        start = time.time()
-    total_to_update = r_serv_tag.scard('maj:v1.5:absolute_path_to_rename')
-    nb_updated = 0
-    if total_to_update > 0:
-        while tag_not_updated:
-            item_path = r_serv_tag.srandmember('maj:v1.5:absolute_path_to_rename')
-            old_tag_item_key = 'tag:{}'.format(item_path)
-            new_item_path = item_path.replace(PASTES_FOLDER, '', 1)
-            new_tag_item_key = 'tag:{}'.format(new_item_path)
-            res = r_serv_metadata.renamenx(old_tag_item_key, new_tag_item_key)
-            if res == 0:
-                tags_key_fusion(old_tag_item_key, new_tag_item_key)
-            nb_updated += 1
-            if r_serv_tag.scard('maj:v1.5:absolute_path_to_rename') == 0:
-                tag_not_updated = False
-            else:
-                progress = int((nb_updated * 100) /total_to_update)
-                print('{}/{}    Tags updated    {}%'.format(nb_updated, total_to_update, progress))
-            r_serv_tag.srem('maj:v1.5:absolute_path_to_rename', item_path)
 
-    end = time.time()
+        #update item metadata tags
+        tag_not_updated = True
+        total_to_update = r_serv_tag.scard('maj:v1.5:absolute_path_to_rename')
+        nb_updated = 0
+        last_progress = 0
+        if total_to_update > 0:
+            while tag_not_updated:
+                item_path = r_serv_tag.srandmember('maj:v1.5:absolute_path_to_rename')
+                old_tag_item_key = 'tag:{}'.format(item_path)
+                new_item_path = item_path.replace(PASTES_FOLDER, '', 1)
+                new_tag_item_key = 'tag:{}'.format(new_item_path)
+                res = r_serv_metadata.renamenx(old_tag_item_key, new_tag_item_key)
+                if res == 0:
+                    tags_key_fusion(old_tag_item_key, new_tag_item_key)
+                nb_updated += 1
+                r_serv_tag.srem('maj:v1.5:absolute_path_to_rename', item_path)
+                if r_serv_tag.scard('maj:v1.5:absolute_path_to_rename') == 0:
+                    tag_not_updated = False
+                else:
+                    progress = int((nb_updated * 100) /total_to_update)
+                    print('{}/{}    Tags updated    {}%'.format(nb_updated, total_to_update, progress))
+                    # update progress stats
+                    if progress != last_progress:
+                        r_serv.set('ail:current_background_script_stat', progress)
+                        last_progress = progress
+
+        end = time.time()
 
 
-    print('Updating ARDB_Tags Done: {} s'.format(end - start))
+        print('Updating ARDB_Tags Done: {} s'.format(end - start))
 
-    r_serv.sadd('ail:update_v1.5', 'tags_background')
+        r_serv.sadd('ail:update_v1.5', 'tags_background')

var/www/modules/settings/Flask_settings.py

@@ -106,9 +106,6 @@ def get_background_update_stats_json():
         else:
             update_stats['background_stats'] = int(update_stats['background_stats'])
 
-        ## DEBUG:
-        update_stats['background_stats'] =12
-
         update_progression = r_serv_db.scard('ail:update_{}'.format(current_update))
         update_nb_scripts = dict_update_stat[current_update]['nb_background_update']
         update_stats['update_stat'] = int(update_progression*100/update_nb_scripts)
@@ -122,7 +119,11 @@ def get_background_update_stats_json():
             else:
                 update_stats['error_message'] = 'Please relaunch the bin/update-background.py script'
         else:
-            update_stats['error'] = False
+            if update_stats['background_name'] is None:
+                update_stats['error'] = True
+                update_stats['error_message'] = 'Please launch the bin/update-background.py script'
+            else:
+                update_stats['error'] = False
 
         return jsonify(update_stats)
 

var/www/modules/settings/templates/header_settings.html

@@ -0,0 +1 @@
+<li id='page-hiddenServices'><a href="{{ url_for('settings.settings_page') }}"><i class="fa fa-cog"></i> Server Management </a></li>

var/www/modules/settings/templates/settings_index.html

@@ -58,7 +58,6 @@
                             {{git_metadata['current_branch']}}
                           </td>
                         </tr>
-                        {%if git_metadata['is_clone']%}
                           <tr
                           {%if git_metadata['new_git_update_available']%}
                             class="table-warning"
@@ -85,11 +84,6 @@
                               {{git_metadata['last_local_tag']}}
                             </td>
                           </tr>
-                        {%else%}
-                        <tr>
-                          <td>This working directory is a fork</td>
-                        </tr>
-                        {%endif%}
                       </tbody>
                     </table>
                   </div>
@@ -196,7 +190,7 @@ update_progress();
 //Interval
 var progress_interval = setInterval(function(){
 	update_progress()
-}, 10000);
+}, 4000);
 </script>
 
 </html>