diff --git a/OVERVIEW.md b/OVERVIEW.md index 882d6b54..02dcbeb9 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -30,7 +30,7 @@ Redis and ARDB overview # Database Map: -## DB2 - TermFreq: (TODO: update/refractor DB) +## DB2 - TermFreq: ##### Set: | Key | Value | diff --git a/update/v1.4/Update-ARDB_Metadata.py b/update/v1.4/Update-ARDB_Metadata.py index 20cfed72..a4edb0f7 100755 --- a/update/v1.4/Update-ARDB_Metadata.py +++ b/update/v1.4/Update-ARDB_Metadata.py @@ -7,6 +7,14 @@ import time import redis import configparser +def update_tracked_terms(main_key, tracked_container_key): + for tracked_item in r_serv_term.smembers(main_key): + all_items = r_serv_term.smembers(tracked_container_key.format(tracked_item)) + for item_path in all_items: + if PASTES_FOLDER in item_path: + new_item_path = item_path.replace(PASTES_FOLDER, '', 1) + r_serv_term.sadd(tracked_container_key.format(tracked_item), new_item_path) + r_serv_term.srem(tracked_container_key.format(tracked_item), item_path) def update_hash_item(has_type): #get all hash items: @@ -70,6 +78,12 @@ if __name__ == '__main__': db=cfg.getint("ARDB_Tags", "db"), decode_responses=True) + r_serv_term = redis.StrictRedis( + host=cfg.get("ARDB_TermFreq", "host"), + port=cfg.getint("ARDB_TermFreq", "port"), + db=cfg.getint("ARDB_TermFreq", "db"), + decode_responses=True) + r_serv_onion = redis.StrictRedis( host=cfg.get("ARDB_Onion", "host"), port=cfg.getint("ARDB_Onion", "port"), @@ -117,6 +131,15 @@ if __name__ == '__main__': end = time.time() + ## update tracked term/set/regex + # update tracked term + update_tracked_terms('TrackedSetTermSet', 'tracked_{}') + # update tracked set + update_tracked_terms('TrackedSetSet', 'set_{}') + # update tracked regex + update_tracked_terms('TrackedRegexSet', 'regex_{}') + ## + print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start)) print() diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py index 86d67af8..1fb83bcb 100644 --- a/var/www/modules/terms/Flask_terms.py +++ b/var/www/modules/terms/Flask_terms.py @@ -281,14 +281,13 @@ def terms_management_query_paste(): p_date = str(paste._get_p_date()) p_date = p_date[0:4]+'/'+p_date[4:6]+'/'+p_date[6:8] p_source = paste.p_source - p_encoding = paste._get_p_encoding() p_size = paste.p_size p_mime = paste.p_mime p_lineinfo = paste.get_lines_info() p_content = paste.get_p_content() if p_content != 0: p_content = p_content[0:400] - paste_info.append({"path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content}) + paste_info.append({"path": path, "date": p_date, "source": p_source, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content}) return jsonify(paste_info) diff --git a/var/www/modules/terms/templates/terms_management.html b/var/www/modules/terms/templates/terms_management.html index fc604391..f4da9031 100644 --- a/var/www/modules/terms/templates/terms_management.html +++ b/var/www/modules/terms/templates/terms_management.html @@ -443,7 +443,6 @@ function bindEventsForCurrentPage() { html_to_add += ""; html_to_add += "Source"; html_to_add += "Date"; - html_to_add += "Encoding"; html_to_add += "Size (Kb)"; html_to_add += "# lines"; html_to_add += "Max length"; @@ -456,7 +455,6 @@ function bindEventsForCurrentPage() { html_to_add += ""; html_to_add += ""+curr_data.source+""; html_to_add += ""+curr_data.date+""; - html_to_add += ""+curr_data.encoding+""; html_to_add += ""+curr_data.size+""; html_to_add += ""+curr_data.lineinfo[0]+""; html_to_add += ""+curr_data.lineinfo[1]+"";