fix: [redis cache] remove old paste_name db

pull/549/head
Terrtia 2021-02-03 14:22:26 +01:00
parent 2fb03baf50
commit 89e95ca030
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
7 changed files with 11 additions and 30 deletions

3
.gitignore vendored
View File

@ -44,6 +44,9 @@ configs/update.cfg
update/current_version update/current_version
files files
# Trackers
bin/trackers/yara/custom-rules/*
# Helper # Helper
bin/helper/gen_cert/rootCA.* bin/helper/gen_cert/rootCA.*
bin/helper/gen_cert/server.* bin/helper/gen_cert/server.*

View File

@ -57,7 +57,7 @@ def compute_most_posted(server, message):
print(redis_progression_name_set) print(redis_progression_name_set)
def compute_provider_info(server_trend, server_pasteName, path): def compute_provider_info(server_trend, path):
redis_all_provider = 'all_provider_set' redis_all_provider = 'all_provider_set'
paste = Paste.Paste(path) paste = Paste.Paste(path)
@ -71,7 +71,6 @@ def compute_provider_info(server_trend, server_pasteName, path):
redis_providers_name_set = 'providers_set_' + paste_date redis_providers_name_set = 'providers_set_' + paste_date
# Add/Update in Redis # Add/Update in Redis
server_pasteName.sadd(paste_baseName, path)
server_trend.sadd(redis_all_provider, paste_provider) server_trend.sadd(redis_all_provider, paste_provider)
num_paste = int(server_trend.hincrby(paste_provider+'_num', paste_date, 1)) num_paste = int(server_trend.hincrby(paste_provider+'_num', paste_date, 1))
@ -137,12 +136,6 @@ if __name__ == '__main__':
db=p.config.get("ARDB_Trending", "db"), db=p.config.get("ARDB_Trending", "db"),
decode_responses=True) decode_responses=True)
r_serv_pasteName = redis.StrictRedis(
host=p.config.get("Redis_Paste_Name", "host"),
port=p.config.get("Redis_Paste_Name", "port"),
db=p.config.get("Redis_Paste_Name", "db"),
decode_responses=True)
# Endless loop getting messages from the input queue # Endless loop getting messages from the input queue
while True: while True:
# Get one message from the input queue # Get one message from the input queue
@ -159,4 +152,4 @@ if __name__ == '__main__':
if len(message.split(';')) > 1: if len(message.split(';')) > 1:
compute_most_posted(r_serv_trend, message) compute_most_posted(r_serv_trend, message)
else: else:
compute_provider_info(r_serv_trend, r_serv_pasteName, message) compute_provider_info(r_serv_trend, message)

View File

@ -147,11 +147,6 @@ host = localhost
port = 6379 port = 6379
db = 1 db = 1
[Redis_Paste_Name]
host = localhost
port = 6379
db = 2
[Redis_Mixer_Cache] [Redis_Mixer_Cache]
host = localhost host = localhost
port = 6381 port = 6381

View File

@ -26,7 +26,6 @@ r_serv_charts = config_loader.get_redis_conn("ARDB_Trending")
r_serv_sentiment = config_loader.get_redis_conn("ARDB_Sentiment") r_serv_sentiment = config_loader.get_redis_conn("ARDB_Sentiment")
r_serv_term = config_loader.get_redis_conn("ARDB_Tracker") r_serv_term = config_loader.get_redis_conn("ARDB_Tracker")
r_serv_cred = config_loader.get_redis_conn("ARDB_TermCred") r_serv_cred = config_loader.get_redis_conn("ARDB_TermCred")
r_serv_pasteName = config_loader.get_redis_conn("Redis_Paste_Name")
r_serv_tags = config_loader.get_redis_conn("ARDB_Tags") r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
r_serv_db = config_loader.get_redis_conn("ARDB_DB") r_serv_db = config_loader.get_redis_conn("ARDB_DB")

View File

@ -27,7 +27,6 @@ import Flask_config
app = Flask_config.app app = Flask_config.app
config_loader = Flask_config.config_loader config_loader = Flask_config.config_loader
baseUrl = Flask_config.baseUrl baseUrl = Flask_config.baseUrl
r_serv_pasteName = Flask_config.r_serv_pasteName
r_serv_metadata = Flask_config.r_serv_metadata r_serv_metadata = Flask_config.r_serv_metadata
max_preview_char = Flask_config.max_preview_char max_preview_char = Flask_config.max_preview_char
max_preview_modal = Flask_config.max_preview_modal max_preview_modal = Flask_config.max_preview_modal
@ -116,17 +115,7 @@ def search():
selected_index = os.path.join(baseindexpath, index_name) selected_index = os.path.join(baseindexpath, index_name)
''' temporary disabled ''' temporary disabled
# Search filename # # TODO: search by filename/item id
for path in r_serv_pasteName.smembers(q[0]):
r.append(path)
paste = Paste.Paste(path)
content = paste.get_p_content()
content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
c.append(content[0:content_range])
curr_date = str(paste._get_p_date())
curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
paste_date.append(curr_date)
paste_size.append(paste._get_p_size())
''' '''
# Search full line # Search full line

View File

@ -32,11 +32,9 @@ import Flask_config
app = Flask_config.app app = Flask_config.app
baseUrl = Flask_config.baseUrl baseUrl = Flask_config.baseUrl
r_serv_pasteName = Flask_config.r_serv_pasteName
r_serv_metadata = Flask_config.r_serv_metadata r_serv_metadata = Flask_config.r_serv_metadata
r_serv_tags = Flask_config.r_serv_tags r_serv_tags = Flask_config.r_serv_tags
r_serv_statistics = Flask_config.r_serv_statistics r_serv_statistics = Flask_config.r_serv_statistics
r_serv_onion = Flask_config.r_serv_onion
max_preview_char = Flask_config.max_preview_char max_preview_char = Flask_config.max_preview_char
max_preview_modal = Flask_config.max_preview_modal max_preview_modal = Flask_config.max_preview_modal
DiffMaxLineLength = Flask_config.DiffMaxLineLength DiffMaxLineLength = Flask_config.DiffMaxLineLength

View File

@ -11,7 +11,11 @@
<div class="card-body pb-0"> <div class="card-body pb-0">
<h5 class="card-title"> <h5 class="card-title">
<a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{dict_domain["id"]}}"> <a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{dict_domain["id"]}}">
{% if 'hl-start' in dict_domain %}
{{dict_domain["id"][:dict_domain['hl-start']]}}<mark>{{dict_domain["id"][dict_domain['hl-start']:dict_domain['hl-end']]}}</mark>{{dict_domain["id"][dict_domain['hl-end']:]}}
{% else %}
{{dict_domain["id"]}} {{dict_domain["id"]}}
{% endif %}
</a> </a>
{% if dict_domain["status"] %} {% if dict_domain["status"] %}