CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

Fixed bug in credential (not sending correctly to ModuleStat) + Fixed date bug in ModuleStats

pull/65/head
Mokaddem 2016-08-08 11:37:18 +02:00
parent 4e56f76f62
commit 14062a549c
2 changed files with 15 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
bin/Credential.py
View File

@ -57,11 +57,19 @@ if __name__ == "__main__":
publisher.warning(to_print) publisher.warning(to_print)
#Send to duplicate #Send to duplicate
p.populate_set_out(filepath, 'Duplicate') p.populate_set_out(filepath, 'Duplicate')
#send to Browse_warning_paste #Send to BrowseWarningPaste
p.populate_set_out('credential;{}'.format(filepath), 'BrowseWarningPaste') p.populate_set_out('credential;{}'.format(filepath), 'BrowseWarningPaste')
#Put in form, count occurences, then send to moduleStats #Put in form, count occurences, then send to moduleStats
creds_sites = {} creds_sites = {}
site_occurence = re.findall(regex_site_for_stats, content)
for site in site_occurence:
site_domain = site[1:-1]
if site_domain in creds_sites.keys():
creds_sites[site_domain] += 1
else:
creds_sites[site_domain] = 1
for url in sites: for url in sites:
faup.decode(url) faup.decode(url)
domain = faup.get()['domain'] domain = faup.get()['domain']
@ -71,7 +79,8 @@ if __name__ == "__main__":
creds_sites[domain] = 1 creds_sites[domain] = 1
for site, num in creds_sites.iteritems(): # Send for each different site to moduleStats for site, num in creds_sites.iteritems(): # Send for each different site to moduleStats
p.populate_set_out('credential;{};{};{}'.format(num, site, paste.p_date), 'ModuleStats') print 'credential;{};{};{}'.format(num, site, paste.p_date)
p.populate_set_out('credential;{};{};{}'.format(num, site, paste.p_date), 'ModuleStats')
if sites_set: if sites_set:
print("=======> Probably on : {}".format(', '.join(sites_set))) print("=======> Probably on : {}".format(', '.join(sites_set)))

5
bin/ModuleStats.py
View File

@ -31,6 +31,7 @@ def compute_most_posted(server, message):
module, num, keyword, paste_date = message.split(';') module, num, keyword, paste_date = message.split(';')
redis_progression_name_set = 'top_'+ module +'_set' redis_progression_name_set = 'top_'+ module +'_set'
# Add/Update in Redis # Add/Update in Redis
prev_score = server.hget(paste_date, module+'-'+keyword) prev_score = server.hget(paste_date, module+'-'+keyword)
if prev_score is not None: if prev_score is not None:
@ -39,7 +40,7 @@ def compute_most_posted(server, message):
ok = server.hset(paste_date, module+'-'+keyword, int(num)) ok = server.hset(paste_date, module+'-'+keyword, int(num))
# Compute Most Posted # Compute Most Posted
date = get_date_range(0) date = get_date_range(0)[0]
# check if this keyword is eligible for progression # check if this keyword is eligible for progression
keyword_total_sum = 0 keyword_total_sum = 0
@ -59,6 +60,8 @@ def compute_most_posted(server, message):
keyw_value = server.hget(paste_date, module+'-'+keyw) keyw_value = server.hget(paste_date, module+'-'+keyw)
if keyw_value is not None: if keyw_value is not None:
member_set.append((keyw, int(keyw_value))) member_set.append((keyw, int(keyw_value)))
else: #No data for this set for today
member_set.append((keyw, int(0)))
member_set.sort(key=lambda tup: tup[1]) member_set.sort(key=lambda tup: tup[1])
if len(member_set) > 0: if len(member_set) > 0:
if member_set[0][1] < keyword_total_sum: if member_set[0][1] < keyword_total_sum: