CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [crawlers] get_all_splash return type

pull/569/head
Terrtia 2021-06-16 10:06:04 +02:00
parent 759ec73f84
commit ec727338e6
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
1 changed files with 11 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
bin/lib/crawlers.py
View File

@ -516,7 +516,7 @@ def api_set_nb_crawlers_to_launch(dict_splash_name):
# TODO: check if is dict # TODO: check if is dict
dict_crawlers_to_launch = {} dict_crawlers_to_launch = {}
all_splash = get_all_splash() all_splash = get_all_splash()
crawlers_to_launch = list(all_splash & set(dict_splash_name.keys())) crawlers_to_launch = list(set(all_splash) & set(dict_splash_name.keys()))
for splash_name in crawlers_to_launch: for splash_name in crawlers_to_launch:
try: try:
nb_to_launch = int(dict_splash_name.get(splash_name, 0)) nb_to_launch = int(dict_splash_name.get(splash_name, 0))
@ -984,26 +984,24 @@ def api_save_splash_manager_url_api(data):
## SPLASH ## ## SPLASH ##
def get_all_splash(r_list=False): def get_all_splash(r_list=False):
res = r_serv_onion.smembers('all_splash') res = r_serv_onion.smembers('all_splash')
if res: if not res:
if r_list: res = set()
return list(res) if r_list:
else: return list(res)
return res
else: else:
return [] return res
def get_splash_proxy(splash_name): def get_splash_proxy(splash_name):
return r_serv_onion.hget('splash:metadata:{}'.format(splash_name), 'proxy') return r_serv_onion.hget('splash:metadata:{}'.format(splash_name), 'proxy')
def get_splash_all_url(splash_name, r_list=False): def get_splash_all_url(splash_name, r_list=False):
res = r_serv_onion.smembers('splash:url:{}'.format(splash_name)) res = r_serv_onion.smembers('splash:url:{}'.format(splash_name))
if res: if not res:
if r_list: res = set()
return list(res) if r_list:
else: return list(res)
return res
else: else:
return [] return res
def get_splash_name_by_url(splash_url): def get_splash_name_by_url(splash_url):
return r_serv_onion.get('splash:map:url:name:{}'.format(splash_url)) return r_serv_onion.get('splash:map:url:name:{}'.format(splash_url))