CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [splash manager] update enpoints + use Splash name to restart docker

pull/559/head
Terrtia 2021-03-04 09:26:28 +01:00
parent fc7a61f67c
commit 1f94c1c693
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
2 changed files with 15 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
bin/Crawler.py
View File

@ -207,7 +207,7 @@ def crawl_onion(url, domain, port, type_service, message, crawler_config):
nb_retry += 1 nb_retry += 1
if nb_retry == 2: if nb_retry == 2:
crawlers.restart_splash_docker(splash_url) crawlers.restart_splash_docker(splash_url, splash_name)
if nb_retry == 6: if nb_retry == 6:
on_error_send_message_back_in_queue(type_service, domain, message) on_error_send_message_back_in_queue(type_service, domain, message)
@ -285,10 +285,12 @@ if __name__ == '__main__':
splash_url = sys.argv[1] splash_url = sys.argv[1]
splash_name = crawlers.get_splash_name_by_url(splash_url) splash_name = crawlers.get_splash_name_by_url(splash_url)
proxy_type = crawlers.get_splash_proxy(splash_name) proxy_name = crawlers.get_splash_proxy(splash_name)
print(splash_name) print(f'SPLASH Name: {splash_name}')
print(proxy_type) print(f'Proxy Name: {proxy_name}')
sys.exit(0)
#rotation_mode = deque(['onion', 'regular']) #rotation_mode = deque(['onion', 'regular'])
all_crawler_queues = crawlers.get_crawler_queue_types_by_splash_name(splash_name) all_crawler_queues = crawlers.get_crawler_queue_types_by_splash_name(splash_name)

18
bin/lib/crawlers.py
View File

@ -825,9 +825,9 @@ def get_splash_url_from_manager_url(splash_manager_url, splash_port):
# else: # else:
# return False # return False
def restart_splash_docker(splash_url): def restart_splash_docker(splash_url, splash_name):
splash_port = splash_url.split(':')[-1] splash_port = splash_url.split(':')[-1]
return _restart_splash_docker(splash_port) return _restart_splash_docker(splash_port, splash_name)
def is_splash_manager_connected(delta_check=30): def is_splash_manager_connected(delta_check=30):
last_check = r_cache.hget('crawler:splash:manager', 'last_check') last_check = r_cache.hget('crawler:splash:manager', 'last_check')
@ -911,7 +911,7 @@ def get_splash_manager_version():
pass pass
def get_all_splash_manager_containers_name(): def get_all_splash_manager_containers_name():
req = requests.get('{}/api/v1/get/splash/name/all'.format(get_splash_manager_url()), headers={"Authorization": get_splash_api_key()}, verify=False) req = requests.get('{}/api/v1/get/splash/all'.format(get_splash_manager_url()), headers={"Authorization": get_splash_api_key()}, verify=False)
if req.status_code == 200: if req.status_code == 200:
return req.json() return req.json()
else: else:
@ -924,8 +924,8 @@ def get_all_splash_manager_proxies():
else: else:
print(req.json()) print(req.json())
def _restart_splash_docker(splash_port): def _restart_splash_docker(splash_port, splash_name):
dict_to_send = {'docker_port': splash_port} dict_to_send = {'port': splash_port, 'name': splash_name}
req = requests.post('{}/api/v1/splash/restart'.format(get_splash_manager_url()), headers={"Authorization": get_splash_api_key()}, verify=False, json=dict_to_send) req = requests.post('{}/api/v1/splash/restart'.format(get_splash_manager_url()), headers={"Authorization": get_splash_api_key()}, verify=False, json=dict_to_send)
if req.status_code == 200: if req.status_code == 200:
return req.json() return req.json()
@ -1079,6 +1079,7 @@ def delete_proxy(proxy_name): # # TODO: force delete (delete all proxy)
## LOADER ## ## LOADER ##
def load_all_splash_containers(): def load_all_splash_containers():
delete_all_splash_containers()
all_splash_containers_name = get_all_splash_manager_containers_name() all_splash_containers_name = get_all_splash_manager_containers_name()
for splash_name in all_splash_containers_name: for splash_name in all_splash_containers_name:
r_serv_onion.sadd('all_splash', splash_name) r_serv_onion.sadd('all_splash', splash_name)
@ -1116,12 +1117,10 @@ def load_all_proxy():
def reload_splash_and_proxies_list(): def reload_splash_and_proxies_list():
if ping_splash_manager(): if ping_splash_manager():
# LOAD SPLASH containers
delete_all_splash_containers()
load_all_splash_containers()
# LOAD PROXIES containers # LOAD PROXIES containers
delete_all_proxies()
load_all_proxy() load_all_proxy()
# LOAD SPLASH containers
load_all_splash_containers()
return True return True
else: else:
return False return False
@ -1148,4 +1147,5 @@ def launch_ail_splash_crawler(splash_url, script_options=''):
if __name__ == '__main__': if __name__ == '__main__':
res = get_splash_manager_version() res = get_splash_manager_version()
#res = restart_splash_docker('127.0.0.1:8050', 'default_splash_tor')
print(res) print(res)