CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [Crawler] faup

pull/559/head
Terrtia 2021-03-05 18:47:38 +01:00
parent 3a347a79b7
commit 6daa750e3b
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
2 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
bin/Crawler.py
View File

@ -287,9 +287,11 @@ if __name__ == '__main__':
splash_name = crawlers.get_splash_name_by_url(splash_url)
proxy_name = crawlers.get_splash_proxy(splash_name)
print(f'SPLASH Name: {splash_name}')
print(f'Proxy Name: {proxy_name}')
print(f'SPLASH Name: {splash_name}')
print(f'Proxy Name: {proxy_name}')
print(f'Crawler Type: {get_splash_crawler_type(splash_name)})
#time.sleep(10)
#sys.exit(0)
#rotation_mode = deque(['onion', 'regular'])
@ -330,7 +332,7 @@ if __name__ == '__main__':
db=p.config.getint("ARDB_Onion", "db"),
decode_responses=True)
faup = Faup()
faup = crawlers.get_faup()
# get HAR files
default_crawler_har = p.config.getboolean("Crawler", "default_crawler_har")

6
bin/lib/crawlers.py
View File

@ -67,6 +67,10 @@ def is_valid_onion_domain(domain):
return True
return False
# TEMP FIX
def get_faup():
return faup
################################################################################
# # TODO: handle prefix cookies
@ -894,7 +898,7 @@ def get_splash_manager_session_uuid():
return res['session_uuid']
else:
print(req.json())
except requests.exceptions.ConnectionError:
except requests.exceptions.ConnectionError, requests.exceptions.MissingSchema:
# splash manager unreachable
update_splash_manager_connection_status(False)