CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [Crawler] faup

pull/559/head
Terrtia 2021-03-05 18:47:38 +01:00
parent 3a347a79b7
commit 6daa750e3b
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
2 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bin/Crawler.py
View File

@ -289,7 +289,9 @@ if __name__ == '__main__':
print(f'SPLASH Name: {splash_name}') print(f'SPLASH Name: {splash_name}')
print(f'Proxy Name: {proxy_name}') print(f'Proxy Name: {proxy_name}')
print(f'Crawler Type: {get_splash_crawler_type(splash_name)})
#time.sleep(10)
#sys.exit(0) #sys.exit(0)
#rotation_mode = deque(['onion', 'regular']) #rotation_mode = deque(['onion', 'regular'])
@ -330,7 +332,7 @@ if __name__ == '__main__':
db=p.config.getint("ARDB_Onion", "db"), db=p.config.getint("ARDB_Onion", "db"),
decode_responses=True) decode_responses=True)
faup = Faup() faup = crawlers.get_faup()
# get HAR files # get HAR files
default_crawler_har = p.config.getboolean("Crawler", "default_crawler_har") default_crawler_har = p.config.getboolean("Crawler", "default_crawler_har")

6
bin/lib/crawlers.py
View File

@ -67,6 +67,10 @@ def is_valid_onion_domain(domain):
return True return True
return False return False
# TEMP FIX
def get_faup():
return faup
################################################################################ ################################################################################
# # TODO: handle prefix cookies # # TODO: handle prefix cookies
@ -894,7 +898,7 @@ def get_splash_manager_session_uuid():
return res['session_uuid'] return res['session_uuid']
else: else:
print(req.json()) print(req.json())
except requests.exceptions.ConnectionError: except requests.exceptions.ConnectionError, requests.exceptions.MissingSchema:
# splash manager unreachable # splash manager unreachable
update_splash_manager_connection_status(False) update_splash_manager_connection_status(False)