CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [Crawler] fix crawler cache info

pull/333/head
Terrtia 2019-01-29 12:09:19 +01:00
parent bb301a870c
commit 2dc0eca4a9
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
1 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
bin/Crawler.py
View File

@ -42,7 +42,7 @@ def crawl_onion(url, domain, date, date_month, message):
# TODO: relaunch docker or send error message # TODO: relaunch docker or send error message
nb_retry += 1 nb_retry += 1
if nb_retry == 30: if nb_retry == 6:
on_error_send_message_back_in_queue(type_hidden_service, domain, message) on_error_send_message_back_in_queue(type_hidden_service, domain, message)
publisher.error('{} SPASH DOWN'.format(splash_url)) publisher.error('{} SPASH DOWN'.format(splash_url))
print('--------------------------------------') print('--------------------------------------')
@ -187,6 +187,8 @@ if __name__ == '__main__':
domain_url = 'http://{}'.format(domain) domain_url = 'http://{}'.format(domain)
print()
print()
print('\033[92m------------------START CRAWLER------------------\033[0m') print('\033[92m------------------START CRAWLER------------------\033[0m')
print('crawler type: {}'.format(type_hidden_service)) print('crawler type: {}'.format(type_hidden_service))
print('\033[92m-------------------------------------------------\033[0m') print('\033[92m-------------------------------------------------\033[0m')
@ -262,7 +264,11 @@ if __name__ == '__main__':
#update crawler status #update crawler status
r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'Waiting') r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'Waiting')
r_cache.hrem('metadata_crawler:{}'.format(splash_port), 'crawling_domain') r_cache.hdel('metadata_crawler:{}'.format(splash_port), 'crawling_domain')
else:
print(' Blacklisted Onion')
print()
print()
else: else:
continue continue