CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [I2P] add default config

pull/260/head
Terrtia 2018-08-24 10:24:03 +02:00
parent 7e24943537
commit ced0b1e350
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
2 changed files with 4 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
bin/Crawler.py
View File

@ -115,22 +115,17 @@ if __name__ == '__main__':
# Recovering the streamed message informations. # Recovering the streamed message informations.
message = r_onion.spop('{}_crawler_queue'.format(type_hidden_service)) message = r_onion.spop('{}_crawler_queue'.format(type_hidden_service))
#message='https://www.myip.com/;/home/aurelien/git/python3/AIL-framework/PASTES/crawled/2018/08/10/onionsnjajzkhm5g.onion49eac19d-d71b-48b5-bc55-9a3c63e5b1e2'
# # FIXME: remove # # FIXME: remove
if message is None: if message is None:
print('get ardb message') print('get ardb message')
message = r_onion.spop('mess_onion') message = r_onion.spop('mess_onion')
print(message)
if message is not None: if message is not None:
splitted = message.split(';') splitted = message.split(';')
if len(splitted) == 2: if len(splitted) == 2:
url, paste = splitted url, paste = splitted
if not '.onion' in url: if not '.onion' in url:
print('not onion') print('not onion')
continue continue
@ -143,7 +138,6 @@ if __name__ == '__main__':
resource_path, query_string, f1, f2, f3, f4 = url_list resource_path, query_string, f1, f2, f3, f4 = url_list
domain = url_list[4] domain = url_list[4]
r_onion.srem('onion_domain_crawler_queue', domain) r_onion.srem('onion_domain_crawler_queue', domain)
#domain = 'myip.com'
domain_url = 'http://{}'.format(domain) domain_url = 'http://{}'.format(domain)

6
bin/packages/config.cfg.sample
View File

@ -225,5 +225,7 @@ db = 0
[Crawler] [Crawler]
crawler_depth_limit = 1 crawler_depth_limit = 1
splash_url = http://127.0.0.1:8050 splash_url_onion = http://127.0.0.1:8050
http_proxy = http://127.0.0.1:9050 splash_url_i2p = http://127.0.0.1:8050
http_proxy_onion = http://127.0.0.1:9050
http_proxy_i2p = http://127.0.0.1:9050