From ced0b1e350e85228a5a6ec9a645047d98f5d14e1 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 24 Aug 2018 10:24:03 +0200 Subject: [PATCH] chg: [I2P] add default config --- bin/Crawler.py | 6 ------ bin/packages/config.cfg.sample | 6 ++++-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/bin/Crawler.py b/bin/Crawler.py index 240ae2a3..ab74c64b 100755 --- a/bin/Crawler.py +++ b/bin/Crawler.py @@ -115,22 +115,17 @@ if __name__ == '__main__': # Recovering the streamed message informations. message = r_onion.spop('{}_crawler_queue'.format(type_hidden_service)) - #message='https://www.myip.com/;/home/aurelien/git/python3/AIL-framework/PASTES/crawled/2018/08/10/onionsnjajzkhm5g.onion49eac19d-d71b-48b5-bc55-9a3c63e5b1e2' # # FIXME: remove if message is None: print('get ardb message') message = r_onion.spop('mess_onion') - print(message) - if message is not None: splitted = message.split(';') if len(splitted) == 2: url, paste = splitted - - if not '.onion' in url: print('not onion') continue @@ -143,7 +138,6 @@ if __name__ == '__main__': resource_path, query_string, f1, f2, f3, f4 = url_list domain = url_list[4] r_onion.srem('onion_domain_crawler_queue', domain) - #domain = 'myip.com' domain_url = 'http://{}'.format(domain) diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index 62ea0887..2ca156d4 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -225,5 +225,7 @@ db = 0 [Crawler] crawler_depth_limit = 1 -splash_url = http://127.0.0.1:8050 -http_proxy = http://127.0.0.1:9050 +splash_url_onion = http://127.0.0.1:8050 +splash_url_i2p = http://127.0.0.1:8050 +http_proxy_onion = http://127.0.0.1:9050 +http_proxy_i2p = http://127.0.0.1:9050