From 8803c8447a49b5d50cda3bc169a52b0ddd1060ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 30 Sep 2014 16:55:16 +0200 Subject: [PATCH] Publish the fetched onions on a ZMQ feed. --- bin/Onion.py | 4 +++- bin/packages/config.cfg.sample | 4 ++++ bin/packages/modules.cfg | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/bin/Onion.py b/bin/Onion.py index 45a8a6aa..c47174bc 100755 --- a/bin/Onion.py +++ b/bin/Onion.py @@ -54,7 +54,8 @@ def fetch(p, r_cache, urls, domains, path): tempfile = process.stdout.read().strip() with open(tempfile, 'r') as f: filename = path + domain + '.gz' - content = base64.standard_b64decode(f.read()) + fetched = f.read() + content = base64.standard_b64decode(fetched) save_path = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "pastes"), filename) @@ -65,6 +66,7 @@ def fetch(p, r_cache, urls, domains, path): ff.write(content) p.populate_set_out(save_path, 'Global') p.populate_set_out(url, 'ValidOnion') + p.populate_set_out(fetched, 'FetchedOnion') yield url os.unlink(tempfile) else: diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index ac8488f8..8e7268bf 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -53,6 +53,10 @@ channel = 102 address = tcp://127.0.0.1:5004 channel = urls +[ZMQ_FetchedOnion] +address = tcp://127.0.0.1:5005 +channel = FetchedOnion + [RedisPubSub] host = localhost port = 6381 diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index c0731fc1..90453a29 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -37,7 +37,7 @@ subscribe = Redis_Mail [Onion] subscribe = Redis_Onion -publish = Redis_Global,Redis_ValidOnion +publish = Redis_Global,Redis_ValidOnion,ZMQ_FetchedOnion [DumpValidOnion] subscribe = Redis_ValidOnion