CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

Publish the fetched onions on a ZMQ feed.

pull/38/head
Raphaël Vinot 2014-09-30 16:55:16 +02:00
parent f15caa70f5
commit 8803c8447a
3 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bin/Onion.py
View File

@ -54,7 +54,8 @@ def fetch(p, r_cache, urls, domains, path):
tempfile = process.stdout.read().strip() tempfile = process.stdout.read().strip()
with open(tempfile, 'r') as f: with open(tempfile, 'r') as f:
filename = path + domain + '.gz' filename = path + domain + '.gz'
content = base64.standard_b64decode(f.read()) fetched = f.read()
content = base64.standard_b64decode(fetched)
save_path = os.path.join(os.environ['AIL_HOME'], save_path = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "pastes"), p.config.get("Directories", "pastes"),
filename) filename)
@ -65,6 +66,7 @@ def fetch(p, r_cache, urls, domains, path):
ff.write(content) ff.write(content)
p.populate_set_out(save_path, 'Global') p.populate_set_out(save_path, 'Global')
p.populate_set_out(url, 'ValidOnion') p.populate_set_out(url, 'ValidOnion')
p.populate_set_out(fetched, 'FetchedOnion')
yield url yield url
os.unlink(tempfile) os.unlink(tempfile)
else: else:

4
bin/packages/config.cfg.sample
View File

@ -53,6 +53,10 @@ channel = 102
address = tcp://127.0.0.1:5004 address = tcp://127.0.0.1:5004
channel = urls channel = urls
[ZMQ_FetchedOnion]
address = tcp://127.0.0.1:5005
channel = FetchedOnion
[RedisPubSub] [RedisPubSub]
host = localhost host = localhost
port = 6381 port = 6381

2
bin/packages/modules.cfg
View File

@ -37,7 +37,7 @@ subscribe = Redis_Mail
[Onion] [Onion]
subscribe = Redis_Onion subscribe = Redis_Onion
publish = Redis_Global,Redis_ValidOnion publish = Redis_Global,Redis_ValidOnion,ZMQ_FetchedOnion
[DumpValidOnion] [DumpValidOnion]
subscribe = Redis_ValidOnion subscribe = Redis_ValidOnion