CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

Update starting script.

pull/38/head
Raphaël Vinot 2014-09-02 15:21:36 +02:00
parent 0c6b09f379
commit 7542eaf739
11 changed files with 84 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
bin/Attributes.py → bin/Attribute.py
View File

0
bin/CreditCards.py → bin/CreditCard.py
View File

0
bin/Duplicates.py → bin/Duplicate.py
View File

3
bin/Helper.py
View File

@ -45,9 +45,8 @@ class PubSub(object):
host=self.config.get('RedisPubSub', 'host'),
port=self.config.get('RedisPubSub', 'port'),
db=self.config.get('RedisPubSub', 'db'))
self.subscriber = r.pubsub()
self.subscriber = r.pubsub(ignore_subscribe_messages=True)
self.subscriber.psubscribe(channel)
self.subscriber.get_message()
elif conn_name.startswith('ZMQ'):
self.zmq_sub = True
context = zmq.Context()

50
bin/LAUNCH.sh
View File

@ -90,30 +90,8 @@ function launching_queues {
screen -dmS "Queue"
sleep 0.1
echo -e $GREEN"\t* Launching redis ZMQ queues"$DEFAULT
screen -S "Queue" -X screen -t "QFeed" bash -c './ZMQ_Feed_Q.py; read x'
sleep 0.1
screen -S "Queue" -X screen -t "QDuplicate" bash -c './ZMQ_Sub_Duplicate_Q.py; read x'
sleep 0.1
screen -S "Queue" -X screen -t "QAttributes" bash -c './ZMQ_Sub_Attributes_Q.py; read x'
sleep 0.1
screen -S "Queue" -X screen -t "Qlines" bash -c './ZMQ_PubSub_Lines_Q.py; read x'
sleep 0.1
screen -S "Queue" -X screen -t "QCateg" bash -c './ZMQ_PubSub_Categ_Q.py; read x'
sleep 0.1
screen -S "Queue" -X screen -t "QTokenize" bash -c './ZMQ_PubSub_Tokenize_Q.py; read x'
sleep 0.1
screen -S "Queue" -X screen -t "Qcreditcard" bash -c './ZMQ_Sub_CreditCards_Q.py; read x'
sleep 0.1
screen -S "Queue" -X screen -t "QOnion" bash -c './ZMQ_Sub_Onion_Q.py; read x'
sleep 0.1
screen -S "Queue" -X screen -t "Qmails" bash -c './ZMQ_Sub_Mails_Q.py; read x'
sleep 0.1
screen -S "Queue" -X screen -t "Qurls" bash -c './ZMQ_Sub_Urls_Q.py; read x'
sleep 0.1
screen -S "Queue" -X screen -t "QCurve" bash -c './ZMQ_Sub_Curve_Q.py; read x'
sleep 0.1
screen -S "Queue" -X screen -t "QIndexer" bash -c './ZMQ_Sub_Indexer_Q.py; read x'
echo -e $GREEN"\t* Launching all the queues"$DEFAULT
screen -S "Queue" -X screen -t "Queues" bash -c './launch_queues.py; read x'
}
function launching_scripts {
@ -122,29 +100,29 @@ function launching_scripts {
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
screen -S "Script" -X screen -t "Feed" bash -c './ZMQ_Feed.py; read x'
screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Duplicate" bash -c './ZMQ_Sub_Duplicate.py; read x'
screen -S "Script" -X screen -t "Duplicate" bash -c './Duplicate.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Attributes" bash -c './ZMQ_Sub_Attributes.py; read x'
screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Lines" bash -c './ZMQ_PubSub_Lines.py; read x'
screen -S "Script" -X screen -t "Line" bash -c './Line.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Categ" bash -c './ZMQ_PubSub_Categ.py; read x'
screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Tokenize" bash -c './ZMQ_PubSub_Tokenize.py; read x'
screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Creditcard" bash -c './ZMQ_Sub_CreditCards.py; read x'
screen -S "Script" -X screen -t "CreditCard" bash -c './CreditCard.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Onion" bash -c './ZMQ_Sub_Onion.py; read x'
screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Mails" bash -c './ZMQ_Sub_Mails.py; read x'
screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Urls" bash -c './ZMQ_Sub_Urls.py; read x'
screen -S "Script" -X screen -t "Url" bash -c './Url.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Curve" bash -c './ZMQ_Sub_Curve.py; read x'
screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Indexer" bash -c './ZMQ_Sub_Indexer.py; read x'
screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
}
#If no params, display the help

0
bin/Lines.py → bin/Line.py
View File

6
bin/Onion.py
View File

@ -35,8 +35,9 @@ from Helper import Process
def fetch(p, r_cache, urls, domains, path):
failed = []
for url, domain in zip(urls, domains):
if r_cache.exists(url):
if r_cache.exists(url) or url in failed:
continue
to_fetch = base64.standard_b64encode(url)
process = subprocess.Popen(["python", './tor_fetcher.py', to_fetch],
@ -64,6 +65,7 @@ def fetch(p, r_cache, urls, domains, path):
yield url
os.unlink(tempfile)
else:
failed.append(url)
print 'Failed at downloading', url
print process.stdout.read()
@ -136,7 +138,7 @@ if __name__ == "__main__":
PST.p_date,
PST.p_name)
for url in fetch(p, r_cache, urls, domains_list, path):
publisher.warning('{}Valid: {}'.format(to_print, url))
publisher.warning('{}Checked {}'.format(to_print, url))
else:
publisher.info('{}Onion related'.format(to_print))

0
bin/Web.py → bin/Url.py
View File

65
bin/launch_queues.py Executable file
View File

@ -0,0 +1,65 @@
#!/usr/bin/env python2
# -*-coding:UTF-8 -*
import ConfigParser
import os
import subprocess
import time
def check_pid(pid):
if pid is None:
# Already seen as finished.
return None
else:
if pid.poll() is not None:
return False
return True
if __name__ == '__main__':
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/modules.cfg')
if not os.path.exists(configfile):
raise Exception('Unable to find the configuration file. \
Did you set environment variables? \
Or activate the virtualenv.')
config = ConfigParser.ConfigParser()
config.read(configfile)
modules = config.sections()
pids = {}
for module in modules:
pin = subprocess.Popen(["python", './QueueIn.py', '-c', module])
pout = subprocess.Popen(["python", './QueueOut.py', '-c', module])
pids[module] = (pin, pout)
is_running = True
try:
while is_running:
time.sleep(5)
is_running = False
for module, p in pids.iteritems():
pin, pout = p
if pin is None:
# already dead
pass
elif not check_pid(pin):
print(module, 'input queue finished.')
pin = None
else:
is_running = True
if pout is None:
# already dead
pass
elif not check_pid(pout):
print(module, 'output queue finished.')
pout = None
else:
is_running = True
pids[module] = (pin, pout)
except KeyboardInterrupt:
for module, p in pids.iteritems():
pin, pout = p
if pin is not None:
pin.kill()
if pout is not None:
pout.kill()

22
bin/run_modules.py
View File

@ -1,22 +0,0 @@
#!/usr/bin/env python2
# -*-coding:UTF-8 -*
import ConfigParser
import os
import subprocess
if __name__ == '__main__':
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/modules.cfg')
if not os.path.exists(configfile):
raise Exception('Unable to find the configuration file. \
Did you set environment variables? \
Or activate the virtualenv.')
config = ConfigParser.ConfigParser()
config.read(configfile)
modules = config.sections()
for module in modules:
subprocess.Popen(["python", './QueueIn.py', '-c', module])
subprocess.Popen(["python", './QueueOut.py', '-c', module])
#subprocess.Popen(["python", './{}.py'.format(module)])

1
var/www/Flask_server.py
View File

@ -55,7 +55,6 @@ def logs():
@app.route("/_stuff", methods=['GET'])
def stuff():
print get_queues(r_serv)
return jsonify(row1=get_queues(r_serv))