put 0MQ subscribers into screens

pull/85/head
Jean-Louis Huynen 2019-02-26 15:45:38 +01:00
parent 1e91b9b59c
commit 787edbc301
No known key found for this signature in database
GPG Key ID: 4CDEBABACE9A5DC9
5 changed files with 137 additions and 28 deletions

View File

@ -38,10 +38,21 @@ filename=logs.log
[RedisGlobal] [RedisGlobal]
host=localhost host=localhost
port=6250 port=6250
#misp_web_url = http://192.168.56.50 misp_web_url = http://0.0.0.0
misp_web_url = http://localhost misp_instances = [{
#zmq_url=tcp://192.168.56.50:50000 "name": "misp1",
zmq_url=tcp://localhost:50000 "url": "http://localhost",
"zmq": "tcp://localhost:50000"}]
#misp_instances = [{
# "name": "misp1",
# "url": "http://localhost",
# "zmq": "tcp://localhost:50000"},
# {
# "name": "misp2",
# "url": "http://10.0.2.4",
# "zmq": "tcp://10.0.2.4:50000"}
# ]
[RedisLIST] [RedisLIST]
db=3 db=3

View File

@ -24,8 +24,6 @@ fi
netstat -an |grep LISTEN |grep 6250 |grep -v tcp6 ; check_redis_port=$? netstat -an |grep LISTEN |grep 6250 |grep -v tcp6 ; check_redis_port=$?
netstat -an |grep LISTEN |grep 8001 |grep -v tcp6 ; check_dashboard_port=$? netstat -an |grep LISTEN |grep 8001 |grep -v tcp6 ; check_dashboard_port=$?
ps auxw |grep zmq_subscriber.py |grep -v grep ; check_zmq_subscriber=$?
ps auxw |grep zmq_dispatcher.py |grep -v grep ; check_zmq_dispatcher=$?
# Configure accordingly, remember: 0.0.0.0 exposes to every active IP interface, play safe and bind it to something you trust and know # Configure accordingly, remember: 0.0.0.0 exposes to every active IP interface, play safe and bind it to something you trust and know
export FLASK_APP=server.py export FLASK_APP=server.py
@ -43,22 +41,6 @@ else
echo -e $RED"\t* NOT starting Redis server, made a very unrealiable check on port 6250, and something seems to be there… please double check if this is good!"$DEFAULT echo -e $RED"\t* NOT starting Redis server, made a very unrealiable check on port 6250, and something seems to be there… please double check if this is good!"$DEFAULT
fi fi
sleep 0.1
if [ "${check_zmq_subscriber}" == "1" ]; then
echo -e $GREEN"\t* Launching zmq subscriber"$DEFAULT
${ENV_PY} ./zmq_subscriber.py &
else
echo -e $RED"\t* NOT starting zmq subscriber, made a rather unrealiable ps -auxw | grep for zmq_subscriber.py, and something seems to be there… please double check if this is good!"$DEFAULT
fi
sleep 0.1
if [ "${check_zmq_dispatcher}" == "1" ]; then
echo -e $GREEN"\t* Launching zmq dispatcher"$DEFAULT
${ENV_PY} ./zmq_dispatcher.py &
else
echo -e $RED"\t* NOT starting zmq dispatcher, made a rather unrealiable ps -auxw | grep for zmq_dispatcher.py, and something seems to be there… please double check if this is good!"$DEFAULT
fi
sleep 0.1 sleep 0.1
if [ "${check_dashboard_port}" == "1" ]; then if [ "${check_dashboard_port}" == "1" ]; then
echo -e $GREEN"\t* Launching flask server"$DEFAULT echo -e $GREEN"\t* Launching flask server"$DEFAULT

42
start_zmq.sh Executable file
View File

@ -0,0 +1,42 @@
#!/usr/bin/env bash
#set -x
GREEN="\\033[1;32m"
DEFAULT="\\033[0;39m"
RED="\\033[1;31m"
# Getting CWD where bash script resides
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
DASH_HOME="${DIR}"
cd ${DASH_HOME}
if [ -e "${DIR}/DASHENV/bin/python" ]; then
echo "dashboard virtualenv seems to exist, good"
ENV_PY="${DIR}/DASHENV/bin/python"
else
echo "Please make sure you have a dashboard environment, au revoir"
exit 1
fi
ps auxw |grep zmq_subscriber.py |grep -v grep ; check_zmq_subscriber=$?
ps auxw |grep zmq_dispatcher.py |grep -v grep ; check_zmq_dispatcher=$?
screen -dmS "Misp_Dashboard"
sleep 0.1
if [ "${check_zmq_subscriber}" == "1" ]; then
echo -e $GREEN"\t* Launching zmq subscribers"$DEFAULT
screen -S "Misp_Dashboard" -X screen -t "zmq-subscribers" bash -c ${ENV_PY}' ./zmq_subscribers.py; read x'
else
echo -e $RED"\t* NOT starting zmq subscribers, made a rather unrealiable ps -auxw | grep for zmq_subscriber.py, and something seems to be there… please double check if this is good!"$DEFAULT
fi
sleep 0.1
if [ "${check_zmq_dispatcher}" == "1" ]; then
echo -e $GREEN"\t* Launching zmq dispatcher"$DEFAULT
screen -S "Misp_Dashboard" -X screen -t "zmq-dispacher" bash -c ${ENV_PY}' ./zmq_dispatcher.py; read x'
else
echo -e $RED"\t* NOT starting zmq dispatcher, made a rather unrealiable ps -auxw | grep for zmq_dispatcher.py, and something seems to be there… please double check if this is good!"$DEFAULT
fi

View File

@ -21,7 +21,6 @@ if not os.path.exists(logDir):
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO) logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
logger = logging.getLogger('zmq_subscriber') logger = logging.getLogger('zmq_subscriber')
ZMQ_URL = cfg.get('RedisGlobal', 'zmq_url')
CHANNEL = cfg.get('RedisLog', 'channel') CHANNEL = cfg.get('RedisLog', 'channel')
LISTNAME = cfg.get('RedisLIST', 'listName') LISTNAME = cfg.get('RedisLIST', 'listName')
@ -41,25 +40,26 @@ def put_in_redis_list(zmq_name, content):
serv_list.lpush(LISTNAME, json.dumps(to_add)) serv_list.lpush(LISTNAME, json.dumps(to_add))
logger.debug('Pushed: {}'.format(json.dumps(to_add))) logger.debug('Pushed: {}'.format(json.dumps(to_add)))
def main(zmqName): def main(zmqName, zmqurl):
context = zmq.Context() context = zmq.Context()
socket = context.socket(zmq.SUB) socket = context.socket(zmq.SUB)
socket.connect(ZMQ_URL) socket.connect(zmqurl)
socket.setsockopt_string(zmq.SUBSCRIBE, '') socket.setsockopt_string(zmq.SUBSCRIBE, '')
while True: while True:
try: try:
content = socket.recv() content = socket.recv()
put_in_redis_list(zmqName, content) put_in_redis_list(zmqName, content)
print(zmqName, content)
except KeyboardInterrupt: except KeyboardInterrupt:
return return
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description='A zmq subscriber. It subscribes to a ZNQ then redispatch it to the misp-dashboard') parser = argparse.ArgumentParser(description='A zmq subscriber. It subscribes to a ZMQ then redispatch it to the misp-dashboard')
parser.add_argument('-n', '--name', required=False, dest='zmqname', help='The ZMQ feed name', default="MISP Standard ZMQ") parser.add_argument('-n', '--name', required=False, dest='zmqname', help='The ZMQ feed name', default="MISP Standard ZMQ")
parser.add_argument('-u', '--url', required=False, dest='zmqurl', help='The URL to connect to', default=ZMQ_URL) parser.add_argument('-u', '--url', required=False, dest='zmqurl', help='The URL to connect to', default="tcp://localhost:50000")
args = parser.parse_args() args = parser.parse_args()
main(args.zmqname) main(args.zmqname, args.zmqurl)

74
zmq_subscribers.py Executable file
View File

@ -0,0 +1,74 @@
#!/usr/bin/env python3
import time, datetime
import logging
import redis
import configparser
import argparse
import os
import subprocess
import sys
import json
import atexit
import signal
import shlex
import pty
import threading
configfile = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config/config.cfg')
cfg = configparser.ConfigParser()
cfg.read(configfile)
logDir = cfg.get('Log', 'directory')
logfilename = cfg.get('Log', 'filename')
logPath = os.path.join(logDir, logfilename)
if not os.path.exists(logDir):
os.makedirs(logDir)
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
logger = logging.getLogger('zmq_subscriber')
CHANNEL = cfg.get('RedisLog', 'channel')
LISTNAME = cfg.get('RedisLIST', 'listName')
serv_list = redis.StrictRedis(
host=cfg.get('RedisGlobal', 'host'),
port=cfg.getint('RedisGlobal', 'port'),
db=cfg.getint('RedisLIST', 'db'))
children = []
def signal_handler(signal, frame):
for child in children:
# We don't resume as we are already attached
cmd = "screen -p"+child+" -X {arg}"
argsc = shlex.split(cmd.format(arg = "kill"))
print("\n\033[1;31m [-] Terminating {child}\033[0;39m".format(child=child))
logger.info('Terminate: {child}'.format(child=child))
subprocess.call(argsc) # kill window
sys.exit(0)
###############
## MAIN LOOP ##
###############
def main():
print("\033[1;31m [+] I am the subscriber's master - kill me to kill'em'all \033[0;39m")
# screen needs a shell and I an no fan of shell=True
(master, slave) = pty.openpty()
try:
for item in json.loads(cfg.get('RedisGlobal', 'misp_instances')):
name = shlex.quote(item.get("name"))
zmq = shlex.quote(item.get("zmq"))
print("\033[1;32m [+] Subscribing to "+zmq+"\033[0;39m")
logger.info('Launching: {child}'.format(child=name))
children.append(name)
subprocess.Popen(["screen", "-r", "Misp_Dashboard", "-X", "screen", "-t", name ,sys.executable, "./zmq_subscriber.py", "-n", name, "-u", zmq], close_fds=True, shell=False, stdin=slave, stdout=slave, stderr=slave)
except ValueError as error:
print("\033[1;31m [!] Fatal exception: {error} \033[0;39m".format(error=error))
logger.error("JSON error: %s", error)
sys.exit(1)
signal.signal(signal.SIGINT, signal_handler)
forever = threading.Event()
forever.wait()
if __name__ == "__main__":
main()