put 0MQ subscribers into screens

config/config.cfg.default

@@ -38,10 +38,21 @@ filename=logs.log
 [RedisGlobal]
 host=localhost
 port=6250
-#misp_web_url = http://192.168.56.50
-misp_web_url = http://localhost
-#zmq_url=tcp://192.168.56.50:50000
-zmq_url=tcp://localhost:50000
+misp_web_url = http://0.0.0.0
+misp_instances = [{
+    "name": "misp1",
+    "url": "http://localhost",
+    "zmq": "tcp://localhost:50000"}] 
+
+#misp_instances = [{
+#    "name": "misp1",
+#    "url": "http://localhost",
+#    "zmq": "tcp://localhost:50000"},
+#    {
+#    "name": "misp2",
+#    "url": "http://10.0.2.4",
+#    "zmq": "tcp://10.0.2.4:50000"}
+#    ] 
 
 [RedisLIST]
 db=3

start_all.sh

@@ -24,8 +24,6 @@ fi
 
 netstat -an |grep LISTEN |grep 6250 |grep -v tcp6 ; check_redis_port=$?
 netstat -an |grep LISTEN |grep 8001 |grep -v tcp6 ; check_dashboard_port=$?
-ps auxw |grep zmq_subscriber.py |grep -v grep ; check_zmq_subscriber=$?
-ps auxw |grep zmq_dispatcher.py |grep -v grep ; check_zmq_dispatcher=$?
 
 # Configure accordingly, remember: 0.0.0.0 exposes to every active IP interface, play safe and bind it to something you trust and know
 export FLASK_APP=server.py
@@ -43,22 +41,6 @@ else
     echo -e $RED"\t* NOT starting Redis server, made a very unrealiable check on port 6250, and something seems to be there… please double check if this is good!"$DEFAULT
 fi
 
-sleep 0.1
-if [ "${check_zmq_subscriber}" == "1" ]; then
-    echo -e $GREEN"\t* Launching zmq subscriber"$DEFAULT
-    ${ENV_PY} ./zmq_subscriber.py &
-else
-    echo -e $RED"\t* NOT starting zmq subscriber, made a rather unrealiable ps -auxw | grep for zmq_subscriber.py, and something seems to be there… please double check if this is good!"$DEFAULT
-fi
-
-sleep 0.1
-if [ "${check_zmq_dispatcher}" == "1" ]; then
-    echo -e $GREEN"\t* Launching zmq dispatcher"$DEFAULT
-    ${ENV_PY} ./zmq_dispatcher.py &
-else
-    echo -e $RED"\t* NOT starting zmq dispatcher, made a rather unrealiable ps -auxw | grep for zmq_dispatcher.py, and something seems to be there… please double check if this is good!"$DEFAULT
-fi
-
 sleep 0.1
 if [ "${check_dashboard_port}" == "1" ]; then
     echo -e $GREEN"\t* Launching flask server"$DEFAULT

start_zmq.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+#set -x
+
+GREEN="\\033[1;32m"
+DEFAULT="\\033[0;39m"
+RED="\\033[1;31m"
+
+# Getting CWD where bash script resides
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+DASH_HOME="${DIR}"
+
+cd ${DASH_HOME}
+
+if [ -e "${DIR}/DASHENV/bin/python" ]; then
+    echo "dashboard virtualenv seems to exist, good"
+    ENV_PY="${DIR}/DASHENV/bin/python"
+else
+    echo "Please make sure you have a dashboard environment, au revoir"
+    exit 1
+fi
+
+ps auxw |grep zmq_subscriber.py |grep -v grep ; check_zmq_subscriber=$?
+ps auxw |grep zmq_dispatcher.py |grep -v grep ; check_zmq_dispatcher=$?
+
+screen -dmS "Misp_Dashboard"
+
+sleep 0.1
+if [ "${check_zmq_subscriber}" == "1" ]; then
+    echo -e $GREEN"\t* Launching zmq subscribers"$DEFAULT
+    screen -S "Misp_Dashboard" -X screen -t "zmq-subscribers" bash -c ${ENV_PY}' ./zmq_subscribers.py; read x'
+else
+    echo -e $RED"\t* NOT starting zmq subscribers, made a rather unrealiable ps -auxw | grep for zmq_subscriber.py, and something seems to be there… please double check if this is good!"$DEFAULT
+fi
+
+sleep 0.1
+if [ "${check_zmq_dispatcher}" == "1" ]; then
+    echo -e $GREEN"\t* Launching zmq dispatcher"$DEFAULT
+    screen -S "Misp_Dashboard" -X screen -t "zmq-dispacher" bash -c ${ENV_PY}' ./zmq_dispatcher.py; read x'
+else
+    echo -e $RED"\t* NOT starting zmq dispatcher, made a rather unrealiable ps -auxw | grep for zmq_dispatcher.py, and something seems to be there… please double check if this is good!"$DEFAULT
+fi

zmq_subscriber.py

@@ -21,7 +21,6 @@ if not os.path.exists(logDir):
 logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
 logger = logging.getLogger('zmq_subscriber')
 
-ZMQ_URL = cfg.get('RedisGlobal', 'zmq_url')
 CHANNEL = cfg.get('RedisLog', 'channel')
 LISTNAME = cfg.get('RedisLIST', 'listName')
 
@@ -41,25 +40,26 @@ def put_in_redis_list(zmq_name, content):
     serv_list.lpush(LISTNAME, json.dumps(to_add))
     logger.debug('Pushed: {}'.format(json.dumps(to_add)))
 
-def main(zmqName):
+def main(zmqName, zmqurl):
     context = zmq.Context()
     socket = context.socket(zmq.SUB)
-    socket.connect(ZMQ_URL)
+    socket.connect(zmqurl)
     socket.setsockopt_string(zmq.SUBSCRIBE, '')
 
     while True:
         try:
             content = socket.recv()
             put_in_redis_list(zmqName, content)
+            print(zmqName, content)
         except KeyboardInterrupt:
             return
 
 
 if __name__ == "__main__":
 
-    parser = argparse.ArgumentParser(description='A zmq subscriber. It subscribes to a ZNQ then redispatch it to the misp-dashboard')
+    parser = argparse.ArgumentParser(description='A zmq subscriber. It subscribes to a ZMQ then redispatch it to the misp-dashboard')
     parser.add_argument('-n', '--name', required=False, dest='zmqname', help='The ZMQ feed name', default="MISP Standard ZMQ")
-    parser.add_argument('-u', '--url', required=False, dest='zmqurl', help='The URL to connect to', default=ZMQ_URL)
+    parser.add_argument('-u', '--url', required=False, dest='zmqurl', help='The URL to connect to', default="tcp://localhost:50000")
     args = parser.parse_args()
 
-    main(args.zmqname)
+    main(args.zmqname, args.zmqurl)

zmq_subscribers.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+
+import time, datetime
+import logging
+import redis
+import configparser
+import argparse
+import os
+import subprocess
+import sys
+import json
+import atexit
+import signal
+import shlex
+import pty
+import threading
+
+configfile = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config/config.cfg')
+cfg = configparser.ConfigParser()
+cfg.read(configfile)
+logDir = cfg.get('Log', 'directory')
+logfilename = cfg.get('Log', 'filename')
+logPath = os.path.join(logDir, logfilename)
+if not os.path.exists(logDir):
+    os.makedirs(logDir)
+logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
+logger = logging.getLogger('zmq_subscriber')
+
+CHANNEL = cfg.get('RedisLog', 'channel')
+LISTNAME = cfg.get('RedisLIST', 'listName')
+
+serv_list = redis.StrictRedis(
+        host=cfg.get('RedisGlobal', 'host'),
+        port=cfg.getint('RedisGlobal', 'port'),
+        db=cfg.getint('RedisLIST', 'db'))
+
+children = []
+
+def signal_handler(signal, frame):
+    for child in children:
+        # We don't resume as we are already attached
+        cmd = "screen -p"+child+" -X {arg}"
+        argsc = shlex.split(cmd.format(arg = "kill"))
+        print("\n\033[1;31m [-] Terminating {child}\033[0;39m".format(child=child))
+        logger.info('Terminate: {child}'.format(child=child))
+        subprocess.call(argsc) # kill window
+    sys.exit(0)
+
+###############
+## MAIN LOOP ##
+###############
+
+def main():
+    print("\033[1;31m [+] I am the subscriber's master - kill me to kill'em'all \033[0;39m")
+    # screen needs a shell and I an no fan of shell=True
+    (master, slave) = pty.openpty()
+    try:
+        for item in json.loads(cfg.get('RedisGlobal', 'misp_instances')):
+            name = shlex.quote(item.get("name"))
+            zmq = shlex.quote(item.get("zmq"))
+            print("\033[1;32m [+] Subscribing to "+zmq+"\033[0;39m")
+            logger.info('Launching: {child}'.format(child=name))
+            children.append(name)
+            subprocess.Popen(["screen", "-r", "Misp_Dashboard", "-X", "screen", "-t", name ,sys.executable, "./zmq_subscriber.py", "-n", name, "-u", zmq], close_fds=True, shell=False, stdin=slave, stdout=slave, stderr=slave)
+    except ValueError as error:
+        print("\033[1;31m [!] Fatal exception: {error} \033[0;39m".format(error=error))
+        logger.error("JSON error: %s", error)
+        sys.exit(1)
+    signal.signal(signal.SIGINT, signal_handler)
+    forever = threading.Event()
+    forever.wait()
+
+if __name__ == "__main__":
+    main()