mirror of https://github.com/MISP/misp-dashboard
put 0MQ subscribers into screens
parent
1e91b9b59c
commit
787edbc301
|
@ -38,10 +38,21 @@ filename=logs.log
|
||||||
[RedisGlobal]
|
[RedisGlobal]
|
||||||
host=localhost
|
host=localhost
|
||||||
port=6250
|
port=6250
|
||||||
#misp_web_url = http://192.168.56.50
|
misp_web_url = http://0.0.0.0
|
||||||
misp_web_url = http://localhost
|
misp_instances = [{
|
||||||
#zmq_url=tcp://192.168.56.50:50000
|
"name": "misp1",
|
||||||
zmq_url=tcp://localhost:50000
|
"url": "http://localhost",
|
||||||
|
"zmq": "tcp://localhost:50000"}]
|
||||||
|
|
||||||
|
#misp_instances = [{
|
||||||
|
# "name": "misp1",
|
||||||
|
# "url": "http://localhost",
|
||||||
|
# "zmq": "tcp://localhost:50000"},
|
||||||
|
# {
|
||||||
|
# "name": "misp2",
|
||||||
|
# "url": "http://10.0.2.4",
|
||||||
|
# "zmq": "tcp://10.0.2.4:50000"}
|
||||||
|
# ]
|
||||||
|
|
||||||
[RedisLIST]
|
[RedisLIST]
|
||||||
db=3
|
db=3
|
||||||
|
|
18
start_all.sh
18
start_all.sh
|
@ -24,8 +24,6 @@ fi
|
||||||
|
|
||||||
netstat -an |grep LISTEN |grep 6250 |grep -v tcp6 ; check_redis_port=$?
|
netstat -an |grep LISTEN |grep 6250 |grep -v tcp6 ; check_redis_port=$?
|
||||||
netstat -an |grep LISTEN |grep 8001 |grep -v tcp6 ; check_dashboard_port=$?
|
netstat -an |grep LISTEN |grep 8001 |grep -v tcp6 ; check_dashboard_port=$?
|
||||||
ps auxw |grep zmq_subscriber.py |grep -v grep ; check_zmq_subscriber=$?
|
|
||||||
ps auxw |grep zmq_dispatcher.py |grep -v grep ; check_zmq_dispatcher=$?
|
|
||||||
|
|
||||||
# Configure accordingly, remember: 0.0.0.0 exposes to every active IP interface, play safe and bind it to something you trust and know
|
# Configure accordingly, remember: 0.0.0.0 exposes to every active IP interface, play safe and bind it to something you trust and know
|
||||||
export FLASK_APP=server.py
|
export FLASK_APP=server.py
|
||||||
|
@ -43,22 +41,6 @@ else
|
||||||
echo -e $RED"\t* NOT starting Redis server, made a very unrealiable check on port 6250, and something seems to be there… please double check if this is good!"$DEFAULT
|
echo -e $RED"\t* NOT starting Redis server, made a very unrealiable check on port 6250, and something seems to be there… please double check if this is good!"$DEFAULT
|
||||||
fi
|
fi
|
||||||
|
|
||||||
sleep 0.1
|
|
||||||
if [ "${check_zmq_subscriber}" == "1" ]; then
|
|
||||||
echo -e $GREEN"\t* Launching zmq subscriber"$DEFAULT
|
|
||||||
${ENV_PY} ./zmq_subscriber.py &
|
|
||||||
else
|
|
||||||
echo -e $RED"\t* NOT starting zmq subscriber, made a rather unrealiable ps -auxw | grep for zmq_subscriber.py, and something seems to be there… please double check if this is good!"$DEFAULT
|
|
||||||
fi
|
|
||||||
|
|
||||||
sleep 0.1
|
|
||||||
if [ "${check_zmq_dispatcher}" == "1" ]; then
|
|
||||||
echo -e $GREEN"\t* Launching zmq dispatcher"$DEFAULT
|
|
||||||
${ENV_PY} ./zmq_dispatcher.py &
|
|
||||||
else
|
|
||||||
echo -e $RED"\t* NOT starting zmq dispatcher, made a rather unrealiable ps -auxw | grep for zmq_dispatcher.py, and something seems to be there… please double check if this is good!"$DEFAULT
|
|
||||||
fi
|
|
||||||
|
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
if [ "${check_dashboard_port}" == "1" ]; then
|
if [ "${check_dashboard_port}" == "1" ]; then
|
||||||
echo -e $GREEN"\t* Launching flask server"$DEFAULT
|
echo -e $GREEN"\t* Launching flask server"$DEFAULT
|
||||||
|
|
|
@ -0,0 +1,42 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
#set -x
|
||||||
|
|
||||||
|
GREEN="\\033[1;32m"
|
||||||
|
DEFAULT="\\033[0;39m"
|
||||||
|
RED="\\033[1;31m"
|
||||||
|
|
||||||
|
# Getting CWD where bash script resides
|
||||||
|
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||||
|
DASH_HOME="${DIR}"
|
||||||
|
|
||||||
|
cd ${DASH_HOME}
|
||||||
|
|
||||||
|
if [ -e "${DIR}/DASHENV/bin/python" ]; then
|
||||||
|
echo "dashboard virtualenv seems to exist, good"
|
||||||
|
ENV_PY="${DIR}/DASHENV/bin/python"
|
||||||
|
else
|
||||||
|
echo "Please make sure you have a dashboard environment, au revoir"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
ps auxw |grep zmq_subscriber.py |grep -v grep ; check_zmq_subscriber=$?
|
||||||
|
ps auxw |grep zmq_dispatcher.py |grep -v grep ; check_zmq_dispatcher=$?
|
||||||
|
|
||||||
|
screen -dmS "Misp_Dashboard"
|
||||||
|
|
||||||
|
sleep 0.1
|
||||||
|
if [ "${check_zmq_subscriber}" == "1" ]; then
|
||||||
|
echo -e $GREEN"\t* Launching zmq subscribers"$DEFAULT
|
||||||
|
screen -S "Misp_Dashboard" -X screen -t "zmq-subscribers" bash -c ${ENV_PY}' ./zmq_subscribers.py; read x'
|
||||||
|
else
|
||||||
|
echo -e $RED"\t* NOT starting zmq subscribers, made a rather unrealiable ps -auxw | grep for zmq_subscriber.py, and something seems to be there… please double check if this is good!"$DEFAULT
|
||||||
|
fi
|
||||||
|
|
||||||
|
sleep 0.1
|
||||||
|
if [ "${check_zmq_dispatcher}" == "1" ]; then
|
||||||
|
echo -e $GREEN"\t* Launching zmq dispatcher"$DEFAULT
|
||||||
|
screen -S "Misp_Dashboard" -X screen -t "zmq-dispacher" bash -c ${ENV_PY}' ./zmq_dispatcher.py; read x'
|
||||||
|
else
|
||||||
|
echo -e $RED"\t* NOT starting zmq dispatcher, made a rather unrealiable ps -auxw | grep for zmq_dispatcher.py, and something seems to be there… please double check if this is good!"$DEFAULT
|
||||||
|
fi
|
|
@ -21,7 +21,6 @@ if not os.path.exists(logDir):
|
||||||
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
|
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
|
||||||
logger = logging.getLogger('zmq_subscriber')
|
logger = logging.getLogger('zmq_subscriber')
|
||||||
|
|
||||||
ZMQ_URL = cfg.get('RedisGlobal', 'zmq_url')
|
|
||||||
CHANNEL = cfg.get('RedisLog', 'channel')
|
CHANNEL = cfg.get('RedisLog', 'channel')
|
||||||
LISTNAME = cfg.get('RedisLIST', 'listName')
|
LISTNAME = cfg.get('RedisLIST', 'listName')
|
||||||
|
|
||||||
|
@ -41,25 +40,26 @@ def put_in_redis_list(zmq_name, content):
|
||||||
serv_list.lpush(LISTNAME, json.dumps(to_add))
|
serv_list.lpush(LISTNAME, json.dumps(to_add))
|
||||||
logger.debug('Pushed: {}'.format(json.dumps(to_add)))
|
logger.debug('Pushed: {}'.format(json.dumps(to_add)))
|
||||||
|
|
||||||
def main(zmqName):
|
def main(zmqName, zmqurl):
|
||||||
context = zmq.Context()
|
context = zmq.Context()
|
||||||
socket = context.socket(zmq.SUB)
|
socket = context.socket(zmq.SUB)
|
||||||
socket.connect(ZMQ_URL)
|
socket.connect(zmqurl)
|
||||||
socket.setsockopt_string(zmq.SUBSCRIBE, '')
|
socket.setsockopt_string(zmq.SUBSCRIBE, '')
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
content = socket.recv()
|
content = socket.recv()
|
||||||
put_in_redis_list(zmqName, content)
|
put_in_redis_list(zmqName, content)
|
||||||
|
print(zmqName, content)
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description='A zmq subscriber. It subscribes to a ZNQ then redispatch it to the misp-dashboard')
|
parser = argparse.ArgumentParser(description='A zmq subscriber. It subscribes to a ZMQ then redispatch it to the misp-dashboard')
|
||||||
parser.add_argument('-n', '--name', required=False, dest='zmqname', help='The ZMQ feed name', default="MISP Standard ZMQ")
|
parser.add_argument('-n', '--name', required=False, dest='zmqname', help='The ZMQ feed name', default="MISP Standard ZMQ")
|
||||||
parser.add_argument('-u', '--url', required=False, dest='zmqurl', help='The URL to connect to', default=ZMQ_URL)
|
parser.add_argument('-u', '--url', required=False, dest='zmqurl', help='The URL to connect to', default="tcp://localhost:50000")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
main(args.zmqname)
|
main(args.zmqname, args.zmqurl)
|
||||||
|
|
|
@ -0,0 +1,74 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import time, datetime
|
||||||
|
import logging
|
||||||
|
import redis
|
||||||
|
import configparser
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import atexit
|
||||||
|
import signal
|
||||||
|
import shlex
|
||||||
|
import pty
|
||||||
|
import threading
|
||||||
|
|
||||||
|
configfile = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config/config.cfg')
|
||||||
|
cfg = configparser.ConfigParser()
|
||||||
|
cfg.read(configfile)
|
||||||
|
logDir = cfg.get('Log', 'directory')
|
||||||
|
logfilename = cfg.get('Log', 'filename')
|
||||||
|
logPath = os.path.join(logDir, logfilename)
|
||||||
|
if not os.path.exists(logDir):
|
||||||
|
os.makedirs(logDir)
|
||||||
|
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
|
||||||
|
logger = logging.getLogger('zmq_subscriber')
|
||||||
|
|
||||||
|
CHANNEL = cfg.get('RedisLog', 'channel')
|
||||||
|
LISTNAME = cfg.get('RedisLIST', 'listName')
|
||||||
|
|
||||||
|
serv_list = redis.StrictRedis(
|
||||||
|
host=cfg.get('RedisGlobal', 'host'),
|
||||||
|
port=cfg.getint('RedisGlobal', 'port'),
|
||||||
|
db=cfg.getint('RedisLIST', 'db'))
|
||||||
|
|
||||||
|
children = []
|
||||||
|
|
||||||
|
def signal_handler(signal, frame):
|
||||||
|
for child in children:
|
||||||
|
# We don't resume as we are already attached
|
||||||
|
cmd = "screen -p"+child+" -X {arg}"
|
||||||
|
argsc = shlex.split(cmd.format(arg = "kill"))
|
||||||
|
print("\n\033[1;31m [-] Terminating {child}\033[0;39m".format(child=child))
|
||||||
|
logger.info('Terminate: {child}'.format(child=child))
|
||||||
|
subprocess.call(argsc) # kill window
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
###############
|
||||||
|
## MAIN LOOP ##
|
||||||
|
###############
|
||||||
|
|
||||||
|
def main():
|
||||||
|
print("\033[1;31m [+] I am the subscriber's master - kill me to kill'em'all \033[0;39m")
|
||||||
|
# screen needs a shell and I an no fan of shell=True
|
||||||
|
(master, slave) = pty.openpty()
|
||||||
|
try:
|
||||||
|
for item in json.loads(cfg.get('RedisGlobal', 'misp_instances')):
|
||||||
|
name = shlex.quote(item.get("name"))
|
||||||
|
zmq = shlex.quote(item.get("zmq"))
|
||||||
|
print("\033[1;32m [+] Subscribing to "+zmq+"\033[0;39m")
|
||||||
|
logger.info('Launching: {child}'.format(child=name))
|
||||||
|
children.append(name)
|
||||||
|
subprocess.Popen(["screen", "-r", "Misp_Dashboard", "-X", "screen", "-t", name ,sys.executable, "./zmq_subscriber.py", "-n", name, "-u", zmq], close_fds=True, shell=False, stdin=slave, stdout=slave, stderr=slave)
|
||||||
|
except ValueError as error:
|
||||||
|
print("\033[1;31m [!] Fatal exception: {error} \033[0;39m".format(error=error))
|
||||||
|
logger.error("JSON error: %s", error)
|
||||||
|
sys.exit(1)
|
||||||
|
signal.signal(signal.SIGINT, signal_handler)
|
||||||
|
forever = threading.Event()
|
||||||
|
forever.wait()
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
Loading…
Reference in New Issue