From 4ce9888f5d76402367972af43c9603f86b3cc64e Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 18 Sep 2019 17:16:45 +0200 Subject: [PATCH] chg: [analyzer] add export analyzer: syslog, unix, udp fix:#27 --- .../analyzer-d4-export/d4_export_syslog.py | 75 +++++++++++++++++ .../analyzer-d4-export/d4_export_udp.py | 75 +++++++++++++++++ .../analyzer-d4-export/d4_export_unix.py | 80 +++++++++++++++++++ 3 files changed, 230 insertions(+) create mode 100755 server/analyzer/analyzer-d4-export/d4_export_syslog.py create mode 100755 server/analyzer/analyzer-d4-export/d4_export_udp.py create mode 100755 server/analyzer/analyzer-d4-export/d4_export_unix.py diff --git a/server/analyzer/analyzer-d4-export/d4_export_syslog.py b/server/analyzer/analyzer-d4-export/d4_export_syslog.py new file mode 100755 index 0000000..6ca78db --- /dev/null +++ b/server/analyzer/analyzer-d4-export/d4_export_syslog.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +import os +import sys +import time +import redis +import socket +import argparse + +import logging +import logging.handlers + +log_level = {'DEBUG': 10, 'INFO': 20, 'WARNING': 30, 'ERROR': 40, 'CRITICAL': 50} + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Export d4 data to stdout') + parser.add_argument('-t', '--type', help='d4 type or extended type' , type=str, dest='type', required=True) + parser.add_argument('-u', '--uuid', help='queue uuid' , type=str, dest='uuid', required=True) + parser.add_argument('-i', '--ip',help='server ip' , type=str, default='127.0.0.1', dest='target_ip') + parser.add_argument('-p', '--port',help='server port' ,type=int, default=514, dest='target_port') + parser.add_argument('-l', '--log_level', help='log level: DEBUG, INFO, WARNING, ERROR, CRITICAL', type=str, default='INFO', dest='req_level') + parser.add_argument('-n', '--newline', help='add new lines', action="store_true") + parser.add_argument('-ri', '--redis_ip',help='redis host' , type=str, default='127.0.0.1', dest='host_redis') + parser.add_argument('-rp', '--redis_port',help='redis port' , type=int, default=6380, dest='port_redis') + args = parser.parse_args() + + if not args.uuid or not args.type or not args.target_port: + parser.print_help() + sys.exit(0) + + host_redis=args.host_redis + port_redis=args.port_redis + newLines = args.newline + req_level = args.req_level + + if req_level not in log_level: + print('ERROR: incorrect log level') + sys.exit(0) + + redis_d4= redis.StrictRedis( + host=host_redis, + port=port_redis, + db=2) + try: + redis_d4.ping() + except redis.exceptions.ConnectionError: + print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis)) + sys.exit(1) + + d4_uuid = args.uuid + d4_type = args.type + data_queue = 'analyzer:{}:{}'.format(d4_type, d4_uuid) + + target_ip = args.target_ip + target_port = args.target_port + addr = (target_ip, target_port) + + syslog_logger = logging.getLogger('D4-SYSLOGOUT') + syslog_logger.setLevel(logging.DEBUG) + client_socket = logging.handlers.SysLogHandler(address = addr) + syslog_logger.addHandler(client_socket) + + while True: + + d4_data = redis_d4.rpop(data_queue) + if d4_data is None: + time.sleep(1) + continue + + if newLines: + d4_data = d4_data + b'\n' + + syslog_logger.log(log_level[req_level], d4_data.decode()) + + client_socket.close() diff --git a/server/analyzer/analyzer-d4-export/d4_export_udp.py b/server/analyzer/analyzer-d4-export/d4_export_udp.py new file mode 100755 index 0000000..b5ba0d9 --- /dev/null +++ b/server/analyzer/analyzer-d4-export/d4_export_udp.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +import os +import sys + +import redis +import time +import datetime + +import argparse +import logging +import logging.handlers + + +import socket + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Export d4 data to stdout') + parser.add_argument('-t', '--type', help='d4 type or extended type' , type=str, dest='type', required=True) + parser.add_argument('-u', '--uuid', help='queue uuid' , type=str, dest='uuid', required=True) + parser.add_argument('-i', '--ip',help='server ip' , type=str, default='127.0.0.1', dest='target_ip') + parser.add_argument('-p', '--port',help='server port' , type=int, dest='target_port', required=True) + parser.add_argument('-l', '--log_level', help='log level: DEBUG, INFO, WARNING, ERROR, CRITICAL', type=str, default='INFO', dest='req_level') + parser.add_argument('-n', '--newline', help='add new lines', action="store_true") + parser.add_argument('-ri', '--redis_ip',help='redis host' , type=str, default='127.0.0.1', dest='host_redis') + parser.add_argument('-rp', '--redis_port',help='redis port' , type=int, default=6380, dest='port_redis') + args = parser.parse_args() + + if not args.uuid or not args.type or not args.target_port: + parser.print_help() + sys.exit(0) + + host_redis=args.host_redis + port_redis=args.port_redis + newLines = args.newline + req_level = args.req_level + + redis_d4= redis.StrictRedis( + host=host_redis, + port=port_redis, + db=2) + try: + redis_d4.ping() + except redis.exceptions.ConnectionError: + print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis)) + sys.exit(1) + + d4_uuid = args.uuid + d4_type = args.type + data_queue = 'analyzer:{}:{}'.format(d4_type, d4_uuid) + + target_ip = args.target_ip + target_port = args.target_port + addr = (target_ip, target_port) + + #Create a UDP socket + client_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + + newLines=True + while True: + + d4_data = redis_d4.rpop(data_queue) + if d4_data is None: + time.sleep(1) + continue + + if newLines: + d4_data = d4_data + b'\n' + + print(d4_data) + client_socket.sendto(d4_data, addr) + + client_socket.close() diff --git a/server/analyzer/analyzer-d4-export/d4_export_unix.py b/server/analyzer/analyzer-d4-export/d4_export_unix.py new file mode 100755 index 0000000..2ef24e9 --- /dev/null +++ b/server/analyzer/analyzer-d4-export/d4_export_unix.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 + +import os +import sys + +import redis +import time +import datetime + +import argparse +import logging +import logging.handlers + + +import socket + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Export d4 data to stdout') + parser.add_argument('-t', '--type', help='d4 type or extended type' , type=str, dest='type', required=True) + parser.add_argument('-u', '--uuid', help='queue uuid' , type=str, dest='uuid', required=True) + parser.add_argument('-s', '--socket',help='socket file' , type=str, dest='socket_file', required=True) + parser.add_argument('-n', '--newline', help='add new lines', action="store_true") + parser.add_argument('-ri', '--redis_ip',help='redis host' , type=str, default='127.0.0.1', dest='host_redis') + parser.add_argument('-rp', '--redis_port',help='redis port' , type=int, default=6380, dest='port_redis') + args = parser.parse_args() + + if not args.uuid or not args.type or not args.socket_file: + parser.print_help() + sys.exit(0) + + host_redis=args.host_redis + port_redis=args.port_redis + newLines = args.newline + + redis_d4= redis.StrictRedis( + host=host_redis, + port=port_redis, + db=2) + try: + redis_d4.ping() + except redis.exceptions.ConnectionError: + print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis)) + sys.exit(1) + + d4_uuid = args.uuid + d4_type = args.type + data_queue = 'analyzer:{}:{}'.format(d4_type, d4_uuid) + + + socket_file = args.socket_file + print("UNIX SOCKET: Connecting...") + if os.path.exists(socket_file): + client = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + client.connect(socket_file) + print("Connected") + else: + print("Couldn't Connect!") + print("ERROR: socket file not found") + print("Done") + + + + newLines=False + while True: + + d4_data = redis_d4.rpop(data_queue) + if d4_data is None: + time.sleep(1) + continue + + if newLines: + d4_data = d4_data + b'\n' + + print(d4_data) + + client.send(d4_data) + + client.close()