PyMISP/examples/suricata_search/suricata_search.py

217 lines
6.7 KiB
Python
Raw Normal View History

#!/usr/bin/env python
# -*- coding: utf-8 -*-
2017-07-03 22:16:38 +02:00
"""
https://github.com/raw-data/pymisp-suricata_search
2017.06.28 start
2017.07.03 fixed args.quiet and status msgs
"""
import argparse
import os
import queue
import sys
from threading import Thread, enumerate
from keys import misp_url, misp_key, misp_verifycert
try:
from pymisp import PyMISP
except ImportError as err:
sys.stderr.write("ERROR: {}\n".format(err))
sys.stderr.write("\t[try] with pip install pymisp\n")
sys.stderr.write("\t[try] with pip3 install pymisp\n")
sys.exit(1)
HEADER = """
#This part might still contain bugs, use and your own risk and report any issues.
#
# MISP export of IDS rules - optimized for suricata
#
# These NIDS rules contain some variables that need to exist in your configuration.
# Make sure you have set:
#
# $HOME_NET - Your internal network range
# $EXTERNAL_NET - The network considered as outside
# $SMTP_SERVERS - All your internal SMTP servers
# $HTTP_PORTS - The ports used to contain HTTP traffic (not required with suricata export)
#
"""
# queue for events matching searched term/s
IDS_EVENTS = queue.Queue()
# queue for downloaded Suricata rules
DOWNLOADED_RULES = queue.Queue()
# Default number of threads to use
THREAD = 4
try:
input = raw_input
except NameError:
pass
def init():
""" init connection to MISP """
return PyMISP(misp_url, misp_key, misp_verifycert, 'json')
def search(misp, quiet, noevent, **kwargs):
""" Start search in MISP """
result = misp.search(**kwargs)
# fetch all events matching **kwargs
track_events = 0
skip_events = list()
for event in result['response']:
event_id = event["Event"].get("id")
track_events += 1
to_ids = False
for attribute in event["Event"]["Attribute"]:
to_ids_event = attribute["to_ids"]
if to_ids_event:
to_ids = True
break
# if there is at least one eligible event to_ids, add event_id
if to_ids:
# check if the event_id is not blacklisted by the user
if isinstance(noevent, list):
if event_id not in noevent[0]:
to_ids_event = (event_id, misp)
IDS_EVENTS.put(to_ids_event)
else:
skip_events.append(event_id)
else:
to_ids_event = (event_id, misp)
IDS_EVENTS.put(to_ids_event)
if not quiet:
print ("\t[i] matching events: {}".format(track_events))
if len(skip_events) > 0:
print ("\t[i] skipped {0} events -> {1}".format(len(skip_events),skip_events))
print ("\t[i] events selected for IDS export: {}".format(IDS_EVENTS.qsize()))
def collect_rules(thread):
""" Dispatch tasks to Suricata_processor worker """
for x in range(int(thread)):
th = Thread(target=suricata_processor, args=(IDS_EVENTS, ))
th.start()
for x in enumerate():
if x.name == "MainThread":
continue
x.join()
def suricata_processor(ids_events):
""" Trigger misp.download_suricata_rule_event """
while not ids_events.empty():
event_id, misp = ids_events.get()
ids_rules = misp.download_suricata_rule_event(event_id).text
for r in ids_rules.split("\n"):
# skip header
if not r.startswith("#"):
if len(r) > 0: DOWNLOADED_RULES.put(r)
def return_rules(output, quiet):
""" Return downloaded rules to user """
rules = set()
while not DOWNLOADED_RULES.empty():
rules.add(DOWNLOADED_RULES.get())
if output is None:
if not quiet:
print ("[+] Displaying rules")
print (HEADER)
for r in rules: print (r)
print ("#")
else:
if not quiet:
print ("[+] Writing rules to {}".format(output))
print ("[+] Generated {} rules".format(len(rules)))
with open(output, 'w') as f:
f.write(HEADER)
f.write("\n".join(r for r in rules))
f.write("\n"+"#")
def format_request(param, term, misp, quiet, output, thread, noevent):
""" Format request and start search """
kwargs = {param: term}
2017-07-03 22:16:38 +02:00
if not quiet:
print ("[+] Searching for: {}".format(kwargs))
search(misp, quiet, noevent, **kwargs)
# collect Suricata rules
collect_rules(thread)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter,
description='Get all attributes that can be converted into Suricata rules, given a parameter and a term to '
'search.',
epilog='''
EXAMPLES:
suricata_search.py -p tags -s 'APT' -o misp_ids.rules -t 5
suricata_search.py -p tags -s 'APT' -o misp_ids.rules -ne 411 357 343
suricata_search.py -p tags -s 'tlp:green, OSINT' -o misp_ids.rules
suricata_search.py -p tags -s 'circl:incident-classification="malware", tlp:green' -o misp_ids.rules
suricata_search.py -p categories -s 'Artifacts dropped' -t 20 -o artifacts_dropped.rules
''')
parser.add_argument("-p", "--param", required=True, help="Parameter to search (e.g. categories, tags, org, etc.).")
parser.add_argument("-s", "--search", required=True, help="Term/s to search.")
parser.add_argument("-q", "--quiet", action='store_true', help="No status messages")
parser.add_argument("-t", "--thread", required=False, help="Number of threads to use", default=THREAD)
parser.add_argument("-ne", "--noevent", nargs='*', required=False, dest='noevent', action='append',
help="Event/s ID to exclude during the search")
parser.add_argument("-o", "--output", help="Output file",required=False)
args = parser.parse_args()
2017-07-03 22:16:38 +02:00
if args.output is not None and os.path.exists(args.output) and not args.quiet:
try:
check = input("[!] Output file {} exists, do you want to continue [Y/n]? ".format(args.output))
if check not in ["Y","y"]:
exit(0)
except KeyboardInterrupt:
sys.exit(0)
if not args.quiet:
print ("[i] Connecting to MISP instance: {}".format(misp_url))
2017-07-03 22:16:38 +02:00
print ("[i] Note: duplicated IDS rules will be removed")
# Based on # of terms, format request
if "," in args.search:
for term in args.search.split(","):
term = term.strip()
misp = init()
format_request(args.param, term, misp, args.quiet, args.output, args.thread, args.noevent)
else:
2017-07-03 22:16:38 +02:00
misp = init()
format_request(args.param, args.search, misp, args.quiet, args.output, args.thread, args.noevent)
# return collected rules
return_rules(args.output, args.quiet)