diff --git a/.travis.yml b/.travis.yml index ec9b16f..c70bed3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,12 +9,12 @@ python: - "3.4" - "3.5" - "3.5-dev" + - "3.6" + - "3.6-dev" - "nightly" install: - - pip install -U nose - - pip install coveralls - - pip install codecov + - pip install -U nose codecov pytest - pip install -U -r REQUIREMENTS - pip install . @@ -35,4 +35,3 @@ script: after_success: - coverage combine .coverage* - codecov - - coveralls diff --git a/README.md b/README.md index ed364ff..7d0c17f 100644 --- a/README.md +++ b/README.md @@ -26,19 +26,26 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en). * [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind. * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address. +* [iprep](misp-modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net. * [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets. +* [shodan](misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module. * [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance. +* [threatminer](misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/). * [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to. * [virustotal](misp_modules/modules/expansion/virustotal.py) - an expansion module to pull known resolutions and malware samples related with an IP/Domain from virusTotal (this modules require a VirusTotal private API key) +* [wikidata](misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module. +* [xforce](misp_modules/modules/expansion/xforceexchange.py) - an IBM X-Force Exchange expansion module. ### Export modules * [CEF](misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF). +* [Lite Export](/misp-modules/blob/master/misp_modules/modules/export_mod/liteexport.py) module to export a lite event. ### Import modules * [Cuckoo JSON](misp_modules/modules/import_mod/cuckooimport.py) Cuckoo JSON import. * [OCR](misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes. +* [OpenIOC](misp_modules/modules/import_mod/openiocimport.py) OpenIOC import based on PyMISP library. * [stiximport](misp_modules/modules/import_mod/stiximport.py) - An import module to process STIX xml/json. * [Email Import](misp_modules/modules/import_mod/email_import.py) Email import module for MISP to import basic metadata. * [VMRay](misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export. @@ -53,7 +60,7 @@ cd misp-modules sudo pip3 install -I -r REQUIREMENTS sudo pip3 install -I . sudo vi /etc/rc.local, add this line: `sudo -u www-data misp-modules -s &` -/usr/local/bin/misp-modules #to start the modules +misp-modules #to start the modules ~~~~ ## How to add your own MISP modules? @@ -352,6 +359,23 @@ Recommended Plugin.Import_ocr_enabled true Enable or disable the ocr In this same menu set any other plugin settings that are required for testing. +## Install misp-module on an offline instance. +First, you need to grab all necessery packages for example like this : + +Use pip wheel to create an archive +~~~ +mkdir misp-modules-offline +pip3 wheel -r REQUIREMENTS shodan --wheel-dir=./misp-modules-offline +tar -cjvf misp-module-bundeled.tar.bz2 ./misp-modules-offline/* +~~~ +On offline machine : +~~~ +mkdir misp-modules-bundle +tar xvf misp-module-bundeled.tar.bz2 -C misp-modules-bundle +cd misp-modules-bundle +ls -1|while read line; do sudo pip3 install --force-reinstall --ignore-installed --upgrade --no-index --no-deps ${line};done +~~~ +Next you can follow standard install procedure. ## How to contribute your own module? diff --git a/REQUIREMENTS b/REQUIREMENTS index 6cda15a..0665aac 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -13,9 +13,10 @@ ipasn-redis asnhistory git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client git+https://github.com/MISP/MISP-STIX-Converter.git#egg=misp_stix_converter -git+https://github.com/CIRCL/PyMISP.git#egg=pymisp +git+https://github.com/MISP/PyMISP.git#egg=pymisp pillow pytesseract SPARQLWrapper domaintools_api pygeoip +bs4 diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 914cb1c..de3c5c3 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -2,4 +2,4 @@ from . import _vmray __all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'ipasn', 'passivetotal', 'sourcecache', - 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki'] + 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer'] diff --git a/misp_modules/modules/expansion/iprep.py b/misp_modules/modules/expansion/iprep.py new file mode 100755 index 0000000..6073052 --- /dev/null +++ b/misp_modules/modules/expansion/iprep.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- + +import json +import requests + + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['text']} +moduleinfo = {'version': '1.0', 'author': 'Keith Faber', + 'description': 'Query IPRep Data for IP Address', + 'module-type': ['expansion']} + +moduleconfig = ['apikey'] + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if request.get('ip-src'): + toquery = request['ip-src'] + elif request.get('ip-dst'): + toquery = request['ip-dst'] + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + + if not request.get('config') and not request['config'].get('apikey'): + misperrors['error'] = 'IPRep api key is missing' + return misperrors + + err, rep = parse_iprep(toquery, request['config'].get('apikey')) + if len(err) > 0: + misperrors['error'] = ','.join(err) + return misperrors + return {'results': rep} + + +def parse_iprep(ip, api): + meta_fields = ['origin', 'Query_Time', 'created_on', 'IP_Lookup_History', 'IPs_in_collection', '_id', 'disclaimer', + 'MaxMind_Free_GeoIP', 'Unique_Lookups', 'query_result'] + rep = [] + err = [] + full_text = '' + url = 'https://www.packetmail.net/iprep.php/%s' % ip + try: + data = requests.get(url, params={'apikey': api}).json() + except: + return ['Error pulling data'], rep + # print '%s' % data + for name, val in data.items(): + if name not in meta_fields: + try: + context = val['context'] + if type(context) is list: + if context[0].get('alert'): + context = ','.join([hit['alert']['signature'] for hit in context]) + elif context[0].get('signature'): + context = ','.join([hit['signature'] for hit in context]) + elif context[0].get('target_port') and context[0].get('protocol'): + context = ','.join( + ['Port Attacked: %s %s' % (hit['target_port'], hit['protocol']) for hit in context]) + elif context[0].get('phishing_kit') and context[0].get('url'): + context = ','.join(['%s (%s)' % (hit['phishing_kit'], hit['url']) for hit in context]) + else: + context = ';'.join(['%s: %s' % (k, v) for k, v in context[0].items()]) + + if val.get('special_note'): + context += '; ' + val['special_note'] + + misp_val = context + full_text += '\n%s' % context + misp_comment = 'IPRep Source %s: %s' % (name, val['last_seen']) + rep.append({'types': mispattributes['output'], 'categories':['External analysis'], 'values': misp_val, 'comment': misp_comment}) + except: + err.append('Error parsing source: %s' % name) + + rep.append({'types': ['freetext'], 'values': full_text , 'comment': 'Free text import of IPRep'}) + return err, rep + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/threatminer.py b/misp_modules/modules/expansion/threatminer.py new file mode 100755 index 0000000..1a4be50 --- /dev/null +++ b/misp_modules/modules/expansion/threatminer.py @@ -0,0 +1,173 @@ +import json +import requests +from requests import HTTPError +import base64 + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['hostname', 'domain', 'ip-src', 'ip-dst', 'md5', 'sha1', 'sha256', 'sha512'], + 'output': ['domain', 'ip-src', 'ip-dst', 'text', 'md5', 'sha1', 'sha256', 'sha512', 'ssdeep', + 'authentihash', 'filename', 'whois-registrant-email', 'url', 'link'] + } + +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '1', 'author': 'KX499', + 'description': 'Get information from ThreatMiner', + 'module-type': ['expansion']} + +desc = '{}: Threatminer - {}' + + +def handler(q=False): + if q is False: + return False + + q = json.loads(q) + + r = {'results': []} + + if 'ip-src' in q: + r['results'] += get_ip(q['ip-src']) + if 'ip-dst' in q: + r['results'] += get_ip(q['ip-dst']) + if 'domain' in q: + r['results'] += get_domain(q['domain']) + if 'hostname' in q: + r['results'] += get_domain(q['hostname']) + if 'md5' in q: + r['results'] += get_hash(q['md5']) + if 'sha1' in q: + r['results'] += get_hash(q['sha1']) + if 'sha256' in q: + r['results'] += get_hash(q['sha256']) + if 'sha512' in q: + r['results'] += get_hash(q['sha512']) + + uniq = [] + for res in r['results']: + if res not in uniq: + uniq.append(res) + r['results'] = uniq + return r + + +def get_domain(q): + ret = [] + for flag in [1, 2, 3, 4, 5, 6]: + req = requests.get('https://www.threatminer.org/domain.php', params={'q': q, 'api': 'True', 'rt': flag}) + if not req.status_code == 200: + continue + results = req.json().get('results') + if not results: + continue + + for result in results: + if flag == 1: #whois + emails = result.get('whois', {}).get('emails') + if not emails: + continue + for em_type, email in emails.items(): + ret.append({'types': ['whois-registrant-email'], 'values': [email], 'comment': desc.format(q, 'whois')}) + if flag == 2: #pdns + ip = result.get('ip') + if ip: + ret.append({'types': ['ip-src', 'ip-dst'], 'values': [ip], 'comment': desc.format(q, 'pdns')}) + if flag == 3: #uri + uri = result.get('uri') + if uri: + ret.append({'types': ['url'], 'values': [uri], 'comment': desc.format(q, 'uri')}) + if flag == 4: #samples + if type(result) is str: + ret.append({'types': ['sha256'], 'values': [result], 'comment': desc.format(q, 'samples')}) + if flag == 5: #subdomains + if type(result) is str: + ret.append({'types': ['domain'], 'values': [result], 'comment': desc.format(q, 'subdomain')}) + if flag == 6: #reports + link = result.get('URL') + if link: + ret.append({'types': ['url'], 'values': [link], 'comment': desc.format(q, 'report')}) + + return ret + + +def get_ip(q): + ret = [] + for flag in [1, 2, 3, 4, 5, 6]: + req = requests.get('https://www.threatminer.org/host.php', params={'q': q, 'api': 'True', 'rt': flag}) + if not req.status_code == 200: + continue + results = req.json().get('results') + if not results: + continue + + for result in results: + if flag == 1: #whois + emails = result.get('whois', {}).get('emails') + if not emails: + continue + for em_type, email in emails.items(): + ret.append({'types': ['whois-registrant-email'], 'values': [email], 'comment': desc.format(q, 'whois')}) + if flag == 2: #pdns + ip = result.get('ip') + if ip: + ret.append({'types': ['ip-src', 'ip-dst'], 'values': [ip], 'comment': desc.format(q, 'pdns')}) + if flag == 3: #uri + uri = result.get('uri') + if uri: + ret.append({'types': ['url'], 'values': [uri], 'comment': desc.format(q, 'uri')}) + if flag == 4: #samples + if type(result) is str: + ret.append({'types': ['sha256'], 'values': [result], 'comment': desc.format(q, 'samples')}) + if flag == 5: #ssl + if type(result) is str: + ret.append({'types': ['x509-fingerprint-sha1'], 'values': [result], 'comment': desc.format(q, 'ssl')}) + if flag == 6: #reports + link = result.get('URL') + if link: + ret.append({'types': ['url'], 'values': [link], 'comment': desc.format(q, 'report')}) + + return ret + + +def get_hash(q): + ret = [] + for flag in [1, 3, 6, 7]: + req = requests.get('https://www.threatminer.org/sample.php', params={'q': q, 'api': 'True', 'rt': flag}) + if not req.status_code == 200: + continue + results = req.json().get('results') + if not results: + continue + + for result in results: + if flag == 1: #meta (filename) + name = result.get('file_name') + if name: + ret.append({'types': ['filename'], 'values': [name], 'comment': desc.format(q, 'file')}) + if flag == 3: #network + domains = result.get('domains') + for dom in domains: + if dom.get('domain'): + ret.append({'types': ['domain'], 'values': [dom['domain']], 'comment': desc.format(q, 'network')}) + + hosts = result.get('hosts') + for h in hosts: + if type(h) is str: + ret.append({'types': ['ip-src', 'ip-dst'], 'values': [h], 'comment': desc.format(q, 'network')}) + if flag == 6: #detections + detections = result.get('av_detections') + for d in detections: + if d.get('detection'): + ret.append({'types': ['text'], 'values': [d['detection']], 'comment': desc.format(q, 'detection')}) + if flag == 7: #report + if type(result) is str: + ret.append({'types': ['sha256'], 'values': [result], 'comment': desc.format(q, 'report')}) + + return ret + + +def introspection(): + return mispattributes + + +def version(): + return moduleinfo diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index c0c7715..44199a1 100755 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -1,23 +1,24 @@ import json import requests -import hashlib -import re +from requests import HTTPError import base64 -import os misperrors = {'error': 'Error'} -mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst"], - 'output':['domain', "ip-src", "ip-dst", "text"] +mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "sha512"], + 'output': ['domain', "ip-src", "ip-dst", "text", "md5", "sha1", "sha256", "sha512", "ssdeep", + "authentihash", "filename"] } # possible module-types: 'expansion', 'hover' or both -moduleinfo = {'version': '', 'author': 'Hannah Ward', +moduleinfo = {'version': '2', 'author': 'Hannah Ward', 'description': 'Get information from virustotal', 'module-type': ['expansion']} # config fields that your code expects from the site admin moduleconfig = ["apikey", "event_limit"] -limit = 5 #Default +limit = 5 # Default +comment = '%s: Enriched via VT' + def handler(q=False): global limit @@ -30,124 +31,172 @@ def handler(q=False): limit = int(q["config"].get("event_limit", 5)) r = {"results": []} - + if "ip-src" in q: - r["results"] += getIP(q["ip-src"], key) + r["results"] += getIP(q["ip-src"], key) if "ip-dst" in q: - r["results"] += getIP(q["ip-dst"], key) + r["results"] += getIP(q["ip-dst"], key) if "domain" in q: - r["results"] += getDomain(q["domain"], key) + r["results"] += getDomain(q["domain"], key) if 'hostname' in q: - r["results"] += getDomain(q['hostname'], key) + r["results"] += getDomain(q['hostname'], key) + if 'md5' in q: + r["results"] += getHash(q['md5'], key) + if 'sha1' in q: + r["results"] += getHash(q['sha1'], key) + if 'sha256' in q: + r["results"] += getHash(q['sha256'], key) + if 'sha512' in q: + r["results"] += getHash(q['sha512'], key) uniq = [] for res in r["results"]: - if res not in uniq: - uniq.append(res) + if res not in uniq: + uniq.append(res) r["results"] = uniq return r -def getIP(ip, key, do_not_recurse = False): + +def getHash(hash, key, do_not_recurse=False): + req = requests.get("https://www.virustotal.com/vtapi/v2/file/report", + params={"allinfo": 1, "apikey": key, 'resource': hash}) + try: + req.raise_for_status() + req = req.json() + except HTTPError as e: + misperrors['error'] = str(e) + return misperrors + + if req["response_code"] == 0: + # Nothing found + return [] + + return getMoreInfo(req, key) + + +def getIP(ip, key, do_not_recurse=False): global limit toReturn = [] - req = requests.get("https://www.virustotal.com/vtapi/v2/ip-address/report", - params = {"ip":ip, "apikey":key} - ).json() + req = requests.get("https://www.virustotal.com/vtapi/v2/ip-address/report", + params={"ip": ip, "apikey": key}) + try: + req.raise_for_status() + req = req.json() + except HTTPError as e: + misperrors['error'] = str(e) + return misperrors + if req["response_code"] == 0: - #Nothing found - return [] - + # Nothing found + return [] + if "resolutions" in req: - for res in req["resolutions"][:limit]: - toReturn.append( {"types":["domain"], "values":[res["hostname"]]}) - #Pivot from here to find all domain info - if not do_not_recurse: - toReturn += getDomain(res["hostname"], key, True) + for res in req["resolutions"][:limit]: + toReturn.append({"types": ["domain"], "values": [res["hostname"]], "comment": comment % ip}) + # Pivot from here to find all domain info + if not do_not_recurse: + toReturn += getDomain(res["hostname"], key, True) toReturn += getMoreInfo(req, key) return toReturn - + + def getDomain(domain, key, do_not_recurse=False): global limit toReturn = [] - req = requests.get("https://www.virustotal.com/vtapi/v2/domain/report", - params = {"domain":domain, "apikey":key} - ).json() + req = requests.get("https://www.virustotal.com/vtapi/v2/domain/report", + params={"domain": domain, "apikey": key}) + try: + req.raise_for_status() + req = req.json() + except HTTPError as e: + misperrors['error'] = str(e) + return misperrors + if req["response_code"] == 0: - #Nothing found - return [] - + # Nothing found + return [] + if "resolutions" in req: - for res in req["resolutions"][:limit]: - toReturn.append( {"types":["ip-dst", "ip-src"], "values":[res["ip_address"]]}) - #Pivot from here to find all info on IPs - if not do_not_recurse: - toReturn += getIP(res["ip_address"], key, True) + for res in req["resolutions"][:limit]: + toReturn.append({"types": ["ip-dst", "ip-src"], "values": [res["ip_address"]], "comment": comment % domain}) + # Pivot from here to find all info on IPs + if not do_not_recurse: + toReturn += getIP(res["ip_address"], key, True) + if "subdomains" in req: + for subd in req["subdomains"]: + toReturn.append({"types": ["domain"], "values": [subd], "comment": comment % domain}) toReturn += getMoreInfo(req, key) return toReturn + def findAll(data, keys): - a = [] - if isinstance(data, dict): - for key in data.keys(): - if key in keys: - a.append(data[key]) - else: - if isinstance(data[key], (dict, list)): - a += findAll(data[key], keys) - if isinstance(data, list): - for i in data: - a += findAll(i, keys) - - return a + a = [] + if isinstance(data, dict): + for key in data.keys(): + if key in keys: + a.append(data[key]) + else: + if isinstance(data[key], (dict, list)): + a += findAll(data[key], keys) + if isinstance(data, list): + for i in data: + a += findAll(i, keys) + + return a + def getMoreInfo(req, key): global limit r = [] - #Get all hashes first + # Get all hashes first hashes = [] hashes = findAll(req, ["md5", "sha1", "sha256", "sha512"]) - r.append({"types":["md5", "sha1", "sha256", "sha512"], "values":hashes}) + r.append({"types": ["md5", "sha1", "sha256", "sha512"], "values": hashes}) for hsh in hashes[:limit]: - #Search VT for some juicy info - data = requests.get("http://www.virustotal.com/vtapi/v2/file/report", - params={"allinfo":1, "apikey":key, "resource":hsh} - ).json() + # Search VT for some juicy info + try: + data = requests.get("http://www.virustotal.com/vtapi/v2/file/report", + params={"allinfo": 1, "apikey": key, "resource": hsh} + ).json() + except: + continue - # Go through each key and check if it exists - if "submission_names" in data: - r.append({'types':["filename"], "values":data["submission_names"]}) + # Go through each key and check if it exists + if "submission_names" in data: + r.append({'types': ["filename"], "values": data["submission_names"], "comment": comment % hsh}) - if "ssdeep" in data: - r.append({'types':["ssdeep"], "values":[data["ssdeep"]]}) + if "ssdeep" in data: + r.append({'types': ["ssdeep"], "values": [data["ssdeep"]], "comment": comment % hsh}) - if "authentihash" in data: - r.append({"types":["authentihash"], "values":[data["authentihash"]]}) + if "authentihash" in data: + r.append({"types": ["authentihash"], "values": [data["authentihash"]], "comment": comment % hsh}) - if "ITW_urls" in data: - r.append({"types":["url"], "values":data["ITW_urls"]}) + if "ITW_urls" in data: + r.append({"types": ["url"], "values": data["ITW_urls"], "comment": comment % hsh}) - #Get the malware sample - sample = requests.get("https://www.virustotal.com/vtapi/v2/file/download", - params = {"hash":hsh, "apikey":key}) - - malsample = sample.content + # Get the malware sample + sample = requests.get("https://www.virustotal.com/vtapi/v2/file/download", + params={"hash": hsh, "apikey": key}) + + malsample = sample.content + + # It is possible for VT to not give us any submission names + if "submission_names" in data: + r.append({"types": ["malware-sample"], + "categories": ["Payload delivery"], + "values": data["submission_names"], + "data": str(base64.b64encode(malsample), 'utf-8') + } + ) - # It is possible for VT to not give us any submission names - if "submission_names" in data: - r.append({"types":["malware-sample"], - "categories":["Payload delivery"], - "values":data["submission_names"], - "data": str(base64.b64encode(malsample), 'utf-8') - } - ) - return r + def introspection(): return mispattributes + def version(): moduleinfo['config'] = moduleconfig return moduleinfo - diff --git a/misp_modules/modules/expansion/xforceexchange.py b/misp_modules/modules/expansion/xforceexchange.py new file mode 100644 index 0000000..459c69a --- /dev/null +++ b/misp_modules/modules/expansion/xforceexchange.py @@ -0,0 +1,101 @@ +import requests +import json +import sys + +BASEurl = "https://api.xforce.ibmcloud.com/" + +extensions = {"ip1": "ipr/%s", + "ip2": "ipr/malware/%s", + "url": "url/%s", + "hash": "malware/%s", + "vuln": "/vulnerabilities/search/%s", + "dns": "resolve/%s"} + +sys.path.append('./') + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-src','ip-dst' 'vulnerability', 'md5', 'sha1', 'sha256'], + 'output': ['ip-src', 'ip-dst', 'text', 'domain']} + +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '1', 'author': 'Joerg Stephan (@johest)', + 'description': 'IBM X-Force Exchange expansion module', + 'module-type': ['expansion', 'hover']} + +# config fields that your code expects from the site admin +moduleconfig = ["apikey", "event_limit"] +limit = 5000 #Default + + + +def MyHeader(key=False): + global limit + if key is False: + return None + + return {"Authorization": "Basic %s " % key, + "Accept": "application/json", + 'User-Agent': 'Mozilla 5.0'} + +def handler(q=False): + global limit + if q is False: + return False + + q = json.loads(q) + + key = q["config"]["apikey"] + limit = int(q["config"].get("event_limit", 5)) + + r = {"results": []} + + if "ip-src" in q: + r["results"] += apicall("dns", q["ip-src"], key) + if "ip-dst" in q: + r["results"] += apicall("dns", q["ip-dst"], key) + if "md5" in q: + r["results"] += apicall("hash", q["md5"], key) + if "sha1" in q: + r["results"] += apicall("hash", q["sha1"], key) + if "sha256" in q: + r["results"] += apicall("hash", q["sha256"], key) + if 'vulnerability' in q: + r["results"] += apicall("vuln", q["vulnerability"], key) + if "domain" in q: + r["results"] += apicall("dns", q["domain"], key) + + uniq = [] + for res in r["results"]: + if res not in uniq: + uniq.append(res) + r["results"] = uniq + return r + +def apicall(indicator_type, indicator, key=False): + try: + myURL = BASEurl + (extensions[str(indicator_type)])%indicator + jsondata = requests.get(myURL, headers=MyHeader(key)).json() + except: + jsondata = None + redata = [] + #print(jsondata) + if not jsondata is None: + if indicator_type is "hash": + if "malware" in jsondata: + lopointer = jsondata["malware"] + redata.append({"type": "text", "values": lopointer["risk"]}) + if indicator_type is "dns": + if "records" in str(jsondata): + lopointer = jsondata["Passive"]["records"] + for dataset in lopointer: + redata.append({"type":"domain", "values": dataset["value"]}) + + return redata + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py index 69f1c00..ee457cf 100644 --- a/misp_modules/modules/export_mod/__init__.py +++ b/misp_modules/modules/export_mod/__init__.py @@ -1 +1 @@ -__all__ = ['testexport','cef_export'] +__all__ = ['testexport','cef_export','liteexport'] diff --git a/misp_modules/modules/export_mod/liteexport.py b/misp_modules/modules/export_mod/liteexport.py new file mode 100755 index 0000000..5d47489 --- /dev/null +++ b/misp_modules/modules/export_mod/liteexport.py @@ -0,0 +1,89 @@ +import json +import base64 + +misperrors = {'error': 'Error'} + +moduleinfo = {'version': '1', + 'author': 'TM', + 'description': 'export lite', + 'module-type': ['export']} + +moduleconfig = ["indent_json_export"] + +mispattributes = {} +outputFileExtension = "json" +responseType = "application/json" + +def handler(q=False): + if q is False: + return False + + request = json.loads(q) + + config = {} + if "config" in request: + config = request["config"] + else: + config = {"indent_json_export" : None} + + if config['indent_json_export'] is not None: + try: + config['indent_json_export'] = int(config['indent_json_export']) + except: + config['indent_json_export'] = None + + if 'data' not in request: + return False + + #~ Misp json structur + liteEvent = {'Event':{}} + + for evt in request['data']: + rawEvent = evt['Event'] + liteEvent['Event']['info'] = rawEvent['info'] + liteEvent['Event']['Attribute'] = [] + + attrs = evt['Attribute'] + for attr in attrs: + if 'Internal reference' not in attr['category']: + liteAttr = {} + liteAttr['category'] = attr['category'] + liteAttr['type'] = attr['type'] + liteAttr['value'] = attr['value'] + liteEvent['Event']['Attribute'].append(liteAttr) + + return {'response' : [], + 'data' : str(base64.b64encode( + bytes( + json.dumps(liteEvent, indent=config['indent_json_export']), + 'utf-8')), + 'utf-8') + } + +def introspection(): + modulesetup = {} + try: + responseType + modulesetup['responseType'] = responseType + except NameError: + pass + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + outputFileExtension + modulesetup['outputFileExtension'] = outputFileExtension + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py index 82505d9..6beeaa2 100644 --- a/misp_modules/modules/import_mod/__init__.py +++ b/misp_modules/modules/import_mod/__init__.py @@ -1,3 +1,4 @@ from . import _vmray -__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport', 'email_import'] +__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport', + 'email_import', 'mispjson', 'openiocimport'] diff --git a/misp_modules/modules/import_mod/mispjson.py b/misp_modules/modules/import_mod/mispjson.py new file mode 100755 index 0000000..f9d52ec --- /dev/null +++ b/misp_modules/modules/import_mod/mispjson.py @@ -0,0 +1,61 @@ +import json +import base64 + +misperrors = {'error': 'Error'} +userConfig = { }; + +inputSource = ['file'] + +moduleinfo = {'version': '0.1', 'author': 'Richard van den Berg', + 'description': 'MISP JSON format import module for merging MISP events', + 'module-type': ['import']} + +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + r = {'results': []} + request = json.loads(q) + try: + mfile = base64.b64decode(request["data"]).decode('utf-8') + misp = json.loads(mfile) + event = misp['response'][0]['Event'] + for a in event["Attribute"]: + tmp = {} + tmp["values"] = a["value"] + tmp["categories"] = a["category"] + tmp["types"] = a["type"] + tmp["to_ids"] = a["to_ids"] + tmp["comment"] = a["comment"] + if a.get("data"): + tmp["data"] = a["data"] + r['results'].append(tmp) + except: + pass + return r + +def introspection(): + modulesetup = {} + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo + +if __name__ == '__main__': + x = open('test.json', 'r') + r = handler(q=x.read()) + print(json.dumps(r)) diff --git a/misp_modules/modules/import_mod/openiocimport.py b/misp_modules/modules/import_mod/openiocimport.py new file mode 100755 index 0000000..d5e39a0 --- /dev/null +++ b/misp_modules/modules/import_mod/openiocimport.py @@ -0,0 +1,96 @@ +import json +import base64 + +from pymisp.tools import openioc + +misperrors = {'error': 'Error'} +userConfig = { + 'not save ioc': { + 'type': 'Boolean', + 'message': 'If you check this box, IOC file will not save as an attachment in MISP' + }, + 'default tag': { + 'type': 'String', + 'message': 'Add tags spaced by a comma (tlp:white,misp:threat-level="no-risk")', + 'validation' : '0' + } + } + +inputSource = ['file'] + +moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot', + 'description': 'Import OpenIOC package', + 'module-type': ['import']} + +moduleconfig = [] + + +def handler(q=False): + # Just in case we have no data + if q is False: + return False + + # The return value + r = {'results': []} + + # Load up that JSON + q = json.loads(q) + + # It's b64 encoded, so decode that stuff + package = base64.b64decode(q.get("data")).decode('utf-8') + + # If something really weird happened + if not package: + return json.dumps({"success": 0}) + + pkg = openioc.load_openioc(package) + + if q.get('config'): + if q['config'].get('not save ioc') == "0": + addFile = { + "values": [q.get('filename')], + "types": ['attachment'], + "categories": ['Support Tool'], + "data" : q.get('data'), + } + # add tag + if q['config'].get('default tag') is not None: + addFile["tags"] = q['config']['default tag'].split(",") + # add file as attachment + r["results"].append(addFile) + + + # return all attributes + for attrib in pkg.attributes: + toAppend = { + "values": [attrib.value], + "types": [attrib.type], + "categories": [attrib.category], + "comment":attrib.comment + } + # add tag + if q.get('config') and q['config'].get('default tag') is not None: + toAppend["tags"] = q['config']['default tag'].split(",") + + r["results"].append(toAppend) + return r + + +def introspection(): + modulesetup = {} + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/tests/openioc.xml b/tests/openioc.xml new file mode 100644 index 0000000..dc7858c --- /dev/null +++ b/tests/openioc.xml @@ -0,0 +1,91 @@ + + + STUXNET VIRUS (METHODOLOGY) + Generic indicator for the stuxnet virus. When loaded, stuxnet spawns lsass.exe in a suspended state. The malware then maps in its own executable section and fixes up the CONTEXT to point to the newly mapped in section. This is a common task performed by malware and allows the malware to execute under the pretense of a known and trusted process. + methodology + Mandiant + 0001-01-01T00:00:00 + + + + + + .stub + + + + mdmcpq3.PNF + + + + mdmeric3.PNF + + + + oem6C.PNF + + + + oem7A.PNF + + + + + fs_rec.sys + + + + mrxsmb.sys + + + + sr.sys + + + + fastfat.sys + + + + + + mrxcls.sys + + + + Realtek Semiconductor Corp + + + + + + mrxnet.sys + + + + Realtek Semiconductor Corp + + + + + + HKEY_LOCAL_MACHINE\SYSTEM\ControlSet001\Services\MRxCls\ImagePath + + + + mrxcls.sys + + + + + + HKEY_LOCAL_MACHINE\SYSTEM\ControlSet001\Services\MRxNet\ImagePath + + + + mrxnet.sys + + + + + \ No newline at end of file diff --git a/tests/test.py b/tests/test.py index 7b7ce8e..b9e4842 100644 --- a/tests/test.py +++ b/tests/test.py @@ -42,6 +42,20 @@ class TestModules(unittest.TestCase): print(response.json()) response.connection.close() + def test_openioc(self): + with open("tests/openioc.xml", "rb") as f: + content = base64.b64encode(f.read()) + data = json.dumps({"module": "openiocimport", + "data": content.decode(), + }) + response = requests.post(self.url + "query", data=data).json() + print(response) + + print("OpenIOC :: {}".format(response)) + values = [x["values"][0] for x in response["results"]] + assert("mrxcls.sys" in values) + assert("mdmcpq3.PNF" in values) + def test_stix(self): with open("tests/stix.xml", "rb") as f: content = base64.b64encode(f.read()) @@ -58,7 +72,7 @@ class TestModules(unittest.TestCase): assert("eu-society.com" in values) def test_email_headers(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": None, "guess_zip_attachment_passwords": None, "extract_urls": None} @@ -106,7 +120,7 @@ class TestModules(unittest.TestCase): self.assertIn("", values) def test_email_attachment_basic(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": None, "guess_zip_attachment_passwords": None, "extract_urls": None} @@ -129,9 +143,8 @@ class TestModules(unittest.TestCase): attch_data = base64.b64decode(i["data"]) self.assertEqual(attch_data, b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') - def test_email_attachment_unpack(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": None, "extract_urls": None} @@ -163,7 +176,7 @@ class TestModules(unittest.TestCase): def test_email_dont_unpack_compressed_doc_attachments(self): """Ensures that compressed """ - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": None, "extract_urls": None} @@ -193,9 +206,8 @@ class TestModules(unittest.TestCase): self.assertEqual(filesum.hexdigest(), '098da5381a90d4a51e6b844c18a0fecf2e364813c2f8b317cfdc51c21f2506a5') - def test_email_attachment_unpack_with_password(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": 'true', "extract_urls": None} @@ -222,9 +234,8 @@ class TestModules(unittest.TestCase): self.assertEqual(attch_data, b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') - def test_email_attachment_password_in_body(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": 'true', "extract_urls": None} @@ -247,7 +258,7 @@ class TestModules(unittest.TestCase): 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') def test_email_attachment_password_in_body_quotes(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": 'true', "extract_urls": None} @@ -275,7 +286,7 @@ class TestModules(unittest.TestCase): 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') def test_email_attachment_password_in_html_body(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": 'true', "extract_urls": None} @@ -305,7 +316,7 @@ class TestModules(unittest.TestCase): query['data'] = decode_email(message) data = json.dumps(query) response = requests.post(self.url + "query", data=data) - #print(response.json()) + # print(response.json()) values = [x["values"] for x in response.json()["results"]] self.assertIn('EICAR.com', values) for i in response.json()['results']: @@ -356,7 +367,7 @@ class TestModules(unittest.TestCase): response = requests.post(self.url + "query", data=data) def test_email_attachment_password_in_subject(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": 'true', "extract_urls": None} @@ -385,9 +396,8 @@ class TestModules(unittest.TestCase): self.assertEqual(attch_data, 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') - def test_email_extract_html_body_urls(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": None, "guess_zip_attachment_passwords": None, "extract_urls": "true"} @@ -415,12 +425,12 @@ without modifying core components. The API is available via a simple REST API wh query['data'] = decode_email(message) data = json.dumps(query) response = requests.post(self.url + "query", data=data) - #print(response.json()) + # print(response.json()) values = [x["values"] for x in response.json()["results"]] self.assertIn("https://github.com/MISP/MISP", values) self.assertIn("https://www.circl.lu/assets/files/misp-training/3.1-MISP-modules.pdf", values) - #def test_domaintools(self): + # def test_domaintools(self): # query = {'config': {'username': 'test_user', 'api_key': 'test_key'}, 'module': 'domaintools', 'domain': 'domaintools.com'} # try: # response = requests.post(self.url + "query", data=json.dumps(query)).json() @@ -429,33 +439,34 @@ without modifying core components. The API is available via a simple REST API wh # response = requests.post(self.url + "query", data=json.dumps(query)).json() # print(response) + def decode_email(message): message64 = base64.b64encode(message.as_bytes()).decode() return message64 def get_base_email(): - headers = {"Received":"via dmail-2008.19 for +INBOX; Tue, 3 Feb 2009 19:29:12 -0600 (CST)", - "Received":"from abc.luxsci.com ([10.10.10.10]) by xyz.luxsci.com (8.13.7/8.13.7) with ESMTP id n141TCa7022588 for ; Tue, 3 Feb 2009 19:29:12 -0600", - "Received":"from [192.168.0.3] (verizon.net [44.44.44.44]) (user=test@sender.com mech=PLAIN bits=2) by abc.luxsci.com (8.13.7/8.13.7) with ESMTP id n141SAfo021855 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=NOT) for ; Tue, 3 Feb 2009 19:28:10 -0600", - "X-Received":"by 192.168.0.45 with SMTP id q4mr156123401yw1g.911.1912342394963; Tue, 3 Feb 2009 19:32:15 -0600 (PST)", - "Message-ID":"<4988EF2D.40804@example.com>", - "Date":"Tue, 03 Feb 2009 20:28:13 -0500", - "From":'"Innocent Person" ', - "User-Agent":'Thunderbird 2.0.0.19 (Windows/20081209)', - "Sender":'"Malicious MailAgent" ', - "References":"", - "In-Reply-To":"", - "Accept-Language":'en-US', - "X-Mailer":'mlx 5.1.7', + headers = {"Received": "via dmail-2008.19 for +INBOX; Tue, 3 Feb 2009 19:29:12 -0600 (CST)", + "Received": "from abc.luxsci.com ([10.10.10.10]) by xyz.luxsci.com (8.13.7/8.13.7) with ESMTP id n141TCa7022588 for ; Tue, 3 Feb 2009 19:29:12 -0600", + "Received": "from [192.168.0.3] (verizon.net [44.44.44.44]) (user=test@sender.com mech=PLAIN bits=2) by abc.luxsci.com (8.13.7/8.13.7) with ESMTP id n141SAfo021855 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=NOT) for ; Tue, 3 Feb 2009 19:28:10 -0600", + "X-Received": "by 192.168.0.45 with SMTP id q4mr156123401yw1g.911.1912342394963; Tue, 3 Feb 2009 19:32:15 -0600 (PST)", + "Message-ID": "<4988EF2D.40804@example.com>", + "Date": "Tue, 03 Feb 2009 20:28:13 -0500", + "From": '"Innocent Person" ', + "User-Agent": 'Thunderbird 2.0.0.19 (Windows/20081209)', + "Sender": '"Malicious MailAgent" ', + "References": "", + "In-Reply-To": "", + "Accept-Language": 'en-US', + "X-Mailer": 'mlx 5.1.7', "Return-Path": "evil_spoofer@example.com", - "Thread-Topic":'This is a thread.', - "Thread-Index":'AQHSR8Us3H3SoaY1oUy9AAwZfMF922bnA9GAgAAi9s4AAGvxAA==', - "Content-Language":'en-US', - "To":'"Testy Testerson" ', - "Cc":'"Second Person" , "Other Friend" , "Last One" ', - "Subject":'Example Message', - "MIME-Version":'1.0'} + "Thread-Topic": 'This is a thread.', + "Thread-Index": 'AQHSR8Us3H3SoaY1oUy9AAwZfMF922bnA9GAgAAi9s4AAGvxAA==', + "Content-Language": 'en-US', + "To": '"Testy Testerson" ', + "Cc": '"Second Person" , "Other Friend" , "Last One" ', + "Subject": 'Example Message', + "MIME-Version": '1.0'} msg = MIMEMultipart() for key, val in headers.items(): msg.add_header(key, val)