From c088b13f0374aeb586e021a2aa21b10358b9b3f1 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 17 May 2018 13:47:49 +0200 Subject: [PATCH 001/196] fix: Using userConfig to define the header instead of moduleconfig --- misp_modules/modules/import_mod/csvimport.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py index 543d67b..85b9c6b 100644 --- a/misp_modules/modules/import_mod/csvimport.py +++ b/misp_modules/modules/import_mod/csvimport.py @@ -3,11 +3,14 @@ import json, os, base64 import pymisp misperrors = {'error': 'Error'} -mispattributes = {'inputSource': ['file'], 'output': ['MISP attributes']} moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'description': 'Import Attributes from a csv file.', 'module-type': ['import']} -moduleconfig = ['header'] +moduleconfig = [] +inputSource = ['file'] +userConfig = {'header': { + 'type': 'String', + 'message': 'Define the header of the csv file, with types (included in MISP attribute types or attribute fields) separated by commas.\nFor fields that do not match these types, please use space or simply nothing between commas.\nFor instance: ip-src,domain, ,timestamp'}} duplicatedFields = {'mispType': {'mispComment': 'comment'}, 'attrField': {'attrComment': 'comment'}} @@ -121,7 +124,18 @@ def handler(q=False): return r def introspection(): - return mispattributes + modulesetup = {} + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup def version(): moduleinfo['config'] = moduleconfig From dba8bd8c5bb0b6cc9e24cd938421240059f5b9f8 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 17 May 2018 16:24:11 +0200 Subject: [PATCH 002/196] fix: Avoid trying to build attributes with not intended fields - Previously: if the header field is not an attribute type, then it was added as an attribute field. PyMISP then used to skip it if needed - Now: Those fields are discarded before they are put in an attribute --- misp_modules/modules/import_mod/csvimport.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py index 85b9c6b..7bea557 100644 --- a/misp_modules/modules/import_mod/csvimport.py +++ b/misp_modules/modules/import_mod/csvimport.py @@ -14,6 +14,7 @@ userConfig = {'header': { duplicatedFields = {'mispType': {'mispComment': 'comment'}, 'attrField': {'attrComment': 'comment'}} +attributesFields = ['type', 'value', 'category', 'to_ids', 'comment', 'distribution'] class CsvParser(): def __init__(self, header): @@ -96,9 +97,12 @@ class CsvParser(): elif h in duplicatedFields['attrField']: # fields that should be considered as attribute fields head.append(duplicatedFields['attrField'].get(h)) - # otherwise, it is an attribute field - else: + # or, it could be an attribute field + elif h in attributesFields: head.append(h) + # otherwise, it is not defined + else: + head.append('') # return list of indexes of the misp types, list of the misp types, remaining fields that will be attribute fields return list2pop, misp, list(reversed(head)) From 1fb72f3c7a645a9674122d54135851a3d1c3997e Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 18 May 2018 11:33:53 +0200 Subject: [PATCH 003/196] add: Added user config to specify if there is a header in the csv to import --- misp_modules/modules/import_mod/csvimport.py | 32 ++++++++++++-------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py index 7bea557..9342a9f 100644 --- a/misp_modules/modules/import_mod/csvimport.py +++ b/misp_modules/modules/import_mod/csvimport.py @@ -10,15 +10,21 @@ moduleconfig = [] inputSource = ['file'] userConfig = {'header': { 'type': 'String', - 'message': 'Define the header of the csv file, with types (included in MISP attribute types or attribute fields) separated by commas.\nFor fields that do not match these types, please use space or simply nothing between commas.\nFor instance: ip-src,domain, ,timestamp'}} + 'message': 'Define the header of the csv file, with types (included in MISP attribute types or attribute fields) separated by commas.\nFor fields that do not match these types, please use space or simply nothing between commas.\nFor instance: ip-src,domain, ,timestamp'}, + 'has_header':{ + 'type': 'Boolean', + 'message': 'Tick this box ONLY if there is a header line, NOT COMMENTED, in the file (which will be skipped atm).' + }} duplicatedFields = {'mispType': {'mispComment': 'comment'}, 'attrField': {'attrComment': 'comment'}} attributesFields = ['type', 'value', 'category', 'to_ids', 'comment', 'distribution'] class CsvParser(): - def __init__(self, header): + def __init__(self, header, has_header): self.header = header + self.fields_number = len(header) + self.has_header = has_header self.attributes = [] def parse_data(self, data): @@ -27,12 +33,12 @@ class CsvParser(): l = line.split('#')[0].strip() if '#' in line else line.strip() if l: return_data.append(l) - self.data = return_data + self.data = return_data[1:] if self.has_header else return_data # find which delimiter is used - self.delimiter, self.length = self.findDelimiter() + self.delimiter = self.find_delimiter() - def findDelimiter(self): - n = len(self.header) + def find_delimiter(self): + n = self.fields_number if n > 1: tmpData = [] for da in self.data: @@ -41,11 +47,11 @@ class CsvParser(): if da.count(d) == (n-1): tmp.append(d) if len(tmp) == 1 and tmp == tmpData: - return tmpData[0], n + return tmpData[0] else: tmpData = tmp else: - return None, 1 + return None def buildAttributes(self): # if there is only 1 field of data @@ -63,7 +69,7 @@ class CsvParser(): datamisp = [] datasplit = data.split(self.delimiter) # in case there is an empty line or an error - if len(datasplit) != self.length: + if len(datasplit) != self.fields_number: continue # pop from the line data that matches with a misp type, using the list of indexes for l in list2pop: @@ -118,9 +124,11 @@ def handler(q=False): if not request.get('config') and not request['config'].get('header'): misperrors['error'] = "Configuration error" return misperrors - config = request['config'].get('header').split(',') - config = [c.strip() for c in config] - csv_parser = CsvParser(config) + header = request['config'].get('header').split(',') + header = [c.strip() for c in header] + has_header = request['config'].get('has_header') + has_header = True if has_header == '1' else False + csv_parser = CsvParser(header, has_header) csv_parser.parse_data(data.split('\n')) # build the attributes csv_parser.buildAttributes() From 2b509a2fd3f7f7df769488def9014a683cef9477 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 18 May 2018 11:38:13 +0200 Subject: [PATCH 004/196] Updated delimiter finder function --- misp_modules/modules/import_mod/csvimport.py | 43 +++++++++++--------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py index 9342a9f..5ccf287 100644 --- a/misp_modules/modules/import_mod/csvimport.py +++ b/misp_modules/modules/import_mod/csvimport.py @@ -19,6 +19,7 @@ userConfig = {'header': { duplicatedFields = {'mispType': {'mispComment': 'comment'}, 'attrField': {'attrComment': 'comment'}} attributesFields = ['type', 'value', 'category', 'to_ids', 'comment', 'distribution'] +delimiters = [',', ';', '|', '/', '\t', ' '] class CsvParser(): def __init__(self, header, has_header): @@ -29,29 +30,31 @@ class CsvParser(): def parse_data(self, data): return_data = [] - for line in data: - l = line.split('#')[0].strip() if '#' in line else line.strip() - if l: - return_data.append(l) + if self.fields_number == 1: + for line in data: + l = line.split('#')[0].strip() + if l: + return_data.append(l) + self.delimiter = None + else: + self.delimiter_count = dict([(d, 0) for d in delimiters]) + for line in data: + l = line.split('#')[0].strip() + if l: + self.parse_delimiter(l) + return_data.append(l) + # find which delimiter is used + self.delimiter = self.find_delimiter() self.data = return_data[1:] if self.has_header else return_data - # find which delimiter is used - self.delimiter = self.find_delimiter() + + def parse_delimiter(self, line): + for d in delimiters: + if line.count(d) >= (self.fields_number - 1): + self.delimiter_count[d] += 1 def find_delimiter(self): - n = self.fields_number - if n > 1: - tmpData = [] - for da in self.data: - tmp = [] - for d in (';', '|', '/', ',', '\t', ' ',): - if da.count(d) == (n-1): - tmp.append(d) - if len(tmp) == 1 and tmp == tmpData: - return tmpData[0] - else: - tmpData = tmp - else: - return None + _, delimiter = max((n, v) for v, n in self.delimiter_count.items()) + return delimiter def buildAttributes(self): # if there is only 1 field of data From 9664127b85b14dc49b4ba0e9fe7ee7e5b2bf9945 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 29 May 2018 21:54:22 +0200 Subject: [PATCH 005/196] add: new expansion module to check hashes against hashdd.com including NSLR dataset. --- misp_modules/modules/expansion/__init__.py | 5 +-- misp_modules/modules/expansion/hashdd.py | 41 ++++++++++++++++++++++ tests/bodyhashdd.json | 1 + tests/{query-cve.sh => query-hashdd.sh} | 2 +- 4 files changed, 44 insertions(+), 5 deletions(-) create mode 100755 misp_modules/modules/expansion/hashdd.py create mode 100644 tests/bodyhashdd.json rename tests/{query-cve.sh => query-hashdd.sh} (61%) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index e40e844..a9389e0 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,6 +1,3 @@ from . import _vmray -__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', - 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', - 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', - 'threatcrowd', 'vulndb', 'crowdstrike_falcon','yara_syntax_validator'] +__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd'] diff --git a/misp_modules/modules/expansion/hashdd.py b/misp_modules/modules/expansion/hashdd.py new file mode 100755 index 0000000..beeaf8e --- /dev/null +++ b/misp_modules/modules/expansion/hashdd.py @@ -0,0 +1,41 @@ +import json +import requests + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['md5'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Alexandre Dulaunoy', 'description': 'An expansion module to check hashes against hashdd.com including NSLR dataset.', 'module-type': ['hover']} +moduleconfig = [] +hashddapi_url = 'https://api.hashdd.com/' + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if not request.get('md5'): + misperrors['error'] = 'MD5 hash value is missing missing' + return misperrors + v = request.get('md5').upper() + r = requests.post(hashddapi_url, data={'hash':v}) + if r.status_code == 200: + state = json.loads(r.text) + if state: + if state.get(v): + summary = state[v]['known_level'] + else: + summary = 'Unknown hash' + else: + misperrors['error'] = '{} API not accessible'.format(hashddapi_url) + return misperrors['error'] + + r = {'results': [{'types': mispattributes['output'], 'values': summary}]} + return r + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/tests/bodyhashdd.json b/tests/bodyhashdd.json new file mode 100644 index 0000000..b6d256c --- /dev/null +++ b/tests/bodyhashdd.json @@ -0,0 +1 @@ +{"module": "hashdd", "md5": "838DE99E82C5B9753BAC96D82C1A8DCB"} diff --git a/tests/query-cve.sh b/tests/query-hashdd.sh similarity index 61% rename from tests/query-cve.sh rename to tests/query-hashdd.sh index 215de4f..4d73663 100755 --- a/tests/query-cve.sh +++ b/tests/query-hashdd.sh @@ -1 +1 @@ -curl -s http://127.0.0.1:6666/query -H "Content-Type: application/json" --data @bodycve.json -X POST +curl -s http://127.0.0.1:6666/query -H "Content-Type: application/json" --data @bodyhashdd.json -X POST From 0af064ac40912d7f74c3d5ae31e1a2186a104ef4 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 29 May 2018 21:57:38 +0200 Subject: [PATCH 006/196] fix: missing cve module test --- tests/query-cve.sh | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/query-cve.sh diff --git a/tests/query-cve.sh b/tests/query-cve.sh new file mode 100644 index 0000000..215de4f --- /dev/null +++ b/tests/query-cve.sh @@ -0,0 +1 @@ +curl -s http://127.0.0.1:6666/query -H "Content-Type: application/json" --data @bodycve.json -X POST From 1e1e69416936e5eef955208a244c270a353800b0 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 30 May 2018 06:56:42 +0200 Subject: [PATCH 007/196] add: mixing modules --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 67ba189..590fbac 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en). * [Farsight DNSDB Passive DNS](misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. * [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind. +* [hashdd](misp_modules/modules/expansion/hashdd.py) - a hover module to check file hashes against [hashdd.com](http://www.hashdd.com) including NSLR dataset. * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address. * [iprep](misp-modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net. * [OTX](misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/). @@ -45,7 +46,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ ### Export modules * [CEF](misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF). -* [GoAML export](misp_modules/modules/export_mod/goamlexport.py) module to export in GoAML format. +* [GoAML export](misp_modules/modules/export_mod/goamlexport.py) module to export in [GoAML format](http://goaml.unodc.org/goaml/en/index.html). * [Lite Export](misp_modules/modules/export_mod/liteexport.py) module to export a lite event. * [Simple PDF export](misp_modules/modules/export_mod/pdfexport.py) module to export in PDF (required: asciidoctor-pdf). * [ThreatConnect](misp_modules/modules/export_mod/threat_connect_export.py) module to export in ThreatConnect CSV format. @@ -56,6 +57,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [CSV import](misp_modules/modules/import_mod/csvimport.py) Customizable CSV import module. * [Cuckoo JSON](misp_modules/modules/import_mod/cuckooimport.py) Cuckoo JSON import. * [Email Import](misp_modules/modules/import_mod/email_import.py) Email import module for MISP to import basic metadata. +* [GoAML import](misp_modules/modules/import_mod/) Module to import [GoAML](http://goaml.unodc.org/goaml/en/index.html) XML format. * [OCR](misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes. * [OpenIOC](misp_modules/modules/import_mod/openiocimport.py) OpenIOC import based on PyMISP library. * [stiximport](misp_modules/modules/import_mod/stiximport.py) - An import module to process STIX xml/json. From 0b0f57b30c349aabe5f43749f33c33829d78b3cd Mon Sep 17 00:00:00 2001 From: Andras Iklody Date: Wed, 6 Jun 2018 08:31:41 +0200 Subject: [PATCH 008/196] Update countrycode.py --- misp_modules/modules/expansion/countrycode.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/countrycode.py b/misp_modules/modules/expansion/countrycode.py index af58fc6..9f22c40 100755 --- a/misp_modules/modules/expansion/countrycode.py +++ b/misp_modules/modules/expansion/countrycode.py @@ -20,10 +20,12 @@ common_tlds = {"com":"Commercial (Worldwide)", "gov":"Government (USA)" } -codes = requests.get("http://www.geognos.com/api/en/countries/info/all.json").json() +codes = False def handler(q=False): global codes + if not codes: + codes = requests.get("http://www.geognos.com/api/en/countries/info/all.json").json() if q is False: return False request = json.loads(q) From e6bac113baaf27981aa431f1f9c218bda49d0e17 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 8 Jun 2018 16:38:41 +0200 Subject: [PATCH 009/196] add onyphe module --- REQUIREMENTS | 1 + misp_modules/modules/expansion/onyphe.py | 65 ++++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 misp_modules/modules/expansion/onyphe.py diff --git a/REQUIREMENTS b/REQUIREMENTS index 9e383d4..0a0c85a 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -14,6 +14,7 @@ asnhistory git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client git+https://github.com/MISP/MISP-STIX-Converter.git#egg=misp_stix_converter git+https://github.com/MISP/PyMISP.git#egg=pymisp +git+https://github.com/sebdraven/pyonyphe#egg=pyonyphe pillow pytesseract SPARQLWrapper diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py new file mode 100644 index 0000000..16a4e94 --- /dev/null +++ b/misp_modules/modules/expansion/onyphe.py @@ -0,0 +1,65 @@ +import json +# -*- coding: utf-8 -*- + +import json +try: + from onyphe import Onyphe +except ImportError: + print("pyonyphe module not installed.") + +misperrors = {'error': 'Error'} + +mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domains'], 'output': ['freetext']} +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', + 'description': 'Query on Onyphe', + 'module-type': ['expansion', 'hover']} + +# config fields that your code expects from the site admin +moduleconfig = ['apikey'] + + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + + if not request.get('config') and not (request['config'].get('apikey')): + misperrors['error'] = 'Onyphe authentication is missing' + return misperrors + + api = Onyphe(request['config'].get('apikey')) + + if not api: + misperrors['error'] = 'Onyphe Error instance api' + + ip = '' + if request.get('ip-src'): + ip = request['ip-src'] + elif request.get('ip-dst'): + ip = request['ip-dst'] + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + + return handle_expansion(api, ip, misperrors) + + +def handle_expansion(api, ip, misperrors): + result = api.ip(ip) + if result['status'] == 'nok': + misperrors['error'] = result['message'] + return misperrors + + return {'results': [{'types': mispattributes['output'], + 'values': json.dumps(result)}]} + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo \ No newline at end of file From 6eeca0fba1efda5605c8fabedc0c9cb2895c0229 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 8 Jun 2018 17:53:50 +0200 Subject: [PATCH 010/196] add pastebin url imports --- .idea/vcs.xml | 6 ++++++ misp_modules/modules/expansion/onyphe.py | 16 +++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 .idea/vcs.xml diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index 16a4e94..ace09ef 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -9,7 +9,7 @@ except ImportError: misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domains'], 'output': ['freetext']} +mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domains'], 'output': ['hostname', 'domain', 'ip-src', 'ip-dst','url']} # possible module-types: 'expansion', 'hover' or both moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', 'description': 'Query on Onyphe', @@ -48,12 +48,22 @@ def handler(q=False): def handle_expansion(api, ip, misperrors): result = api.ip(ip) + if result['status'] == 'nok': misperrors['error'] = result['message'] return misperrors - return {'results': [{'types': mispattributes['output'], - 'values': json.dumps(result)}]} + categories = list(set([item['@category'] for item in result['results']])) + + result_filtered = [] + urls_pasties = [] + for r in result['results']: + if r['@category'] == 'pastries': + if r['@type'] == 'pastebin': + urls_pasties.append('https://pastebin.com/raw/%s' % r['key']) + result_filtered.append({'type': ['url'], 'values': urls_pasties}) + + return result_filtered def introspection(): From f18f8fe05ac85b9d5c3ce312b5c43024ff9d5a3b Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 8 Jun 2018 18:01:58 +0200 Subject: [PATCH 011/196] correct a bug --- misp_modules/modules/expansion/onyphe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index ace09ef..ce68262 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -55,13 +55,13 @@ def handle_expansion(api, ip, misperrors): categories = list(set([item['@category'] for item in result['results']])) - result_filtered = [] + result_filtered = {"results": []} urls_pasties = [] for r in result['results']: if r['@category'] == 'pastries': if r['@type'] == 'pastebin': urls_pasties.append('https://pastebin.com/raw/%s' % r['key']) - result_filtered.append({'type': ['url'], 'values': urls_pasties}) + result_filtered['results'].append({'type': ['url'], 'values': urls_pasties}) return result_filtered From 3ec15358977ba79e6dae485da1811d02e8dc6ab1 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 8 Jun 2018 18:09:59 +0200 Subject: [PATCH 012/196] correct key in map result --- misp_modules/modules/expansion/onyphe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index ce68262..4542fc7 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -61,7 +61,7 @@ def handle_expansion(api, ip, misperrors): if r['@category'] == 'pastries': if r['@type'] == 'pastebin': urls_pasties.append('https://pastebin.com/raw/%s' % r['key']) - result_filtered['results'].append({'type': ['url'], 'values': urls_pasties}) + result_filtered['results'].append({'types': ['url'], 'values': urls_pasties}) return result_filtered From cad35b5332d083004b90f103796bc2ceda7d7660 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 8 Jun 2018 18:11:04 +0200 Subject: [PATCH 013/196] error indentation --- misp_modules/modules/expansion/onyphe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index 4542fc7..b42c86b 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -61,7 +61,7 @@ def handle_expansion(api, ip, misperrors): if r['@category'] == 'pastries': if r['@type'] == 'pastebin': urls_pasties.append('https://pastebin.com/raw/%s' % r['key']) - result_filtered['results'].append({'types': ['url'], 'values': urls_pasties}) + result_filtered['results'].append({'types': ['url'], 'values': urls_pasties}) return result_filtered From 04032d110c2803ba22d81c4950c574632f2cf09c Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 8 Jun 2018 18:31:08 +0200 Subject: [PATCH 014/196] add as number with onyphe --- misp_modules/modules/expansion/onyphe.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index b42c86b..854fc87 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -55,14 +55,17 @@ def handle_expansion(api, ip, misperrors): categories = list(set([item['@category'] for item in result['results']])) - result_filtered = {"results": []} + result_filtered = {"results": []} urls_pasties = [] + asn_list = [] for r in result['results']: if r['@category'] == 'pastries': if r['@type'] == 'pastebin': urls_pasties.append('https://pastebin.com/raw/%s' % r['key']) + elif r['@category'] == 'synscan': + ans_list = r['asn'] result_filtered['results'].append({'types': ['url'], 'values': urls_pasties}) - + result_filtered['results'].append({'types': ['AS'], 'values': asn_list}) return result_filtered From 735e6260588bce574dad615c597bba2ff97a017e Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 11 Jun 2018 10:41:05 +0200 Subject: [PATCH 015/196] add as number with onyphe --- misp_modules/modules/expansion/onyphe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index 854fc87..53f0eba 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -63,9 +63,9 @@ def handle_expansion(api, ip, misperrors): if r['@type'] == 'pastebin': urls_pasties.append('https://pastebin.com/raw/%s' % r['key']) elif r['@category'] == 'synscan': - ans_list = r['asn'] + asn_list.append(r['asn']) result_filtered['results'].append({'types': ['url'], 'values': urls_pasties}) - result_filtered['results'].append({'types': ['AS'], 'values': asn_list}) + result_filtered['results'].append({'types': ['AS'], 'values': list(set(asn_list))}) return result_filtered From ef035d051bb5002693838ea04849ada8710d468d Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 11 Jun 2018 10:54:06 +0200 Subject: [PATCH 016/196] add category --- misp_modules/modules/expansion/onyphe.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index 53f0eba..2a0b913 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -64,7 +64,8 @@ def handle_expansion(api, ip, misperrors): urls_pasties.append('https://pastebin.com/raw/%s' % r['key']) elif r['@category'] == 'synscan': asn_list.append(r['asn']) - result_filtered['results'].append({'types': ['url'], 'values': urls_pasties}) + result_filtered['results'].append({'types': ['url'], 'values': urls_pasties, + 'category': 'External analysis'}) result_filtered['results'].append({'types': ['AS'], 'values': list(set(asn_list))}) return result_filtered From 0a543ca0d52515f0e9b5b12a2ae1231772ad46a1 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 11 Jun 2018 10:55:44 +0200 Subject: [PATCH 017/196] change type --- misp_modules/modules/expansion/onyphe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index 2a0b913..33aa69d 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -65,7 +65,7 @@ def handle_expansion(api, ip, misperrors): elif r['@category'] == 'synscan': asn_list.append(r['asn']) result_filtered['results'].append({'types': ['url'], 'values': urls_pasties, - 'category': 'External analysis'}) + 'category': ['External analysis']}) result_filtered['results'].append({'types': ['AS'], 'values': list(set(asn_list))}) return result_filtered From f069cd9bf47cbb16a53ca9bfa01c70fbe7bbe514 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 11 Jun 2018 10:56:40 +0200 Subject: [PATCH 018/196] change keys --- misp_modules/modules/expansion/onyphe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index 33aa69d..9f4f942 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -65,7 +65,7 @@ def handle_expansion(api, ip, misperrors): elif r['@category'] == 'synscan': asn_list.append(r['asn']) result_filtered['results'].append({'types': ['url'], 'values': urls_pasties, - 'category': ['External analysis']}) + 'categories': ['External analysis']}) result_filtered['results'].append({'types': ['AS'], 'values': list(set(asn_list))}) return result_filtered From 7580c63433f1b79a5146ff6e1d333a1c73900f55 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 11 Jun 2018 10:59:06 +0200 Subject: [PATCH 019/196] add category for AS number --- misp_modules/modules/expansion/onyphe.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index 9f4f942..ea59ea2 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -66,7 +66,8 @@ def handle_expansion(api, ip, misperrors): asn_list.append(r['asn']) result_filtered['results'].append({'types': ['url'], 'values': urls_pasties, 'categories': ['External analysis']}) - result_filtered['results'].append({'types': ['AS'], 'values': list(set(asn_list))}) + result_filtered['results'].append({'types': ['AS'], 'values': list(set(asn_list)), + 'categories': ['Network activity']}) return result_filtered From 2cbe6ac9cad62d42daa0c13b2be6eb25fae87132 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 11 Jun 2018 11:24:55 +0200 Subject: [PATCH 020/196] Updated requirements to avoid version issues in the MISP packer installation script --- REQUIREMENTS | 3 --- 1 file changed, 3 deletions(-) diff --git a/REQUIREMENTS b/REQUIREMENTS index 9e383d4..43dc97d 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -1,5 +1,3 @@ -stix -cybox tornado dnspython requests @@ -12,7 +10,6 @@ pyeupi ipasn-redis asnhistory git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client -git+https://github.com/MISP/MISP-STIX-Converter.git#egg=misp_stix_converter git+https://github.com/MISP/PyMISP.git#egg=pymisp pillow pytesseract From 2e0e63fad60050e163ad0e6aa898e775fab04d84 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 11 Jun 2018 11:25:17 +0200 Subject: [PATCH 021/196] add targeting os --- misp_modules/modules/expansion/onyphe.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index ea59ea2..30bb179 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -58,16 +58,22 @@ def handle_expansion(api, ip, misperrors): result_filtered = {"results": []} urls_pasties = [] asn_list = [] + os_list = [] for r in result['results']: if r['@category'] == 'pastries': if r['@type'] == 'pastebin': urls_pasties.append('https://pastebin.com/raw/%s' % r['key']) elif r['@category'] == 'synscan': asn_list.append(r['asn']) + os_list.append(r['os']) result_filtered['results'].append({'types': ['url'], 'values': urls_pasties, 'categories': ['External analysis']}) result_filtered['results'].append({'types': ['AS'], 'values': list(set(asn_list)), 'categories': ['Network activity']}) + + result_filtered['results'].append({'types': ['target-machine'], + 'values': list(set(os_list)), + 'categories': ['Targeting data']}) return result_filtered From d9ee5286e3134a71a1e2f19f24c01fe4c30bdf6a Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 11 Jun 2018 11:59:00 +0200 Subject: [PATCH 022/196] add domains --- misp_modules/modules/expansion/onyphe.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index 30bb179..ac127ab 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -59,21 +59,41 @@ def handle_expansion(api, ip, misperrors): urls_pasties = [] asn_list = [] os_list = [] + domains_resolver = [] + domains_forward = [] for r in result['results']: if r['@category'] == 'pastries': if r['@type'] == 'pastebin': urls_pasties.append('https://pastebin.com/raw/%s' % r['key']) elif r['@category'] == 'synscan': asn_list.append(r['asn']) - os_list.append(r['os']) + os_target = r['os'] + if os_target != 'Unknown': + os_list.append(r['os']) + elif r['@category'] == 'resolver' and r['@type'] =='reverse': + domains_resolver.append(r['reverse']) + elif r['@category'] == 'resolver' and r['@type'] =='forward': + domains_forward.append(r['forward']) + result_filtered['results'].append({'types': ['url'], 'values': urls_pasties, 'categories': ['External analysis']}) + result_filtered['results'].append({'types': ['AS'], 'values': list(set(asn_list)), 'categories': ['Network activity']}) result_filtered['results'].append({'types': ['target-machine'], 'values': list(set(os_list)), 'categories': ['Targeting data']}) + + result_filtered['results'].append({'types': ['domains'], + 'values': list(set(domains_resolver)), + 'categories': ['Network activity'], + 'comments': ['resolver to %s' % ip]}) + + result_filtered['results'].append({'types': ['domains'], + 'values': list(set(domains_resolver)), + 'categories': ['Network activity'], + 'comments': ['forward to %s' % ip]}) return result_filtered From 59b49f9d20c0e7cd02e7a7df747a7a0ca2332d7f Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 11 Jun 2018 12:00:46 +0200 Subject: [PATCH 023/196] add domains forward --- misp_modules/modules/expansion/onyphe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index ac127ab..cb4a443 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -91,7 +91,7 @@ def handle_expansion(api, ip, misperrors): 'comments': ['resolver to %s' % ip]}) result_filtered['results'].append({'types': ['domains'], - 'values': list(set(domains_resolver)), + 'values': list(set(domains_forward)), 'categories': ['Network activity'], 'comments': ['forward to %s' % ip]}) return result_filtered From e0631c9651ed5bcb31ad6af9529dea53119b1d56 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 11 Jun 2018 12:02:34 +0200 Subject: [PATCH 024/196] correct typo --- misp_modules/modules/expansion/onyphe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index cb4a443..7cd249d 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -85,12 +85,12 @@ def handle_expansion(api, ip, misperrors): 'values': list(set(os_list)), 'categories': ['Targeting data']}) - result_filtered['results'].append({'types': ['domains'], + result_filtered['results'].append({'types': ['domain'], 'values': list(set(domains_resolver)), 'categories': ['Network activity'], 'comments': ['resolver to %s' % ip]}) - result_filtered['results'].append({'types': ['domains'], + result_filtered['results'].append({'types': ['domain'], 'values': list(set(domains_forward)), 'categories': ['Network activity'], 'comments': ['forward to %s' % ip]}) From 43402fde26b4ccc6ac576691d36f60c8a5a9909a Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 11 Jun 2018 12:28:40 +0200 Subject: [PATCH 025/196] correct typo --- misp_modules/modules/expansion/onyphe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index 7cd249d..c99de19 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -88,12 +88,12 @@ def handle_expansion(api, ip, misperrors): result_filtered['results'].append({'types': ['domain'], 'values': list(set(domains_resolver)), 'categories': ['Network activity'], - 'comments': ['resolver to %s' % ip]}) + 'comment': ['resolver to %s' % ip]}) result_filtered['results'].append({'types': ['domain'], 'values': list(set(domains_forward)), 'categories': ['Network activity'], - 'comments': ['forward to %s' % ip]}) + 'comment': ['forward to %s' % ip]}) return result_filtered From f6b8655f64e8d0a2e1200e3b1f44c590ccb70f34 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 11 Jun 2018 12:29:51 +0200 Subject: [PATCH 026/196] correct type of comments --- misp_modules/modules/expansion/onyphe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index c99de19..9dad860 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -88,12 +88,12 @@ def handle_expansion(api, ip, misperrors): result_filtered['results'].append({'types': ['domain'], 'values': list(set(domains_resolver)), 'categories': ['Network activity'], - 'comment': ['resolver to %s' % ip]}) + 'comment': 'resolver to %s' % ip }) result_filtered['results'].append({'types': ['domain'], 'values': list(set(domains_forward)), 'categories': ['Network activity'], - 'comment': ['forward to %s' % ip]}) + 'comment': 'forward to %s' % ip}) return result_filtered From 755d907580d0f4462445af673cc20d385bccd096 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 11 Jun 2018 13:21:21 +0200 Subject: [PATCH 027/196] pep 8 compliant --- misp_modules/modules/expansion/onyphe.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index 9dad860..7494867 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -61,6 +61,7 @@ def handle_expansion(api, ip, misperrors): os_list = [] domains_resolver = [] domains_forward = [] + for r in result['results']: if r['@category'] == 'pastries': if r['@type'] == 'pastebin': From 14695bbeb9ba7fce9f8495aadedfc0b7386d28ed Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 11 Jun 2018 13:34:45 +0200 Subject: [PATCH 028/196] correct codecov --- misp_modules/modules/expansion/onyphe.py | 39 ++++++++++++------------ 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index 7494867..86abe7a 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -19,31 +19,32 @@ moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', moduleconfig = ['apikey'] - def handler(q=False): - if q is False: - return False - request = json.loads(q) + if q: - if not request.get('config') and not (request['config'].get('apikey')): - misperrors['error'] = 'Onyphe authentication is missing' - return misperrors + request = json.loads(q) - api = Onyphe(request['config'].get('apikey')) + if not request.get('config') and not (request['config'].get('apikey')): + misperrors['error'] = 'Onyphe authentication is missing' + return misperrors - if not api: - misperrors['error'] = 'Onyphe Error instance api' + api = Onyphe(request['config'].get('apikey')) - ip = '' - if request.get('ip-src'): - ip = request['ip-src'] - elif request.get('ip-dst'): - ip = request['ip-dst'] + if not api: + misperrors['error'] = 'Onyphe Error instance api' + + ip = '' + if request.get('ip-src'): + ip = request['ip-src'] + elif request.get('ip-dst'): + ip = request['ip-dst'] + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + + return handle_expansion(api, ip, misperrors) else: - misperrors['error'] = "Unsupported attributes type" - return misperrors - - return handle_expansion(api, ip, misperrors) + return False def handle_expansion(api, ip, misperrors): From a6717b53eff6ed2b076e90a3a5258eb934b7f17e Mon Sep 17 00:00:00 2001 From: sebdraven Date: Mon, 11 Jun 2018 13:56:37 +0200 Subject: [PATCH 029/196] Delete vcs.xml --- .idea/vcs.xml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .idea/vcs.xml diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file From 479e66cc9b574de1c7fc8a44b4819f674a11943c Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 11 Jun 2018 17:03:23 +0200 Subject: [PATCH 030/196] fix: Removed STIX related libraries, files, documentation, etc. --- README.md | 1 - setup.py | 2 - tests/stix.xml | 331 ------------------------------------------------- tests/test.py | 15 --- 4 files changed, 349 deletions(-) delete mode 100644 tests/stix.xml diff --git a/README.md b/README.md index 590fbac..8b96814 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,6 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [GoAML import](misp_modules/modules/import_mod/) Module to import [GoAML](http://goaml.unodc.org/goaml/en/index.html) XML format. * [OCR](misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes. * [OpenIOC](misp_modules/modules/import_mod/openiocimport.py) OpenIOC import based on PyMISP library. -* [stiximport](misp_modules/modules/import_mod/stiximport.py) - An import module to process STIX xml/json. * [ThreatAnalyzer](misp_modules/modules/import_mod/threatanalyzer_import.py) - An import module to process ThreatAnalyzer archive.zip/analysis.json sandbox exports. * [VMRay](misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export. diff --git a/setup.py b/setup.py index 3bdc180..f6c3a64 100644 --- a/setup.py +++ b/setup.py @@ -33,8 +33,6 @@ setup( 'pyeupi', 'ipasn-redis', 'asnhistory', - 'stix', - 'cybox', 'pillow', 'pytesseract', 'shodan', diff --git a/tests/stix.xml b/tests/stix.xml deleted file mode 100644 index a4a60d8..0000000 --- a/tests/stix.xml +++ /dev/null @@ -1,331 +0,0 @@ - - - - CNC Server 1 - - - 82.146.166.56 - - - - - CNC Server 2 - - - 209.239.79.47 - - - - - CNC Server 3 - - - 41.213.121.180 - - - - - Watering Hole Wordpress - - - eu-society.com - - - - - Watering Hole Wordpress - - - aromatravel.org - - - - - Watering Hole Wordpress - - - bss.servebbs.com - - - - - - - Watering Hole Detected - URL Watchlist - - - - C2 List - - - C2 List - - - C2 List - - - - - - CnC Beaconing Detected - C2 - - - - - - - - - - - - - - - Malware CnC Channels - - Advantage - - - - Hosting - - - - - - - - - - - - - Fingerprinting and whitelisting during watering-hole operations - - Theft - Credential Theft - - - - Domain Registration - - - C2 List - - - C2 List - - - C2 List - - - - - - - - - - Spear-phishing in tandem with 0-day exploits - - Unauthorized Access - - - - - - - Infiltration of organisations via third party supplier/partner - - Unauthorized Access - - - - - - - Custom recon tool to compromise and identify credentials of the network - - Theft - Credential Theft - - - - - - - Multiple means of C2 communications given the diversity of the attacker toolset - - Advantage - - - - - - - rootkit communicates during the same time as network activity, encoded with an XOR key - - Advantage - - - - - - - Kernel-centric rootkit waits for network trigger before launching - - Advantage - - - - - - - Kernel centric exfiltration over TCP/UDP/DNS/ICMP/HTTP - - Theft - - - - - - - Exfiltration over HTTP/HTTPS - - Theft - - - - - - - Use of previously undocumented functions in their Kernel centric attacks - - Advantage - - - - - - - - - - - - - - - - - Privilage Escalation Vulnerability - - CVE-2013-5065 - - - - - - The Epic Turla Campaign - The Epic Turla Campaign - - Advantage - Political - - - - - - - - - - SNAKE Campaign - The SNAKE Campaign - - Advantage - Political - - - - - - - - - - - - SNAKE - -The group behind the SNAKE campaign are a top tier nation-state threat. Their capabilities extend from subtle watering-hole attacks to sophisticated server rootkits – virtually undetectable by conventional security products. -This threat actor group has been operating continuously for over a decade, infiltrating governments and strategic private sector networks in that time. The most notorious of their early campaigns led to a breach of classified US military systems, an extensive clean-up called ‘Operation Buckshot Yankee’, and led to the creation of the US Cyber Command. -Whilst the sophisticated rootkit is used for persistent access to networks, the group also leverage more straight-forward capabilities for gaining an initial toe-hold on targets. This includes the use of watering-hole attacks and basic remote access tools. - - -The group behind the SNAKE campaign are a top tier nation-state threat. Their capabilities extend from subtle watering-hole attacks to sophisticated server rootkits – virtually undetectable by conventional security products. - - - - - - SNAKE - - - Turla - - - WRAITH - - - - - - Russia - - - Moscow - - - - - snake@gmail.com - twitter.com/snake - - - Russian - - - - - Political - - - Expert - - - Advantage - Political - - - Theft - Intellectual Property - - - - diff --git a/tests/test.py b/tests/test.py index 5a56f5a..d32bd00 100644 --- a/tests/test.py +++ b/tests/test.py @@ -57,21 +57,6 @@ class TestModules(unittest.TestCase): assert("mrxcls.sys" in values) assert("mdmcpq3.PNF" in values) - def test_stix(self): - with open("tests/stix.xml", "rb") as f: - content = base64.b64encode(f.read()) - data = json.dumps({"module": "stiximport", - "data": content.decode('utf-8'), - }) - response = requests.post(self.url + "query", data=data).json() - - print("STIX :: {}".format(response)) - values = [x["values"][0] for x in response["results"]] - - assert("209.239.79.47" in values) - assert("41.213.121.180" in values) - assert("eu-society.com" in values) - def test_email_headers(self): query = {"module": "email_import"} query["config"] = {"unzip_attachments": None, From 023c35f5d85ed9c094a0ee0efb67e0241691bc45 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 14 Jun 2018 16:47:11 +0200 Subject: [PATCH 031/196] add onyphe full module and code the stub --- misp_modules/modules/expansion/onyphe_full.py | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 misp_modules/modules/expansion/onyphe_full.py diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py new file mode 100644 index 0000000..1b121a9 --- /dev/null +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -0,0 +1,126 @@ +import json +# -*- coding: utf-8 -*- + +import json +try: + from onyphe import Onyphe +except ImportError: + print("pyonyphe module not installed.") + +misperrors = {'error': 'Error'} + +mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'], 'output': ['hostname', 'domain', 'ip-src', 'ip-dst','url']} +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', + 'description': 'Query on Onyphe', + 'module-type': ['expansion', 'hover']} + +# config fields that your code expects from the site admin +moduleconfig = ['apikey'] + + +def handler(q=False): + if q: + + request = json.loads(q) + + if not request.get('config') and not (request['config'].get('apikey')): + misperrors['error'] = 'Onyphe authentication is missing' + return misperrors + + api = Onyphe(request['config'].get('apikey')) + + if not api: + misperrors['error'] = 'Onyphe Error instance api' + + ip = '' + if request.get('ip-src'): + ip = request['ip-src'] + return handle_ip(api ,ip, misperrors) + elif request.get('ip-dst'): + ip = request['ip-dst'] + return handle_ip(api,ip,misperrors) + elif request.get('domain'): + domain = request['domain'] + elif request.get('hostname'): + hostname = request['hostname'] + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + + + else: + return False + + +def handle_domain(api, domain, misperrors): + pass + +def handle_ip(api, ip, misperrors): + result_filtered = {"results": []} + + r,status_ok = expand_syscan(api,ip,misperrors) + + if status_ok: + result_filtered['results'].append(r) + else: + return r + + r, status_ok = expand_datascan(api,misperrors, ip=ip) + + if status_ok: + result_filtered['results'].append(r) + else: + return r + + r, status_ok = expand_forward(api, ip,misperrors) + + if status_ok: + result_filtered['results'].append(r) + else: + return r + + r, status_ok = expand_reverse(api, ip,misperrors) + + if status_ok: + result_filtered['results'].append(r) + else: + return r + + return result_filtered + + +def expand_syscan(api, ip, misperror): + status_ok = False + r = None + + return r,status_ok + + +def expand_datascan(api, misperror,**kwargs): + status_ok = False + r = None + + return r,status_ok + + +def expand_reverse(api, ip, misperror): + status_ok = False + r = None + + return r,status_ok + + +def expand_forward(api, ip, misperror): + status_ok = False + r = None + + return r,status_ok + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo \ No newline at end of file From 8ae7210aef0ab5c17232ed60da89e889f305617e Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 11:07:33 +0200 Subject: [PATCH 032/196] add onyphe full module --- misp_modules/modules/expansion/onyphe_full.py | 71 +++++++++++++++---- 1 file changed, 58 insertions(+), 13 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 1b121a9..7047a30 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -56,31 +56,33 @@ def handler(q=False): def handle_domain(api, domain, misperrors): pass + def handle_ip(api, ip, misperrors): result_filtered = {"results": []} - r,status_ok = expand_syscan(api,ip,misperrors) + r, status_ok = expand_syscan(api, ip, misperrors) + + if status_ok: + result_filtered['results'].append(r) + else: + misperrors['error'] = "Error syscan result" + return misperrors + + r, status_ok = expand_datascan(api, misperrors, ip=ip) if status_ok: result_filtered['results'].append(r) else: return r - r, status_ok = expand_datascan(api,misperrors, ip=ip) + r, status_ok = expand_forward(api, ip, misperrors) if status_ok: result_filtered['results'].append(r) else: return r - r, status_ok = expand_forward(api, ip,misperrors) - - if status_ok: - result_filtered['results'].append(r) - else: - return r - - r, status_ok = expand_reverse(api, ip,misperrors) + r, status_ok = expand_reverse(api, ip, misperrors) if status_ok: result_filtered['results'].append(r) @@ -94,7 +96,7 @@ def expand_syscan(api, ip, misperror): status_ok = False r = None - return r,status_ok + return r, status_ok def expand_datascan(api, misperror,**kwargs): @@ -108,14 +110,57 @@ def expand_reverse(api, ip, misperror): status_ok = False r = None - return r,status_ok + return r, status_ok def expand_forward(api, ip, misperror): status_ok = False r = None - return r,status_ok + return r, status_ok + + +def expand_pastries(api, misperror, **kwargs): + status_ok = False + r = [] + ip = None + domain = None + result = None + urls_pasties = [] + domains = [] + ips = [] + if 'ip' in kwargs: + ip = kwargs.get('ip') + result = api.pastries(ip) + + if 'domain' in kwargs: + domain = kwargs.get('domain') + result = api.pastries(domain) + + if result['status'] =='ok': + status_ok = True + for item in result['results']: + if item['@category'] == 'pastries': + if item['@type'] == 'pastebin': + urls_pasties.append('https://pastebin.com/raw/%s' % item['key']) + + if 'domain' in item: + domains.extend(item['domain']) + if 'ip' in item: + ips.extend(item['ips']) + if 'hostname' in item: + domains.extend(item['hostname']) + + r.append({'types': ['url'], 'values': urls_pasties, + 'categories': ['External analysis']}) + r.append({'types': ['domains'], 'values': list(set(domains)), + 'categories': ['Network activity']}) + + r.append({'types': ['ip-dst'], 'values': list(set(ips)), + 'categories': ['Network activity']}) + + return r, status_ok + def introspection(): return mispattributes From c14d05adefa94498b6e37fc1656481717a1a74b9 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 12:32:54 +0200 Subject: [PATCH 033/196] test patries expansion --- misp_modules/modules/expansion/onyphe_full.py | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 7047a30..220fb50 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -62,32 +62,39 @@ def handle_ip(api, ip, misperrors): r, status_ok = expand_syscan(api, ip, misperrors) + # if status_ok: + # result_filtered['results'].append(r) + # else: + # misperrors['error'] = "Error syscan result" + + r,status_ok = expand_pastries(api,misperrors,ip=ip) + if status_ok: result_filtered['results'].append(r) else: - misperrors['error'] = "Error syscan result" + misperrors['error'] = 'Error pastries result' return misperrors - r, status_ok = expand_datascan(api, misperrors, ip=ip) - - if status_ok: - result_filtered['results'].append(r) - else: - return r - - r, status_ok = expand_forward(api, ip, misperrors) - - if status_ok: - result_filtered['results'].append(r) - else: - return r - - r, status_ok = expand_reverse(api, ip, misperrors) - - if status_ok: - result_filtered['results'].append(r) - else: - return r + # r, status_ok = expand_datascan(api, misperrors, ip=ip) + # + # if status_ok: + # result_filtered['results'].append(r) + # else: + # return r + # + # r, status_ok = expand_forward(api, ip, misperrors) + # + # if status_ok: + # result_filtered['results'].append(r) + # else: + # return r + # + # r, status_ok = expand_reverse(api, ip, misperrors) + # + # if status_ok: + # result_filtered['results'].append(r) + # else: + # return r return result_filtered From 7195f33f5dccdba730f6937433b92d2b3d4c15fe Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 12:34:07 +0200 Subject: [PATCH 034/196] correct error keys --- misp_modules/modules/expansion/onyphe_full.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 220fb50..cad7135 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -154,7 +154,7 @@ def expand_pastries(api, misperror, **kwargs): if 'domain' in item: domains.extend(item['domain']) if 'ip' in item: - ips.extend(item['ips']) + ips.extend(item['ip']) if 'hostname' in item: domains.extend(item['hostname']) From e8aefde2ee50364e61f95fddce2dead8d7feab21 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 12:36:32 +0200 Subject: [PATCH 035/196] add logs --- misp_modules/modules/expansion/onyphe_full.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index cad7135..a13df9b 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -95,7 +95,7 @@ def handle_ip(api, ip, misperrors): # result_filtered['results'].append(r) # else: # return r - + print(result_filtered) return result_filtered From 7a3c4b10846bd149d15b2d2e256c8ef9f00d8d8b Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 12:38:41 +0200 Subject: [PATCH 036/196] change add in results --- misp_modules/modules/expansion/onyphe_full.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index a13df9b..c003672 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -70,7 +70,7 @@ def handle_ip(api, ip, misperrors): r,status_ok = expand_pastries(api,misperrors,ip=ip) if status_ok: - result_filtered['results'].append(r) + result_filtered['results'].extend(r) else: misperrors['error'] = 'Error pastries result' return misperrors From 5426ec5380c8395ce3b7c17677c29e7938aabee7 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 12:40:52 +0200 Subject: [PATCH 037/196] change key access domains --- misp_modules/modules/expansion/onyphe_full.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index c003672..e910dc4 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -160,7 +160,7 @@ def expand_pastries(api, misperror, **kwargs): r.append({'types': ['url'], 'values': urls_pasties, 'categories': ['External analysis']}) - r.append({'types': ['domains'], 'values': list(set(domains)), + r.append({'types': ['domain'], 'values': list(set(domains)), 'categories': ['Network activity']}) r.append({'types': ['ip-dst'], 'values': list(set(ips)), From e1bc67afad5c2e7f5e04d5ac9e629200f2cc232e Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 14:41:57 +0200 Subject: [PATCH 038/196] add expansion synscan --- misp_modules/modules/expansion/onyphe_full.py | 42 ++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index e910dc4..1a34d54 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -62,12 +62,12 @@ def handle_ip(api, ip, misperrors): r, status_ok = expand_syscan(api, ip, misperrors) - # if status_ok: - # result_filtered['results'].append(r) - # else: - # misperrors['error'] = "Error syscan result" + if status_ok: + result_filtered['results'].append(r) + else: + misperrors['error'] = "Error syscan result" - r,status_ok = expand_pastries(api,misperrors,ip=ip) + r, status_ok = expand_pastries(api,misperrors,ip=ip) if status_ok: result_filtered['results'].extend(r) @@ -101,7 +101,37 @@ def handle_ip(api, ip, misperrors): def expand_syscan(api, ip, misperror): status_ok = False - r = None + r = [] + asn_list = [] + os_list = [] + geoloc = [] + orgs = [] + results = api.synscan(ip) + + if results['status'] == 'ok': + status_ok = True + for elem in results['result']: + asn_list.append(elem['asn']) + os_list = elem['os'] + geoloc.append(elem['location']) + orgs.append(elem['organization']) + if os_list != 'Unknown': + os_list.append(elem['os']) + + r.append({'types': ['target-machine'], + 'values': list(set(os_list)), + 'categories': ['Targeting data']}) + + r.append({'types': ['target-location'], + 'values': list(set(geoloc)), + 'categories': ['Targeting data']}) + + r.append({'types': ['target-org'], + 'values': list(set(orgs)), + 'categories': ['Targeting data']}) + + r.append({'types': ['AS'], 'values': list(set(asn_list)), + 'categories': ['Network activity']}) return r, status_ok From 9427c766038cbac8a572fcd4004c9b92916ac5b4 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 14:45:06 +0200 Subject: [PATCH 039/196] error keys --- misp_modules/modules/expansion/onyphe_full.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 1a34d54..f6c334e 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -110,7 +110,7 @@ def expand_syscan(api, ip, misperror): if results['status'] == 'ok': status_ok = True - for elem in results['result']: + for elem in results['results']: asn_list.append(elem['asn']) os_list = elem['os'] geoloc.append(elem['location']) From 3a4294391fa83024779b546a94c4a07555e78177 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 14:48:18 +0200 Subject: [PATCH 040/196] error type --- misp_modules/modules/expansion/onyphe_full.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index f6c334e..9b9c06c 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -112,10 +112,10 @@ def expand_syscan(api, ip, misperror): status_ok = True for elem in results['results']: asn_list.append(elem['asn']) - os_list = elem['os'] + os_target = elem['os'] geoloc.append(elem['location']) orgs.append(elem['organization']) - if os_list != 'Unknown': + if os_target != 'Unknown': os_list.append(elem['os']) r.append({'types': ['target-machine'], @@ -131,7 +131,7 @@ def expand_syscan(api, ip, misperror): 'categories': ['Targeting data']}) r.append({'types': ['AS'], 'values': list(set(asn_list)), - 'categories': ['Network activity']}) + 'categories': ['Network activity']}) return r, status_ok From d1e72676f18bb44fe770b9031fb720ee96727ca4 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 14:50:48 +0200 Subject: [PATCH 041/196] error method --- misp_modules/modules/expansion/onyphe_full.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 9b9c06c..27ed2f7 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -63,7 +63,7 @@ def handle_ip(api, ip, misperrors): r, status_ok = expand_syscan(api, ip, misperrors) if status_ok: - result_filtered['results'].append(r) + result_filtered['results'].extend(r) else: misperrors['error'] = "Error syscan result" From 7eba7c0386cd29cc8b350b6134841e0557a9e940 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 14:53:08 +0200 Subject: [PATCH 042/196] error loops --- misp_modules/modules/expansion/onyphe_full.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 27ed2f7..0dad6fd 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -118,19 +118,19 @@ def expand_syscan(api, ip, misperror): if os_target != 'Unknown': os_list.append(elem['os']) - r.append({'types': ['target-machine'], + r.append({'types': ['target-machine'], 'values': list(set(os_list)), 'categories': ['Targeting data']}) - r.append({'types': ['target-location'], + r.append({'types': ['target-location'], 'values': list(set(geoloc)), 'categories': ['Targeting data']}) - r.append({'types': ['target-org'], + r.append({'types': ['target-org'], 'values': list(set(orgs)), 'categories': ['Targeting data']}) - r.append({'types': ['AS'], 'values': list(set(asn_list)), + r.append({'types': ['AS'], 'values': list(set(asn_list)), 'categories': ['Network activity']}) return r, status_ok From 915747073a2f917b76261106f6ecb6021febfef6 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 15:05:00 +0200 Subject: [PATCH 043/196] add comment of attributes --- misp_modules/modules/expansion/onyphe_full.py | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 0dad6fd..eaf7a37 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -115,23 +115,32 @@ def expand_syscan(api, ip, misperror): os_target = elem['os'] geoloc.append(elem['location']) orgs.append(elem['organization']) - if os_target != 'Unknown': + if os_target != 'Unknown' and os_target != 'Undefined': os_list.append(elem['os']) r.append({'types': ['target-machine'], 'values': list(set(os_list)), - 'categories': ['Targeting data']}) + 'categories': ['Targeting data'], + 'comment':'OS found on %s with synscan of Onyphe' % ip}) r.append({'types': ['target-location'], 'values': list(set(geoloc)), - 'categories': ['Targeting data']}) + 'categories': ['Targeting data'], + 'comment': 'geolocalisation of %s found with synscan of Onyphe' + % ip + }) r.append({'types': ['target-org'], - 'values': list(set(orgs)), - 'categories': ['Targeting data']}) + 'values': list(set(orgs)), + 'categories': ['Targeting data'], + 'comment': 'Organisations of %s found with synscan of Onyphe' + }) - r.append({'types': ['AS'], 'values': list(set(asn_list)), - 'categories': ['Network activity']}) + r.append({'types': ['AS'], + 'values': list(set(asn_list)), + 'categories': ['Network activity'], + 'comment': 'As number of %s found with synscan of Onyphe' + }) return r, status_ok @@ -191,10 +200,12 @@ def expand_pastries(api, misperror, **kwargs): r.append({'types': ['url'], 'values': urls_pasties, 'categories': ['External analysis']}) r.append({'types': ['domain'], 'values': list(set(domains)), - 'categories': ['Network activity']}) + 'categories': ['Network activity'], + 'comment': 'domains found in pasties of Onyphe'}) r.append({'types': ['ip-dst'], 'values': list(set(ips)), - 'categories': ['Network activity']}) + 'categories': ['Network activity'], + 'comment': 'IPs found in pasties of Onyphe'}) return r, status_ok From d0f42c1772cdef93e3bb260e99b4cd0b930e1772 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 15:07:55 +0200 Subject: [PATCH 044/196] add comment of attributes --- misp_modules/modules/expansion/onyphe_full.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index eaf7a37..ba27e11 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -197,11 +197,13 @@ def expand_pastries(api, misperror, **kwargs): if 'hostname' in item: domains.extend(item['hostname']) - r.append({'types': ['url'], 'values': urls_pasties, - 'categories': ['External analysis']}) + r.append({'types': ['url'], + 'values': urls_pasties, + 'categories': ['External analysis'], + 'comment':'URLs of pasties where %s has found' % ip}) r.append({'types': ['domain'], 'values': list(set(domains)), 'categories': ['Network activity'], - 'comment': 'domains found in pasties of Onyphe'}) + 'comment': 'Domains found in pasties of Onyphe'}) r.append({'types': ['ip-dst'], 'values': list(set(ips)), 'categories': ['Network activity'], From a24b529868fc1f4b2067be8226a21f8c562bd8b9 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 15:33:21 +0200 Subject: [PATCH 045/196] add forward infos --- misp_modules/modules/expansion/onyphe_full.py | 53 +++++++++++++------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index ba27e11..fca5af0 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -9,7 +9,9 @@ except ImportError: misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'], 'output': ['hostname', 'domain', 'ip-src', 'ip-dst','url']} +mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'], + 'output': ['hostname', 'domain', 'ip-src', 'ip-dst','url']} + # possible module-types: 'expansion', 'hover' or both moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', 'description': 'Query on Onyphe', @@ -82,12 +84,13 @@ def handle_ip(api, ip, misperrors): # else: # return r # - # r, status_ok = expand_forward(api, ip, misperrors) - # - # if status_ok: - # result_filtered['results'].append(r) - # else: - # return r + r, status_ok = expand_forward(api, ip, misperrors) + + if status_ok: + result_filtered['results'].extend(r) + else: + misperrors['error'] = 'Error forward result' + return # # r, status_ok = expand_reverse(api, ip, misperrors) # @@ -119,14 +122,14 @@ def expand_syscan(api, ip, misperror): os_list.append(elem['os']) r.append({'types': ['target-machine'], - 'values': list(set(os_list)), - 'categories': ['Targeting data'], - 'comment':'OS found on %s with synscan of Onyphe' % ip}) + 'values': list(set(os_list)), + 'categories': ['Targeting data'], + 'comment': 'OS found on %s with synscan of Onyphe' % ip}) r.append({'types': ['target-location'], - 'values': list(set(geoloc)), - 'categories': ['Targeting data'], - 'comment': 'geolocalisation of %s found with synscan of Onyphe' + 'values': list(set(geoloc)), + 'categories': ['Targeting data'], + 'comment': 'Geolocalisation of %s found with synscan of Onyphe' % ip }) @@ -149,7 +152,7 @@ def expand_datascan(api, misperror,**kwargs): status_ok = False r = None - return r,status_ok + return r, status_ok def expand_reverse(api, ip, misperror): @@ -161,8 +164,28 @@ def expand_reverse(api, ip, misperror): def expand_forward(api, ip, misperror): status_ok = False - r = None + r = [] + results = api.forward(ip) + domains_forward = [] + + domains = [] + if results['status'] == 'ok': + status_ok = True + + for elem in results['results']: + domains_forward.append(elem['forward']) + domains.append(elem['domain']) + + r.append({'types': ['domain'], + 'values': list(set(domains)), + 'categories': ['Network activity'], + 'comment': 'Domains of %s from forward service of Onyphe' % ip}) + + r.append({'types': ['domain'], + 'values': list(set(domains_forward)), + 'categories': ['Network activity'], + 'comment': 'Forward Domains of %s from forward service of Onyphe' % ip}) return r, status_ok From 0d120af64715b5fe5a191a8d6fa22a05e5a4c3b6 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 16:24:17 +0200 Subject: [PATCH 046/196] add reverse infos --- misp_modules/modules/expansion/onyphe_full.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index fca5af0..a2876a1 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -158,7 +158,29 @@ def expand_datascan(api, misperror,**kwargs): def expand_reverse(api, ip, misperror): status_ok = False r = None + status_ok = False + r = [] + results = api.forward(ip) + domains_reverse = [] + + domains = [] + if results['status'] == 'ok': + status_ok = True + + for elem in results['results']: + domains_reverse.append(elem['forward']) + domains.append(elem['domain']) + + r.append({'types': ['domain'], + 'values': list(set(domains)), + 'categories': ['Network activity'], + 'comment': 'Domains of %s from forward service of Onyphe' % ip}) + + r.append({'types': ['domain'], + 'values': list(set(domains_reverse)), + 'categories': ['Network activity'], + 'comment': 'Reverse Domains of %s from forward service of Onyphe' % ip}) return r, status_ok From 4a8a79c56062e77aab1f64e9602f192961064824 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 16:26:09 +0200 Subject: [PATCH 047/196] add reverse infos --- misp_modules/modules/expansion/onyphe_full.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index a2876a1..f340f92 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -169,7 +169,7 @@ def expand_reverse(api, ip, misperror): status_ok = True for elem in results['results']: - domains_reverse.append(elem['forward']) + domains_reverse.append(elem['reverse']) domains.append(elem['domain']) r.append({'types': ['domain'], From d4be9d9fda875582485a9230fdd7f1cf1566fabd Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 16:29:04 +0200 Subject: [PATCH 048/196] add reverse infos --- misp_modules/modules/expansion/onyphe_full.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index f340f92..19fb207 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -92,12 +92,14 @@ def handle_ip(api, ip, misperrors): misperrors['error'] = 'Error forward result' return # - # r, status_ok = expand_reverse(api, ip, misperrors) - # - # if status_ok: - # result_filtered['results'].append(r) - # else: - # return r + r, status_ok = expand_reverse(api, ip, misperrors) + + if status_ok: + result_filtered['results'].extend(r) + else: + misperrors['error'] = 'Error reverse result' + return misperrors + print(result_filtered) return result_filtered From b56f8cfa36071d62a9f422c0d828868ecb9c4c94 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 16:30:56 +0200 Subject: [PATCH 049/196] add reverse infos --- misp_modules/modules/expansion/onyphe_full.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 19fb207..9f4e16d 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -162,7 +162,7 @@ def expand_reverse(api, ip, misperror): r = None status_ok = False r = [] - results = api.forward(ip) + results = api.reverse(ip) domains_reverse = [] From 04e932cce063d10759a0e708ed3cb8c43d68ab55 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 17:47:11 +0200 Subject: [PATCH 050/196] add datascan expansion --- misp_modules/modules/expansion/onyphe_full.py | 67 ++++++++++++++++--- 1 file changed, 58 insertions(+), 9 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 9f4e16d..8b8e0a2 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -77,13 +77,13 @@ def handle_ip(api, ip, misperrors): misperrors['error'] = 'Error pastries result' return misperrors - # r, status_ok = expand_datascan(api, misperrors, ip=ip) - # - # if status_ok: - # result_filtered['results'].append(r) - # else: - # return r - # + r, status_ok = expand_datascan(api, misperrors, ip=ip) + + if status_ok: + result_filtered['results'].append(r) + else: + return r + r, status_ok = expand_forward(api, ip, misperrors) if status_ok: @@ -139,12 +139,13 @@ def expand_syscan(api, ip, misperror): 'values': list(set(orgs)), 'categories': ['Targeting data'], 'comment': 'Organisations of %s found with synscan of Onyphe' + % ip }) r.append({'types': ['AS'], 'values': list(set(asn_list)), 'categories': ['Network activity'], - 'comment': 'As number of %s found with synscan of Onyphe' + 'comment': 'As number of %s found with synscan of Onyphe' % ip }) return r, status_ok @@ -152,7 +153,55 @@ def expand_syscan(api, ip, misperror): def expand_datascan(api, misperror,**kwargs): status_ok = False - r = None + r = [] + ip = '' + query ='' + asn_list = [] + geoloc = [] + orgs = [] + ports = [] + if 'ip' in kwargs: + query = kwargs.get('ip') + else: + query = kwargs.get('domain') + + results = api.datascan(query) + + if results['status'] == 'ok': + for elem in results['results']: + asn_list.append(elem['asn']) + os_target = elem['os'] + geoloc.append(elem['location']) + orgs.append(elem['organization']) + ports.append(elem['port']) + + r.append({'types': ['port'], + 'values': list(set(ports)), + 'categories': ['Other'], + 'comment': 'Ports of %s found with datascan of Onyphe' + % ip + }) + + r.append({'types': ['target-location'], + 'values': list(set(geoloc)), + 'categories': ['Targeting data'], + 'comment': 'Geolocalisation of %s found with synscan of Onyphe' + % ip + }) + + r.append({'types': ['target-org'], + 'values': list(set(orgs)), + 'categories': ['Targeting data'], + 'comment': 'Organisations of %s found with synscan of Onyphe' + % ip + }) + + r.append({'types': ['AS'], + 'values': list(set(asn_list)), + 'categories': ['Network activity'], + 'comment': 'As number of %s found with synscan of Onyphe' % ip + }) + return r, status_ok From 2afd2b8aaf901a3dd89e58070917d27bbb490b02 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 17:50:28 +0200 Subject: [PATCH 051/196] correct bug --- misp_modules/modules/expansion/onyphe_full.py | 1 - 1 file changed, 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 8b8e0a2..1f6c580 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -170,7 +170,6 @@ def expand_datascan(api, misperror,**kwargs): if results['status'] == 'ok': for elem in results['results']: asn_list.append(elem['asn']) - os_target = elem['os'] geoloc.append(elem['location']) orgs.append(elem['organization']) ports.append(elem['port']) From 9195887f986cfa123fb87d3971c1cb290e8145ce Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 17:51:46 +0200 Subject: [PATCH 052/196] pep 8 --- misp_modules/modules/expansion/onyphe_full.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 1f6c580..4ca9a26 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -200,8 +200,6 @@ def expand_datascan(api, misperror,**kwargs): 'categories': ['Network activity'], 'comment': 'As number of %s found with synscan of Onyphe' % ip }) - - return r, status_ok From 153d8bd340e407e058fa97b0c2b11b04e70d5b94 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 17:56:19 +0200 Subject: [PATCH 053/196] add logs --- misp_modules/modules/expansion/onyphe_full.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 4ca9a26..acdaced 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -22,6 +22,7 @@ moduleconfig = ['apikey'] def handler(q=False): + print(q) if q: request = json.loads(q) From fe00f099f618e2628560552a6592c61030207a37 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 17:59:49 +0200 Subject: [PATCH 054/196] add logs --- misp_modules/modules/expansion/onyphe_full.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index acdaced..66c208f 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -22,7 +22,6 @@ moduleconfig = ['apikey'] def handler(q=False): - print(q) if q: request = json.loads(q) @@ -83,7 +82,8 @@ def handle_ip(api, ip, misperrors): if status_ok: result_filtered['results'].append(r) else: - return r + misperrors['error'] = 'Error datascan result' + return misperrors r, status_ok = expand_forward(api, ip, misperrors) @@ -91,8 +91,8 @@ def handle_ip(api, ip, misperrors): result_filtered['results'].extend(r) else: misperrors['error'] = 'Error forward result' - return - # + return misperrors + r, status_ok = expand_reverse(api, ip, misperrors) if status_ok: From 4166475f9e7d9906e691b581469dc8ac343c11e2 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 18:02:12 +0200 Subject: [PATCH 055/196] add logs --- misp_modules/modules/expansion/onyphe_full.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 66c208f..6982ccb 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -82,7 +82,7 @@ def handle_ip(api, ip, misperrors): if status_ok: result_filtered['results'].append(r) else: - misperrors['error'] = 'Error datascan result' + misperrors['error'] = 'Error datascan result %s' % status_ok return misperrors r, status_ok = expand_forward(api, ip, misperrors) @@ -90,7 +90,7 @@ def handle_ip(api, ip, misperrors): if status_ok: result_filtered['results'].extend(r) else: - misperrors['error'] = 'Error forward result' + misperrors['error'] = 'Error forward result %s' % status_ok return misperrors r, status_ok = expand_reverse(api, ip, misperrors) @@ -167,6 +167,7 @@ def expand_datascan(api, misperror,**kwargs): query = kwargs.get('domain') results = api.datascan(query) + print(results) if results['status'] == 'ok': for elem in results['results']: From a9b7a10c4117472f745337c7491d6ab4dfe76521 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 18:03:34 +0200 Subject: [PATCH 056/196] set status after requests --- misp_modules/modules/expansion/onyphe_full.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 6982ccb..7168618 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -167,9 +167,10 @@ def expand_datascan(api, misperror,**kwargs): query = kwargs.get('domain') results = api.datascan(query) - print(results) + if results['status'] == 'ok': + status_ok = False for elem in results['results']: asn_list.append(elem['asn']) geoloc.append(elem['location']) From e712a31760ef29dfd41a1a911e459a3be1a4cc6d Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 18:04:12 +0200 Subject: [PATCH 057/196] set status after requests --- misp_modules/modules/expansion/onyphe_full.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 7168618..1187648 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -170,7 +170,7 @@ def expand_datascan(api, misperror,**kwargs): if results['status'] == 'ok': - status_ok = False + status_ok = True for elem in results['results']: asn_list.append(elem['asn']) geoloc.append(elem['location']) From 1d1fd365693308a8eecb8ca82f62c06a43d73058 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 20 Jun 2018 18:05:28 +0200 Subject: [PATCH 058/196] change method to concat methods --- misp_modules/modules/expansion/onyphe_full.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 1187648..9466d8a 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -80,7 +80,7 @@ def handle_ip(api, ip, misperrors): r, status_ok = expand_datascan(api, misperrors, ip=ip) if status_ok: - result_filtered['results'].append(r) + result_filtered['results'].extend(r) else: misperrors['error'] = 'Error datascan result %s' % status_ok return misperrors From e230c88c155c4956e2478c394c78335eb1dfcee9 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 22 Jun 2018 11:59:09 +0200 Subject: [PATCH 059/196] add threat list expansion --- misp_modules/modules/expansion/onyphe_full.py | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 9466d8a..931271c 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -82,7 +82,7 @@ def handle_ip(api, ip, misperrors): if status_ok: result_filtered['results'].extend(r) else: - misperrors['error'] = 'Error datascan result %s' % status_ok + misperrors['error'] = 'Error datascan result ' return misperrors r, status_ok = expand_forward(api, ip, misperrors) @@ -90,7 +90,7 @@ def handle_ip(api, ip, misperrors): if status_ok: result_filtered['results'].extend(r) else: - misperrors['error'] = 'Error forward result %s' % status_ok + misperrors['error'] = 'Error forward result' return misperrors r, status_ok = expand_reverse(api, ip, misperrors) @@ -101,7 +101,14 @@ def handle_ip(api, ip, misperrors): misperrors['error'] = 'Error reverse result' return misperrors - print(result_filtered) + r, status_ok = expand_threatlist(api, misperrors, ip=ip) + + if status_ok: + result_filtered['results'].extend(r) + else: + misperrors['error'] = 'Error threat list' + return misperrors + return result_filtered @@ -168,7 +175,6 @@ def expand_datascan(api, misperror,**kwargs): results = api.datascan(query) - if results['status'] == 'ok': status_ok = True for elem in results['results']: @@ -308,6 +314,32 @@ def expand_pastries(api, misperror, **kwargs): return r, status_ok +def expand_threatlist(api, misperror,**kwargs): + status_ok = False + r = [] + + query = None + + threat_list = [] + + if 'ip' in kwargs: + query = kwargs.get('ip') + else: + query = kwargs.get('domain') + + results = api.threatlist(query) + if results['status'] == 'ok': + status_ok = True + threat_list = ['seen %s on %s ' % (item['seen_date'], item['threatlist']) + for item in results['results']] + + r.append({'types': ['comment'], + 'categories': ['Other'], + 'values': [threat_list] + }) + + return r,status_ok + def introspection(): return mispattributes From e9c18b3d5fe01dcad008d04831f50e7d860c7efa Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 22 Jun 2018 13:03:09 +0200 Subject: [PATCH 060/196] correct comments --- misp_modules/modules/expansion/onyphe_full.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 931271c..02375c7 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -187,27 +187,27 @@ def expand_datascan(api, misperror,**kwargs): 'values': list(set(ports)), 'categories': ['Other'], 'comment': 'Ports of %s found with datascan of Onyphe' - % ip + % query }) r.append({'types': ['target-location'], 'values': list(set(geoloc)), 'categories': ['Targeting data'], 'comment': 'Geolocalisation of %s found with synscan of Onyphe' - % ip + % query }) r.append({'types': ['target-org'], 'values': list(set(orgs)), 'categories': ['Targeting data'], 'comment': 'Organisations of %s found with synscan of Onyphe' - % ip + % query }) r.append({'types': ['AS'], 'values': list(set(asn_list)), 'categories': ['Network activity'], - 'comment': 'As number of %s found with synscan of Onyphe' % ip + 'comment': 'As number of %s found with synscan of Onyphe' % query }) return r, status_ok From 8d03354399f689295c68fee17a4786e9bc74f8ef Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 22 Jun 2018 15:12:10 +0200 Subject: [PATCH 061/196] correct bugs --- misp_modules/modules/expansion/onyphe_full.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 02375c7..5d7011f 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -335,7 +335,7 @@ def expand_threatlist(api, misperror,**kwargs): r.append({'types': ['comment'], 'categories': ['Other'], - 'values': [threat_list] + 'values': threat_list }) return r,status_ok From 96c829470dd205d6dbf007d3e19db9a74fb552d9 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 22 Jun 2018 15:14:44 +0200 Subject: [PATCH 062/196] add comment --- misp_modules/modules/expansion/onyphe_full.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 5d7011f..9e7ae8c 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -335,7 +335,8 @@ def expand_threatlist(api, misperror,**kwargs): r.append({'types': ['comment'], 'categories': ['Other'], - 'values': threat_list + 'values': threat_list, + 'comment': '%s is present in threatlist' % query }) return r,status_ok From 83999d6402030fa5b78630f49cdf1f5d65db5e82 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 22 Jun 2018 15:57:52 +0200 Subject: [PATCH 063/196] add domain expansion --- misp_modules/modules/expansion/onyphe_full.py | 42 ++++++++++++++----- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 9e7ae8c..39f3d42 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -49,13 +49,36 @@ def handler(q=False): else: misperrors['error'] = "Unsupported attributes type" return misperrors - - else: return False def handle_domain(api, domain, misperrors): + result_filtered = {"results": []} + + r, status_ok = expand_pastries(api, misperrors, domain=domain) + + if status_ok: + result_filtered['results'].extend(r) + else: + misperrors['error'] = 'Error pastries result' + return misperrors + + r, status_ok = expand_datascan(api, misperrors, domain=domain) + + if status_ok: + result_filtered['results'].extend(r) + else: + misperrors['error'] = 'Error datascan result ' + return misperrors + + r, status_ok = expand_threatlist(api, misperrors, domain=domain) + + if status_ok: + result_filtered['results'].extend(r) + else: + misperrors['error'] = 'Error threat list' + return misperrors pass @@ -271,19 +294,18 @@ def expand_forward(api, ip, misperror): def expand_pastries(api, misperror, **kwargs): status_ok = False r = [] - ip = None - domain = None + + query = None result = None urls_pasties = [] domains = [] ips = [] if 'ip' in kwargs: - ip = kwargs.get('ip') - result = api.pastries(ip) - + query = kwargs.get('ip') if 'domain' in kwargs: - domain = kwargs.get('domain') - result = api.pastries(domain) + query = kwargs.get('domain') + + api.pastries(query) if result['status'] =='ok': status_ok = True @@ -302,7 +324,7 @@ def expand_pastries(api, misperror, **kwargs): r.append({'types': ['url'], 'values': urls_pasties, 'categories': ['External analysis'], - 'comment':'URLs of pasties where %s has found' % ip}) + 'comment':'URLs of pasties where %s has found' % query}) r.append({'types': ['domain'], 'values': list(set(domains)), 'categories': ['Network activity'], 'comment': 'Domains found in pasties of Onyphe'}) From de6a81d4883afd56a91e8af0fadc683aeac0a3ba Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 22 Jun 2018 16:04:14 +0200 Subject: [PATCH 064/196] correct bugs --- misp_modules/modules/expansion/onyphe_full.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 39f3d42..8a33e3c 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -305,7 +305,7 @@ def expand_pastries(api, misperror, **kwargs): if 'domain' in kwargs: query = kwargs.get('domain') - api.pastries(query) + result = api.pastries(query) if result['status'] =='ok': status_ok = True From 396b71ef3b7481c22474fc5f6d821950a5c63268 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 22 Jun 2018 16:06:34 +0200 Subject: [PATCH 065/196] add domain to expand --- misp_modules/modules/expansion/onyphe_full.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 8a33e3c..bde4d4a 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -44,8 +44,10 @@ def handler(q=False): return handle_ip(api,ip,misperrors) elif request.get('domain'): domain = request['domain'] + return handle_domain(api, domain, misperrors) elif request.get('hostname'): hostname = request['hostname'] + return handle_domain(api, hostname, misperrors) else: misperrors['error'] = "Unsupported attributes type" return misperrors From 87b07b89b54ec1e569f14e1e2e29f4f5de18f2e9 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 22 Jun 2018 16:15:34 +0200 Subject: [PATCH 066/196] add search --- misp_modules/modules/expansion/onyphe_full.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index bde4d4a..6ac4750 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -193,12 +193,13 @@ def expand_datascan(api, misperror,**kwargs): geoloc = [] orgs = [] ports = [] + if 'ip' in kwargs: query = kwargs.get('ip') + results = api.datascan(query) else: query = kwargs.get('domain') - - results = api.datascan(query) + results = api.search_datascan('domain:%s' % query) if results['status'] == 'ok': status_ok = True @@ -304,10 +305,10 @@ def expand_pastries(api, misperror, **kwargs): ips = [] if 'ip' in kwargs: query = kwargs.get('ip') + result = api.pastries(query) if 'domain' in kwargs: query = kwargs.get('domain') - - result = api.pastries(query) + result = api.search_pastries('domain:%s' % query) if result['status'] =='ok': status_ok = True @@ -348,10 +349,11 @@ def expand_threatlist(api, misperror,**kwargs): if 'ip' in kwargs: query = kwargs.get('ip') + results = api.threatlist(query) else: query = kwargs.get('domain') + results = api.search_threatlist('domain:%s' % query) - results = api.threatlist(query) if results['status'] == 'ok': status_ok = True threat_list = ['seen %s on %s ' % (item['seen_date'], item['threatlist']) From 785aac3e6b6d8474f4916f48fde3b16f9d49c4e3 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 22 Jun 2018 16:18:23 +0200 Subject: [PATCH 067/196] add return handle domains --- misp_modules/modules/expansion/onyphe_full.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 6ac4750..7a05d12 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -81,7 +81,8 @@ def handle_domain(api, domain, misperrors): else: misperrors['error'] = 'Error threat list' return misperrors - pass + + return result_filtered def handle_ip(api, ip, misperrors): From 59b7688bdcbdc3d1d1e085ec6e8862c0c8585b33 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 28 Jun 2018 16:00:14 +0800 Subject: [PATCH 068/196] - Added initial PDF support, nothing is processed yet - Test to replace PIL with wand --- misp_modules/modules/import_mod/ocr.py | 27 +++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/import_mod/ocr.py b/misp_modules/modules/import_mod/ocr.py index aafe653..17d634c 100755 --- a/misp_modules/modules/import_mod/ocr.py +++ b/misp_modules/modules/import_mod/ocr.py @@ -1,8 +1,11 @@ import json import base64 +import magic from PIL import Image +from wand.image import Image as WImage + from pytesseract import image_to_string from io import BytesIO misperrors = {'error': 'Error'} @@ -10,7 +13,7 @@ userConfig = { }; inputSource = ['file'] -moduleinfo = {'version': '0.1', 'author': 'Alexandre Dulaunoy', +moduleinfo = {'version': '0.2', 'author': 'Alexandre Dulaunoy', 'description': 'Optical Character Recognition (OCR) module for MISP', 'module-type': ['import']} @@ -22,10 +25,28 @@ def handler(q=False): return False r = {'results': []} request = json.loads(q) - image = base64.b64decode(request["data"]) + document = base64.b64decode(request["data"]) + if magic.from_buffer(document, mime=True).split("/")[1] == 'pdf': + print("PDF Detected") + with WImage(blob=document) as pdf: + pages=len(pdf.sequence) + img = WImage(width=pdf.width, height=pdf.height * pages) + for p in range(pages): + img.composite(pdf.sequence[p], top=pdf.height * i, left=0) + image = document + image_file = BytesIO(image) image_file.seek(0) - ocrized = image_to_string(Image.open(image_file)) + + try: + im = WImage(blob=image_file) + except IOError: + misperrors['error'] = "Corrupt or not an image file." + return misperrors + + + ocrized = image_to_string(im) + freetext = {} freetext['values'] = ocrized freetext['types'] = ['freetext'] From 7c691af8075c5c4138cf57b0b902df998067cab8 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 28 Jun 2018 10:39:40 +0200 Subject: [PATCH 069/196] Updated the list of expansion modules --- misp_modules/modules/expansion/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index a9389e0..02c40ef 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,3 +1,3 @@ from . import _vmray -__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd'] +__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange'] From b1c90b411eb5792a6f78e15e1ab104055b51d5dc Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 28 Jun 2018 10:41:32 +0200 Subject: [PATCH 070/196] add: Sigma syntax validator expansion module --> Checks sigma rules syntax - Updated the expansion modules list as well - Updated the requirements list --- REQUIREMENTS | 1 + misp_modules/modules/expansion/__init__.py | 2 +- .../expansion/sigma_syntax_validator.py | 35 +++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 misp_modules/modules/expansion/sigma_syntax_validator.py diff --git a/REQUIREMENTS b/REQUIREMENTS index 9404855..a8baf52 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -20,3 +20,4 @@ pygeoip bs4 oauth2 yara +sigmatools diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 02c40ef..b49c1dc 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,3 +1,3 @@ from . import _vmray -__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange'] +__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator'] diff --git a/misp_modules/modules/expansion/sigma_syntax_validator.py b/misp_modules/modules/expansion/sigma_syntax_validator.py new file mode 100644 index 0000000..0d5226f --- /dev/null +++ b/misp_modules/modules/expansion/sigma_syntax_validator.py @@ -0,0 +1,35 @@ +import json +try: + import yaml + from sigma.parser import SigmaParser + from sigma.config import SigmaConfiguration +except ModuleNotFoundError: + print("sigma or yaml is missing, use 'pip3 install sigmatools' to install it.") + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['sigma'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'module-type': ['expansion', 'hover'], + 'description': 'An expansion hover module to perform a syntax check on sigma rules'} +moduleconfig = [] + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if not request.get('sigma'): + misperrors['error'] = 'Sigma rule missing' + return misperrors + config = SigmaConfiguration() + try: + parser = SigmaParser(yaml.load(request.get('sigma')), config) + result = ("Syntax valid: {}".format(parser.values)) + except Exception as e: + result = ("Syntax error: {}".format(str(e))) + return {'results': [{'types': mispattributes['output'], 'values': result}]} + +def introspection(): + return mispattributes + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 7dd8e988c0b0c6af51c9234b938f8ea7291d8a04 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 28 Jun 2018 10:51:40 +0200 Subject: [PATCH 071/196] Updated the list of modules (removed stiximport) --- misp_modules/modules/import_mod/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py index 886eaf7..0a732e2 100644 --- a/misp_modules/modules/import_mod/__init__.py +++ b/misp_modules/modules/import_mod/__init__.py @@ -1,4 +1,3 @@ from . import _vmray -__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport', 'goamlimport', - 'email_import', 'mispjson', 'openiocimport', 'threatanalyzer_import', 'csvimport'] +__all__ = ['vmray_import', 'testimport', 'ocr', 'cuckooimport', 'goamlimport', 'email_import', 'mispjson', 'openiocimport', 'threatanalyzer_import', 'csvimport'] From 7885017981a122761ce1613858ff904115eb10cc Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 28 Jun 2018 16:59:03 +0800 Subject: [PATCH 072/196] - fixed typo move image back in scope --- misp_modules/modules/import_mod/ocr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/import_mod/ocr.py b/misp_modules/modules/import_mod/ocr.py index 17d634c..0748d35 100755 --- a/misp_modules/modules/import_mod/ocr.py +++ b/misp_modules/modules/import_mod/ocr.py @@ -32,8 +32,8 @@ def handler(q=False): pages=len(pdf.sequence) img = WImage(width=pdf.width, height=pdf.height * pages) for p in range(pages): - img.composite(pdf.sequence[p], top=pdf.height * i, left=0) - image = document + img.composite(pdf.sequence[p], top=pdf.height * p, left=0) + image = document image_file = BytesIO(image) image_file.seek(0) From e9ee09eec61fbca7fbd956fdcd8ef2a224e585a5 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 28 Jun 2018 11:27:35 +0200 Subject: [PATCH 073/196] Updated README to add sigma & some other missing modules --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8b96814..14840ea 100644 --- a/README.md +++ b/README.md @@ -30,15 +30,22 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind. * [hashdd](misp_modules/modules/expansion/hashdd.py) - a hover module to check file hashes against [hashdd.com](http://www.hashdd.com) including NSLR dataset. * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address. -* [iprep](misp-modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net. +* [iprep](misp_modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net. +* [onyphe](misp_modules/modules/expansion/onyphe.py) - a modules to process queries on Onyphe. +* [onyphe_full](misp_modules/modules/expansion/onyphe_full.py) - a modules to process full queries on Onyphe. * [OTX](misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/). * [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets. * [rbl](misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute. +* [reversedns](misp_modules/modules/expansion/reversedns.py) - Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes. * [shodan](misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module. +* [Sigma syntax validator](misp_modules/modules/expansion/sigma_syntax_validator.py) - Sigma syntax validator. * [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance. * [ThreatCrowd](misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/). * [threatminer](misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/). * [virustotal](misp_modules/modules/expansion/virustotal.py) - an expansion module to pull known resolutions and malware samples related with an IP/Domain from virusTotal (this modules require a VirusTotal private API key) +* [VMray](misp_modules/modules/expansion/vmray_submit.py) - a module to submit a sample to VMray. +* [VulnDB](misp_modules/modules/expansion/vulndb.py) - a module to query [VulnDB](https://www.riskbasedsecurity.com/). +* [whois](misp_modules/modules/expansion) - a module to query a local instance of [uwhois](https://github.com/rafiot/uwhoisd). * [wikidata](misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module. * [xforce](misp_modules/modules/expansion/xforceexchange.py) - an IBM X-Force Exchange expansion module. * [YARA syntax validator](misp_modules/modules/expansion/yara_syntax_validator.py) - YARA syntax validator. From 60a3fbe28204c5178b2bceed2b3551a27e8c6ce4 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 28 Jun 2018 23:20:38 +0800 Subject: [PATCH 074/196] - added wand requirement - fixed missing return png byte-stream - move module import to handler to catch and report errorz --- REQUIREMENTS | 1 + misp_modules/modules/import_mod/ocr.py | 41 +++++++++++++++++++------- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/REQUIREMENTS b/REQUIREMENTS index 9404855..c116763 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -14,6 +14,7 @@ git+https://github.com/MISP/PyMISP.git#egg=pymisp git+https://github.com/sebdraven/pyonyphe#egg=pyonyphe pillow pytesseract +wand SPARQLWrapper domaintools_api pygeoip diff --git a/misp_modules/modules/import_mod/ocr.py b/misp_modules/modules/import_mod/ocr.py index 0748d35..a30bba0 100755 --- a/misp_modules/modules/import_mod/ocr.py +++ b/misp_modules/modules/import_mod/ocr.py @@ -1,12 +1,5 @@ import json import base64 -import magic - -from PIL import Image - -from wand.image import Image as WImage - -from pytesseract import image_to_string from io import BytesIO misperrors = {'error': 'Error'} userConfig = { }; @@ -21,6 +14,32 @@ moduleconfig = [] def handler(q=False): + # try to import modules and return errors if module not found + try: + import magic + except ImportError: + misperrors['error'] = "Please pip(3) install magic" + return misperrors + + try: + from PIL import Image + except ImportError: + misperrors['error'] = "Please pip(3) install pillow" + return misperrors + + try: + # Official ImageMagick module + from wand.image import Image as WImage + except ImportError: + misperrors['error'] = "Please pip(3) install wand" + return misperrors + + try: + from pytesseract import image_to_string + except ImportError: + misperrors['error'] = "Please pip(3) install pytesseract" + return misperrors + if q is False: return False r = {'results': []} @@ -32,14 +51,16 @@ def handler(q=False): pages=len(pdf.sequence) img = WImage(width=pdf.width, height=pdf.height * pages) for p in range(pages): - img.composite(pdf.sequence[p], top=pdf.height * p, left=0) - image = document + image = img.composite(pdf.sequence[p], top=pdf.height * p, left=0) + image = img.make_blob('png') + else: + image = document image_file = BytesIO(image) image_file.seek(0) try: - im = WImage(blob=image_file) + im = Image.open(image_file) except IOError: misperrors['error'] = "Corrupt or not an image file." return misperrors From fbb3617f256d19e272d95e7a6f2c9e745acdfe94 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Fri, 29 Jun 2018 12:01:17 +0800 Subject: [PATCH 075/196] - Quick comment ToDo: Avoid using Magic in future releases --- misp_modules/modules/import_mod/ocr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/import_mod/ocr.py b/misp_modules/modules/import_mod/ocr.py index a30bba0..2248306 100755 --- a/misp_modules/modules/import_mod/ocr.py +++ b/misp_modules/modules/import_mod/ocr.py @@ -45,7 +45,7 @@ def handler(q=False): r = {'results': []} request = json.loads(q) document = base64.b64decode(request["data"]) - if magic.from_buffer(document, mime=True).split("/")[1] == 'pdf': + if magic.from_buffer(document, mime=True).split("/")[1] == 'pdf': # Eventually this could be replaced with wand.obj.format print("PDF Detected") with WImage(blob=document) as pdf: pages=len(pdf.sequence) From c7c93b53e8522c6318f9571d1c78d7f5a4b8b25f Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Fri, 29 Jun 2018 12:02:08 +0800 Subject: [PATCH 076/196] - Set tornado timeout to 300 seconds. --- misp_modules/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/__init__.py b/misp_modules/__init__.py index 1c1713b..3bb7253 100644 --- a/misp_modules/__init__.py +++ b/misp_modules/__init__.py @@ -193,7 +193,7 @@ class QueryModule(tornado.web.RequestHandler): if dict_payload.get('timeout'): timeout = datetime.timedelta(seconds=int(dict_payload.get('timeout'))) else: - timeout = datetime.timedelta(seconds=30) + timeout = datetime.timedelta(seconds=300) response = yield tornado.gen.with_timeout(timeout, self.run_request(jsonpayload)) self.write(response) except tornado.gen.TimeoutError: From ff793bc221aff4ab70911115b9ea3b18eb7be1e1 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Fri, 29 Jun 2018 11:17:03 +0200 Subject: [PATCH 077/196] threatanalyzer_import - order of category tuned --- misp_modules/modules/import_mod/threatanalyzer_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/import_mod/threatanalyzer_import.py b/misp_modules/modules/import_mod/threatanalyzer_import.py index 757f849..83d8291 100755 --- a/misp_modules/modules/import_mod/threatanalyzer_import.py +++ b/misp_modules/modules/import_mod/threatanalyzer_import.py @@ -69,7 +69,7 @@ def handler(q=False): results.append({ 'values': current_sample_filename, 'data': base64.b64encode(file_data).decode(), - 'type': 'malware-sample', 'categories': ['Artifacts dropped', 'Payload delivery'], 'to_ids': True, 'comment': ''}) + 'type': 'malware-sample', 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': True, 'comment': ''}) if 'Analysis/analysis.json' in zip_file_name: with zf.open(zip_file_name, mode='r', pwd=None) as fp: From 60f772b9050ccd9ae3678dd38bbd9003de2df3c1 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 11:27:36 +0200 Subject: [PATCH 078/196] add new module dnstrails --- misp_modules/modules/expansion/dnstrails.py | 25 +++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 misp_modules/modules/expansion/dnstrails.py diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py new file mode 100644 index 0000000..d77c8d1 --- /dev/null +++ b/misp_modules/modules/expansion/dnstrails.py @@ -0,0 +1,25 @@ +import logging +import sys + +log = logging.getLogger('dnstrails') +log.setLevel(logging.DEBUG) +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +log.addHandler(ch) + +misperrors = {'error': 'Error'} +mispattributes = { + 'input': ['hostname', 'domain', 'ip-src', 'ip-dst'], + 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'dns-soa-email'] +} + +moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', + 'description': 'Query on securitytrails.com', + 'module-type': ['expansion', 'hover']} + +# config fields that your code expects from the site admin +moduleconfig = ['apikey'] + + From 035606a21acfed52a951c56c3ffe5a551d3d4d74 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 11:47:11 +0200 Subject: [PATCH 079/196] add link pydnstrain in requirements --- REQUIREMENTS | 1 + 1 file changed, 1 insertion(+) diff --git a/REQUIREMENTS b/REQUIREMENTS index a8baf52..11393b0 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -12,6 +12,7 @@ asnhistory git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client git+https://github.com/MISP/PyMISP.git#egg=pymisp git+https://github.com/sebdraven/pyonyphe#egg=pyonyphe +git+https://github.com/sebdraven/pydnstrails#egg=pydnstrails pillow pytesseract SPARQLWrapper From cfe971a27185a1f64fc2927f3ffbe9b74a98ffcc Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 15:50:26 +0200 Subject: [PATCH 080/196] add expand domains --- misp_modules/modules/expansion/dnstrails.py | 140 ++++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index d77c8d1..3655269 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -1,5 +1,7 @@ +import json import logging import sys +from dnstrails import DnsTrails log = logging.getLogger('dnstrails') log.setLevel(logging.DEBUG) @@ -23,3 +25,141 @@ moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', moduleconfig = ['apikey'] +def handler(q=False): + if q: + + request = json.loads(q) + + if not request.get('config') and not (request['config'].get('apikey')): + misperrors['error'] = 'DNS authentication is missing' + return misperrors + + api = DnsTrails(request['config'].get('apikey')) + + if not api: + misperrors['error'] = 'Onyphe Error instance api' + + ip = "" + dns_name = "" + + ip = '' + if request.get('ip-src'): + ip = request['ip-src'] + return handle_ip(api, ip, misperrors) + elif request.get('ip-dst'): + ip = request['ip-dst'] + return handle_ip(api, ip, misperrors) + elif request.get('domain'): + domain = request['domain'] + return handle_domain(api, domain, misperrors) + elif request.get('hostname'): + hostname = request['hostname'] + return handle_domain(api, hostname, misperrors) + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + else: + return False + + +def handle_domain(api, domain, misperrors): + result_filtered = {"results": []} + + r, status_ok = expand_domain_info(api, misperrors, domain) + + if status_ok: + result_filtered['results'].extend(r) + else: + misperrors['error'] = 'Error pastries result' + return misperrors + + return result_filtered + +def handle_ip(api, ip, misperrors): + pass + + +def expand_domain_info(api, misperror,domain): + r = [] + status_ok = False + ns_servers = [] + list_ipv4 = [] + list_ipv6 = [] + servers_mx = [] + soa_hostnames = [] + + results = api.domain(domain) + + if results: + if 'current_dns' in results: + if 'values' in results['current_dns']['ns']: + ns_servers = [ns_entry['nameserver'] for ns_entry in + results['current_dns']['ns']['values'] + if 'nameserver' in ns_entry] + if 'values' in results['current_dns']['a']: + list_ipv4 = [a_entry['ip'] for a_entry in + results['current_dns']['a']['values'] if + 'ip' in a_entry] + + if 'values' in results['current_dns']['aaaa']: + list_ipv6 = [ipv6_entry['ipv6'] for ipv6_entry in + results['current_ns']['aaaa']['values'] if + 'ipv6' in ipv6_entry] + + if 'values' in results['current_dns']['mx']: + servers_mx = [mx_entry['hostname'] for mx_entry in + results['current_dns']['mx']['values'] if + 'hostname' in mx_entry] + if 'values' in results['current_dns']['soa']: + soa_hostnames = [soa_entry['email'] for soa_entry in + results['current_dns']['soa']['values'] if + 'email' in soa_entry] + + if ns_servers: + r.append({'type': ['domain'], + 'values': ns_servers, + 'Category': ['Network Activity'], + 'comment': 'List of name servers of %s first seen %s ' % + (domain, results['current_dns']['ns']['first_seen']) + }) + + if list_ipv4: + r.append({'type': ['domain|ip'], + 'values': ['%s|%s' % (domain, ipv4) for ipv4 in list_ipv4], + 'Category': ['Network Activity'], + 'comment': ' List ipv4 of %s first seen %s' % + (domain, + results['current_dns']['a']['first_seen']) + + }) + if list_ipv6: + r.append({'type': ['domain|ip'], + 'values': ['%s|%s' % (domain, ipv6) for ipv6 in + list_ipv6], + 'Category': ['Network Activity'], + 'comment': ' List ipv6 of %s first seen %s' % + (domain, + results['current_dns']['aaaa']['first_seen']) + + }) + + if servers_mx: + r.append({'type': ['domain'], + 'values': servers_mx, + 'Category': ['Network Activity'], + 'comment': ' List mx of %s first seen %s' % + (domain, + results['current_dns']['mx']['first_seen']) + + }) + if soa_hostnames: + r.append({'type': ['domain'], + 'values': soa_hostnames, + 'Category': ['Network Activity'], + 'comment': ' List soa of %s first seen %s' % + (domain, + results['current_dns']['soa']['first_seen']) + }) + + + return r, status_ok From 09c52788b8f2479f1b0571be7514c66220d42baf Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 16:11:24 +0200 Subject: [PATCH 081/196] add methods --- misp_modules/modules/expansion/dnstrails.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 3655269..d357ed8 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -163,3 +163,11 @@ def expand_domain_info(api, misperror,domain): return r, status_ok + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo \ No newline at end of file From f3962d2d0599ef689d2d52b458c36c9cf123fea5 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 16:17:32 +0200 Subject: [PATCH 082/196] add status ! --- misp_modules/modules/expansion/dnstrails.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index d357ed8..5d5e9d4 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -70,7 +70,7 @@ def handle_domain(api, domain, misperrors): if status_ok: result_filtered['results'].extend(r) else: - misperrors['error'] = 'Error pastries result' + misperrors['error'] = 'Error dns result' return misperrors return result_filtered @@ -91,6 +91,7 @@ def expand_domain_info(api, misperror,domain): results = api.domain(domain) if results: + status_ok = True if 'current_dns' in results: if 'values' in results['current_dns']['ns']: ns_servers = [ns_entry['nameserver'] for ns_entry in From 0275e3ecd8b9e976a12f2de2bfa1ac146f4b3fbe Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 16:20:35 +0200 Subject: [PATCH 083/196] changes keys --- misp_modules/modules/expansion/dnstrails.py | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 5d5e9d4..41151e6 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -19,7 +19,7 @@ mispattributes = { moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', 'description': 'Query on securitytrails.com', - 'module-type': ['expansion', 'hover']} + 'module-types': ['expansion', 'hover']} # config fields that your code expects from the site admin moduleconfig = ['apikey'] @@ -56,7 +56,7 @@ def handler(q=False): hostname = request['hostname'] return handle_domain(api, hostname, misperrors) else: - misperrors['error'] = "Unsupported attributes type" + misperrors['error'] = "Unsupported attributes types" return misperrors else: return False @@ -117,27 +117,27 @@ def expand_domain_info(api, misperror,domain): 'email' in soa_entry] if ns_servers: - r.append({'type': ['domain'], + r.append({'types': ['domain'], 'values': ns_servers, - 'Category': ['Network Activity'], + 'categories': ['Network Activity'], 'comment': 'List of name servers of %s first seen %s ' % (domain, results['current_dns']['ns']['first_seen']) }) if list_ipv4: - r.append({'type': ['domain|ip'], + r.append({'types': ['domain|ip'], 'values': ['%s|%s' % (domain, ipv4) for ipv4 in list_ipv4], - 'Category': ['Network Activity'], + 'categories': ['Network Activity'], 'comment': ' List ipv4 of %s first seen %s' % (domain, results['current_dns']['a']['first_seen']) }) if list_ipv6: - r.append({'type': ['domain|ip'], + r.append({'types': ['domain|ip'], 'values': ['%s|%s' % (domain, ipv6) for ipv6 in list_ipv6], - 'Category': ['Network Activity'], + 'categories': ['Network Activity'], 'comment': ' List ipv6 of %s first seen %s' % (domain, results['current_dns']['aaaa']['first_seen']) @@ -145,18 +145,18 @@ def expand_domain_info(api, misperror,domain): }) if servers_mx: - r.append({'type': ['domain'], + r.append({'types': ['domain'], 'values': servers_mx, - 'Category': ['Network Activity'], + 'categories': ['Network Activity'], 'comment': ' List mx of %s first seen %s' % (domain, results['current_dns']['mx']['first_seen']) }) if soa_hostnames: - r.append({'type': ['domain'], + r.append({'types': ['domain'], 'values': soa_hostnames, - 'Category': ['Network Activity'], + 'categories': ['Network Activity'], 'comment': ' List soa of %s first seen %s' % (domain, results['current_dns']['soa']['first_seen']) From 2d1adf4aa959aa9874ef4dc7b0ec79a133f3fbc2 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 16:30:47 +0200 Subject: [PATCH 084/196] change categories --- misp_modules/modules/expansion/dnstrails.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 41151e6..d2fd5e1 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -119,7 +119,7 @@ def expand_domain_info(api, misperror,domain): if ns_servers: r.append({'types': ['domain'], 'values': ns_servers, - 'categories': ['Network Activity'], + 'categories': ['Network activity'], 'comment': 'List of name servers of %s first seen %s ' % (domain, results['current_dns']['ns']['first_seen']) }) @@ -127,7 +127,8 @@ def expand_domain_info(api, misperror,domain): if list_ipv4: r.append({'types': ['domain|ip'], 'values': ['%s|%s' % (domain, ipv4) for ipv4 in list_ipv4], - 'categories': ['Network Activity'], + 'categories': ['Network activity'], + 'comment': ' List ipv4 of %s first seen %s' % (domain, results['current_dns']['a']['first_seen']) @@ -137,7 +138,7 @@ def expand_domain_info(api, misperror,domain): r.append({'types': ['domain|ip'], 'values': ['%s|%s' % (domain, ipv6) for ipv6 in list_ipv6], - 'categories': ['Network Activity'], + 'categories': ['Network activity'], 'comment': ' List ipv6 of %s first seen %s' % (domain, results['current_dns']['aaaa']['first_seen']) @@ -147,7 +148,7 @@ def expand_domain_info(api, misperror,domain): if servers_mx: r.append({'types': ['domain'], 'values': servers_mx, - 'categories': ['Network Activity'], + 'categories': ['Network activity'], 'comment': ' List mx of %s first seen %s' % (domain, results['current_dns']['mx']['first_seen']) @@ -156,7 +157,7 @@ def expand_domain_info(api, misperror,domain): if soa_hostnames: r.append({'types': ['domain'], 'values': soa_hostnames, - 'categories': ['Network Activity'], + 'categories': ['Network activity'], 'comment': ' List soa of %s first seen %s' % (domain, results['current_dns']['soa']['first_seen']) From 64847a8a04253f68b016e0283dd06cf5972a98ca Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 17:19:21 +0200 Subject: [PATCH 085/196] add expand subdomains --- misp_modules/modules/expansion/dnstrails.py | 25 ++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index d2fd5e1..ffb4055 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -2,6 +2,7 @@ import json import logging import sys from dnstrails import DnsTrails +from dnstrails import APIError log = logging.getLogger('dnstrails') log.setLevel(logging.DEBUG) @@ -163,9 +164,31 @@ def expand_domain_info(api, misperror,domain): results['current_dns']['soa']['first_seen']) }) - return r, status_ok + +def expand_subdomains(api, domain): + + r = [] + status_ok = False + + try: + results = api.subdomains(domain) + + if results: + status_ok = True + if 'subdomains' in results: + r.append({ + 'type': ['domain'], + 'values': ['%s.%s' % (sub,domain) for sub in results['subdomains']], + } + + ) + except APIError as e: + misperrors['error'] = e + return r, status_ok + + def introspection(): return mispattributes From 0965def6bfd9d4e6def23d8b442e7898238b534c Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 17:22:19 +0200 Subject: [PATCH 086/196] add expand subdomains --- misp_modules/modules/expansion/dnstrails.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index ffb4055..abef1e5 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -1,8 +1,9 @@ import json import logging import sys -from dnstrails import DnsTrails + from dnstrails import APIError +from dnstrails import DnsTrails log = logging.getLogger('dnstrails') log.setLevel(logging.DEBUG) @@ -68,6 +69,14 @@ def handle_domain(api, domain, misperrors): r, status_ok = expand_domain_info(api, misperrors, domain) + if status_ok: + result_filtered['results'].extend(r) + else: + misperrors['error'] = 'Error dns result' + return misperrors + + r, status_ok = expand_subdomains(api, domain) + if status_ok: result_filtered['results'].extend(r) else: From 78d6de9b7a47d4a997cfa996a9ae7f7b8014ab95 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 17:25:37 +0200 Subject: [PATCH 087/196] add categories and comments --- misp_modules/modules/expansion/dnstrails.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index abef1e5..1fd7500 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -190,6 +190,8 @@ def expand_subdomains(api, domain): r.append({ 'type': ['domain'], 'values': ['%s.%s' % (sub,domain) for sub in results['subdomains']], + 'categories': ['Network activity'], + 'comment': 'subdomains of %s' % domain } ) From f1c6095914381aa41feed5a2043989665e04eabc Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 17:26:56 +0200 Subject: [PATCH 088/196] typo --- misp_modules/modules/expansion/dnstrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 1fd7500..79afeb3 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -188,7 +188,7 @@ def expand_subdomains(api, domain): status_ok = True if 'subdomains' in results: r.append({ - 'type': ['domain'], + 'types': ['domain'], 'values': ['%s.%s' % (sub,domain) for sub in results['subdomains']], 'categories': ['Network activity'], 'comment': 'subdomains of %s' % domain From 34da5cdb767232bdf03c087f2368eb104f0b88d2 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 17:57:11 +0200 Subject: [PATCH 089/196] add expand whois --- misp_modules/modules/expansion/dnstrails.py | 53 ++++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 79afeb3..d1c276a 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -16,7 +16,10 @@ log.addHandler(ch) misperrors = {'error': 'Error'} mispattributes = { 'input': ['hostname', 'domain', 'ip-src', 'ip-dst'], - 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'dns-soa-email'] + 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'dns-soa-email', + 'whois-registrant-email', 'whois-registrant-phone', + 'whois-registrant-name', + 'whois-registrar', 'whois-creation-date', 'domain'] } moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', @@ -77,6 +80,14 @@ def handle_domain(api, domain, misperrors): r, status_ok = expand_subdomains(api, domain) + if status_ok: + result_filtered['results'].extend(r) + else: + misperrors['error'] = 'Error dns result' + return misperrors + + r, status_ok = expand_whois(api, domain) + if status_ok: result_filtered['results'].extend(r) else: @@ -181,6 +192,7 @@ def expand_subdomains(api, domain): r = [] status_ok = False + try: results = api.subdomains(domain) @@ -200,10 +212,47 @@ def expand_subdomains(api, domain): return r, status_ok +def expand_whois(api, domain): + r = [] + status_ok = False + + try: + results = api.whois(domain) + + if results: + status_ok = True + item_registrant = __select_registrant_item(results) + + r.append({ + 'types': ['whois-registrant-email', 'whois-registrant-phone', + 'whois-registrant-name', 'whois-registrar', + 'whois-creation-date'], + 'values': [item_registrant['email'], + item_registrant['telephone'], + item_registrant['name'], results['registrarName'], + results['creationDate']], + 'categories': ['attribution'], + 'comment': 'whois information of %s by securitytrails' % domain + } + + ) + + except APIError as e: + misperrors['error'] = e + + return r, status_ok + def introspection(): return mispattributes def version(): moduleinfo['config'] = moduleconfig - return moduleinfo \ No newline at end of file + return moduleinfo + + +def __select_registrant_item(entry): + if 'contacts' in entry: + for c in entry['contacts']: + if c['type'] == 'registrant': + return entry From ef3837077e1e5da2d1f7bf4f40730e10be6aea1f Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Sat, 30 Jun 2018 00:58:25 +0800 Subject: [PATCH 090/196] - Some more comments - Removed libmagic, wand can handle it better --- misp_modules/modules/import_mod/ocr.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/misp_modules/modules/import_mod/ocr.py b/misp_modules/modules/import_mod/ocr.py index 2248306..441adc4 100755 --- a/misp_modules/modules/import_mod/ocr.py +++ b/misp_modules/modules/import_mod/ocr.py @@ -1,8 +1,9 @@ import json import base64 from io import BytesIO + misperrors = {'error': 'Error'} -userConfig = { }; +userConfig = {}; inputSource = ['file'] @@ -15,12 +16,6 @@ moduleconfig = [] def handler(q=False): # try to import modules and return errors if module not found - try: - import magic - except ImportError: - misperrors['error'] = "Please pip(3) install magic" - return misperrors - try: from PIL import Image except ImportError: @@ -45,13 +40,18 @@ def handler(q=False): r = {'results': []} request = json.loads(q) document = base64.b64decode(request["data"]) - if magic.from_buffer(document, mime=True).split("/")[1] == 'pdf': # Eventually this could be replaced with wand.obj.format - print("PDF Detected") + document = WImage(blob=document) + if document.format == 'PDF': with WImage(blob=document) as pdf: + # Get number of pages pages=len(pdf.sequence) + print(f"PDF with {pages} page(s) detected") + # Create new image object where the height will be the number of pages. With huge PDFs this will overflow, break, consume silly memory etc… img = WImage(width=pdf.width, height=pdf.height * pages) + # Cycle through pages and stitch it together to one big file for p in range(pages): image = img.composite(pdf.sequence[p], top=pdf.height * p, left=0) + # Create a png blob image = img.make_blob('png') else: image = document From 2f5dd9928e89fb4adf9c7c1849e003f2e8b9a360 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Sat, 30 Jun 2018 11:38:26 +0800 Subject: [PATCH 091/196] - content was already a wand.obj --- misp_modules/modules/import_mod/ocr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/import_mod/ocr.py b/misp_modules/modules/import_mod/ocr.py index 441adc4..f37ba9b 100755 --- a/misp_modules/modules/import_mod/ocr.py +++ b/misp_modules/modules/import_mod/ocr.py @@ -42,7 +42,7 @@ def handler(q=False): document = base64.b64decode(request["data"]) document = WImage(blob=document) if document.format == 'PDF': - with WImage(blob=document) as pdf: + with document as pdf: # Get number of pages pages=len(pdf.sequence) print(f"PDF with {pages} page(s) detected") From ffce2aa5cc465823ae558953c2b46fc1fe88cef5 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Sat, 30 Jun 2018 11:52:12 +0800 Subject: [PATCH 092/196] - Added logger functionality for debug sessions --- misp_modules/modules/import_mod/ocr.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/import_mod/ocr.py b/misp_modules/modules/import_mod/ocr.py index f37ba9b..fc7acf7 100755 --- a/misp_modules/modules/import_mod/ocr.py +++ b/misp_modules/modules/import_mod/ocr.py @@ -2,6 +2,16 @@ import json import base64 from io import BytesIO +import logging + +log = logging.getLogger('ocr') +log.setLevel(logging.DEBUG) +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +log.addHandler(ch) + misperrors = {'error': 'Error'} userConfig = {}; @@ -45,14 +55,16 @@ def handler(q=False): with document as pdf: # Get number of pages pages=len(pdf.sequence) - print(f"PDF with {pages} page(s) detected") + log.debug(f"PDF with {pages} page(s) detected") # Create new image object where the height will be the number of pages. With huge PDFs this will overflow, break, consume silly memory etc… img = WImage(width=pdf.width, height=pdf.height * pages) # Cycle through pages and stitch it together to one big file for p in range(pages): + log.debug(f"Stitching page {p}") image = img.composite(pdf.sequence[p], top=pdf.height * p, left=0) # Create a png blob image = img.make_blob('png') + log.debug(f"Final image size is {pdf.width}x{pdf.height*p}") else: image = document From 184065cf741818d9a38a2fb885a77b7b441bb02e Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Sat, 30 Jun 2018 11:58:44 +0800 Subject: [PATCH 093/196] - Forgot to import sys --- misp_modules/modules/import_mod/ocr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/import_mod/ocr.py b/misp_modules/modules/import_mod/ocr.py index fc7acf7..b52722f 100755 --- a/misp_modules/modules/import_mod/ocr.py +++ b/misp_modules/modules/import_mod/ocr.py @@ -1,3 +1,4 @@ +import sys import json import base64 from io import BytesIO From 9f0313a97e0cf4e13cf4af580feddfe64591709f Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Sat, 30 Jun 2018 12:01:21 +0800 Subject: [PATCH 094/196] - Fixed log output --- misp_modules/modules/import_mod/ocr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/import_mod/ocr.py b/misp_modules/modules/import_mod/ocr.py index b52722f..15d660b 100755 --- a/misp_modules/modules/import_mod/ocr.py +++ b/misp_modules/modules/import_mod/ocr.py @@ -61,11 +61,11 @@ def handler(q=False): img = WImage(width=pdf.width, height=pdf.height * pages) # Cycle through pages and stitch it together to one big file for p in range(pages): - log.debug(f"Stitching page {p}") + log.debug(f"Stitching page {p+1}") image = img.composite(pdf.sequence[p], top=pdf.height * p, left=0) # Create a png blob image = img.make_blob('png') - log.debug(f"Final image size is {pdf.width}x{pdf.height*p}") + log.debug(f"Final image size is {pdf.width}x{pdf.height*(p+1)}") else: image = document From 549f32547d474d7d3a33651cf5dc8d9ce8a8720b Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Sun, 1 Jul 2018 22:08:42 +0800 Subject: [PATCH 095/196] - Reverted to <3.6 compatibility --- misp_modules/modules/import_mod/ocr.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/import_mod/ocr.py b/misp_modules/modules/import_mod/ocr.py index 15d660b..f14212b 100755 --- a/misp_modules/modules/import_mod/ocr.py +++ b/misp_modules/modules/import_mod/ocr.py @@ -56,16 +56,16 @@ def handler(q=False): with document as pdf: # Get number of pages pages=len(pdf.sequence) - log.debug(f"PDF with {pages} page(s) detected") + log.debug("PDF with {} page(s) detected".format(pages)) # Create new image object where the height will be the number of pages. With huge PDFs this will overflow, break, consume silly memory etc… img = WImage(width=pdf.width, height=pdf.height * pages) # Cycle through pages and stitch it together to one big file for p in range(pages): - log.debug(f"Stitching page {p+1}") + log.debug("Stitching page {}".format(p+1)) image = img.composite(pdf.sequence[p], top=pdf.height * p, left=0) # Create a png blob image = img.make_blob('png') - log.debug(f"Final image size is {pdf.width}x{pdf.height*(p+1)}") + log.debug("Final image size is {}x{}".format(pdf.width, pdf.height*(p+1))) else: image = document From 08d8459e1a6a41723bcc179bc6cead1b4ec966f1 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 2 Jul 2018 11:38:33 +0200 Subject: [PATCH 096/196] add: STIX2 pattern syntax validator --- README.md | 3 +- misp_modules/modules/expansion/__init__.py | 2 +- .../stix2_pattern_syntax_validator.py | 39 +++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 misp_modules/modules/expansion/stix2_pattern_syntax_validator.py diff --git a/README.md b/README.md index 14840ea..95019da 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [shodan](misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module. * [Sigma syntax validator](misp_modules/modules/expansion/sigma_syntax_validator.py) - Sigma syntax validator. * [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance. +* [STIX2 pattern syntax validator](misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - a module to check a STIX2 pattern syntax. * [ThreatCrowd](misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/). * [threatminer](misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/). * [virustotal](misp_modules/modules/expansion/virustotal.py) - an expansion module to pull known resolutions and malware samples related with an IP/Domain from virusTotal (this modules require a VirusTotal private API key) @@ -380,7 +381,7 @@ Recommended Plugin.Import_ocr_enabled true Enable or disable the ocr In this same menu set any other plugin settings that are required for testing. ## Install misp-module on an offline instance. -First, you need to grab all necessery packages for example like this : +First, you need to grab all necessary packages for example like this : Use pip wheel to create an archive ~~~ diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index b49c1dc..5e0d65e 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,3 +1,3 @@ from . import _vmray -__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator'] +__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator'] diff --git a/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py b/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py new file mode 100644 index 0000000..92e48c5 --- /dev/null +++ b/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py @@ -0,0 +1,39 @@ +import json +from stix2patterns.validator import run_validator + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['stix2-pattern'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'module-type': ['expansion', 'hover'], + 'description': 'An expansion hover module to perform a syntax check on stix2 patterns.'} +moduleconfig = [] + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if not request.get('stix2-pattern'): + misperrors['error'] = 'STIX2 pattern missing' + return misperrors + pattern = request.get('stix2-pattern') + syntax_errors = [] + for p in pattern[2:-2].split(' AND '): + syntax_validator = run_validator("[{}]".format(p)) + if syntax_validator: + for error in syntax_validator: + syntax_errors.append(error) + if syntax_errors: + s = 's' if len(syntax_errors) > 1 else '' + s_errors = "" + for error in syntax_errors: + s_errors += "{}\n".format(error[6:]) + result = "Syntax error{}: \n{}".format(s, s_errors[:-1]) + else: + result = "Syntax valid" + return {'results': [{'types': mispattributes['output'], 'values': result}]} + +def introspection(): + return mispattributes + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 5ff8bad85b58e2adcd0b9a151fbd52d9d1d849ec Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 2 Jul 2018 12:07:21 +0200 Subject: [PATCH 097/196] add: stix2 pattern validator requirements --- REQUIREMENTS | 1 + 1 file changed, 1 insertion(+) diff --git a/REQUIREMENTS b/REQUIREMENTS index 1365ec2..6f68e4b 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -22,3 +22,4 @@ bs4 oauth2 yara sigmatools +stix2-patterns From 90e42c03058c83f40b41734df6cb8c858e58c9c7 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 2 Jul 2018 12:14:21 +0200 Subject: [PATCH 098/196] fix: Put the stix2-pattern library import in a try statement --> Error more easily caught --- .../modules/expansion/stix2_pattern_syntax_validator.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py b/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py index 92e48c5..bf5d408 100644 --- a/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py +++ b/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py @@ -1,5 +1,8 @@ import json -from stix2patterns.validator import run_validator +try: + from stix2patterns.validator import run_validator +except ModuleNotFoundError: + print("stix2 patterns python library is missing, use 'pip3 install stix2-patterns' to install it.") misperrors = {'error': 'Error'} mispattributes = {'input': ['stix2-pattern'], 'output': ['text']} From 562a6b13084b68938a23d2dd01886d2aad9fda7b Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Tue, 3 Jul 2018 08:27:54 +0200 Subject: [PATCH 099/196] - Removed test modules from view - Moved skeleton expansion module to it's proper place --- {modules => misp_modules/modules}/expansion/module.py.skeleton | 0 misp_modules/modules/export_mod/__init__.py | 2 +- misp_modules/modules/import_mod/__init__.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename {modules => misp_modules/modules}/expansion/module.py.skeleton (100%) diff --git a/modules/expansion/module.py.skeleton b/misp_modules/modules/expansion/module.py.skeleton similarity index 100% rename from modules/expansion/module.py.skeleton rename to misp_modules/modules/expansion/module.py.skeleton diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py index 0034f5d..7c8d18b 100644 --- a/misp_modules/modules/export_mod/__init__.py +++ b/misp_modules/modules/export_mod/__init__.py @@ -1 +1 @@ -__all__ = ['testexport','cef_export','liteexport','goamlexport','threat_connect_export','pdfexport','threatStream_misp_export'] +__all__ = ['cef_export','liteexport','goamlexport','threat_connect_export','pdfexport','threatStream_misp_export'] diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py index 0a732e2..5190abb 100644 --- a/misp_modules/modules/import_mod/__init__.py +++ b/misp_modules/modules/import_mod/__init__.py @@ -1,3 +1,3 @@ from . import _vmray -__all__ = ['vmray_import', 'testimport', 'ocr', 'cuckooimport', 'goamlimport', 'email_import', 'mispjson', 'openiocimport', 'threatanalyzer_import', 'csvimport'] +__all__ = ['vmray_import', 'ocr', 'cuckooimport', 'goamlimport', 'email_import', 'mispjson', 'openiocimport', 'threatanalyzer_import', 'csvimport'] From 2a8fb76e8483ee4df174b94275b0b3c883583aaa Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 14:56:20 +0200 Subject: [PATCH 100/196] add logs --- misp_modules/modules/expansion/dnstrails.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index d1c276a..cb00262 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -236,6 +236,9 @@ def expand_whois(api, domain): } ) + # TODO File "modules/expansion/dnstrails.py", line 230, in expand_whois + # 'values': [item_registrant['email'], + # TypeError: 'NoneType' object is not subscriptable except APIError as e: misperrors['error'] = e @@ -254,5 +257,6 @@ def version(): def __select_registrant_item(entry): if 'contacts' in entry: for c in entry['contacts']: + print(c) if c['type'] == 'registrant': return entry From f710162beda5cee2005cff6df50d30a09c1af25d Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 14:59:39 +0200 Subject: [PATCH 101/196] change errors --- misp_modules/modules/expansion/dnstrails.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index cb00262..bf16601 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -83,7 +83,7 @@ def handle_domain(api, domain, misperrors): if status_ok: result_filtered['results'].extend(r) else: - misperrors['error'] = 'Error dns result' + misperrors['error'] = 'Error subdomains result' return misperrors r, status_ok = expand_whois(api, domain) @@ -91,7 +91,7 @@ def handle_domain(api, domain, misperrors): if status_ok: result_filtered['results'].extend(r) else: - misperrors['error'] = 'Error dns result' + misperrors['error'] = 'Error whois result' return misperrors return result_filtered From e1a1648f14c1834892af9b036be494bf251b2b18 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 15:01:04 +0200 Subject: [PATCH 102/196] add logs --- misp_modules/modules/expansion/dnstrails.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index bf16601..9787a15 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -242,6 +242,7 @@ def expand_whois(api, domain): except APIError as e: misperrors['error'] = e + print(e) return r, status_ok From 714c15f079c91b82c796239238859501c84f3ea8 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 15:05:10 +0200 Subject: [PATCH 103/196] change return value --- misp_modules/modules/expansion/dnstrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 9787a15..5f2cc63 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -260,4 +260,4 @@ def __select_registrant_item(entry): for c in entry['contacts']: print(c) if c['type'] == 'registrant': - return entry + return c From 1223d93d52faea1105e8bd289691e33bc12e31a8 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 15:07:54 +0200 Subject: [PATCH 104/196] change name keys --- misp_modules/modules/expansion/dnstrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 5f2cc63..5a0cfcc 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -230,7 +230,7 @@ def expand_whois(api, domain): 'values': [item_registrant['email'], item_registrant['telephone'], item_registrant['name'], results['registrarName'], - results['creationDate']], + results['createdDate']], 'categories': ['attribution'], 'comment': 'whois information of %s by securitytrails' % domain } From 1d100833a4cade1d942e91c9f176ef0aea6aded7 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 15:12:27 +0200 Subject: [PATCH 105/196] concat results --- misp_modules/modules/expansion/dnstrails.py | 29 +++++++++++---------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 5a0cfcc..6f52dff 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -222,27 +222,28 @@ def expand_whois(api, domain): if results: status_ok = True item_registrant = __select_registrant_item(results) - - r.append({ - 'types': ['whois-registrant-email', 'whois-registrant-phone', + types = ['whois-registrant-email', 'whois-registrant-phone', 'whois-registrant-name', 'whois-registrar', - 'whois-creation-date'], - 'values': [item_registrant['email'], - item_registrant['telephone'], - item_registrant['name'], results['registrarName'], - results['createdDate']], - 'categories': ['attribution'], - 'comment': 'whois information of %s by securitytrails' % domain - } + 'whois-creation-date'] + values = [item_registrant['email'], + item_registrant['telephone'], + item_registrant['name'], results['registrarName'], + results['createdDate']] + for t, v in zip(types, values): + r.append({ + 'types': t, + 'values': v, + 'categories': ['attribution'], + 'comment': 'whois information of %s by securitytrails' % domain + } - ) + ) # TODO File "modules/expansion/dnstrails.py", line 230, in expand_whois # 'values': [item_registrant['email'], # TypeError: 'NoneType' object is not subscriptable except APIError as e: misperrors['error'] = e - print(e) return r, status_ok @@ -258,6 +259,6 @@ def version(): def __select_registrant_item(entry): if 'contacts' in entry: for c in entry['contacts']: - print(c) + if c['type'] == 'registrant': return c From b677cd5fc7ef1a69b938ff8020e15c42142b5dfd Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 15:16:02 +0200 Subject: [PATCH 106/196] change categories --- misp_modules/modules/expansion/dnstrails.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 6f52dff..8d17fa5 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -229,15 +229,14 @@ def expand_whois(api, domain): item_registrant['telephone'], item_registrant['name'], results['registrarName'], results['createdDate']] - for t, v in zip(types, values): - r.append({ + + r = [{ 'types': t, 'values': v, - 'categories': ['attribution'], + 'categories': ['Attribution'], 'comment': 'whois information of %s by securitytrails' % domain - } + } for t, v in zip(types, values)] - ) # TODO File "modules/expansion/dnstrails.py", line 230, in expand_whois # 'values': [item_registrant['email'], # TypeError: 'NoneType' object is not subscriptable From 21794249d0339b5910e6db09292119b51066b488 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 15:17:37 +0200 Subject: [PATCH 107/196] add logs --- misp_modules/modules/expansion/dnstrails.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 8d17fa5..8ebecda 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -209,6 +209,7 @@ def expand_subdomains(api, domain): ) except APIError as e: misperrors['error'] = e + return r, status_ok @@ -243,6 +244,7 @@ def expand_whois(api, domain): except APIError as e: misperrors['error'] = e + print(e) return r, status_ok From 495c720d0fecb6552757d026dc175598814c5e4b Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 16:31:39 +0200 Subject: [PATCH 108/196] add history ipv4 --- misp_modules/modules/expansion/dnstrails.py | 75 ++++++++++++++++----- 1 file changed, 58 insertions(+), 17 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 8ebecda..e0b78fa 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -9,7 +9,8 @@ log = logging.getLogger('dnstrails') log.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) log.addHandler(ch) @@ -94,13 +95,22 @@ def handle_domain(api, domain, misperrors): misperrors['error'] = 'Error whois result' return misperrors + r, status_ok = expand_history_ipv4(api, domain) + + if status_ok: + result_filtered['results'].extend(r) + else: + misperrors['error'] = 'Error history ipv4' + return misperrors + return result_filtered + def handle_ip(api, ip, misperrors): pass -def expand_domain_info(api, misperror,domain): +def expand_domain_info(api, misperror, domain): r = [] status_ok = False ns_servers = [] @@ -130,31 +140,33 @@ def expand_domain_info(api, misperror,domain): if 'values' in results['current_dns']['mx']: servers_mx = [mx_entry['hostname'] for mx_entry in - results['current_dns']['mx']['values'] if - 'hostname' in mx_entry] + results['current_dns']['mx']['values'] if + 'hostname' in mx_entry] if 'values' in results['current_dns']['soa']: soa_hostnames = [soa_entry['email'] for soa_entry in - results['current_dns']['soa']['values'] if - 'email' in soa_entry] + results['current_dns']['soa']['values'] if + 'email' in soa_entry] if ns_servers: r.append({'types': ['domain'], 'values': ns_servers, 'categories': ['Network activity'], 'comment': 'List of name servers of %s first seen %s ' % - (domain, results['current_dns']['ns']['first_seen']) - }) + (domain, + results['current_dns']['ns']['first_seen']) + }) if list_ipv4: r.append({'types': ['domain|ip'], - 'values': ['%s|%s' % (domain, ipv4) for ipv4 in list_ipv4], + 'values': ['%s|%s' % (domain, ipv4) for ipv4 in + list_ipv4], 'categories': ['Network activity'], 'comment': ' List ipv4 of %s first seen %s' % (domain, results['current_dns']['a']['first_seen']) - }) + }) if list_ipv6: r.append({'types': ['domain|ip'], 'values': ['%s|%s' % (domain, ipv6) for ipv6 in @@ -188,11 +200,9 @@ def expand_domain_info(api, misperror,domain): def expand_subdomains(api, domain): - r = [] status_ok = False - try: results = api.subdomains(domain) @@ -201,7 +211,8 @@ def expand_subdomains(api, domain): if 'subdomains' in results: r.append({ 'types': ['domain'], - 'values': ['%s.%s' % (sub,domain) for sub in results['subdomains']], + 'values': ['%s.%s' % (sub, domain) + for sub in results['subdomains']], 'categories': ['Network activity'], 'comment': 'subdomains of %s' % domain } @@ -224,7 +235,7 @@ def expand_whois(api, domain): status_ok = True item_registrant = __select_registrant_item(results) types = ['whois-registrant-email', 'whois-registrant-phone', - 'whois-registrant-name', 'whois-registrar', + 'whois-registrant-name', 'whois-registrar', 'whois-creation-date'] values = [item_registrant['email'], item_registrant['telephone'], @@ -232,10 +243,10 @@ def expand_whois(api, domain): results['createdDate']] r = [{ - 'types': t, - 'values': v, + 'types': t, + 'values': v, 'categories': ['Attribution'], - 'comment': 'whois information of %s by securitytrails' % domain + 'comment': 'whois information of %s by securitytrails' % domain } for t, v in zip(types, values)] # TODO File "modules/expansion/dnstrails.py", line 230, in expand_whois @@ -248,6 +259,36 @@ def expand_whois(api, domain): return r, status_ok + +def expand_history_ipv4(api, domain): + r = [] + status_ok = False + + try: + results = api.history_dns_ipv4(domain) + + if results: + status_ok = True + if 'records' in results: + for record in results['records']: + if 'values' in record: + r.append( + {'type': ['domain|ip'], + 'values': ['%s|%s' % (domain, record['ip'])], + 'categories': ['Newtwork activity'], + 'comment': 'last seen: %s first seen: %s' % + (record['last_seen'], + record['first_seen']) + } + ) + + except APIError as e: + misperrors['error'] = e + print(e) + + return r, status_ok + + def introspection(): return mispattributes From 602da3d1a3baa52005678025b2a320d87ada5286 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 16:35:01 +0200 Subject: [PATCH 109/196] control return of records --- misp_modules/modules/expansion/dnstrails.py | 27 +++++++++++---------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index e0b78fa..d8d9a53 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -234,20 +234,21 @@ def expand_whois(api, domain): if results: status_ok = True item_registrant = __select_registrant_item(results) - types = ['whois-registrant-email', 'whois-registrant-phone', - 'whois-registrant-name', 'whois-registrar', - 'whois-creation-date'] - values = [item_registrant['email'], - item_registrant['telephone'], - item_registrant['name'], results['registrarName'], - results['createdDate']] + if item_registrant: + types = ['whois-registrant-email', 'whois-registrant-phone', + 'whois-registrant-name', 'whois-registrar', + 'whois-creation-date'] + values = [item_registrant['email'], + item_registrant['telephone'], + item_registrant['name'], results['registrarName'], + results['createdDate']] - r = [{ - 'types': t, - 'values': v, - 'categories': ['Attribution'], - 'comment': 'whois information of %s by securitytrails' % domain - } for t, v in zip(types, values)] + r = [{ + 'types': t, + 'values': v, + 'categories': ['Attribution'], + 'comment': 'whois information of %s by securitytrails' % domain + } for t, v in zip(types, values)] # TODO File "modules/expansion/dnstrails.py", line 230, in expand_whois # 'values': [item_registrant['email'], From e9747a3379dbaaf225eec6411b8d2ca9fa7ad936 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 16:41:44 +0200 Subject: [PATCH 110/196] add time sleep in each request --- misp_modules/modules/expansion/dnstrails.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index d8d9a53..b352bd7 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -1,6 +1,7 @@ import json import logging import sys +import time from dnstrails import APIError from dnstrails import DnsTrails @@ -79,6 +80,7 @@ def handle_domain(api, domain, misperrors): misperrors['error'] = 'Error dns result' return misperrors + time.sleep(1) r, status_ok = expand_subdomains(api, domain) if status_ok: @@ -87,6 +89,7 @@ def handle_domain(api, domain, misperrors): misperrors['error'] = 'Error subdomains result' return misperrors + time.sleep(1) r, status_ok = expand_whois(api, domain) if status_ok: @@ -95,6 +98,7 @@ def handle_domain(api, domain, misperrors): misperrors['error'] = 'Error whois result' return misperrors + time.sleep(1) r, status_ok = expand_history_ipv4(api, domain) if status_ok: From 26950ea7de576906ae4ab5ae38b3f1cf4874d23e Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 16:51:31 +0200 Subject: [PATCH 111/196] change loop --- misp_modules/modules/expansion/dnstrails.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index b352bd7..f950500 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -277,15 +277,16 @@ def expand_history_ipv4(api, domain): if 'records' in results: for record in results['records']: if 'values' in record: - r.append( - {'type': ['domain|ip'], - 'values': ['%s|%s' % (domain, record['ip'])], - 'categories': ['Newtwork activity'], - 'comment': 'last seen: %s first seen: %s' % - (record['last_seen'], - record['first_seen']) - } - ) + for item in record['values']: + r.append( + {'type': ['domain|ip'], + 'values': ['%s|%s' % (domain, item['ip'])], + 'categories': ['Newtwork activity'], + 'comment': 'last seen: %s first seen: %s' % + (record['last_seen'], + record['first_seen']) + } + ) except APIError as e: misperrors['error'] = e From 9e6162a4349a14eb66cc83ad57eadb5f6ee456d9 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 16:53:06 +0200 Subject: [PATCH 112/196] change type --- misp_modules/modules/expansion/dnstrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index f950500..c30c747 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -281,7 +281,7 @@ def expand_history_ipv4(api, domain): r.append( {'type': ['domain|ip'], 'values': ['%s|%s' % (domain, item['ip'])], - 'categories': ['Newtwork activity'], + 'categories': ['Network activity'], 'comment': 'last seen: %s first seen: %s' % (record['last_seen'], record['first_seen']) From f2333a4978e016e2a870e3cf1c7b1e8fcf0e25e0 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 16:55:13 +0200 Subject: [PATCH 113/196] change type --- misp_modules/modules/expansion/dnstrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index c30c747..a176489 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -279,7 +279,7 @@ def expand_history_ipv4(api, domain): if 'values' in record: for item in record['values']: r.append( - {'type': ['domain|ip'], + {'types': ['domain|ip'], 'values': ['%s|%s' % (domain, item['ip'])], 'categories': ['Network activity'], 'comment': 'last seen: %s first seen: %s' % From 3a96e189ed5f45780056226566da0fcdef0ab096 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 08:43:23 +0200 Subject: [PATCH 114/196] add ipv6 and ipv4 --- misp_modules/modules/expansion/dnstrails.py | 41 ++++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index a176489..02e1346 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -265,7 +265,7 @@ def expand_whois(api, domain): return r, status_ok -def expand_history_ipv4(api, domain): +def expand_history_ipv4_ipv6(api, domain): r = [] status_ok = False @@ -274,19 +274,14 @@ def expand_history_ipv4(api, domain): if results: status_ok = True - if 'records' in results: - for record in results['records']: - if 'values' in record: - for item in record['values']: - r.append( - {'types': ['domain|ip'], - 'values': ['%s|%s' % (domain, item['ip'])], - 'categories': ['Network activity'], - 'comment': 'last seen: %s first seen: %s' % - (record['last_seen'], - record['first_seen']) - } - ) + r.extend(__history_ip(results, domain)) + + time.sleep(1) + results = api.history_dns_aaaa(domain) + + if results: + status_ok = True + r.extend(__history_ip(results, domain)) except APIError as e: misperrors['error'] = e @@ -295,6 +290,24 @@ def expand_history_ipv4(api, domain): return r, status_ok +def __history_ip(results, domain): + r = [] + if 'records' in results: + for record in results['records']: + if 'values' in record: + for item in record['values']: + r.append( + {'types': ['domain|ip'], + 'values': ['%s|%s' % (domain, item['ip'])], + 'categories': ['Network activity'], + 'comment': 'last seen: %s first seen: %s' % + (record['last_seen'], + record['first_seen']) + } + ) + + return r + def introspection(): return mispattributes From 41635d43c772f15bb693260504b67df2eb88727a Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 08:49:59 +0200 Subject: [PATCH 115/196] correct typo --- misp_modules/modules/expansion/dnstrails.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 02e1346..c601481 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -99,7 +99,7 @@ def handle_domain(api, domain, misperrors): return misperrors time.sleep(1) - r, status_ok = expand_history_ipv4(api, domain) + r, status_ok = expand_history_ipv4_ipv6(api, domain) if status_ok: result_filtered['results'].extend(r) @@ -139,7 +139,7 @@ def expand_domain_info(api, misperror, domain): if 'values' in results['current_dns']['aaaa']: list_ipv6 = [ipv6_entry['ipv6'] for ipv6_entry in - results['current_ns']['aaaa']['values'] if + results['current_dns']['aaaa']['values'] if 'ipv6' in ipv6_entry] if 'values' in results['current_dns']['mx']: From 42c362d2fd6a629184b8887ff7876b4f7a8f52de Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 09:00:23 +0200 Subject: [PATCH 116/196] refactoring expand_whois --- misp_modules/modules/expansion/dnstrails.py | 68 +++++++++++++++++---- 1 file changed, 55 insertions(+), 13 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index c601481..7d18042 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -239,20 +239,62 @@ def expand_whois(api, domain): status_ok = True item_registrant = __select_registrant_item(results) if item_registrant: - types = ['whois-registrant-email', 'whois-registrant-phone', - 'whois-registrant-name', 'whois-registrar', - 'whois-creation-date'] - values = [item_registrant['email'], - item_registrant['telephone'], - item_registrant['name'], results['registrarName'], - results['createdDate']] - r = [{ - 'types': t, - 'values': v, - 'categories': ['Attribution'], - 'comment': 'whois information of %s by securitytrails' % domain - } for t, v in zip(types, values)] + if 'email' in item_registrant: + r.append( + { + 'types': ['whois-registrant-email'], + 'values': [item_registrant['email']], + 'categories': ['Attribution'], + 'comment': 'Whois information of %s by securitytrails' + % domain + } + ) + + if 'telephone' in item_registrant: + r.append( + { + 'types': ['whois-registrant-phone'], + 'values': [item_registrant['telephone']], + 'categories': ['Attribution'], + 'comment': 'Whois information of %s by securitytrails' + % domain + } + ) + + if 'name' in item_registrant: + r.append( + { + 'types': ['whois-registrant-name'], + 'values': [item_registrant['name']], + 'categories': ['Attribution'], + 'comment': 'Whois information of %s by securitytrails' + % domain + } + ) + + if 'registrarName' in item_registrant: + r.append( + { + 'types': ['whois-registrar'], + 'values': [item_registrant['registrarName']], + 'categories': ['Attribution'], + 'comment': 'Whois information of %s by securitytrails' + % domain + } + ) + + if 'createdDate' in item_registrant: + r.append( + { + 'types': ['whois-creation-date'], + 'values': [item_registrant['createdDate']], + 'categories': ['Attribution'], + 'comment': 'Whois information of %s by securitytrails' + % domain + } + ) + # TODO File "modules/expansion/dnstrails.py", line 230, in expand_whois # 'values': [item_registrant['email'], From dcdb6e589556a09a562f78a9ceb6684078eef0b0 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 09:02:47 +0200 Subject: [PATCH 117/196] switch type ip --- misp_modules/modules/expansion/dnstrails.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 7d18042..ee9f1ad 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -323,7 +323,7 @@ def expand_history_ipv4_ipv6(api, domain): if results: status_ok = True - r.extend(__history_ip(results, domain)) + r.extend(__history_ip(results, domain, type_ip='ipv6')) except APIError as e: misperrors['error'] = e @@ -332,7 +332,7 @@ def expand_history_ipv4_ipv6(api, domain): return r, status_ok -def __history_ip(results, domain): +def __history_ip(results, domain, type_ip='ip'): r = [] if 'records' in results: for record in results['records']: @@ -340,7 +340,7 @@ def __history_ip(results, domain): for item in record['values']: r.append( {'types': ['domain|ip'], - 'values': ['%s|%s' % (domain, item['ip'])], + 'values': ['%s|%s' % (domain, item[type_ip])], 'categories': ['Network activity'], 'comment': 'last seen: %s first seen: %s' % (record['last_seen'], From 54d996cb00b47046558c5ad30c12bda5066fe9aa Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 09:39:09 +0200 Subject: [PATCH 118/196] add history dns --- misp_modules/modules/expansion/dnstrails.py | 42 +++++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index ee9f1ad..24e252b 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -295,11 +295,6 @@ def expand_whois(api, domain): } ) - - # TODO File "modules/expansion/dnstrails.py", line 230, in expand_whois - # 'values': [item_registrant['email'], - # TypeError: 'NoneType' object is not subscriptable - except APIError as e: misperrors['error'] = e print(e) @@ -332,6 +327,37 @@ def expand_history_ipv4_ipv6(api, domain): return r, status_ok +def expand_history_dns(api, domain): + r = [] + status_ok = False + + try: + + results = api.history_dns_ns(domain) + if results: + status_ok = True + + if 'records' in results: + for record in results['records']: + if 'values' in record: + for item in record['values']: + r.append( + {'types': ['domain|ip'], + 'values': [ + '%s|%s' % (domain, item['nameserver'])], + 'categories': ['Network activity'], + 'comment': 'history DNS of %s last seen: %s first seen: %s' % + (domain, record['last_seen'], + record['first_seen']) + } + ) + + except APIError as e: + misperrors['error'] = e + + return r, status_ok + + def __history_ip(results, domain, type_ip='ip'): r = [] if 'records' in results: @@ -342,14 +368,16 @@ def __history_ip(results, domain, type_ip='ip'): {'types': ['domain|ip'], 'values': ['%s|%s' % (domain, item[type_ip])], 'categories': ['Network activity'], - 'comment': 'last seen: %s first seen: %s' % - (record['last_seen'], + 'comment': 'History IP on securitytrails %s ' + 'last seen: %s first seen: %s' % + (domain, record['last_seen'], record['first_seen']) } ) return r + def introspection(): return mispattributes From 43a49dafc6d993b7a9142e7f16993eb209c5038a Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 09:48:14 +0200 Subject: [PATCH 119/196] add history dns and handler exception --- misp_modules/modules/expansion/dnstrails.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 24e252b..89e45ec 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -77,7 +77,7 @@ def handle_domain(api, domain, misperrors): if status_ok: result_filtered['results'].extend(r) else: - misperrors['error'] = 'Error dns result' + misperrors['error'] = misperrors['error'] + ' Error DNS result' return misperrors time.sleep(1) @@ -86,7 +86,7 @@ def handle_domain(api, domain, misperrors): if status_ok: result_filtered['results'].extend(r) else: - misperrors['error'] = 'Error subdomains result' + misperrors['error'] = misperrors['error'] + ' Error subdomains result' return misperrors time.sleep(1) @@ -95,7 +95,7 @@ def handle_domain(api, domain, misperrors): if status_ok: result_filtered['results'].extend(r) else: - misperrors['error'] = 'Error whois result' + misperrors['error'] = misperrors['error'] + ' Error whois result' return misperrors time.sleep(1) @@ -104,9 +104,18 @@ def handle_domain(api, domain, misperrors): if status_ok: result_filtered['results'].extend(r) else: - misperrors['error'] = 'Error history ipv4' + misperrors['error'] = misperrors['error'] + ' Error history ipv4' return misperrors + time.sleep(1) + r, status_ok = expand_history_dns(api, domain) + + if status_ok: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors[ + 'error'] + ' Error in expand History DNS' + return misperrors return result_filtered From f47a64b3647fde0cf70d582dac831beada066779 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 11:24:49 +0200 Subject: [PATCH 120/196] add history mx and soa --- misp_modules/modules/expansion/dnstrails.py | 49 +++++++++++++++------ 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 89e45ec..0e79ad0 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -346,20 +346,23 @@ def expand_history_dns(api, domain): if results: status_ok = True - if 'records' in results: - for record in results['records']: - if 'values' in record: - for item in record['values']: - r.append( - {'types': ['domain|ip'], - 'values': [ - '%s|%s' % (domain, item['nameserver'])], - 'categories': ['Network activity'], - 'comment': 'history DNS of %s last seen: %s first seen: %s' % - (domain, record['last_seen'], - record['first_seen']) - } - ) + r.extend(__history_dns(results, domain, 'nameserver', 'ns')) + + time.sleep(1) + + results = api.history_dns_soa(results, domain) + + if results: + status_ok = True + r.extend(__history_dns(results, domain, 'email', 'soa')) + + time.sleep(1) + + results = api.history_dns_mx(domain) + + if results: + status_ok = True + r.extend(__history_dns(results, domain, 'host', 'mx')) except APIError as e: misperrors['error'] = e @@ -387,6 +390,24 @@ def __history_ip(results, domain, type_ip='ip'): return r +def __history_dns(results, domain, type_serv, service): + r = [] + + if 'records' in results: + for record in results['records']: + if 'values' in record: + for item in record['values']: + r.append( + {'types': ['domain|ip'], + 'values': [item[type_serv]], + 'categories': ['Network activity'], + 'comment': 'history %s of %s last seen: %s first seen: %s' % + (service, domain, record['last_seen'], + record['first_seen']) + } + ) + return r + def introspection(): return mispattributes From 74c611d2fb38c084084c0aca2a59cfb5189b5379 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 11:37:07 +0200 Subject: [PATCH 121/196] correct call function --- misp_modules/modules/expansion/dnstrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 0e79ad0..86a01a7 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -350,7 +350,7 @@ def expand_history_dns(api, domain): time.sleep(1) - results = api.history_dns_soa(results, domain) + results = api.history_dns_soa(domain) if results: status_ok = True From 560dacbf7e7491ac5bd75d62f91aa1d3d1aacf3f Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 11:40:22 +0200 Subject: [PATCH 122/196] add logs to debug --- misp_modules/modules/expansion/dnstrails.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 86a01a7..1a3f720 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -397,6 +397,7 @@ def __history_dns(results, domain, type_serv, service): for record in results['records']: if 'values' in record: for item in record['values']: + print(item) r.append( {'types': ['domain|ip'], 'values': [item[type_serv]], From 64e7f9c8b628115339d848b1ed57918167dc7a4b Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 11:47:10 +0200 Subject: [PATCH 123/196] change history dns --- misp_modules/modules/expansion/dnstrails.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 1a3f720..3343fa4 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -396,11 +396,23 @@ def __history_dns(results, domain, type_serv, service): if 'records' in results: for record in results['records']: if 'values' in record: - for item in record['values']: - print(item) + values = record['values'] + if type(values) is list: + + for item in record['values']: + r.append( + {'types': ['domain|ip'], + 'values': [item[type_serv]], + 'categories': ['Network activity'], + 'comment': 'history %s of %s last seen: %s first seen: %s' % + (service, domain, record['last_seen'], + record['first_seen']) + } + ) + else: r.append( {'types': ['domain|ip'], - 'values': [item[type_serv]], + 'values': [values[type_serv]], 'categories': ['Network activity'], 'comment': 'history %s of %s last seen: %s first seen: %s' % (service, domain, record['last_seen'], From 45c473aef504562b6bea2b37d8f671f827338ddf Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 11:52:10 +0200 Subject: [PATCH 124/196] change status --- misp_modules/modules/expansion/dnstrails.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 3343fa4..cab4778 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -344,7 +344,7 @@ def expand_history_dns(api, domain): results = api.history_dns_ns(domain) if results: - status_ok = True + r.extend(__history_dns(results, domain, 'nameserver', 'ns')) @@ -353,7 +353,6 @@ def expand_history_dns(api, domain): results = api.history_dns_soa(domain) if results: - status_ok = True r.extend(__history_dns(results, domain, 'email', 'soa')) time.sleep(1) @@ -367,6 +366,8 @@ def expand_history_dns(api, domain): except APIError as e: misperrors['error'] = e + status_ok = True + return r, status_ok From 45decc728dcd0391f0b1dfad08684e9844757b7f Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 11:55:31 +0200 Subject: [PATCH 125/196] debug --- misp_modules/modules/expansion/dnstrails.py | 71 +++++++++++---------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index cab4778..5ff9e25 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -72,42 +72,43 @@ def handler(q=False): def handle_domain(api, domain, misperrors): result_filtered = {"results": []} - r, status_ok = expand_domain_info(api, misperrors, domain) + # r, status_ok = expand_domain_info(api, misperrors, domain) + # + # if status_ok: + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + ' Error DNS result' + # return misperrors + # + # time.sleep(1) + # r, status_ok = expand_subdomains(api, domain) + # + # if status_ok: + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + ' Error subdomains result' + # return misperrors + # + # time.sleep(1) + # r, status_ok = expand_whois(api, domain) + # + # if status_ok: + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + ' Error whois result' + # return misperrors + # + # time.sleep(1) + # r, status_ok = expand_history_ipv4_ipv6(api, domain) + # + # if status_ok: + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + ' Error history ipv4' + # return misperrors + # + # time.sleep(1) - if status_ok: - result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + ' Error DNS result' - return misperrors - - time.sleep(1) - r, status_ok = expand_subdomains(api, domain) - - if status_ok: - result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + ' Error subdomains result' - return misperrors - - time.sleep(1) - r, status_ok = expand_whois(api, domain) - - if status_ok: - result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + ' Error whois result' - return misperrors - - time.sleep(1) - r, status_ok = expand_history_ipv4_ipv6(api, domain) - - if status_ok: - result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + ' Error history ipv4' - return misperrors - - time.sleep(1) r, status_ok = expand_history_dns(api, domain) if status_ok: From 386d38c88f7bacda2f905a8ff6f4a60c861074d1 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 11:55:59 +0200 Subject: [PATCH 126/196] add debug --- misp_modules/modules/expansion/dnstrails.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 5ff9e25..b97997a 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -112,6 +112,7 @@ def handle_domain(api, domain, misperrors): r, status_ok = expand_history_dns(api, domain) if status_ok: + print(r) result_filtered['results'].extend(r) else: misperrors['error'] = misperrors[ From 80e71f582c901bd866cc321f2cbe3f6bcfdd0935 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 11:58:42 +0200 Subject: [PATCH 127/196] debug ipv4 or ipv6 --- misp_modules/modules/expansion/dnstrails.py | 26 +++++++++++---------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index b97997a..9e3a80f 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -99,25 +99,27 @@ def handle_domain(api, domain, misperrors): # return misperrors # # time.sleep(1) - # r, status_ok = expand_history_ipv4_ipv6(api, domain) + r, status_ok = expand_history_ipv4_ipv6(api, domain) # - # if status_ok: - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + ' Error history ipv4' - # return misperrors - # - # time.sleep(1) - - r, status_ok = expand_history_dns(api, domain) if status_ok: print(r) result_filtered['results'].extend(r) else: - misperrors['error'] = misperrors[ - 'error'] + ' Error in expand History DNS' + misperrors['error'] = misperrors['error'] + ' Error history ipv4' return misperrors + + # time.sleep(1) + + # r, status_ok = expand_history_dns(api, domain) + # + # if status_ok: + # print(r) + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors[ + # 'error'] + ' Error in expand History DNS' + # return misperrors return result_filtered From fb262b451fe970a6e1b86fd191c8db2d459dd450 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 12:00:59 +0200 Subject: [PATCH 128/196] debug whois --- misp_modules/modules/expansion/dnstrails.py | 26 ++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 9e3a80f..5ceb3e7 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -90,24 +90,24 @@ def handle_domain(api, domain, misperrors): # return misperrors # # time.sleep(1) - # r, status_ok = expand_whois(api, domain) - # - # if status_ok: - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + ' Error whois result' - # return misperrors - # - # time.sleep(1) - r, status_ok = expand_history_ipv4_ipv6(api, domain) - # + r, status_ok = expand_whois(api, domain) if status_ok: - print(r) result_filtered['results'].extend(r) else: - misperrors['error'] = misperrors['error'] + ' Error history ipv4' + misperrors['error'] = misperrors['error'] + ' Error whois result' return misperrors + # + # time.sleep(1) + # r, status_ok = expand_history_ipv4_ipv6(api, domain) + # # + # + # if status_ok: + # print(r) + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + ' Error history ipv4' + # return misperrors # time.sleep(1) From dbeec4682e24482d319097d647cc20cddca9f320 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 12:02:31 +0200 Subject: [PATCH 129/196] add logs --- misp_modules/modules/expansion/dnstrails.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 5ceb3e7..d1e6eaa 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -93,6 +93,7 @@ def handle_domain(api, domain, misperrors): r, status_ok = expand_whois(api, domain) if status_ok: + print(r) result_filtered['results'].extend(r) else: misperrors['error'] = misperrors['error'] + ' Error whois result' From f0a4c7190889192ca5723443218a248a758ab9be Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 12:08:01 +0200 Subject: [PATCH 130/196] add a test to check if the list is not empty --- misp_modules/modules/expansion/dnstrails.py | 82 +++++++++++---------- 1 file changed, 42 insertions(+), 40 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index d1e6eaa..eab1ae7 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -72,55 +72,57 @@ def handler(q=False): def handle_domain(api, domain, misperrors): result_filtered = {"results": []} - # r, status_ok = expand_domain_info(api, misperrors, domain) + r, status_ok = expand_domain_info(api, misperrors, domain) # - # if status_ok: - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + ' Error DNS result' - # return misperrors - # - # time.sleep(1) - # r, status_ok = expand_subdomains(api, domain) - # - # if status_ok: - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + ' Error subdomains result' - # return misperrors - # - # time.sleep(1) + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error DNS result' + return misperrors + + time.sleep(1) + r, status_ok = expand_subdomains(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error subdomains result' + return misperrors + + time.sleep(1) r, status_ok = expand_whois(api, domain) if status_ok: - print(r) - result_filtered['results'].extend(r) + if r: + result_filtered['results'].extend(r) else: misperrors['error'] = misperrors['error'] + ' Error whois result' return misperrors - # - # time.sleep(1) - # r, status_ok = expand_history_ipv4_ipv6(api, domain) - # # - # - # if status_ok: - # print(r) - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + ' Error history ipv4' - # return misperrors - # time.sleep(1) - - # r, status_ok = expand_history_dns(api, domain) + time.sleep(1) + r, status_ok = expand_history_ipv4_ipv6(api, domain) # - # if status_ok: - # print(r) - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors[ - # 'error'] + ' Error in expand History DNS' - # return misperrors + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error history ipv4' + return misperrors + + time.sleep(1) + + r, status_ok = expand_history_dns(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors[ + 'error'] + ' Error in expand History DNS' + return misperrors return result_filtered From a8ae6e06e9d23b835c724fa6099290fb09415534 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 12:09:34 +0200 Subject: [PATCH 131/196] add a test to check if the list is not empty --- misp_modules/modules/expansion/dnstrails.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index eab1ae7..5e85db5 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -123,6 +123,8 @@ def handle_domain(api, domain, misperrors): misperrors['error'] = misperrors[ 'error'] + ' Error in expand History DNS' return misperrors + print(result_filtered) + print(misperrors) return result_filtered From 3a2aab6d7150f2ece96aad73b4b929996bdd8fa7 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 12:41:54 +0200 Subject: [PATCH 132/196] rename misp modules --- .../modules/expansion/{dnstrails.py => securitytrails.py} | 2 -- 1 file changed, 2 deletions(-) rename misp_modules/modules/expansion/{dnstrails.py => securitytrails.py} (99%) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/securitytrails.py similarity index 99% rename from misp_modules/modules/expansion/dnstrails.py rename to misp_modules/modules/expansion/securitytrails.py index 5e85db5..6940907 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -353,8 +353,6 @@ def expand_history_dns(api, domain): results = api.history_dns_ns(domain) if results: - - r.extend(__history_dns(results, domain, 'nameserver', 'ns')) time.sleep(1) From 51067039daf8255e2a874d1bae7bcda43107b369 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 13:03:47 +0200 Subject: [PATCH 133/196] correct typo --- misp_modules/modules/expansion/securitytrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 6940907..e97b493 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -26,7 +26,7 @@ mispattributes = { moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', 'description': 'Query on securitytrails.com', - 'module-types': ['expansion', 'hover']} + 'module-type': ['expansion', 'hover']} # config fields that your code expects from the site admin moduleconfig = ['apikey'] From a62078aad1681ea63b07e7c7a8020508b1133189 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 11 Jul 2018 23:43:42 +0200 Subject: [PATCH 134/196] add: Experimental expansion module to display the SIEM signatures from a sigma rule --- README.md | 1 + misp_modules/modules/expansion/__init__.py | 2 +- .../modules/expansion/sigma_queries.py | 50 +++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 misp_modules/modules/expansion/sigma_queries.py diff --git a/README.md b/README.md index 95019da..c1bc7a0 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [rbl](misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute. * [reversedns](misp_modules/modules/expansion/reversedns.py) - Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes. * [shodan](misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module. +* [Sigma queries](misp_modules/modules/expansion/sigma_queries.py) - Experimental expansion module querying a sigma rule to convert it into all the available SIEM signatures. * [Sigma syntax validator](misp_modules/modules/expansion/sigma_syntax_validator.py) - Sigma syntax validator. * [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance. * [STIX2 pattern syntax validator](misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - a module to check a STIX2 pattern syntax. diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 5e0d65e..cda3af5 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,3 +1,3 @@ from . import _vmray -__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator'] +__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries'] diff --git a/misp_modules/modules/expansion/sigma_queries.py b/misp_modules/modules/expansion/sigma_queries.py new file mode 100644 index 0000000..e37df23 --- /dev/null +++ b/misp_modules/modules/expansion/sigma_queries.py @@ -0,0 +1,50 @@ +import sys, os, io, json +try: + from sigma.parser import SigmaCollectionParser + from sigma.config import SigmaConfiguration + from sigma.backends import getBackend, BackendOptions +except ModuleNotFoundError: + print("sigma or yaml is missing, use 'pip3 install sigmatools' to install it.") + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['sigma'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'module-type': ['expansion', 'hover'], + 'description': 'An expansion hover module to display the result of sigma queries.'} +moduleconfig = [] +sigma_targets = ('es-dsl', 'es-qs', 'graylog', 'kibana', 'xpack-watcher', 'logpoint', 'splunk', 'grep', 'wdatp', 'splunkxml', 'arcsight', 'qualys') + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if not request.get('sigma'): + misperrors['error'] = 'Sigma rule missing' + return misperrors + config = SigmaConfiguration() + backend_options = BackendOptions(None) + f = io.TextIOWrapper(io.BytesIO(request.get('sigma').encode()), encoding='utf-8') + parser = SigmaCollectionParser(f, config, None) + targets = [] + old_stdout = sys.stdout + result = io.StringIO() + sys.stdout = result + for t in sigma_targets: + backend = getBackend(t)(config, backend_options, None) + try: + parser.generate(backend) + backend.finalize() + print("#NEXT") + targets.append(t) + except: + continue + sys.stdout = old_stdout + results = result.getvalue()[:-5].split('#NEXT') + d_result = {t: r.strip() for t,r in zip(targets, results)} + return {'results': [{'types': mispattributes['output'], 'values': d_result}]} + +def introspection(): + return mispattributes + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 3fd58537f66cbf3145e2fe335561c8a31c874c67 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Thu, 12 Jul 2018 11:43:24 +0200 Subject: [PATCH 135/196] remove Python 3.4 and Python 3.7 added --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0a3a912..a998141 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,11 +6,12 @@ services: cache: pip python: - - "3.4" - "3.5" - "3.5-dev" - "3.6" - "3.6-dev" + - "3.7" + - "3.7-dev" install: - pip install -U nose codecov pytest From 576b3c9b9bdb2c8c66268c44b041646a014bcbac Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 13:40:51 +0200 Subject: [PATCH 136/196] history whois dns --- .../modules/expansion/securitytrails.py | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index e97b493..e23184d 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -340,7 +340,7 @@ def expand_history_ipv4_ipv6(api, domain): except APIError as e: misperrors['error'] = e - print(e) + return [], False return r, status_ok @@ -372,12 +372,71 @@ def expand_history_dns(api, domain): except APIError as e: misperrors['error'] = e + return [], False status_ok = True return r, status_ok +def expand_history_whois(api, domain): + r = [] + status_ok = False + try: + results = api.history_whois(domain) + + if results: + + if 'items' in results['results']: + for item in results['results']['items']: + item_registrant = __select_registrant_item(item) + + r.extend( + { + 'type': ['domain'], + 'values': item['nameServers'], + 'categories': ['Network activity'], + 'comment': 'Whois history Name Servers of %s ' + 'Status: %s ' % (domain, item['status']) + + } + ) + if 'email' in item_registrant: + r.append( + { + 'types': ['whois-registrant-email'], + 'values': [item_registrant['email']], + 'categories': ['Attribution'], + 'comment': 'Whois history registrant email of %s' + 'Status: %s' % ( + domain, item['status']) + } + ) + + if 'telephone' in item_registrant: + r.append( + { + 'types': ['whois-registrant-phone'], + 'values': [item_registrant['telephone']], + 'categories': ['Attribution'], + 'comment': 'Whois history registrant phone of %s' + 'Status: %s' % ( + domain, item['status']) + } + ) + + + + + except APIError as e: + misperrors['error'] = e + return [], False + + + + return r, status_ok + + def __history_ip(results, domain, type_ip='ip'): r = [] if 'records' in results: From 4b0daee6f1389467d5bbae26242e71e281d84d28 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:04:37 +0200 Subject: [PATCH 137/196] test whois history --- .../modules/expansion/securitytrails.py | 98 ++++++++++--------- 1 file changed, 51 insertions(+), 47 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index e23184d..eb93756 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -72,59 +72,63 @@ def handler(q=False): def handle_domain(api, domain, misperrors): result_filtered = {"results": []} - r, status_ok = expand_domain_info(api, misperrors, domain) + # r, status_ok = expand_domain_info(api, misperrors, domain) + # # + # if status_ok: + # if r: + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + ' Error DNS result' + # return misperrors # - if status_ok: - if r: - result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + ' Error DNS result' - return misperrors - - time.sleep(1) - r, status_ok = expand_subdomains(api, domain) - - if status_ok: - if r: - result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + ' Error subdomains result' - return misperrors - - time.sleep(1) - r, status_ok = expand_whois(api, domain) - - if status_ok: - if r: - result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + ' Error whois result' - return misperrors - - time.sleep(1) - r, status_ok = expand_history_ipv4_ipv6(api, domain) + # time.sleep(1) + # r, status_ok = expand_subdomains(api, domain) # + # if status_ok: + # if r: + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + ' Error subdomains result' + # return misperrors + # + # time.sleep(1) + # r, status_ok = expand_whois(api, domain) + # + # if status_ok: + # if r: + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + ' Error whois result' + # return misperrors + # + # time.sleep(1) + # r, status_ok = expand_history_ipv4_ipv6(api, domain) + # # + # + # if status_ok: + # if r: + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + ' Error history ipv4' + # return misperrors + # + # time.sleep(1) + # + # r, status_ok = expand_history_dns(api, domain) + # + # if status_ok: + # if r: + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors[ + # 'error'] + ' Error in expand History DNS' + # return misperrors + + r, status_ok = expand_history_whois(api, domain) if status_ok: if r: result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + ' Error history ipv4' - return misperrors - - time.sleep(1) - - r, status_ok = expand_history_dns(api, domain) - - if status_ok: - if r: - result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors[ - 'error'] + ' Error in expand History DNS' - return misperrors - print(result_filtered) - print(misperrors) return result_filtered From 41587bd56823c2799dce55c68f982acc2f567c6f Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:14:43 +0200 Subject: [PATCH 138/196] correct typo --- misp_modules/modules/expansion/securitytrails.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index eb93756..043ded3 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -391,8 +391,8 @@ def expand_history_whois(api, domain): if results: - if 'items' in results['results']: - for item in results['results']['items']: + if 'items' in results['result']: + for item in results['result']['items']: item_registrant = __select_registrant_item(item) r.extend( From fb595c08aac4d7297fc169183879324bee1890b7 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:16:19 +0200 Subject: [PATCH 139/196] add logs --- misp_modules/modules/expansion/securitytrails.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 043ded3..bf35c7b 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -506,4 +506,5 @@ def __select_registrant_item(entry): for c in entry['contacts']: if c['type'] == 'registrant': + print(c) return c From 731c06a939778b3beeb188375073bbe765503797 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:17:16 +0200 Subject: [PATCH 140/196] add logs --- misp_modules/modules/expansion/securitytrails.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index bf35c7b..8ae5dc1 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -502,9 +502,10 @@ def version(): def __select_registrant_item(entry): + print(entry) if 'contacts' in entry: for c in entry['contacts']: - + print(c) if c['type'] == 'registrant': print(c) return c From 9063da88cdc6fad8b1ab2e21a9522eac87189118 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:27:59 +0200 Subject: [PATCH 141/196] correct key and return of functions --- .../modules/expansion/securitytrails.py | 74 ++++++++++--------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 8ae5dc1..845646a 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -394,40 +394,41 @@ def expand_history_whois(api, domain): if 'items' in results['result']: for item in results['result']['items']: item_registrant = __select_registrant_item(item) - - r.extend( - { - 'type': ['domain'], - 'values': item['nameServers'], - 'categories': ['Network activity'], - 'comment': 'Whois history Name Servers of %s ' - 'Status: %s ' % (domain, item['status']) - - } - ) - if 'email' in item_registrant: - r.append( + if item_registrant: + r.extend( { - 'types': ['whois-registrant-email'], - 'values': [item_registrant['email']], - 'categories': ['Attribution'], - 'comment': 'Whois history registrant email of %s' - 'Status: %s' % ( - domain, item['status']) + 'type': ['domain'], + 'values': item['nameServers'], + 'categories': ['Network activity'], + 'comment': 'Whois history Name Servers of %s ' + 'Status: %s ' % ( + domain, item['status']) + } ) + if 'email' in item_registrant: + r.append( + { + 'types': ['whois-registrant-email'], + 'values': [item_registrant['email']], + 'categories': ['Attribution'], + 'comment': 'Whois history registrant email of %s' + 'Status: %s' % ( + domain, item['status']) + } + ) - if 'telephone' in item_registrant: - r.append( - { - 'types': ['whois-registrant-phone'], - 'values': [item_registrant['telephone']], - 'categories': ['Attribution'], - 'comment': 'Whois history registrant phone of %s' - 'Status: %s' % ( - domain, item['status']) - } - ) + if 'telephone' in item_registrant: + r.append( + { + 'types': ['whois-registrant-phone'], + 'values': [item_registrant['telephone']], + 'categories': ['Attribution'], + 'comment': 'Whois history registrant phone of %s' + 'Status: %s' % ( + domain, item['status']) + } + ) @@ -502,10 +503,11 @@ def version(): def __select_registrant_item(entry): - print(entry) + if 'contacts' in entry: - for c in entry['contacts']: - print(c) - if c['type'] == 'registrant': - print(c) - return c + return list(filter(lambda x: x['type'] == 'registrant', + entry['contacts']))[0] + + if 'contact' in entry: + return list(filter(lambda x: x['type'] == 'registrant', + entry['contact']))[0] From 28f45ce94e9124e331700392a614373a83190e8a Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Thu, 12 Jul 2018 14:29:04 +0200 Subject: [PATCH 142/196] remove the never release Python code in Travis --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a998141..d7f452e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,6 @@ python: - "3.5-dev" - "3.6" - "3.6-dev" - - "3.7" - "3.7-dev" install: From 844b25b4cdcaa9f293bb7bb8db9aea4082219631 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:32:56 +0200 Subject: [PATCH 143/196] correct out of bound returns --- .../modules/expansion/securitytrails.py | 44 +++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 845646a..87ea1cd 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -266,7 +266,7 @@ def expand_whois(api, domain): r.append( { 'types': ['whois-registrant-email'], - 'values': [item_registrant['email']], + 'values': [item_registrant[0]['email']], 'categories': ['Attribution'], 'comment': 'Whois information of %s by securitytrails' % domain @@ -277,7 +277,7 @@ def expand_whois(api, domain): r.append( { 'types': ['whois-registrant-phone'], - 'values': [item_registrant['telephone']], + 'values': [item_registrant[0]['telephone']], 'categories': ['Attribution'], 'comment': 'Whois information of %s by securitytrails' % domain @@ -288,7 +288,7 @@ def expand_whois(api, domain): r.append( { 'types': ['whois-registrant-name'], - 'values': [item_registrant['name']], + 'values': [item_registrant[0]['name']], 'categories': ['Attribution'], 'comment': 'Whois information of %s by securitytrails' % domain @@ -299,7 +299,7 @@ def expand_whois(api, domain): r.append( { 'types': ['whois-registrar'], - 'values': [item_registrant['registrarName']], + 'values': [item_registrant[0]['registrarName']], 'categories': ['Attribution'], 'comment': 'Whois information of %s by securitytrails' % domain @@ -310,7 +310,7 @@ def expand_whois(api, domain): r.append( { 'types': ['whois-creation-date'], - 'values': [item_registrant['createdDate']], + 'values': [item_registrant[0]['createdDate']], 'categories': ['Attribution'], 'comment': 'Whois information of %s by securitytrails' % domain @@ -394,23 +394,24 @@ def expand_history_whois(api, domain): if 'items' in results['result']: for item in results['result']['items']: item_registrant = __select_registrant_item(item) - if item_registrant: - r.extend( - { - 'type': ['domain'], - 'values': item['nameServers'], - 'categories': ['Network activity'], - 'comment': 'Whois history Name Servers of %s ' - 'Status: %s ' % ( + r.extend( + { + 'type': ['domain'], + 'values': item['nameServers'], + 'categories': ['Network activity'], + 'comment': 'Whois history Name Servers of %s ' + 'Status: %s ' % ( domain, item['status']) - } - ) - if 'email' in item_registrant: + } + ) + if item_registrant: + + if 'email' in item_registrant[0]: r.append( { 'types': ['whois-registrant-email'], - 'values': [item_registrant['email']], + 'values': [item_registrant[0]['email']], 'categories': ['Attribution'], 'comment': 'Whois history registrant email of %s' 'Status: %s' % ( @@ -422,7 +423,7 @@ def expand_history_whois(api, domain): r.append( { 'types': ['whois-registrant-phone'], - 'values': [item_registrant['telephone']], + 'values': [item_registrant[0]['telephone']], 'categories': ['Attribution'], 'comment': 'Whois history registrant phone of %s' 'Status: %s' % ( @@ -430,9 +431,6 @@ def expand_history_whois(api, domain): } ) - - - except APIError as e: misperrors['error'] = e return [], False @@ -506,8 +504,8 @@ def __select_registrant_item(entry): if 'contacts' in entry: return list(filter(lambda x: x['type'] == 'registrant', - entry['contacts']))[0] + entry['contacts'])) if 'contact' in entry: return list(filter(lambda x: x['type'] == 'registrant', - entry['contact']))[0] + entry['contact'])) From d56bf550389fc23b44980e6335cba63c30238a94 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:33:52 +0200 Subject: [PATCH 144/196] add logs --- misp_modules/modules/expansion/securitytrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 87ea1cd..bb1c1b3 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -501,7 +501,7 @@ def version(): def __select_registrant_item(entry): - + print(entry) if 'contacts' in entry: return list(filter(lambda x: x['type'] == 'registrant', entry['contacts'])) From 9de201375b8221015fabdada7c511cf3cf42d11d Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:37:09 +0200 Subject: [PATCH 145/196] add logs --- misp_modules/modules/expansion/securitytrails.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index bb1c1b3..b113f1f 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -501,11 +501,13 @@ def version(): def __select_registrant_item(entry): - print(entry) + if 'contacts' in entry: return list(filter(lambda x: x['type'] == 'registrant', entry['contacts'])) if 'contact' in entry: + print(entry) + print('\r\n') return list(filter(lambda x: x['type'] == 'registrant', entry['contact'])) From a0cf9de590c5e126219877959c51f04621f2bd0d Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:38:38 +0200 Subject: [PATCH 146/196] add logs --- misp_modules/modules/expansion/securitytrails.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index b113f1f..0cbd6f8 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -509,5 +509,7 @@ def __select_registrant_item(entry): if 'contact' in entry: print(entry) print('\r\n') - return list(filter(lambda x: x['type'] == 'registrant', - entry['contact'])) + res = list(filter(lambda x: x['type'] == 'registrant', + entry['contact'])) + print(res) + return res From 86d94278162f0397fe0b092661555d1629cd31b1 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:42:33 +0200 Subject: [PATCH 147/196] add logs --- misp_modules/modules/expansion/securitytrails.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 0cbd6f8..77bdd2c 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -394,6 +394,7 @@ def expand_history_whois(api, domain): if 'items' in results['result']: for item in results['result']['items']: item_registrant = __select_registrant_item(item) + print(item_registrant) r.extend( { 'type': ['domain'], @@ -435,8 +436,6 @@ def expand_history_whois(api, domain): misperrors['error'] = e return [], False - - return r, status_ok From aa89a7fc4d2b69ca9e0b738aa890402e72c20bae Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:44:19 +0200 Subject: [PATCH 148/196] add logs --- misp_modules/modules/expansion/securitytrails.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 77bdd2c..68a7206 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -128,6 +128,7 @@ def handle_domain(api, domain, misperrors): if status_ok: if r: + print(r) result_filtered['results'].extend(r) return result_filtered From 86d236f859c8d6065d078853d512eb243927fddc Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:47:34 +0200 Subject: [PATCH 149/196] add status_ok to true --- misp_modules/modules/expansion/securitytrails.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 68a7206..8395c13 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -128,8 +128,12 @@ def handle_domain(api, domain, misperrors): if status_ok: if r: - print(r) + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + \ + ' Error in expand History Whois' + return result_filtered @@ -395,7 +399,7 @@ def expand_history_whois(api, domain): if 'items' in results['result']: for item in results['result']['items']: item_registrant = __select_registrant_item(item) - print(item_registrant) + r.extend( { 'type': ['domain'], @@ -436,7 +440,7 @@ def expand_history_whois(api, domain): except APIError as e: misperrors['error'] = e return [], False - + status_ok = True return r, status_ok @@ -507,9 +511,7 @@ def __select_registrant_item(entry): entry['contacts'])) if 'contact' in entry: - print(entry) - print('\r\n') res = list(filter(lambda x: x['type'] == 'registrant', entry['contact'])) - print(res) + return res From 0b0137829a03629b8972e6fd3c3a6d00bfb5b756 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:48:15 +0200 Subject: [PATCH 150/196] add logs --- misp_modules/modules/expansion/securitytrails.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 8395c13..c3d82a2 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -441,6 +441,7 @@ def expand_history_whois(api, domain): misperrors['error'] = e return [], False status_ok = True + print(r) return r, status_ok From 2f5381d7b2f38b6488dfbac004ea364a6d74b724 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:49:51 +0200 Subject: [PATCH 151/196] add logs --- misp_modules/modules/expansion/securitytrails.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index c3d82a2..36703f5 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -130,9 +130,10 @@ def handle_domain(api, domain, misperrors): if r: result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + \ - ' Error in expand History Whois' + else: + misperrors['error'] = misperrors['error'] + \ + ' Error in expand History Whois' + return misperrors return result_filtered From 0341bdc398b978a031b60ac6c32a7550f93a3efd Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:52:01 +0200 Subject: [PATCH 152/196] error call functions --- misp_modules/modules/expansion/securitytrails.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 36703f5..4b2cac6 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -401,9 +401,9 @@ def expand_history_whois(api, domain): for item in results['result']['items']: item_registrant = __select_registrant_item(item) - r.extend( + r.append( { - 'type': ['domain'], + 'types': ['domain'], 'values': item['nameServers'], 'categories': ['Network activity'], 'comment': 'Whois history Name Servers of %s ' From db35c9b0917c2bae5491a3a927943c69bde944fd Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:55:56 +0200 Subject: [PATCH 153/196] correct index error --- misp_modules/modules/expansion/securitytrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 4b2cac6..14cb8b8 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -426,7 +426,7 @@ def expand_history_whois(api, domain): } ) - if 'telephone' in item_registrant: + if 'telephone' in item_registrant[0]: r.append( { 'types': ['whois-registrant-phone'], From 5a422c2e5b370a23f1fa9668837a3e4c6f59626f Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:57:37 +0200 Subject: [PATCH 154/196] add whois expand to test --- .../modules/expansion/securitytrails.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 14cb8b8..d83eeed 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -91,17 +91,17 @@ def handle_domain(api, domain, misperrors): # misperrors['error'] = misperrors['error'] + ' Error subdomains result' # return misperrors # - # time.sleep(1) - # r, status_ok = expand_whois(api, domain) - # - # if status_ok: - # if r: - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + ' Error whois result' - # return misperrors - # - # time.sleep(1) + time.sleep(1) + r, status_ok = expand_whois(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error whois result' + return misperrors + + time.sleep(1) # r, status_ok = expand_history_ipv4_ipv6(api, domain) # # # @@ -268,7 +268,7 @@ def expand_whois(api, domain): item_registrant = __select_registrant_item(results) if item_registrant: - if 'email' in item_registrant: + if 'email' in item_registrant[0]: r.append( { 'types': ['whois-registrant-email'], @@ -279,7 +279,7 @@ def expand_whois(api, domain): } ) - if 'telephone' in item_registrant: + if 'telephone' in item_registrant[0]: r.append( { 'types': ['whois-registrant-phone'], @@ -290,7 +290,7 @@ def expand_whois(api, domain): } ) - if 'name' in item_registrant: + if 'name' in item_registrant[0]: r.append( { 'types': ['whois-registrant-name'], @@ -301,7 +301,7 @@ def expand_whois(api, domain): } ) - if 'registrarName' in item_registrant: + if 'registrarName' in item_registrant[0]: r.append( { 'types': ['whois-registrar'], @@ -312,7 +312,7 @@ def expand_whois(api, domain): } ) - if 'createdDate' in item_registrant: + if 'createdDate' in item_registrant[0]: r.append( { 'types': ['whois-creation-date'], From 3eda71219365a6ae489cd39cf7c7d084f07effcd Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:58:48 +0200 Subject: [PATCH 155/196] add whois expand to test --- .../modules/expansion/securitytrails.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index d83eeed..ab8087c 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -124,16 +124,16 @@ def handle_domain(api, domain, misperrors): # 'error'] + ' Error in expand History DNS' # return misperrors - r, status_ok = expand_history_whois(api, domain) - - if status_ok: - if r: - - result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + \ - ' Error in expand History Whois' - return misperrors + # r, status_ok = expand_history_whois(api, domain) + # + # if status_ok: + # if r: + # + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + \ + # ' Error in expand History Whois' + # return misperrors return result_filtered From 7f52a15d16c3ead17734180b55a078b452ce9122 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:59:50 +0200 Subject: [PATCH 156/196] add logs --- misp_modules/modules/expansion/securitytrails.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index ab8087c..3332026 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -509,9 +509,9 @@ def version(): def __select_registrant_item(entry): if 'contacts' in entry: - return list(filter(lambda x: x['type'] == 'registrant', - entry['contacts'])) - + res = list(filter(lambda x: x['type'] == 'registrant', + entry['contacts'])) + print(res) if 'contact' in entry: res = list(filter(lambda x: x['type'] == 'registrant', entry['contact'])) From 966f9603a9ec20ffa6055bb12cbf2a46a4793347 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 15:02:46 +0200 Subject: [PATCH 157/196] add return --- misp_modules/modules/expansion/securitytrails.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 3332026..4b276da 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -267,7 +267,7 @@ def expand_whois(api, domain): status_ok = True item_registrant = __select_registrant_item(results) if item_registrant: - + print(item_registrant) if 'email' in item_registrant[0]: r.append( { @@ -511,7 +511,7 @@ def __select_registrant_item(entry): if 'contacts' in entry: res = list(filter(lambda x: x['type'] == 'registrant', entry['contacts'])) - print(res) + return res if 'contact' in entry: res = list(filter(lambda x: x['type'] == 'registrant', entry['contact'])) From a41cf59e0c28141fda01efa6aee840cd5fade869 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 17 Jul 2018 15:05:15 +0200 Subject: [PATCH 158/196] add searching domains --- .../modules/expansion/securitytrails.py | 42 +++++++++++++++++-- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 4b276da..b4d9609 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -442,7 +442,7 @@ def expand_history_whois(api, domain): misperrors['error'] = e return [], False status_ok = True - print(r) + return r, status_ok @@ -497,6 +497,40 @@ def __history_dns(results, domain, type_serv, service): ) return r + +def expand_searching_domain(api, ip): + r = [] + status_ok = False + + try: + results = api.searching_domains(ip) + + if results: + if 'records' in results: + res = [(r['host_provider'], r['hostname'], r['whois']) + for r in results['records']] + + for host_provider, hostname, whois in res: + comment = 'domain for %s by %s' % (ip, host_provider[0]) + if whois['registrant']: + comment = comment + ' registrar %s' % whois['registrar'] + + r.append( + { + 'types': ['domain'], + 'category': ['Network activity'], + 'values': [hostname], + 'comment': comment + + } + ) + status_ok = True + except APIError as e: + misperrors['error'] = e + return [], False + + return r, status_ok + def introspection(): return mispattributes @@ -507,13 +541,13 @@ def version(): def __select_registrant_item(entry): - + res = None if 'contacts' in entry: res = list(filter(lambda x: x['type'] == 'registrant', entry['contacts'])) - return res + if 'contact' in entry: res = list(filter(lambda x: x['type'] == 'registrant', entry['contact'])) - return res + return res From 999ae1f6f0ec2264a38ee557f9a4a7e4a533d188 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 17 Jul 2018 17:09:01 +0200 Subject: [PATCH 159/196] add searching domains --- .../modules/expansion/securitytrails.py | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index b4d9609..4aaca61 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -91,17 +91,17 @@ def handle_domain(api, domain, misperrors): # misperrors['error'] = misperrors['error'] + ' Error subdomains result' # return misperrors # - time.sleep(1) - r, status_ok = expand_whois(api, domain) - - if status_ok: - if r: - result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + ' Error whois result' - return misperrors - - time.sleep(1) + # time.sleep(1) + # r, status_ok = expand_whois(api, domain) + # + # if status_ok: + # if r: + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + ' Error whois result' + # return misperrors + # + # time.sleep(1) # r, status_ok = expand_history_ipv4_ipv6(api, domain) # # # @@ -135,11 +135,23 @@ def handle_domain(api, domain, misperrors): # ' Error in expand History Whois' # return misperrors + return result_filtered def handle_ip(api, ip, misperrors): - pass + result_filtered = {"results": []} + + r, status_ok = expand_searching_domain(api, ip) + + if status_ok: + if r: + result_filtered['result'].extend(r) + else: + misperrors['error'] += ' Error in expand searching domain' + return misperrors + + return result_filtered def expand_domain_info(api, misperror, domain): From 431c1511a37d700fd6cd1e70f23fb18493bd6b4e Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 17 Jul 2018 17:20:30 +0200 Subject: [PATCH 160/196] correct param --- misp_modules/modules/expansion/securitytrails.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 4aaca61..53a0111 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -150,7 +150,7 @@ def handle_ip(api, ip, misperrors): else: misperrors['error'] += ' Error in expand searching domain' return misperrors - + return result_filtered @@ -515,7 +515,7 @@ def expand_searching_domain(api, ip): status_ok = False try: - results = api.searching_domains(ip) + results = api.searching_domains(ipv4=ip) if results: if 'records' in results: From 2706c4a82a6294341d276eeb151428ad4c5fe047 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 17 Jul 2018 17:21:38 +0200 Subject: [PATCH 161/196] correct key --- misp_modules/modules/expansion/securitytrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 53a0111..b6b3e65 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -524,7 +524,7 @@ def expand_searching_domain(api, ip): for host_provider, hostname, whois in res: comment = 'domain for %s by %s' % (ip, host_provider[0]) - if whois['registrant']: + if whois['registrar']: comment = comment + ' registrar %s' % whois['registrar'] r.append( From c785cae89b409a174714e8d012b4b6cccd062567 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 17 Jul 2018 17:22:48 +0200 Subject: [PATCH 162/196] correct key --- misp_modules/modules/expansion/securitytrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index b6b3e65..b2790fa 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -146,7 +146,7 @@ def handle_ip(api, ip, misperrors): if status_ok: if r: - result_filtered['result'].extend(r) + result_filtered['results'].extend(r) else: misperrors['error'] += ' Error in expand searching domain' return misperrors From 9d603344c2cb0c0560995252f8a9d3bdd50c51f4 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 17 Jul 2018 18:32:50 +0200 Subject: [PATCH 163/196] add searching_stats --- .../modules/expansion/securitytrails.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index b2790fa..5294a62 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -543,6 +543,31 @@ def expand_searching_domain(api, ip): return r, status_ok + +def expand_search_stats(api, ip, misperror): + r = [] + status_ok = False + + try: + result = api.searching_stats(ipv4=ip) + if result and 'top_organizations' in result: + comment = '' + for reg in result['top_organizations']: + comment += 'Organization %s used %s count: %s' % (reg['key'], + ip, + reg['count']) + r.append({'types': ['comment'], + 'categories': ['Other'], + 'values': comment, + }) + status_ok = True + except APIError as e: + misperrors['error'] = e + return [], False + + return r, status_ok + + def introspection(): return mispattributes From 8cbeda40a5043eb2494d1d41c9431dabfce9d067 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 17 Jul 2018 18:42:01 +0200 Subject: [PATCH 164/196] add searching_stats --- misp_modules/modules/expansion/securitytrails.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 5294a62..f8f822e 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -151,6 +151,17 @@ def handle_ip(api, ip, misperrors): misperrors['error'] += ' Error in expand searching domain' return misperrors + time.sleep(1) + + r, status_ok = expand_search_stats(api, ip, misperrors) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] += ' Error in expand searching stats' + return misperrors + return result_filtered @@ -549,7 +560,7 @@ def expand_search_stats(api, ip, misperror): status_ok = False try: - result = api.searching_stats(ipv4=ip) + result = api.search_stats(ipv4=ip) if result and 'top_organizations' in result: comment = '' for reg in result['top_organizations']: From 88859a0ba7123bd9683c9bdededb1a769a0546b7 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 17 Jul 2018 18:43:52 +0200 Subject: [PATCH 165/196] add logs --- misp_modules/modules/expansion/securitytrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index f8f822e..63f08c9 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -573,7 +573,7 @@ def expand_search_stats(api, ip, misperror): }) status_ok = True except APIError as e: - misperrors['error'] = e + misperrors['error'] = e.value return [], False return r, status_ok From f2df6dc538244210715c4ae45202871d8f3987ee Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 18 Jul 2018 10:47:42 +0200 Subject: [PATCH 166/196] last commit for release --- .../modules/expansion/securitytrails.py | 167 +++++++----------- 1 file changed, 65 insertions(+), 102 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 63f08c9..8c96f76 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -72,69 +72,66 @@ def handler(q=False): def handle_domain(api, domain, misperrors): result_filtered = {"results": []} - # r, status_ok = expand_domain_info(api, misperrors, domain) - # # - # if status_ok: - # if r: - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + ' Error DNS result' - # return misperrors - # - # time.sleep(1) - # r, status_ok = expand_subdomains(api, domain) - # - # if status_ok: - # if r: - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + ' Error subdomains result' - # return misperrors - # - # time.sleep(1) - # r, status_ok = expand_whois(api, domain) - # - # if status_ok: - # if r: - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + ' Error whois result' - # return misperrors - # - # time.sleep(1) - # r, status_ok = expand_history_ipv4_ipv6(api, domain) - # # - # - # if status_ok: - # if r: - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + ' Error history ipv4' - # return misperrors - # - # time.sleep(1) - # - # r, status_ok = expand_history_dns(api, domain) - # - # if status_ok: - # if r: - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors[ - # 'error'] + ' Error in expand History DNS' - # return misperrors + r, status_ok = expand_domain_info(api, misperrors, domain) - # r, status_ok = expand_history_whois(api, domain) - # - # if status_ok: - # if r: - # - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + \ - # ' Error in expand History Whois' - # return misperrors + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error DNS result' + return misperrors + time.sleep(1) + r, status_ok = expand_subdomains(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error subdomains result' + return misperrors + + time.sleep(1) + r, status_ok = expand_whois(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error whois result' + return misperrors + + time.sleep(1) + r, status_ok = expand_history_ipv4_ipv6(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error history ipv4' + return misperrors + + time.sleep(1) + + r, status_ok = expand_history_dns(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors[ + 'error'] + ' Error in expand History DNS' + return misperrors + + r, status_ok = expand_history_whois(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + \ + ' Error in expand History Whois' + return misperrors return result_filtered @@ -151,17 +148,6 @@ def handle_ip(api, ip, misperrors): misperrors['error'] += ' Error in expand searching domain' return misperrors - time.sleep(1) - - r, status_ok = expand_search_stats(api, ip, misperrors) - - if status_ok: - if r: - result_filtered['results'].extend(r) - else: - misperrors['error'] += ' Error in expand searching stats' - return misperrors - return result_filtered @@ -274,7 +260,8 @@ def expand_subdomains(api, domain): ) except APIError as e: - misperrors['error'] = e + misperrors['error'] = e.value + return [], False return r, status_ok @@ -347,8 +334,8 @@ def expand_whois(api, domain): ) except APIError as e: - misperrors['error'] = e - print(e) + misperrors['error'] = e.value + return [], False return r, status_ok @@ -372,7 +359,7 @@ def expand_history_ipv4_ipv6(api, domain): r.extend(__history_ip(results, domain, type_ip='ipv6')) except APIError as e: - misperrors['error'] = e + misperrors['error'] = e.value return [], False return r, status_ok @@ -404,7 +391,7 @@ def expand_history_dns(api, domain): r.extend(__history_dns(results, domain, 'host', 'mx')) except APIError as e: - misperrors['error'] = e + misperrors['error'] = e.value return [], False status_ok = True @@ -462,7 +449,7 @@ def expand_history_whois(api, domain): ) except APIError as e: - misperrors['error'] = e + misperrors['error'] = e.value return [], False status_ok = True @@ -548,30 +535,6 @@ def expand_searching_domain(api, ip): } ) status_ok = True - except APIError as e: - misperrors['error'] = e - return [], False - - return r, status_ok - - -def expand_search_stats(api, ip, misperror): - r = [] - status_ok = False - - try: - result = api.search_stats(ipv4=ip) - if result and 'top_organizations' in result: - comment = '' - for reg in result['top_organizations']: - comment += 'Organization %s used %s count: %s' % (reg['key'], - ip, - reg['count']) - r.append({'types': ['comment'], - 'categories': ['Other'], - 'values': comment, - }) - status_ok = True except APIError as e: misperrors['error'] = e.value return [], False From c8e20d90879b024eb068cc74a21c848a8324e024 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 18 Jul 2018 10:51:47 +0200 Subject: [PATCH 167/196] remove print --- misp_modules/modules/expansion/securitytrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 8c96f76..bf1c5b1 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -277,7 +277,7 @@ def expand_whois(api, domain): status_ok = True item_registrant = __select_registrant_item(results) if item_registrant: - print(item_registrant) + if 'email' in item_registrant[0]: r.append( { From 804e59ed8d6c48b7383de4327d45fae352197cbe Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 18 Jul 2018 10:58:51 +0200 Subject: [PATCH 168/196] change type of status --- misp_modules/modules/expansion/securitytrails.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index bf1c5b1..325fa13 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -418,7 +418,7 @@ def expand_history_whois(api, domain): 'categories': ['Network activity'], 'comment': 'Whois history Name Servers of %s ' 'Status: %s ' % ( - domain, item['status']) + domain, ' '.join(item['status'])) } ) @@ -432,7 +432,8 @@ def expand_history_whois(api, domain): 'categories': ['Attribution'], 'comment': 'Whois history registrant email of %s' 'Status: %s' % ( - domain, item['status']) + domain, + ' '.join(item['status'])) } ) @@ -444,7 +445,8 @@ def expand_history_whois(api, domain): 'categories': ['Attribution'], 'comment': 'Whois history registrant phone of %s' 'Status: %s' % ( - domain, item['status']) + domain, + ' '.join(item['status'])) } ) From 1fcc16efb7113acf532ba42f09444f7ff1c16b57 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 18 Jul 2018 22:19:52 +0200 Subject: [PATCH 169/196] securitytrails.com expansion module added --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c1bc7a0..cbf9550 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets. * [rbl](misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute. * [reversedns](misp_modules/modules/expansion/reversedns.py) - Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes. +* [securitytrails](misp_modules/modules/expansion/securitytrails.py) - an expansion module for [securitytrails](https://securitytrails.com/). * [shodan](misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module. * [Sigma queries](misp_modules/modules/expansion/sigma_queries.py) - Experimental expansion module querying a sigma rule to convert it into all the available SIEM signatures. * [Sigma syntax validator](misp_modules/modules/expansion/sigma_syntax_validator.py) - Sigma syntax validator. From 2f27ff12446d36ebd6353029dcc17805ee64865e Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Fri, 27 Jul 2018 14:44:06 +0200 Subject: [PATCH 170/196] ta_import - support for TheatAnalyzer 6.1 --- .../import_mod/threatanalyzer_import.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/misp_modules/modules/import_mod/threatanalyzer_import.py b/misp_modules/modules/import_mod/threatanalyzer_import.py index 83d8291..a6358ab 100755 --- a/misp_modules/modules/import_mod/threatanalyzer_import.py +++ b/misp_modules/modules/import_mod/threatanalyzer_import.py @@ -46,15 +46,19 @@ def handler(q=False): with zf.open(zip_file_name, mode='r', pwd=None) as fp: file_data = fp.read() for line in file_data.decode().split('\n'): - if line: + if not line: + continue + if line.count('|') == 3: l_fname, l_size, l_md5, l_created = line.split('|') - l_fname = cleanup_filepath(l_fname) - if l_fname: - if l_size == 0: - pass # FIXME create an attribute for the filename/path - else: - # file is a non empty sample, upload the sample later - modified_files_mapping[l_md5] = l_fname + if line.count('|') == 4: + l_fname, l_size, l_md5, l_sha256, l_created = line.split('|') + l_fname = cleanup_filepath(l_fname) + if l_fname: + if l_size == 0: + pass # FIXME create an attribute for the filename/path + else: + # file is a non empty sample, upload the sample later + modified_files_mapping[l_md5] = l_fname # now really process the data for zip_file_name in zf.namelist(): # Get all files in the zip file From 63c32520623a35ff40de12bdd40b033a9bfd1edc Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 30 Jul 2018 14:22:40 +0200 Subject: [PATCH 171/196] fix: Put the report location parsing in a try/catch statement as it is an optional field --- misp_modules/modules/export_mod/goamlexport.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/export_mod/goamlexport.py b/misp_modules/modules/export_mod/goamlexport.py index c277640..961a11b 100644 --- a/misp_modules/modules/export_mod/goamlexport.py +++ b/misp_modules/modules/export_mod/goamlexport.py @@ -87,9 +87,12 @@ class GoAmlGeneration(object): person_to_parse = [person_uuid for person_uuid in self.uuids.get('person') if person_uuid not in self.parsed_uuids.get('person')] if len(person_to_parse) == 1: self.itterate('person', 'reporting_person', person_to_parse[0], 'header') - location_to_parse = [location_uuid for location_uuid in self.uuids.get('geolocation') if location_uuid not in self.parsed_uuids.get('geolocation')] - if len(location_to_parse) == 1: - self.itterate('geolocation', 'location', location_to_parse[0], 'header') + try: + location_to_parse = [location_uuid for location_uuid in self.uuids.get('geolocation') if location_uuid not in self.parsed_uuids.get('geolocation')] + if len(location_to_parse) == 1: + self.itterate('geolocation', 'location', location_to_parse[0], 'header') + except TypeError: + pass self.xml['data'] += "" def itterate(self, object_type, aml_type, uuid, xml_part): From 8d4e2025f760279eca24a640cd732291bb0c88af Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Fri, 3 Aug 2018 13:58:53 +0200 Subject: [PATCH 172/196] ta_import - bugfixes for TA 6.1 --- .../import_mod/threatanalyzer_import.py | 42 +++++++++++-------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/misp_modules/modules/import_mod/threatanalyzer_import.py b/misp_modules/modules/import_mod/threatanalyzer_import.py index a6358ab..916628e 100755 --- a/misp_modules/modules/import_mod/threatanalyzer_import.py +++ b/misp_modules/modules/import_mod/threatanalyzer_import.py @@ -15,7 +15,7 @@ misperrors = {'error': 'Error'} userConfig = {} inputSource = ['file'] -moduleinfo = {'version': '0.7', 'author': 'Christophe Vandeplas', +moduleinfo = {'version': '0.8', 'author': 'Christophe Vandeplas', 'description': 'Import for ThreatAnalyzer archive.zip/analysis.json files', 'module-type': ['import']} @@ -73,7 +73,7 @@ def handler(q=False): results.append({ 'values': current_sample_filename, 'data': base64.b64encode(file_data).decode(), - 'type': 'malware-sample', 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': True, 'comment': ''}) + 'type': 'malware-sample', 'categories': ['Artifacts dropped', 'Payload delivery'], 'to_ids': True, 'comment': ''}) if 'Analysis/analysis.json' in zip_file_name: with zf.open(zip_file_name, mode='r', pwd=None) as fp: @@ -88,7 +88,7 @@ def handler(q=False): results.append({ 'values': sample_filename, 'data': base64.b64encode(file_data).decode(), - 'type': 'malware-sample', 'categories': ['Artifacts dropped', 'Payload delivery'], 'to_ids': True, 'comment': ''}) + 'type': 'malware-sample', 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': True, 'comment': ''}) except Exception as e: # no 'sample' in archive, might be an url analysis, just ignore pass @@ -113,7 +113,15 @@ def process_analysis_json(analysis_json): for process in analysis_json['analysis']['processes']['process']: # print_json(process) if 'connection_section' in process and 'connection' in process['connection_section']: + # compensate for absurd behavior of the data format: if one entry = immediately the dict, if multiple entries = list containing dicts + # this will always create a list, even with only one item + if isinstance(process['connection_section']['connection'], dict): + process['connection_section']['connection'] = [process['connection_section']['connection']] + # iterate over each entry for connection_section_connection in process['connection_section']['connection']: + if 'name_to_ip' in connection_section_connection: # TA 6.1 data format + connection_section_connection['@remote_ip'] = connection_section_connection['name_to_ip']['@result_addresses'] + connection_section_connection['@remote_hostname'] = connection_section_connection['name_to_ip']['@request_name'] connection_section_connection['@remote_ip'] = cleanup_ip(connection_section_connection['@remote_ip']) connection_section_connection['@remote_hostname'] = cleanup_hostname(connection_section_connection['@remote_hostname']) @@ -124,7 +132,7 @@ def process_analysis_json(analysis_json): # connection_section_connection['@remote_hostname'], # connection_section_connection['@remote_ip']) # ) - yield({'values': val, 'type': 'domain|ip', 'categories': 'Network activity', 'to_ids': True, 'comment': ''}) + yield({'values': val, 'type': 'domain|ip', 'categories': ['Network activity'], 'to_ids': True, 'comment': ''}) elif connection_section_connection['@remote_ip']: # print("connection_section_connection ip-dst: {} IDS:yes".format( # connection_section_connection['@remote_ip']) @@ -136,20 +144,19 @@ def process_analysis_json(analysis_json): # ) yield({'values': connection_section_connection['@remote_hostname'], 'type': 'hostname', 'to_ids': True, 'comment': ''}) if 'http_command' in connection_section_connection: - for http_command in connection_section_connection['http_command']: - # print('connection_section_connection HTTP COMMAND: {}\t{}'.format( - # http_command['@method'], # comment - # http_command['@url']) # url - # ) - val = cleanup_url(http_command['@url']) - if val: - yield({'values': val, 'type': 'url', 'categories': 'Network activity', 'to_ids': True, 'comment': http_command['@method']}) + # print('connection_section_connection HTTP COMMAND: {}\t{}'.format( + # connection_section_connection['http_command']['@method'], # comment + # connection_section_connection['http_command']['@url']) # url + # ) + val = cleanup_url(connection_section_connection['http_command']['@url']) + if val: + yield({'values': val, 'type': 'url', 'categories': ['Network activity'], 'to_ids': True, 'comment': connection_section_connection['http_command']['@method']}) if 'http_header' in connection_section_connection: for http_header in connection_section_connection['http_header']: if 'User-Agent:' in http_header['@header']: val = http_header['@header'][len('User-Agent: '):] - yield({'values': val, 'type': 'user-agent', 'categories': 'Network activity', 'to_ids': False, 'comment': ''}) + yield({'values': val, 'type': 'user-agent', 'categories': ['Network activity'], 'to_ids': False, 'comment': ''}) elif 'Host:' in http_header['@header']: val = http_header['@header'][len('Host: '):] if ':' in val: @@ -162,7 +169,7 @@ def process_analysis_json(analysis_json): if val_hostname and val_port: val_combined = '{}|{}'.format(val_hostname, val_port) # print({'values': val_combined, 'type': 'hostname|port', 'to_ids': True, 'comment': ''}) - yield({'values': val_combined, 'type': 'hostname|port', 'to_ids': True, 'comment': ''}) + yield({'values': val_combined, 'type': 'hostname|port', 'categories': ['Network activity'], 'to_ids': True, 'comment': ''}) elif val_ip and val_port: val_combined = '{}|{}'.format(val_ip, val_port) # print({'values': val_combined, 'type': 'ip-dst|port', 'to_ids': True, 'comment': ''}) @@ -207,7 +214,7 @@ def process_analysis_json(analysis_json): # networkoperation_section_dns_request_by_name['@request_name'], # networkoperation_section_dns_request_by_name['@result_addresses']) # ) - yield({'values': val, 'type': 'domain|ip', 'categories': 'Network activity', 'to_ids': True, 'comment': ''}) + yield({'values': val, 'type': 'domain|ip', 'categories': ['Network activity'], 'to_ids': True, 'comment': ''}) elif networkoperation_section_dns_request_by_name['@request_name']: # print("networkoperation_section_dns_request_by_name hostname: {} IDS:yes".format( # networkoperation_section_dns_request_by_name['@request_name']) @@ -231,14 +238,14 @@ def process_analysis_json(analysis_json): # networkpacket_section_connect_to_computer['@remote_port']) # ) val_combined = "{}|{}".format(networkpacket_section_connect_to_computer['@remote_hostname'], networkpacket_section_connect_to_computer['@remote_ip']) - yield({'values': val_combined, 'type': 'hostname|ip', 'to_ids': True, 'comment': ''}) + yield({'values': val_combined, 'type': 'domain|ip', 'to_ids': True, 'comment': ''}) elif networkpacket_section_connect_to_computer['@remote_hostname']: # print("networkpacket_section_connect_to_computer hostname: {} IDS:yes COMMENT:port {}".format( # networkpacket_section_connect_to_computer['@remote_hostname'], # networkpacket_section_connect_to_computer['@remote_port']) # ) val_combined = "{}|{}".format(networkpacket_section_connect_to_computer['@remote_hostname'], networkpacket_section_connect_to_computer['@remote_port']) - yield({'values': val_combined, 'type': 'hostname|port', 'to_ids': True, 'comment': ''}) + yield({'values': val_combined, 'type': 'hostname|port', 'categories': ['Network activity'], 'to_ids': True, 'comment': ''}) elif networkpacket_section_connect_to_computer['@remote_ip']: # print("networkpacket_section_connect_to_computer ip-dst: {} IDS:yes COMMENT:port {}".format( # networkpacket_section_connect_to_computer['@remote_ip'], @@ -524,3 +531,4 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig return moduleinfo + From 57af98720d950d4880436fb888dca702bb7aba76 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 7 Aug 2018 18:13:25 +0200 Subject: [PATCH 173/196] fix: [cleanup] Quick clean up on exception type --- misp_modules/modules/expansion/rbl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/rbl.py b/misp_modules/modules/expansion/rbl.py index da8c5fb..49de421 100644 --- a/misp_modules/modules/expansion/rbl.py +++ b/misp_modules/modules/expansion/rbl.py @@ -6,7 +6,7 @@ try: resolver = dns.resolver.Resolver() resolver.timeout = 0.2 resolver.lifetime = 0.2 -except: +except ModuleNotFoundError: print("dnspython3 is missing, use 'pip install dnspython3' to install it.") sys.exit(0) From bb6002a3ff48432915d96e97351e092b78a3c149 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 7 Aug 2018 18:14:29 +0200 Subject: [PATCH 174/196] fix: [cleanup] Quick clean up on yaml load function --- misp_modules/modules/expansion/sigma_syntax_validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/sigma_syntax_validator.py b/misp_modules/modules/expansion/sigma_syntax_validator.py index 0d5226f..6452654 100644 --- a/misp_modules/modules/expansion/sigma_syntax_validator.py +++ b/misp_modules/modules/expansion/sigma_syntax_validator.py @@ -21,7 +21,7 @@ def handler(q=False): return misperrors config = SigmaConfiguration() try: - parser = SigmaParser(yaml.load(request.get('sigma')), config) + parser = SigmaParser(yaml.safe_load(request.get('sigma')), config) result = ("Syntax valid: {}".format(parser.values)) except Exception as e: result = ("Syntax error: {}".format(str(e))) From 0666a60b3dbc1a5f2b54e546ff70b08e7f359fba Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 7 Aug 2018 18:15:15 +0200 Subject: [PATCH 175/196] fix: [cleanup] Quick clean up on exception type --- misp_modules/modules/expansion/yara_syntax_validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yara_syntax_validator.py b/misp_modules/modules/expansion/yara_syntax_validator.py index 4953c41..57c71ea 100644 --- a/misp_modules/modules/expansion/yara_syntax_validator.py +++ b/misp_modules/modules/expansion/yara_syntax_validator.py @@ -2,7 +2,7 @@ import json import requests try: import yara -except: +except ModuleNotFoundError: print("yara is missing, use 'pip3 install yara' to install it.") misperrors = {'error': 'Error'} From 61232ad93e5bda5dd98796a964c74b8a8d179d2d Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 8 Aug 2018 17:00:10 +0200 Subject: [PATCH 176/196] new: Expansion hover module to check spamhaus DBL for a domain name --- .../modules/expansion/dbl_spamhaus.py | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 misp_modules/modules/expansion/dbl_spamhaus.py diff --git a/misp_modules/modules/expansion/dbl_spamhaus.py b/misp_modules/modules/expansion/dbl_spamhaus.py new file mode 100644 index 0000000..f78cb74 --- /dev/null +++ b/misp_modules/modules/expansion/dbl_spamhaus.py @@ -0,0 +1,60 @@ +import json +import datetime +from collections import defaultdict + +try: + import dns.resolver + resolver = dns.resolver.Resolver() + resolver.timeout = 0.2 + resolver.lifetime = 0.2 +except ModuleNotFoundError: + print("dnspython3 is missing, use 'pip install dnspython3' to install it.") + sys.exit(0) + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['domain', 'domain|ip', 'hostname', 'hostname|port'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Christian Studer', + 'description': 'Checks Spamhaus DBL for a domain name.', + 'module-type': ['expansion', 'hover']} +moduleconfig = [] + +dbl = 'dbl.spamhaus.org' +dbl_mapping = {'127.0.1.2': 'spam domain', + '127.0.1.4': 'phish domain', + '127.0.1.5': 'malware domain', + '127.0.1.6': 'botnet C&C domain', + '127.0.1.102': 'abused legit spam', + '127.0.1.103': 'abused spammed redirector domain', + '127.0.1.104': 'abused legit phish', + '127.0.1.105': 'abused legit malware', + '127.0.1.106': 'abused legit botnet C&C', + '127.0.1.255': 'IP queries prohibited!'} + +def fetch_requested_value(request): + for attribute_type in mispattributes['input']: + if request.get(attribute_type): + return request[attribute_type].split('|')[0] + return None + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + requested_value = fetch_requested_value(request) + if requested_value is None: + misperrors['error'] = "Unsupported attributes type" + return misperrors + query = "{}.{}".format(requested_value, dbl) + try: + query_result = resolver.query(query, 'A')[0] + result = "{} - {}".format(requested_value, dbl_mapping[str(query_result)]) + except Exception as e: + result = e + return {'results': [{'types': mispattributes.get('output'), 'values': result}]} + +def introspection(): + return mispattributes + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 90baa1dd5a1c931e02a1a91dda351dde7983c3b6 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 8 Aug 2018 17:05:22 +0200 Subject: [PATCH 177/196] add: Added DBL spamhaus module documentation and in expansion init file --- README.md | 1 + misp_modules/modules/expansion/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cbf9550..ad4b098 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to. * [CrowdStrike Falcon](misp_modules/modules/expansion/crowdstrike_falcon.py) - an expansion module to expand using CrowdStrike Falcon Intel Indicator API. * [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE). +* [DBL Spamhaus](misp_modules/modules/expansion/dbl_spamhaus.py) - a hover module to check Spamhaus DBL for a domain name. * [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes. * [DomainTools](misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois. * [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en). diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index cda3af5..c6e81a7 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,3 +1,3 @@ from . import _vmray -__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries'] +__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus'] From bdbf5388934478d79fee2c5c3ae21b0642d3cd78 Mon Sep 17 00:00:00 2001 From: David J Date: Fri, 10 Aug 2018 16:00:01 -0500 Subject: [PATCH 178/196] Create urlscan.py --- misp_modules/modules/expansion/urlscan.py | 269 ++++++++++++++++++++++ 1 file changed, 269 insertions(+) create mode 100644 misp_modules/modules/expansion/urlscan.py diff --git a/misp_modules/modules/expansion/urlscan.py b/misp_modules/modules/expansion/urlscan.py new file mode 100644 index 0000000..8f4067c --- /dev/null +++ b/misp_modules/modules/expansion/urlscan.py @@ -0,0 +1,269 @@ +import json +import requests +import logging +import sys +import time +# Need base64 if encoding data for attachments, but disabled for now +# import base64 + +log = logging.getLogger('urlscan') +log.setLevel(logging.DEBUG) +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +log.addHandler(ch) + +moduleinfo = { + 'version': '0.1', + 'author': 'Dave Johnson', + 'description': 'Module to query urlscan.io', + 'module-type': ['expansion'] + } + +moduleconfig = ['apikey'] +misperrors = {'error': 'Error'} +mispattributes = { + 'input': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url'], + 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url', 'text', 'link'] + } + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if (request.get('config')): + if (request['config'].get('apikey') is None): + misperrors['error'] = 'urlscan apikey is missing' + return misperrors + client = urlscanAPI(request['config']['apikey']) + + r = {'results': []} + + if 'ip-src' in request: + r['results'] += lookup_indicator(client, request['ip-src']) + if 'ip-dst' in request: + r['results'] += lookup_indicator(client, request['ip-dst']) + if 'domain' in request: + r['results'] += lookup_indicator(client, request['domain']) + if 'hostname' in request: + r['results'] += lookup_indicator(client, request['hostname']) + if 'url' in request: + r['results'] += lookup_indicator(client, request['url']) + + uniq = [] + for item in r['results']: + if item not in uniq: + uniq.append(item) + r['results'] = uniq + return r + + +def lookup_indicator(client, query): + result = client.search_url(query) + log.debug('RESULTS: ' + json.dumps(result)) + r = [] + if result.get('page'): + if result['page'].get('domain'): + misp_val = result['page']['domain'] + misp_comment = "Domain associated with {} (source: urlscan.io)".format(query) + r.append({'types': 'domain', + 'categories': ['Network activity'], + 'values': misp_val, + 'comment': misp_comment}) + + if result['page'].get('ip'): + misp_val = result['page']['ip'] + misp_comment = "IP associated with {} (source: urlscan.io)".format(query) + r.append({'types': 'ip-dst', + 'categories': ['Network activity'], + 'values': misp_val, + 'comment': misp_comment}) + + if result['page'].get('country'): + misp_val = 'Country: ' + result['page']['country'] + if result['page'].get('city'): + misp_val += ', City: ' + result['page']['city'] + misp_comment = "Location associated with {} (source: urlscan.io)".format(query) + r.append({'types': 'text', + 'categories': ['External analysis'], + 'values': misp_val, + 'comment': misp_comment}) + + if result['page'].get('asn'): + misp_val = result['page']['asn'] + misp_comment = "ASN associated with {} (source: urlscan.io)".format(query) + r.append({'types': 'AS', 'categories': ['Network activity'], 'values': misp_val, 'comment': misp_comment}) + + if result['page'].get('asnname'): + misp_val = result['page']['asnname'] + misp_comment = "ASN name associated with {} (source: urlscan.io)".format(query) + r.append({'types': 'text', + 'categories': ['External analysis'], + 'values': misp_val, + 'comment': misp_comment}) + + if result.get('stats'): + if result['stats'].get('malicious'): + log.debug('There is something in results > stats > malicious') + threat_list = set() + + if 'matches' in result['meta']['processors']['gsb']['data']: + for item in result['meta']['processors']['gsb']['data']['matches']: + if item['threatType']: + threat_list.add(item['threatType']) + + threat_list = ', '.join(threat_list) + log.debug('threat_list values are: \'' + threat_list + '\'') + + if threat_list: + misp_val = '{} threat(s) detected'.format(threat_list) + misp_comment = '{} malicious indicator(s) were present on ' \ + '{} (source: urlscan.io)'.format(result['stats']['malicious'], query, threat_list) + r.append({'types': 'text', + 'categories': ['External analysis'], + 'values': misp_val, + 'comment': misp_comment}) + + if result.get('lists'): + if result['lists'].get('urls'): + for url in result['lists']['urls']: + url = url.lower() + if 'office' in url: + misp_val = 'Possible Microsoft Office themed phishing page' + misp_comment = 'There was resource containing an \'Office\' string in the URL.' + elif 'o365' in url or '0365' in url: + misp_val = 'Possible Microsoft O365 themed phishing page' + misp_comment = 'There was resource containing an \'O365\' string in the URL.' + elif 'microsoft' in url: + misp_val = 'Possible Microsoft themed phishing page' + misp_comment = 'There was resource containing an \'Office\' string in the URL.' + elif 'paypal' in url: + misp_val = 'Possible PayPal themed phishing page' + misp_comment = 'There was resource containing a \'PayPal\' string in the URL.' + elif 'onedrive' in url: + misp_val = 'Possible OneDrive themed phishing page' + misp_comment = 'There was resource containing a \'OneDrive\' string in the URL.' + elif 'docusign' in url: + misp_val = 'Possible DocuSign themed phishing page' + misp_comment = 'There was resource containing a \'DocuSign\' string in the URL' + r.append({'types': 'text', + 'categories': ['External analysis'], + 'values': misp_val, + 'comment': misp_comment}) + + if result.get('task'): + if result['task'].get('reportURL'): + misp_val = result['task']['reportURL'] + misp_comment = 'Link to full report (source: urlscan.io)' + r.append({'types': 'link', + 'categories': ['External analysis'], + 'values': misp_val, + 'comment': misp_comment}) + + if result['task'].get('screenshotURL'): + image_url = result['task']['screenshotURL'] + misp_comment = 'Link to screenshot (source: urlscan.io)' + r.append({'types': 'link', + 'categories': ['External analysis'], + 'values': image_url, + 'comment': misp_comment}) + ### TO DO ### + ### Add ability to add an in-line screenshot of the target website into an attribute + # screenshot = requests.get(image_url).content + # r.append({'types': ['attachment'], + # 'categories': ['External analysis'], + # 'values': image_url, + # 'image': str(base64.b64encode(screenshot), 'utf-8'), + # 'comment': 'Screenshot of website'}) + + if result['task'].get('domURL'): + misp_val = result['task']['domURL'] + misp_comment = 'Link to DOM (source: urlscan.io)' + r.append({'types': 'link', + 'categories': ['External analysis'], + 'values': misp_val, + 'comment': misp_comment}) + + return r + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo + + +class urlscanAPI(): + def __init__(self, apikey=None, uuid=None): + self.key = apikey + self.uuid = uuid + + def request(self, query): + log.debug('From request function with the parameter: ' + query) + payload = {'url': query} + headers = {'API-Key': self.key, + 'Content-Type': "application/json", + 'Cache-Control': "no-cache"} + + # Troubleshooting problems with initial search request + log.debug('PAYLOAD: ' + json.dumps(payload)) + log.debug('HEADERS: ' + json.dumps(headers)) + + search_url_string = "https://urlscan.io/api/v1/scan/" + response = requests.request("POST", + search_url_string, + data=json.dumps(payload), + headers=headers) + + # HTTP 400 - Bad Request + if response.status_code == 400: + raise Exception('HTTP Error 400 - Bad Request') + + # HTTP 404 - Not found + if response.status_code == 404: + raise Exception('HTTP Error 404 - These are not the droids you\'re looking for') + + # Any other status code + if response.status_code != 200: + raise Exception('HTTP Error ' + str(response.status_code)) + + if response.text: + response = json.loads(response.content.decode("utf-8")) + time.sleep(3) + self.uuid = response['uuid'] + + # Strings for to check for errors on the results page + # Null response string for any unavailable resources + null_response_string = '"status": 404' + # Redirect string accounting for 301/302/303/307/308 status codes + redirect_string = '"status": 30' + # Normal response string with 200 status code + normal_response_string = '"status": 200' + + results_url_string = "https://urlscan.io/api/v1/result/" + self.uuid + log.debug('Results URL: ' + results_url_string) + + # Need to wait for results to process and check if they are valid + tries = 10 + while tries >= 0: + results = requests.request("GET", results_url_string) + log.debug('Made a GET request') + results = results.content.decode("utf-8") + # checking if there is a 404 status code and no available resources + if null_response_string in results and \ + redirect_string not in results and \ + normal_response_string not in results: + log.debug('Results not processed. Please check again later.') + time.sleep(3) + tries -= 1 + else: + return json.loads(results) + raise Exception('Results contained a 404 status error and could not be processed.') + + def search_url(self, query): + log.debug('From search_url with parameter: ' + query) + return self.request(query) From a697f653822b893d95a4142a4d88cbf66821f208 Mon Sep 17 00:00:00 2001 From: David J Date: Tue, 14 Aug 2018 10:51:15 -0500 Subject: [PATCH 179/196] Add error handling for DNS failures, reduce imports, and simplify misp_comments --- misp_modules/modules/expansion/urlscan.py | 78 +++++++++++------------ 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/misp_modules/modules/expansion/urlscan.py b/misp_modules/modules/expansion/urlscan.py index 8f4067c..a0adc25 100644 --- a/misp_modules/modules/expansion/urlscan.py +++ b/misp_modules/modules/expansion/urlscan.py @@ -3,8 +3,6 @@ import requests import logging import sys import time -# Need base64 if encoding data for attachments, but disabled for now -# import base64 log = logging.getLogger('urlscan') log.setLevel(logging.DEBUG) @@ -15,18 +13,19 @@ ch.setFormatter(formatter) log.addHandler(ch) moduleinfo = { - 'version': '0.1', - 'author': 'Dave Johnson', - 'description': 'Module to query urlscan.io', - 'module-type': ['expansion'] - } + 'version': '0.1', + 'author': 'Dave Johnson', + 'description': 'Module to query urlscan.io', + 'module-type': ['expansion'] +} moduleconfig = ['apikey'] misperrors = {'error': 'Error'} mispattributes = { - 'input': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url'], - 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url', 'text', 'link'] - } + 'input': ['hostname', 'domain', 'url'], + 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url', 'text', 'link'] +} + def handler(q=False): if q is False: @@ -51,8 +50,15 @@ def handler(q=False): if 'url' in request: r['results'] += lookup_indicator(client, request['url']) + # Return any errors generated from lookup to the UI and remove duplicates + uniq = [] + log.debug(r['results']) for item in r['results']: + log.debug(item) + if 'error' in item: + misperrors['error'] = item['error'] + return misperrors if item not in uniq: uniq.append(item) r['results'] = uniq @@ -63,10 +69,19 @@ def lookup_indicator(client, query): result = client.search_url(query) log.debug('RESULTS: ' + json.dumps(result)) r = [] + misp_comment = "{}: Enriched via the urlscan module".format(query) + + # Determine if the page is reachable + for request in result['data']['requests']: + if request['response'].get('failed'): + if request['response']['failed']['errorText']: + log.debug('The page could not load') + r.append( + {'error': 'Domain could not be resolved: {}'.format(request['response']['failed']['errorText'])}) + if result.get('page'): if result['page'].get('domain'): misp_val = result['page']['domain'] - misp_comment = "Domain associated with {} (source: urlscan.io)".format(query) r.append({'types': 'domain', 'categories': ['Network activity'], 'values': misp_val, @@ -74,17 +89,15 @@ def lookup_indicator(client, query): if result['page'].get('ip'): misp_val = result['page']['ip'] - misp_comment = "IP associated with {} (source: urlscan.io)".format(query) r.append({'types': 'ip-dst', 'categories': ['Network activity'], 'values': misp_val, 'comment': misp_comment}) if result['page'].get('country'): - misp_val = 'Country: ' + result['page']['country'] + misp_val = 'country: ' + result['page']['country'] if result['page'].get('city'): - misp_val += ', City: ' + result['page']['city'] - misp_comment = "Location associated with {} (source: urlscan.io)".format(query) + misp_val += ', city: ' + result['page']['city'] r.append({'types': 'text', 'categories': ['External analysis'], 'values': misp_val, @@ -92,12 +105,10 @@ def lookup_indicator(client, query): if result['page'].get('asn'): misp_val = result['page']['asn'] - misp_comment = "ASN associated with {} (source: urlscan.io)".format(query) - r.append({'types': 'AS', 'categories': ['Network activity'], 'values': misp_val, 'comment': misp_comment}) + r.append({'types': 'AS', 'categories': ['External analysis'], 'values': misp_val, 'comment': misp_comment}) if result['page'].get('asnname'): misp_val = result['page']['asnname'] - misp_comment = "ASN name associated with {} (source: urlscan.io)".format(query) r.append({'types': 'text', 'categories': ['External analysis'], 'values': misp_val, @@ -118,8 +129,6 @@ def lookup_indicator(client, query): if threat_list: misp_val = '{} threat(s) detected'.format(threat_list) - misp_comment = '{} malicious indicator(s) were present on ' \ - '{} (source: urlscan.io)'.format(result['stats']['malicious'], query, threat_list) r.append({'types': 'text', 'categories': ['External analysis'], 'values': misp_val, @@ -130,23 +139,17 @@ def lookup_indicator(client, query): for url in result['lists']['urls']: url = url.lower() if 'office' in url: - misp_val = 'Possible Microsoft Office themed phishing page' - misp_comment = 'There was resource containing an \'Office\' string in the URL.' + misp_val = "Possible Office-themed phishing" elif 'o365' in url or '0365' in url: - misp_val = 'Possible Microsoft O365 themed phishing page' - misp_comment = 'There was resource containing an \'O365\' string in the URL.' + misp_val = "Possible O365-themed phishing" elif 'microsoft' in url: - misp_val = 'Possible Microsoft themed phishing page' - misp_comment = 'There was resource containing an \'Office\' string in the URL.' + misp_val = "Possible Microsoft-themed phishing" elif 'paypal' in url: - misp_val = 'Possible PayPal themed phishing page' - misp_comment = 'There was resource containing a \'PayPal\' string in the URL.' + misp_val = "Possible PayPal-themed phishing" elif 'onedrive' in url: - misp_val = 'Possible OneDrive themed phishing page' - misp_comment = 'There was resource containing a \'OneDrive\' string in the URL.' + misp_val = "Possible OneDrive-themed phishing" elif 'docusign' in url: - misp_val = 'Possible DocuSign themed phishing page' - misp_comment = 'There was resource containing a \'DocuSign\' string in the URL' + misp_val = "Possible DocuSign-themed phishing" r.append({'types': 'text', 'categories': ['External analysis'], 'values': misp_val, @@ -155,7 +158,6 @@ def lookup_indicator(client, query): if result.get('task'): if result['task'].get('reportURL'): misp_val = result['task']['reportURL'] - misp_comment = 'Link to full report (source: urlscan.io)' r.append({'types': 'link', 'categories': ['External analysis'], 'values': misp_val, @@ -163,7 +165,6 @@ def lookup_indicator(client, query): if result['task'].get('screenshotURL'): image_url = result['task']['screenshotURL'] - misp_comment = 'Link to screenshot (source: urlscan.io)' r.append({'types': 'link', 'categories': ['External analysis'], 'values': image_url, @@ -177,14 +178,6 @@ def lookup_indicator(client, query): # 'image': str(base64.b64encode(screenshot), 'utf-8'), # 'comment': 'Screenshot of website'}) - if result['task'].get('domURL'): - misp_val = result['task']['domURL'] - misp_comment = 'Link to DOM (source: urlscan.io)' - r.append({'types': 'link', - 'categories': ['External analysis'], - 'values': misp_val, - 'comment': misp_comment}) - return r @@ -262,6 +255,7 @@ class urlscanAPI(): tries -= 1 else: return json.loads(results) + raise Exception('Results contained a 404 status error and could not be processed.') def search_url(self, query): From 7deeb95820e1e4099208496025807e6ae6ef1164 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Tue, 21 Aug 2018 11:13:08 +0200 Subject: [PATCH 180/196] fix: ta_import - bugfixes --- .../import_mod/threatanalyzer_import.py | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/misp_modules/modules/import_mod/threatanalyzer_import.py b/misp_modules/modules/import_mod/threatanalyzer_import.py index 916628e..2e3a507 100755 --- a/misp_modules/modules/import_mod/threatanalyzer_import.py +++ b/misp_modules/modules/import_mod/threatanalyzer_import.py @@ -15,7 +15,7 @@ misperrors = {'error': 'Error'} userConfig = {} inputSource = ['file'] -moduleinfo = {'version': '0.8', 'author': 'Christophe Vandeplas', +moduleinfo = {'version': '0.9', 'author': 'Christophe Vandeplas', 'description': 'Import for ThreatAnalyzer archive.zip/analysis.json files', 'module-type': ['import']} @@ -45,7 +45,7 @@ def handler(q=False): if re.match(r"Analysis/proc_\d+/modified_files/mapping\.log", zip_file_name): with zf.open(zip_file_name, mode='r', pwd=None) as fp: file_data = fp.read() - for line in file_data.decode().split('\n'): + for line in file_data.decode("utf-8", 'ignore').split('\n'): if not line: continue if line.count('|') == 3: @@ -55,7 +55,8 @@ def handler(q=False): l_fname = cleanup_filepath(l_fname) if l_fname: if l_size == 0: - pass # FIXME create an attribute for the filename/path + results.append({'values': l_fname, 'type': 'filename', 'to_ids': True, + 'categories': ['Artifacts dropped', 'Payload delivery'], 'comment': ''}) else: # file is a non empty sample, upload the sample later modified_files_mapping[l_md5] = l_fname @@ -144,13 +145,14 @@ def process_analysis_json(analysis_json): # ) yield({'values': connection_section_connection['@remote_hostname'], 'type': 'hostname', 'to_ids': True, 'comment': ''}) if 'http_command' in connection_section_connection: - # print('connection_section_connection HTTP COMMAND: {}\t{}'.format( - # connection_section_connection['http_command']['@method'], # comment - # connection_section_connection['http_command']['@url']) # url - # ) - val = cleanup_url(connection_section_connection['http_command']['@url']) - if val: - yield({'values': val, 'type': 'url', 'categories': ['Network activity'], 'to_ids': True, 'comment': connection_section_connection['http_command']['@method']}) + for http_command in connection_section_connection['http_command']: + # print('connection_section_connection HTTP COMMAND: {}\t{}'.format( + # connection_section_connection['http_command']['@method'], # comment + # connection_section_connection['http_command']['@url']) # url + # ) + val = cleanup_url(http_command['@url']) + if val: + yield({'values': val, 'type': 'url', 'categories': ['Network activity'], 'to_ids': True, 'comment': http_command['@method']}) if 'http_header' in connection_section_connection: for http_header in connection_section_connection['http_header']: @@ -453,9 +455,9 @@ def cleanup_filepath(item): '\\AppData\\Roaming\\Adobe\\Acrobat\\9.0\\UserCache.bin', '\\AppData\\Roaming\\Macromedia\\Flash Player\\macromedia.com\\support\\flashplayer\\sys\\settings.sol', - '\\AppData\\Roaming\Adobe\\Flash Player\\NativeCache\\', + '\\AppData\\Roaming\\Adobe\\Flash Player\\NativeCache\\', 'C:\\Windows\\AppCompat\\Programs\\', - 'C:\~' # caused by temp file created by MS Office when opening malicious doc/xls/... + 'C:\\~' # caused by temp file created by MS Office when opening malicious doc/xls/... } if list_in_string(noise_substrings, item): return None @@ -531,4 +533,3 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig return moduleinfo - From d15cbe58fe47d137554883084e02540b7f2ab9a8 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 30 Aug 2018 20:41:49 +0200 Subject: [PATCH 181/196] fix: Quick cleanup --- misp_modules/modules/export_mod/goamlexport.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/export_mod/goamlexport.py b/misp_modules/modules/export_mod/goamlexport.py index 961a11b..f09f2e6 100644 --- a/misp_modules/modules/export_mod/goamlexport.py +++ b/misp_modules/modules/export_mod/goamlexport.py @@ -1,4 +1,5 @@ -import json, datetime, base64 +import json +import base64 from pymisp import MISPEvent from collections import defaultdict, Counter @@ -67,7 +68,7 @@ class GoAmlGeneration(object): try: report_code.append(obj.get_attributes_by_relation('report-code')[0].value.split(' ')[0]) currency_code.append(obj.get_attributes_by_relation('currency-code')[0].value) - except: + except IndexError: print('report_code or currency_code error') self.uuids, self.report_codes, self.currency_codes = uuids, report_code, currency_code From 35f3a5e43f92eff195f98fc312fa53f922155090 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 30 Aug 2018 20:45:29 +0200 Subject: [PATCH 182/196] fix: Quick cleanup --- misp_modules/modules/expansion/rbl.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/rbl.py b/misp_modules/modules/expansion/rbl.py index 49de421..7aac103 100644 --- a/misp_modules/modules/expansion/rbl.py +++ b/misp_modules/modules/expansion/rbl.py @@ -96,13 +96,12 @@ def handler(q=False): txt = resolver.query(query,'TXT') listed.append(query) info.append(str(txt[0])) - except: + except Exception: continue result = {} for l, i in zip(listed, info): result[l] = i - r = {'results': [{'types': mispattributes.get('output'), 'values': json.dumps(result)}]} - return r + return {'results': [{'types': mispattributes.get('output'), 'values': json.dumps(result)}]} def introspection(): return mispattributes From b0be965e576a1fb8e02d36bc4a34ecde5d967e4b Mon Sep 17 00:00:00 2001 From: SuRb0 <1809870+surbo@users.noreply.github.com> Date: Thu, 30 Aug 2018 19:41:34 -0500 Subject: [PATCH 183/196] Update urlscan.py Added hash to the search so you can take advantage of the new file down load function on urlscan.io. You can use this to pivot on file hashes and find out domains that hosting the same malicious file. --- misp_modules/modules/expansion/urlscan.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/urlscan.py b/misp_modules/modules/expansion/urlscan.py index a0adc25..ecd1a50 100644 --- a/misp_modules/modules/expansion/urlscan.py +++ b/misp_modules/modules/expansion/urlscan.py @@ -22,8 +22,8 @@ moduleinfo = { moduleconfig = ['apikey'] misperrors = {'error': 'Error'} mispattributes = { - 'input': ['hostname', 'domain', 'url'], - 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url', 'text', 'link'] + 'input': ['hostname', 'domain', 'url', 'hash'], + 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url', 'text', 'link', 'hash'] } @@ -49,6 +49,8 @@ def handler(q=False): r['results'] += lookup_indicator(client, request['hostname']) if 'url' in request: r['results'] += lookup_indicator(client, request['url']) + f 'hash' in request: + r['results'] += lookup_indicator(client, request['hash']) # Return any errors generated from lookup to the UI and remove duplicates From 179430d69db5e45fa0c0e815a934fd87b69135c1 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 31 Aug 2018 21:38:53 +0200 Subject: [PATCH 184/196] fix: Some cleanup and output types fixed - hashes types specified in output --- misp_modules/modules/expansion/virustotal.py | 304 ++++++++----------- 1 file changed, 133 insertions(+), 171 deletions(-) mode change 100755 => 100644 misp_modules/modules/expansion/virustotal.py diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py old mode 100755 new mode 100644 index 3997ee6..404f621 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -2,201 +2,163 @@ import json import requests from requests import HTTPError import base64 +from collections import defaultdict misperrors = {'error': 'Error'} mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "sha512"], 'output': ['domain', "ip-src", "ip-dst", "text", "md5", "sha1", "sha256", "sha512", "ssdeep", - "authentihash", "filename"] - } + "authentihash", "filename"]} # possible module-types: 'expansion', 'hover' or both -moduleinfo = {'version': '2', 'author': 'Hannah Ward', +moduleinfo = {'version': '3', 'author': 'Hannah Ward', 'description': 'Get information from virustotal', 'module-type': ['expansion']} # config fields that your code expects from the site admin moduleconfig = ["apikey", "event_limit"] -limit = 5 # Default -comment = '%s: Enriched via VT' +comment = '{}: Enriched via VirusTotal' +hash_types = ["md5", "sha1", "sha256", "sha512"] +class VirusTotalRequest(object): + def __init__(self, config): + self.apikey = config['apikey'] + self.limit = int(config.get('event_limit', 5)) + self.base_url = "https://www.virustotal.com/vtapi/v2/{}/report" + self.results = defaultdict(set) + self.to_return = [] + self.input_types_mapping = {'ip-src': self.get_ip, 'ip-dst': self.get_ip, + 'domain': self.get_domain, 'hostname': self.get_domain, + 'md5': self.get_hash, 'sha1': self.get_hash, + 'sha256': self.get_hash, 'sha512': self.get_hash} + self.output_types_mapping = {'submission_names': 'filename', 'ssdeep': 'ssdeep', + 'authentihash': 'authentihash', 'ITW_urls': 'url'} + + def parse_request(self, attribute_type, attribute_value): + error = self.input_types_mapping[attribute_type](attribute_value) + if error is not None: + return error + for key, values in self.results.items(): + if isinstance(key, tuple): + types, comment = key + self.to_return.append({'types': list(types), 'values': list(values), 'comment': comment}) + else: + self.to_return.append({'types': key, 'values': list(values)}) + return self.to_return + + def get_domain(self, domain, do_not_recurse=False): + req = requests.get(self.base_url.format('domain'), params={'domain': domain, 'apikey': self.apikey}) + try: + req.raise_for_status() + req = req.json() + except HTTPError as e: + return str(e) + if req["response_code"] == 0: + # Nothing found + return [] + if "resolutions" in req: + for res in req["resolutions"][:self.limit]: + ip_address = res["ip_address"] + self.results[(("ip-dst", "ip-src"), comment.format(domain))].add(ip_address) + # Pivot from here to find all domain info + if not do_not_recurse: + error = self.get_ip(ip_address, True) + if error is not None: + return error + self.get_more_info(req) + + def get_hash(self, _hash): + req = requests.get(self.base_url.format('file'), params={'resource': _hash, 'apikey': self.apikey, 'allinfo': 1}) + try: + req.raise_for_status() + req = req.json() + except HTTPError as e: + return str(e) + if req["response_code"] == 0: + # Nothing found + return [] + self.get_more_info(req) + + def get_ip(self, ip, do_not_recurse=False): + req = requests.get(self.base_url.format('ip-address'), params={'ip': ip, 'apikey': self.apikey}) + try: + req.raise_for_status() + req = req.json() + except HTTPError as e: + return str(e) + if req["response_code"] == 0: + # Nothing found + return [] + if "resolutions" in req: + for res in req["resolutions"][:self.limit]: + hostname = res["hostname"] + self.results[(("domain",), comment.format(ip))].add(hostname) + # Pivot from here to find all domain info + if not do_not_recurse: + error = self.get_domain(hostname, True) + if error is not None: + return error + self.get_more_info(req) + + def find_all(self, data): + hashes = [] + if isinstance(data, dict): + for key, value in data.items(): + if key in hash_types: + print(key) + self.results[key].add(value) + hashes.append(value) + else: + if isinstance(value, (dict, list)): + hashes.extend(self.find_all(value)) + elif isinstance(data, list): + for d in data: + hashes.extend(self.find_all(d)) + return hashes + + def get_more_info(self, req): + # Get all hashes first + hashes = self.find_all(req) + for h in hashes[:self.limit]: + # Search VT for some juicy info + try: + data = requests.get(self.base_url.format('file'), params={'resource': h, 'apikey': apikey, 'allinfo': 1}).json() + except Exception: + continue + # Go through euch key and check if it exists + for VT_type, MISP_type in self.output_types_mapping.items(): + if VT_type in data: + self.results[((MISP_type,), comment.format(h))].add(data[VT_type]) + # Get the malware sample + sample = requests.get(self.base_url[:-6].format('file/download'), params={'hash': h, 'apikey': apikey}) + malsample = sample.content + # It is possible for VT to not give us any submission names + if "submission_names" in data: + self.to_return.append({"types": ["malware-sample"], "categories": ["Payload delivery"], + "values": data["submimssion_names"], "data": str(base64.b64encore(malsample), 'utf-8')}) def handler(q=False): - global limit if q is False: return False - q = json.loads(q) - - key = q["config"]["apikey"] - limit = int(q["config"].get("event_limit", 5)) - - r = {"results": []} - - if "ip-src" in q: - r["results"] += getIP(q["ip-src"], key) - if "ip-dst" in q: - r["results"] += getIP(q["ip-dst"], key) - if "domain" in q: - r["results"] += getDomain(q["domain"], key) - if 'hostname' in q: - r["results"] += getDomain(q['hostname'], key) - if 'md5' in q: - r["results"] += getHash(q['md5'], key) - if 'sha1' in q: - r["results"] += getHash(q['sha1'], key) - if 'sha256' in q: - r["results"] += getHash(q['sha256'], key) - if 'sha512' in q: - r["results"] += getHash(q['sha512'], key) - - uniq = [] - for res in r["results"]: - if res not in uniq: - uniq.append(res) - r["results"] = uniq - return r - - -def getHash(hash, key, do_not_recurse=False): - req = requests.get("https://www.virustotal.com/vtapi/v2/file/report", - params={"allinfo": 1, "apikey": key, 'resource': hash}) - try: - req.raise_for_status() - req = req.json() - except HTTPError as e: - misperrors['error'] = str(e) + if not q.get('config') or not q['config'].get('apikey'): + misperrors['error']: "A VirusTotal api key is required for this module." return misperrors - - if req["response_code"] == 0: - # Nothing found - return [] - - return getMoreInfo(req, key) - - -def getIP(ip, key, do_not_recurse=False): - global limit - toReturn = [] - req = requests.get("https://www.virustotal.com/vtapi/v2/ip-address/report", - params={"ip": ip, "apikey": key}) - try: - req.raise_for_status() - req = req.json() - except HTTPError as e: - misperrors['error'] = str(e) + del q['module'] + query = VirusTotalRequest(q.pop('config')) + r = query.parse_request(*list(q.items())[0]) + if isinstance(r, str): + misperrors['error'] = r return misperrors + return {'results': r} - if req["response_code"] == 0: - # Nothing found - return [] - - if "resolutions" in req: - for res in req["resolutions"][:limit]: - toReturn.append({"types": ["domain"], "values": [res["hostname"]], "comment": comment % ip}) - # Pivot from here to find all domain info - if not do_not_recurse: - toReturn += getDomain(res["hostname"], key, True) - - toReturn += getMoreInfo(req, key) - return toReturn - - -def getDomain(domain, key, do_not_recurse=False): - global limit - toReturn = [] - req = requests.get("https://www.virustotal.com/vtapi/v2/domain/report", - params={"domain": domain, "apikey": key}) - try: - req.raise_for_status() - req = req.json() - except HTTPError as e: - misperrors['error'] = str(e) - return misperrors - - if req["response_code"] == 0: - # Nothing found - return [] - - if "resolutions" in req: - for res in req["resolutions"][:limit]: - toReturn.append({"types": ["ip-dst", "ip-src"], "values": [res["ip_address"]], "comment": comment % domain}) - # Pivot from here to find all info on IPs - if not do_not_recurse: - toReturn += getIP(res["ip_address"], key, True) - if "subdomains" in req: - for subd in req["subdomains"]: - toReturn.append({"types": ["domain"], "values": [subd], "comment": comment % domain}) - toReturn += getMoreInfo(req, key) - return toReturn - - -def findAll(data, keys): - a = [] - if isinstance(data, dict): - for key in data.keys(): - if key in keys: - a.append(data[key]) - else: - if isinstance(data[key], (dict, list)): - a += findAll(data[key], keys) - if isinstance(data, list): - for i in data: - a += findAll(i, keys) - - return a - - -def getMoreInfo(req, key): - global limit - r = [] - # Get all hashes first - hashes = [] - hashes = findAll(req, ["md5", "sha1", "sha256", "sha512"]) - r.append({"types": ["freetext"], "values": hashes}) - for hsh in hashes[:limit]: - # Search VT for some juicy info - try: - data = requests.get("http://www.virustotal.com/vtapi/v2/file/report", - params={"allinfo": 1, "apikey": key, "resource": hsh} - ).json() - except: - continue - - # Go through each key and check if it exists - if "submission_names" in data: - r.append({'types': ["filename"], "values": data["submission_names"], "comment": comment % hsh}) - - if "ssdeep" in data: - r.append({'types': ["ssdeep"], "values": [data["ssdeep"]], "comment": comment % hsh}) - - if "authentihash" in data: - r.append({"types": ["authentihash"], "values": [data["authentihash"]], "comment": comment % hsh}) - - if "ITW_urls" in data: - r.append({"types": ["url"], "values": data["ITW_urls"], "comment": comment % hsh}) - - # Get the malware sample - sample = requests.get("https://www.virustotal.com/vtapi/v2/file/download", - params={"hash": hsh, "apikey": key}) - - malsample = sample.content - - # It is possible for VT to not give us any submission names - if "submission_names" in data: - r.append({"types": ["malware-sample"], - "categories": ["Payload delivery"], - "values": data["submission_names"], - "data": str(base64.b64encode(malsample), 'utf-8') - } - ) - - return r - +def get_ip(ip, key): + params = {'ip': ip, 'apikey': key} + req = requests.get('https://www.virustotal.com/vtapi/v2/ip-address/report', params=params) + return json.dumps(req.json(), indent=2) def introspection(): return mispattributes - def version(): moduleinfo['config'] = moduleconfig return moduleinfo From 2af947a2deba46ddf2e92cc8f1b0c08be07e1429 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 3 Sep 2018 10:23:05 +0200 Subject: [PATCH 185/196] fix: Removed print --- misp_modules/modules/expansion/virustotal.py | 1 - 1 file changed, 1 deletion(-) diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index 404f621..de91e09 100644 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -104,7 +104,6 @@ class VirusTotalRequest(object): if isinstance(data, dict): for key, value in data.items(): if key in hash_types: - print(key) self.results[key].add(value) hashes.append(value) else: From 936e30b15b97643828b95011bbf7e349acd9f146 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 3 Sep 2018 12:03:42 +0200 Subject: [PATCH 186/196] fix: Multiple attributes parsing support - Fixing one of my previous changes not processing multiple attributes parsing --- misp_modules/modules/expansion/virustotal.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index de91e09..4ebbd43 100644 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -33,10 +33,14 @@ class VirusTotalRequest(object): self.output_types_mapping = {'submission_names': 'filename', 'ssdeep': 'ssdeep', 'authentihash': 'authentihash', 'ITW_urls': 'url'} - def parse_request(self, attribute_type, attribute_value): - error = self.input_types_mapping[attribute_type](attribute_value) - if error is not None: - return error + def parse_request(self, q): + for attribute_type, attribute_value in q.items(): + try: + error = self.input_types_mapping[attribute_type](attribute_value) + except KeyError: + continue + if error is not None: + return error for key, values in self.results.items(): if isinstance(key, tuple): types, comment = key @@ -144,7 +148,7 @@ def handler(q=False): return misperrors del q['module'] query = VirusTotalRequest(q.pop('config')) - r = query.parse_request(*list(q.items())[0]) + r = query.parse_request(q) if isinstance(r, str): misperrors['error'] = r return misperrors From 0ab38feade0e2ead2628c19b12a7ba42709b5141 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 3 Sep 2018 13:17:48 +0200 Subject: [PATCH 187/196] fix: Cleaned up test function not used anymore --- misp_modules/modules/expansion/virustotal.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index 4ebbd43..cd4efcd 100644 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -154,11 +154,6 @@ def handler(q=False): return misperrors return {'results': r} -def get_ip(ip, key): - params = {'ip': ip, 'apikey': key} - req = requests.get('https://www.virustotal.com/vtapi/v2/ip-address/report', params=params) - return json.dumps(req.json(), indent=2) - def introspection(): return mispattributes From 33181bc52baee54433c2cf710ad5f4d6c38c4e61 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 3 Sep 2018 14:29:42 +0200 Subject: [PATCH 188/196] fix: Fixed quick variable issue --- misp_modules/modules/expansion/virustotal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index cd4efcd..a917f6f 100644 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -124,7 +124,7 @@ class VirusTotalRequest(object): for h in hashes[:self.limit]: # Search VT for some juicy info try: - data = requests.get(self.base_url.format('file'), params={'resource': h, 'apikey': apikey, 'allinfo': 1}).json() + data = requests.get(self.base_url.format('file'), params={'resource': h, 'apikey': self.apikey, 'allinfo': 1}).json() except Exception: continue # Go through euch key and check if it exists @@ -132,7 +132,7 @@ class VirusTotalRequest(object): if VT_type in data: self.results[((MISP_type,), comment.format(h))].add(data[VT_type]) # Get the malware sample - sample = requests.get(self.base_url[:-6].format('file/download'), params={'hash': h, 'apikey': apikey}) + sample = requests.get(self.base_url[:-6].format('file/download'), params={'hash': h, 'apikey': self.apikey}) malsample = sample.content # It is possible for VT to not give us any submission names if "submission_names" in data: From cdf2f434ce66c2a1c21ecfceb13ac200eeaccf9c Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 3 Sep 2018 14:30:33 +0200 Subject: [PATCH 189/196] fix: Avoiding adding attributes that are already in the event --- misp_modules/modules/expansion/virustotal.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index a917f6f..614b8d0 100644 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -34,7 +34,9 @@ class VirusTotalRequest(object): 'authentihash': 'authentihash', 'ITW_urls': 'url'} def parse_request(self, q): + req_values = set() for attribute_type, attribute_value in q.items(): + req_values.add(attribute_value) try: error = self.input_types_mapping[attribute_type](attribute_value) except KeyError: @@ -42,11 +44,13 @@ class VirusTotalRequest(object): if error is not None: return error for key, values in self.results.items(): - if isinstance(key, tuple): - types, comment = key - self.to_return.append({'types': list(types), 'values': list(values), 'comment': comment}) - else: - self.to_return.append({'types': key, 'values': list(values)}) + values = values.difference(req_values) + if values: + if isinstance(key, tuple): + types, comment = key + self.to_return.append({'types': list(types), 'values': list(values), 'comment': comment}) + else: + self.to_return.append({'types': key, 'values': list(values)}) return self.to_return def get_domain(self, domain, do_not_recurse=False): From ba728f712076986d425c7dbcca0357b4488158d4 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 3 Sep 2018 14:43:51 +0200 Subject: [PATCH 190/196] fix: Fixed 1 variable misuse + cleaned up variable names - Fixed use of 'domain' variable instead of 'email' - Cleaned up variable names to avoid redefinition of built-in variables --- misp_modules/modules/expansion/otx.py | 35 +++++++++++++-------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/misp_modules/modules/expansion/otx.py b/misp_modules/modules/expansion/otx.py index 214e7f0..86685eb 100755 --- a/misp_modules/modules/expansion/otx.py +++ b/misp_modules/modules/expansion/otx.py @@ -32,16 +32,15 @@ def valid_ip(ip): def findAll(data, keys): a = [] if isinstance(data, dict): - for key in data.keys(): + for key, value in data.items(): if key == keys: - a.append(data[key]) + a.append(value) else: - if isinstance(data[key], (dict, list)): - a += findAll(data[key], keys) + if isinstance(value, (dict, list)): + a.extend(findAll(value, keys)) if isinstance(data, list): for i in data: - a += findAll(i, keys) - + a.extend(findAll(i, keys)) return a def valid_email(email): @@ -82,10 +81,10 @@ def handler(q=False): return r -def getHash(hash, key): +def getHash(_hash, key): ret = [] - req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/file/analysis/" + hash).text) + req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/file/analysis/" + _hash).text) for ip in findAll(req, "dst"): if not isBlacklisted(ip) and valid_ip(ip): @@ -102,8 +101,8 @@ def getIP(ip, key): ret = [] req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/ip/malware/" + ip + "?limit=1000").text ) - for hash in findAll(req, "hash"): - ret.append({"types": ["sha256"], "values": [hash]}) + for _hash in findAll(req, "hash"): + ret.append({"types": ["sha256"], "values": [_hash]}) req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/ip/passive_dns/" + ip).text ) @@ -122,21 +121,21 @@ def getDomain(domain, key): req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/domain/malware/" + domain + "?limit=1000").text ) - for hash in findAll(req, "hash"): - ret.append({"types": ["sha256"], "values": [hash]}) + for _hash in findAll(req, "hash"): + ret.append({"types": ["sha256"], "values": [_hash]}) req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/domain/whois/" + domain).text) - for domain in findAll(req, "domain"): - ret.append({"types": ["hostname"], "values": [domain]}) + for _domain in findAll(req, "domain"): + ret.append({"types": ["hostname"], "values": [_domain]}) for email in findAll(req, "value"): if valid_email(email): - ret.append({"types": ["email"], "values": [domain]}) + ret.append({"types": ["email"], "values": [email]}) - for domain in findAll(req, "hostname"): - if "." in domain and not isBlacklisted(domain): - ret.append({"types": ["hostname"], "values": [domain]}) + for _domain in findAll(req, "hostname"): + if "." in _domain and not isBlacklisted(_domain): + ret.append({"types": ["hostname"], "values": [_domain]}) req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/hostname/passive_dns/" + domain).text) for ip in findAll(req, "address"): From ef781f59f8e9b489695879b70b3f1eee477adb79 Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Thu, 6 Sep 2018 14:05:55 +0200 Subject: [PATCH 191/196] fixed typo via #220 --- misp_modules/modules/expansion/urlscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/urlscan.py b/misp_modules/modules/expansion/urlscan.py index ecd1a50..31c9230 100644 --- a/misp_modules/modules/expansion/urlscan.py +++ b/misp_modules/modules/expansion/urlscan.py @@ -49,7 +49,7 @@ def handler(q=False): r['results'] += lookup_indicator(client, request['hostname']) if 'url' in request: r['results'] += lookup_indicator(client, request['url']) - f 'hash' in request: + if 'hash' in request: r['results'] += lookup_indicator(client, request['hash']) # Return any errors generated from lookup to the UI and remove duplicates From 26647a164bba1bf7b14cca6e89108e84c1ca9955 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 7 Sep 2018 17:43:46 +0200 Subject: [PATCH 192/196] fix: Fixed indentation error --- misp_modules/modules/expansion/virustotal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index 614b8d0..2889d7b 100644 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -128,7 +128,7 @@ class VirusTotalRequest(object): for h in hashes[:self.limit]: # Search VT for some juicy info try: - data = requests.get(self.base_url.format('file'), params={'resource': h, 'apikey': self.apikey, 'allinfo': 1}).json() + data = requests.get(self.base_url.format('file'), params={'resource': h, 'apikey': self.apikey, 'allinfo': 1}).json() except Exception: continue # Go through euch key and check if it exists From 48fcf9a85eae5ed2a52dfd65f79dddc0180435e9 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 7 Sep 2018 17:49:28 +0200 Subject: [PATCH 193/196] fix: Fixed syntax error --- misp_modules/modules/expansion/virustotal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index 2889d7b..524bc49 100644 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -148,7 +148,7 @@ def handler(q=False): return False q = json.loads(q) if not q.get('config') or not q['config'].get('apikey'): - misperrors['error']: "A VirusTotal api key is required for this module." + misperrors['error'] = "A VirusTotal api key is required for this module." return misperrors del q['module'] query = VirusTotalRequest(q.pop('config')) From a18db2ed1d2985ef55a6fc4cf7161fbb74273ee9 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 7 Sep 2018 17:56:25 +0200 Subject: [PATCH 194/196] fix: Fixed exception type --- misp_modules/modules/expansion/yara_syntax_validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yara_syntax_validator.py b/misp_modules/modules/expansion/yara_syntax_validator.py index 57c71ea..18e2de0 100644 --- a/misp_modules/modules/expansion/yara_syntax_validator.py +++ b/misp_modules/modules/expansion/yara_syntax_validator.py @@ -2,7 +2,7 @@ import json import requests try: import yara -except ModuleNotFoundError: +except (OSError, ModuleNotFoundError): print("yara is missing, use 'pip3 install yara' to install it.") misperrors = {'error': 'Error'} From cfbd63f14ec8c348a749bb37f0ea543605535aa8 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 7 Sep 2018 18:06:01 +0200 Subject: [PATCH 195/196] fix: Fixed exception type for python 3.5 --- misp_modules/modules/expansion/yara_syntax_validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yara_syntax_validator.py b/misp_modules/modules/expansion/yara_syntax_validator.py index 18e2de0..c68d934 100644 --- a/misp_modules/modules/expansion/yara_syntax_validator.py +++ b/misp_modules/modules/expansion/yara_syntax_validator.py @@ -2,7 +2,7 @@ import json import requests try: import yara -except (OSError, ModuleNotFoundError): +except (OSError, ImportError): print("yara is missing, use 'pip3 install yara' to install it.") misperrors = {'error': 'Error'} From 5c718c5379ccb7bfa32bdd12a53b5bc9d9134106 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Sat, 8 Sep 2018 02:53:15 +0200 Subject: [PATCH 196/196] fix: Making python 3.5 happy with the exception type ImportError --- misp_modules/modules/expansion/dbl_spamhaus.py | 2 +- misp_modules/modules/expansion/rbl.py | 2 +- misp_modules/modules/expansion/sigma_queries.py | 2 +- misp_modules/modules/expansion/sigma_syntax_validator.py | 2 +- .../modules/expansion/stix2_pattern_syntax_validator.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/misp_modules/modules/expansion/dbl_spamhaus.py b/misp_modules/modules/expansion/dbl_spamhaus.py index f78cb74..306ea21 100644 --- a/misp_modules/modules/expansion/dbl_spamhaus.py +++ b/misp_modules/modules/expansion/dbl_spamhaus.py @@ -7,7 +7,7 @@ try: resolver = dns.resolver.Resolver() resolver.timeout = 0.2 resolver.lifetime = 0.2 -except ModuleNotFoundError: +except ImportError: print("dnspython3 is missing, use 'pip install dnspython3' to install it.") sys.exit(0) diff --git a/misp_modules/modules/expansion/rbl.py b/misp_modules/modules/expansion/rbl.py index 7aac103..6626760 100644 --- a/misp_modules/modules/expansion/rbl.py +++ b/misp_modules/modules/expansion/rbl.py @@ -6,7 +6,7 @@ try: resolver = dns.resolver.Resolver() resolver.timeout = 0.2 resolver.lifetime = 0.2 -except ModuleNotFoundError: +except ImportError: print("dnspython3 is missing, use 'pip install dnspython3' to install it.") sys.exit(0) diff --git a/misp_modules/modules/expansion/sigma_queries.py b/misp_modules/modules/expansion/sigma_queries.py index e37df23..d263245 100644 --- a/misp_modules/modules/expansion/sigma_queries.py +++ b/misp_modules/modules/expansion/sigma_queries.py @@ -3,7 +3,7 @@ try: from sigma.parser import SigmaCollectionParser from sigma.config import SigmaConfiguration from sigma.backends import getBackend, BackendOptions -except ModuleNotFoundError: +except ImportError: print("sigma or yaml is missing, use 'pip3 install sigmatools' to install it.") misperrors = {'error': 'Error'} diff --git a/misp_modules/modules/expansion/sigma_syntax_validator.py b/misp_modules/modules/expansion/sigma_syntax_validator.py index 6452654..e5cc335 100644 --- a/misp_modules/modules/expansion/sigma_syntax_validator.py +++ b/misp_modules/modules/expansion/sigma_syntax_validator.py @@ -3,7 +3,7 @@ try: import yaml from sigma.parser import SigmaParser from sigma.config import SigmaConfiguration -except ModuleNotFoundError: +except ImportError: print("sigma or yaml is missing, use 'pip3 install sigmatools' to install it.") misperrors = {'error': 'Error'} diff --git a/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py b/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py index bf5d408..78307a4 100644 --- a/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py +++ b/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py @@ -1,7 +1,7 @@ import json try: from stix2patterns.validator import run_validator -except ModuleNotFoundError: +except ImportError: print("stix2 patterns python library is missing, use 'pip3 install stix2-patterns' to install it.") misperrors = {'error': 'Error'}