From c567d1e6f28e7fe3c9dcd2da0ae10cc89deb821b Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Mon, 21 Nov 2016 10:59:30 +0000 Subject: [PATCH 01/12] Moved to misp_stix_converter --- misp_modules/modules/import_mod/stiximport.py | 215 +----------------- setup.py | 3 +- tests/test.py | 11 +- 3 files changed, 22 insertions(+), 207 deletions(-) diff --git a/misp_modules/modules/import_mod/stiximport.py b/misp_modules/modules/import_mod/stiximport.py index 16a2f22..99e891c 100755 --- a/misp_modules/modules/import_mod/stiximport.py +++ b/misp_modules/modules/import_mod/stiximport.py @@ -1,19 +1,21 @@ import json -from stix.core import STIXPackage import re import base64 import hashlib import tempfile +import os + +from pymisp.tools import stix misperrors = {'error': 'Error'} userConfig = {} inputSource = ['file'] -moduleinfo = {'version': '0.1', 'author': 'Hannah Ward', +moduleinfo = {'version': '0.2', 'author': 'Hannah Ward', 'description': 'Import some stix stuff', 'module-type': ['import']} -moduleconfig = ["max_size"] +moduleconfig = [] def handler(q=False): @@ -34,211 +36,18 @@ def handler(q=False): if not package: return json.dumps({"success": 0}) - # Get the maxsize from the config - # Default to 10MB - # (I believe the max_size arg is given in bytes) - # Check if we were given a configuration - memsize = q.get("config", None) + tfile = tempfile.NamedTemporaryFile(mode="w", prefix="STIX", delete=False) + tfile.write(package) + tfile.close() - # If we were, find out if there's a memsize field - if memsize: - memsize = memsize.get("max_size", 10 * 1024) - else: - memsize = 10 * 1024 + pkg = stix.load_stix(tfile.name) - # Load up the package into STIX - package = loadPackage(package, memsize) + for attrib in pkg.attributes: + r["results"].append({ "values" : [attrib.value] , "types": [attrib.type], "categories": [attrib.category]}) - # Build all the observables - if package.observables: - for obs in package.observables: - r["results"].append(buildObservable(obs)) - - # And now the threat actors - if package.threat_actors: - for ta in package.threat_actors: - r["results"].append(buildActor(ta)) - - # Aaaand the indicators - if package.indicators: - for ind in package.indicators: - r["results"] += buildIndicator(ind) - - # Are you seeing a pattern? - if package.exploit_targets: - for et in package.exploit_targets: - r["results"].append(buildExploitTarget(et)) - - # LOADING STUFF - if package.campaigns: - for cpn in package.campaigns: - r["results"].append(buildCampaign(cpn)) - - # Clean up results - # Don't send on anything that didn't have a value - r["results"] = [x for x in r["results"] if isinstance(x, dict) and len(x["values"]) != 0] + os.unlink(tfile.name) return r -# Quick and dirty regex for IP addresses -ipre = re.compile("([0-9]{1,3}.){3}[0-9]{1,3}") - - -def buildCampaign(cpn): - """ - Extract a campaign name - """ - return {"values": [cpn.title], "types": ["campaign-name"]} - - -def buildExploitTarget(et): - """ - Extract CVEs from exploit targets - """ - - r = {"values": [], "types": ["vulnerability"]} - - if et.vulnerabilities: - for v in et.vulnerabilities: - if v.cve_id: - r["values"].append(v.cve_id) - return r - - - -def identifyHash(hsh): - """ - What's that hash!? - """ - - possible_hashes = [] - - hashes = [x for x in hashlib.algorithms_guaranteed] - - for h in hashes: - if len(str(hsh)) == len(hashlib.new(h).hexdigest()): - possible_hashes.append(h) - possible_hashes.append("filename|{}".format(h)) - return possible_hashes - - -def buildIndicator(ind): - """ - Extract hashes - and other fun things - like that - """ - r = [] - # Try to get hashes. I hate stix - if ind.observables: - for i in ind.observables: - if i.observable_composition: - for j in i.observable_composition.observables: - r.append(buildObservable(j)) - r.append(buildObservable(i)) - return r - - -def buildActor(ta): - """ - Extract the name - and comment of a - threat actor - """ - - r = {"values": [ta.title], "types": ["threat-actor"]} - - return r - - -def buildObservable(o): - """ - Take a STIX observable - and extract the value - and category - """ - # Life is easier with json - if not isinstance(o, dict): - o = json.loads(o.to_json()) - # Make a new record to store values in - r = {"values": []} - - # Get the object properties. This contains all the - # fun stuff like values - if "observable_composition" in o: - # May as well be useless - return r - - if not o.get('object'): - return r - - props = o["object"]["properties"] - - # If it has an address_value field, it's gonna be an address - # Kinda obvious really - if "address_value" in props: - - # We've got ourselves a nice little address - value = props["address_value"] - - if isinstance(value, dict): - # Sometimes it's embedded in a dictionary - value = value["value"] - - # Is it an IP? - if ipre.match(str(value)): - # Yes! - r["values"].append(value) - r["types"] = ["ip-src", "ip-dst"] - else: - # Probably a domain yo - r["values"].append(value) - r["types"] = ["domain", "hostname"] - - if "hashes" in props: - for hsh in props["hashes"]: - r["values"].append(hsh["simple_hash_value"]["value"]) - r["types"] = identifyHash(hsh["simple_hash_value"]["value"]) - - elif "xsi:type" in props: - # Cybox. Ew. - try: - type_ = props["xsi:type"] - val = props["value"] - - if type_ == "LinkObjectType": - r["types"] = ["link"] - r["values"].append(val) - else: - print("Ignoring {}".format(type_)) - except: - pass - return r - - -def loadPackage(data, memsize=1024): - # Write the stix package to a tmp file - - temp = tempfile.SpooledTemporaryFile(max_size=int(memsize), mode="w+") - - temp.write(data) - - # Back to the beginning so we can read it again - temp.seek(0) - try: - # Try loading it into every format we know of - try: - package = STIXPackage().from_xml(temp) - except: - # We have to seek back again - temp.seek(0) - package = STIXPackage().from_json(temp) - except Exception: - print("Failed to load package") - raise ValueError("COULD NOT LOAD STIX PACKAGE!") - temp.close() - return package - - def introspection(): modulesetup = {} try: diff --git a/setup.py b/setup.py index 8ad517e..0010c05 100644 --- a/setup.py +++ b/setup.py @@ -38,5 +38,6 @@ setup( 'pillow', 'pytesseract', 'shodan', - ] + 'misp_stix_converter' + ], ) diff --git a/tests/test.py b/tests/test.py index d506595..faf5173 100644 --- a/tests/test.py +++ b/tests/test.py @@ -37,10 +37,15 @@ class TestModules(unittest.TestCase): content = base64.b64encode(f.read()) data = json.dumps({"module": "stiximport", "data": content.decode('utf-8'), - "config": {"max_size": "15000"}, }) - response = requests.post(self.url + "query", data=data) - print('STIX', response.json()) + response = requests.post(self.url + "query", data=data).json() + + print("STIX :: {}".format(response)) + values = [x["values"][0] for x in response["results"]] + + assert("209.239.79.47" in values) + assert("41.213.121.180" in values) + assert("eu-society.com" in values) def test_virustotal(self): # This can't actually be tested without disclosing a private From b35c0ae4de4d2044e7800731018b3201ca90eb87 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 21 Nov 2016 12:09:44 +0100 Subject: [PATCH 02/12] VMRay import module added --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fbc9400..b523140 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [OCR](misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes. * [stiximport](misp_modules/modules/import_mod/stiximport.py) - An import module to process STIX xml/json +* [VMRay](misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export ## How to install and start MISP modules? From 0dfea440014b4e1701fd42a20c45f4d8992c00bb Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Mon, 21 Nov 2016 11:57:04 +0000 Subject: [PATCH 03/12] Use SpooledTemp, not NamedTemp file --- misp_modules/modules/import_mod/stiximport.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/misp_modules/modules/import_mod/stiximport.py b/misp_modules/modules/import_mod/stiximport.py index 99e891c..4905cd2 100755 --- a/misp_modules/modules/import_mod/stiximport.py +++ b/misp_modules/modules/import_mod/stiximport.py @@ -2,8 +2,6 @@ import json import re import base64 import hashlib -import tempfile -import os from pymisp.tools import stix @@ -36,16 +34,11 @@ def handler(q=False): if not package: return json.dumps({"success": 0}) - tfile = tempfile.NamedTemporaryFile(mode="w", prefix="STIX", delete=False) - tfile.write(package) - tfile.close() - - pkg = stix.load_stix(tfile.name) + pkg = stix.load_stix(package) for attrib in pkg.attributes: r["results"].append({ "values" : [attrib.value] , "types": [attrib.type], "categories": [attrib.category]}) - os.unlink(tfile.name) return r def introspection(): From 1f49f362050aeaeefc4b562c50d327406fd26704 Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Mon, 21 Nov 2016 13:05:07 +0000 Subject: [PATCH 04/12] Removed unneeded modules --- misp_modules/modules/import_mod/stiximport.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/misp_modules/modules/import_mod/stiximport.py b/misp_modules/modules/import_mod/stiximport.py index 4905cd2..39ac9dc 100755 --- a/misp_modules/modules/import_mod/stiximport.py +++ b/misp_modules/modules/import_mod/stiximport.py @@ -1,7 +1,5 @@ import json -import re import base64 -import hashlib from pymisp.tools import stix From 454c450abd4ca75ee0bb66246354558478e5ab19 Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Mon, 21 Nov 2016 13:13:24 +0000 Subject: [PATCH 05/12] Don't cache anything pls travis --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index fbe3487..5653c1e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,5 @@ language: python -cache: pip - services: - redis-server From 35538b4e87f711df5fd6ed59dca4a5e36cc15f0c Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Mon, 21 Nov 2016 14:08:20 +0000 Subject: [PATCH 06/12] Added pymisp to reqs --- REQUIREMENTS | 1 + 1 file changed, 1 insertion(+) diff --git a/REQUIREMENTS b/REQUIREMENTS index 92a84d7..6bb1d47 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -12,6 +12,7 @@ pyeupi ipasn-redis asnhistory git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client +git+https://github.com/MISP/PyMISP.git#egg=pymisp pillow pytesseract SPARQLWrapper From eb538810acee730f8b29f54242e010ea04682610 Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Mon, 21 Nov 2016 14:31:48 +0000 Subject: [PATCH 07/12] Maybe it'll take the git repo now? --- REQUIREMENTS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REQUIREMENTS b/REQUIREMENTS index 6bb1d47..23e1cbe 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -12,7 +12,7 @@ pyeupi ipasn-redis asnhistory git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client -git+https://github.com/MISP/PyMISP.git#egg=pymisp +git+https://github.com/MISP/PyMISP.git pillow pytesseract SPARQLWrapper From 5372f5029891124a0a8a806bd724f3b658c13684 Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Mon, 21 Nov 2016 14:43:29 +0000 Subject: [PATCH 08/12] Travis should now use the master branch --- .travis.yml | 6 +++++- REQUIREMENTS | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5653c1e..3462f08 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,11 @@ install: - pip install codecov - pip install -U -r REQUIREMENTS - pip install . - + # Force travis to use the right pymisp + - git clone https://github.com/MISP/PyMISP.git + - cd PyMISP + - python setup.py install + - cd .. script: - coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ & - pid=$! diff --git a/REQUIREMENTS b/REQUIREMENTS index 23e1cbe..92a84d7 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -12,7 +12,6 @@ pyeupi ipasn-redis asnhistory git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client -git+https://github.com/MISP/PyMISP.git pillow pytesseract SPARQLWrapper From d60f4ef4958ad67abe401175b09fa5f65bb9f1d4 Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Mon, 21 Nov 2016 14:49:05 +0000 Subject: [PATCH 09/12] Use the CIRCL pymisp. Silly @rafiot ;) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3462f08..15858f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ install: - pip install -U -r REQUIREMENTS - pip install . # Force travis to use the right pymisp - - git clone https://github.com/MISP/PyMISP.git + - git clone https://github.com/CIRCL/PyMISP.git - cd PyMISP - python setup.py install - cd .. From 22f4edd25d541e769d8320210c2ec5a6e5227c2e Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Mon, 21 Nov 2016 15:07:56 +0000 Subject: [PATCH 10/12] Ok we'll use the dep from misp-stix-converter. Surely this'll work? --- .travis.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 15858f4..5653c1e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,11 +16,7 @@ install: - pip install codecov - pip install -U -r REQUIREMENTS - pip install . - # Force travis to use the right pymisp - - git clone https://github.com/CIRCL/PyMISP.git - - cd PyMISP - - python setup.py install - - cd .. + script: - coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ & - pid=$! From daa66dd32f13231a5d9de54c4508051187fcdda3 Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Mon, 21 Nov 2016 15:20:57 +0000 Subject: [PATCH 11/12] Use git for everything we can --- REQUIREMENTS | 2 ++ setup.py | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/REQUIREMENTS b/REQUIREMENTS index 92a84d7..16f5512 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -12,6 +12,8 @@ pyeupi ipasn-redis asnhistory git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client +git+https://github.com/MISP/MISP-STIX-Converter.git#egg=misp_stix_converter +git+https://github.com/CIRCL/PyMISP.git#egg=pymisp pillow pytesseract SPARQLWrapper diff --git a/setup.py b/setup.py index 0010c05..3bdc180 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,5 @@ setup( 'pillow', 'pytesseract', 'shodan', - 'misp_stix_converter' ], ) From 1e303b7af503e19d4e756d0d5d3441f8092e42a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 29 Nov 2016 13:49:00 +0100 Subject: [PATCH 12/12] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b523140..87dba7d 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,8 @@ sudo apt-get install python3-dev python3-pip libpq5 cd /usr/local/src/ sudo git clone https://github.com/MISP/misp-modules.git cd misp-modules -sudo pip3 install --upgrade -r REQUIREMENTS -sudo pip3 install --upgrade . +sudo pip3 install -I -r REQUIREMENTS +sudo pip3 install -I . sudo vi /etc/rc.local, add this line: `sudo -u www-data misp-modules -s &` ~~~~