From 3544ef6de061d2e0d6bcb33e2fa5362beb9a3810 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 8 Jan 2021 10:43:06 +0100 Subject: [PATCH 001/101] Update .gitignore update .gitignore to env pycharm --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3d994af..e4adeb2 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,7 @@ misp_modules.egg-info/ docs/expansion* docs/import_mod* docs/export_mod* -site* \ No newline at end of file +site* + +#pycharm env +.idea/* \ No newline at end of file From 7781a0cae7586843344eec8f7c1a0142d3727fd4 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Feb 2021 10:18:52 +0100 Subject: [PATCH 002/101] add new module new module yeti --- REQUIREMENTS | 4 ++-- misp_modules/modules/expansion/yeti.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 misp_modules/modules/expansion/yeti.py diff --git a/REQUIREMENTS b/REQUIREMENTS index 73b002a..92175f2 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -8,7 +8,7 @@ -e git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails -e git+https://github.com/sebdraven/pyonyphe@1ce15581beebb13e841193a08a2eb6f967855fcb#egg=pyonyphe --e git+https://github.com/stricaud/faup.git#egg=pyfaup&subdirectory=src/lib/bindings/python +#-e git+https://github.com/stricaud/faup.git#egg=pyfaup&subdirectory=src/lib/bindings/python aiohttp==3.4.4 antlr4-python3-runtime==4.8 ; python_version >= '3' apiosintds==1.8.3 @@ -44,7 +44,7 @@ importlib-metadata==1.6.0 ; python_version < '3.8' isodate==0.6.0 jbxapi==3.4.0 jsonschema==3.2.0 -lief==0.10.1 + lxml==4.5.0 maclookup==1.0.3 maxminddb==1.5.2 diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py new file mode 100644 index 0000000..5742a08 --- /dev/null +++ b/misp_modules/modules/expansion/yeti.py @@ -0,0 +1,17 @@ +import json + +import json +try: + import pyeti +except ImportError: + print("pyeti module not installed.") + +misperrors = {'error': 'Error'} + +mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'], + 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url']} +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', + 'description': 'Query on yeti', + 'module-type': ['expansion', 'hover']} + From 66fc121dbe0dbb7a69a62bfdaf98838a4f7a0bf3 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Feb 2021 11:17:40 +0100 Subject: [PATCH 003/101] Update yeti.py add config and struct --- misp_modules/modules/expansion/yeti.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 5742a08..0dd2275 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -1,6 +1,7 @@ import json import json + try: import pyeti except ImportError: @@ -15,3 +16,23 @@ moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', 'description': 'Query on yeti', 'module-type': ['expansion', 'hover']} +moduleconfig = ['apikey', 'url'] + + +class Yeti: + + def __init__(self, url, key): + self.api = pyeti.YetiApi(url, api_key=key) + self.dict = {'Ip': 'ip-src', 'Domain': 'domain', 'Hostname': 'hostname'} + + def search(self, value): + obs = self.api.observable_search(value=value) + if obs: + return obs + + def +def handler(q=False): + if q is False: + return False + request = json.loads(q) + attribute = request['attribute'] From 10e9b6db12f48eae397a592942a595b2c68b01cf Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Feb 2021 11:21:29 +0100 Subject: [PATCH 004/101] Update REQUIREMENTS correct conflic --- REQUIREMENTS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/REQUIREMENTS b/REQUIREMENTS index 92175f2..3abe64c 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -8,7 +8,7 @@ -e git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails -e git+https://github.com/sebdraven/pyonyphe@1ce15581beebb13e841193a08a2eb6f967855fcb#egg=pyonyphe -#-e git+https://github.com/stricaud/faup.git#egg=pyfaup&subdirectory=src/lib/bindings/python +-e git+https://github.com/stricaud/faup.git#egg=pyfaup&subdirectory=src/lib/bindings/python aiohttp==3.4.4 antlr4-python3-runtime==4.8 ; python_version >= '3' apiosintds==1.8.3 @@ -44,7 +44,7 @@ importlib-metadata==1.6.0 ; python_version < '3.8' isodate==0.6.0 jbxapi==3.4.0 jsonschema==3.2.0 - +lief==0.10.0 lxml==4.5.0 maclookup==1.0.3 maxminddb==1.5.2 From 619d64808444a51e49db5a5864283530b8f66c54 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Feb 2021 11:37:34 +0100 Subject: [PATCH 005/101] Update yeti.py correct import --- misp_modules/modules/expansion/yeti.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 0dd2275..c2aed98 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -1,7 +1,5 @@ import json -import json - try: import pyeti except ImportError: @@ -30,7 +28,7 @@ class Yeti: if obs: return obs - def + def handler(q=False): if q is False: return False From b29b3ded28d26d43c5ebda403e95145d3cb24d96 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Feb 2021 11:47:27 +0100 Subject: [PATCH 006/101] Update yeti.py add method version --- misp_modules/modules/expansion/yeti.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index c2aed98..bc30fdd 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -34,3 +34,8 @@ def handler(q=False): return False request = json.loads(q) attribute = request['attribute'] + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo \ No newline at end of file From 1def6e3f06393b77604763ed9af3c770165cfbcd Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Feb 2021 12:02:08 +0100 Subject: [PATCH 007/101] Update yeti.py add introspection method --- misp_modules/modules/expansion/yeti.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index bc30fdd..863bcd9 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -38,4 +38,7 @@ def handler(q=False): def version(): moduleinfo['config'] = moduleconfig - return moduleinfo \ No newline at end of file + return moduleinfo + +def introspection(): + return mispattributes \ No newline at end of file From 1209cd3a759f4c54613e323b70ad6154264b8ca7 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Mar 2021 11:00:19 +0100 Subject: [PATCH 008/101] yeti pluggin get_entities and get_neighboors --- .gitignore | 5 +++- misp_modules/modules/expansion/yeti.py | 39 +++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index e4adeb2..323f87a 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,7 @@ docs/export_mod* site* #pycharm env -.idea/* \ No newline at end of file +.idea/* + +#venv +venv* \ No newline at end of file diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 863bcd9..8991aa5 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -17,23 +17,54 @@ moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', moduleconfig = ['apikey', 'url'] -class Yeti: +class Yeti(pyeti.YetiApi): def __init__(self, url, key): - self.api = pyeti.YetiApi(url, api_key=key) + super(Yeti, self).__init__(url, key) self.dict = {'Ip': 'ip-src', 'Domain': 'domain', 'Hostname': 'hostname'} def search(self, value): - obs = self.api.observable_search(value=value) + obs = self.observable_search(value=value) if obs: - return obs + return obs[0] + def get_neighboors(self, obs_id): + neighboors = self.neighbors_observables(obs_id) + if neighboors and 'objs' in neighboors: + for n in neighboors: + yield n + + def get_tags(self, value): + obs = self.search(value) + if obs: + for t in obs['tags']: + yield t + + def get_entity(self, obs_id): + companies = self.observable_to_company(obs_id) + actors = self.observable_to_actor(obs_id) + campaigns = self.observable_to_campaign(obs_id) + exploit_kit = self.observable_to_exploitkit(obs_id) + exploit = self.observable_to_exploit(obs_id) + ind = self.observable_to_indicator(obs_id) + + res = [] + res.extend(companies) + res.extend(actors) + res.extend(campaigns) + res.extend(exploit) + res.extend(exploit_kit) + res.extend(ind) + + for r in res: + yield r['name'] def handler(q=False): if q is False: return False request = json.loads(q) attribute = request['attribute'] + print(attribute) def version(): From 0f31893fdb99c1bb7930efd4067fab8f98612588 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Mar 2021 11:06:12 +0100 Subject: [PATCH 009/101] Update yeti.py add logs --- misp_modules/modules/expansion/yeti.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 8991aa5..78b7b4b 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -63,6 +63,7 @@ def handler(q=False): if q is False: return False request = json.loads(q) + print(request) attribute = request['attribute'] print(attribute) From e7cb15a0c43dcfbd0132c8a41f69a16114ad07a3 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Mar 2021 11:22:53 +0100 Subject: [PATCH 010/101] Update yeti.py add ip-dst to enrich --- misp_modules/modules/expansion/yeti.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 78b7b4b..8c6f68a 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -63,9 +63,21 @@ def handler(q=False): if q is False: return False request = json.loads(q) - print(request) - attribute = request['attribute'] - print(attribute) + + if 'url' in request: + yeti_url = request['url'] + if 'apikey' in request: + apikey = request['apikey'] + if apikey and yeti_url: + yeti_client = Yeti(yeti_url,apikey) + if request.get('ip-dst'): + obs_value = request['ip-dst'] + + if yeti_client: + obs=yeti_client.search(obs_value) + print(obs) + else: + misperrors['error'] = 'Yeti Config Error' def version(): From 3fdce84ff793ca74d7c3e68ac4f7a7d0ad5202bc Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Mar 2021 11:24:43 +0100 Subject: [PATCH 011/101] Update yeti.py add log --- misp_modules/modules/expansion/yeti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 8c6f68a..d8f47bb 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -63,7 +63,7 @@ def handler(q=False): if q is False: return False request = json.loads(q) - + print(request) if 'url' in request: yeti_url = request['url'] if 'apikey' in request: From e2a1ade14ac049ed11af95ba1627eec1d0f98ee1 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Mar 2021 11:28:50 +0100 Subject: [PATCH 012/101] Update yeti.py change path to access config settings --- misp_modules/modules/expansion/yeti.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index d8f47bb..110d6eb 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -64,10 +64,12 @@ def handler(q=False): return False request = json.loads(q) print(request) - if 'url' in request: - yeti_url = request['url'] - if 'apikey' in request: - apikey = request['apikey'] + apikey = None + yeti_url = None + if 'config' in request and 'url' in request['config']: + yeti_url = request['config']['url'] + if 'config' in request and 'apikey' in request['config']: + apikey = request['config']['apikey'] if apikey and yeti_url: yeti_client = Yeti(yeti_url,apikey) if request.get('ip-dst'): From 800020d6a23abd686050514e6499023115db4cae Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Mar 2021 11:34:01 +0100 Subject: [PATCH 013/101] Update yeti.py change inherit --- misp_modules/modules/expansion/yeti.py | 31 +++++++++++++++----------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 110d6eb..78c4928 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -17,19 +17,19 @@ moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', moduleconfig = ['apikey', 'url'] -class Yeti(pyeti.YetiApi): +class Yeti(): def __init__(self, url, key): super(Yeti, self).__init__(url, key) self.dict = {'Ip': 'ip-src', 'Domain': 'domain', 'Hostname': 'hostname'} - + self.yeti_client = pyeti.YetiApi(url, key) def search(self, value): - obs = self.observable_search(value=value) + obs = self.yeti_client.observable_search(value=value) if obs: return obs[0] def get_neighboors(self, obs_id): - neighboors = self.neighbors_observables(obs_id) + neighboors = self.yeti_client.neighbors_observables(obs_id) if neighboors and 'objs' in neighboors: for n in neighboors: yield n @@ -41,12 +41,12 @@ class Yeti(pyeti.YetiApi): yield t def get_entity(self, obs_id): - companies = self.observable_to_company(obs_id) - actors = self.observable_to_actor(obs_id) - campaigns = self.observable_to_campaign(obs_id) - exploit_kit = self.observable_to_exploitkit(obs_id) - exploit = self.observable_to_exploit(obs_id) - ind = self.observable_to_indicator(obs_id) + companies = self.yeti_client.observable_to_company(obs_id) + actors = self.yeti_client.observable_to_actor(obs_id) + campaigns = self.yeti_client.observable_to_campaign(obs_id) + exploit_kit = self.yeti_client.observable_to_exploitkit(obs_id) + exploit = self.yeti_client.observable_to_exploit(obs_id) + ind = self.yeti_client.observable_to_indicator(obs_id) res = [] res.extend(companies) @@ -62,10 +62,15 @@ class Yeti(pyeti.YetiApi): def handler(q=False): if q is False: return False - request = json.loads(q) - print(request) + + apikey = None yeti_url = None + yeti_client = None + + request = json.loads(q) + print(request) + if 'config' in request and 'url' in request['config']: yeti_url = request['config']['url'] if 'config' in request and 'apikey' in request['config']: @@ -76,7 +81,7 @@ def handler(q=False): obs_value = request['ip-dst'] if yeti_client: - obs=yeti_client.search(obs_value) + obs= yeti_client.search(obs_value) print(obs) else: misperrors['error'] = 'Yeti Config Error' From 6aff43cf992c9e0acfd2b7059b56fb6ee301f0d5 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Mar 2021 11:37:04 +0100 Subject: [PATCH 014/101] Update yeti.py Correct bugs --- misp_modules/modules/expansion/yeti.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 78c4928..950c8d6 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -20,7 +20,6 @@ moduleconfig = ['apikey', 'url'] class Yeti(): def __init__(self, url, key): - super(Yeti, self).__init__(url, key) self.dict = {'Ip': 'ip-src', 'Domain': 'domain', 'Hostname': 'hostname'} self.yeti_client = pyeti.YetiApi(url, key) def search(self, value): @@ -70,7 +69,7 @@ def handler(q=False): request = json.loads(q) print(request) - + if 'config' in request and 'url' in request['config']: yeti_url = request['config']['url'] if 'config' in request and 'apikey' in request['config']: From e3f23793e04b830942fbb9c6b00d54ecc0db1482 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Mar 2021 11:40:11 +0100 Subject: [PATCH 015/101] Update yeti.py modify call yeti --- misp_modules/modules/expansion/yeti.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 950c8d6..eaf27ab 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -21,7 +21,8 @@ class Yeti(): def __init__(self, url, key): self.dict = {'Ip': 'ip-src', 'Domain': 'domain', 'Hostname': 'hostname'} - self.yeti_client = pyeti.YetiApi(url, key) + self.yeti_client = pyeti.YetiApi(url=url, api_key=key) + def search(self, value): obs = self.yeti_client.observable_search(value=value) if obs: From cb008124c327f662c55ee842a4052a9dbef151b2 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Mar 2021 15:06:13 +0100 Subject: [PATCH 016/101] Update yeti.py add neighboors iocs to add the event --- misp_modules/modules/expansion/yeti.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index eaf27ab..ccc614d 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -20,9 +20,9 @@ moduleconfig = ['apikey', 'url'] class Yeti(): def __init__(self, url, key): - self.dict = {'Ip': 'ip-src', 'Domain': 'domain', 'Hostname': 'hostname'} + self.dict = {'Ip': 'ip-dst', 'Domain': 'domain', 'Hostname': 'hostname', 'Url': 'url'} self.yeti_client = pyeti.YetiApi(url=url, api_key=key) - + def search(self, value): obs = self.yeti_client.observable_search(value=value) if obs: @@ -81,10 +81,24 @@ def handler(q=False): obs_value = request['ip-dst'] if yeti_client: - obs= yeti_client.search(obs_value) - print(obs) + obs = yeti_client.search(obs_value) + values = [] + types = [] + to_push = {"results": []} + for obs in yeti_client.get_neighboors(obs['id']): + values.append(obs['value']) + types.append(yeti_client.dict[obs['type']]) + to_push['results'].append( + {'types': types, + 'values': values, + 'categories': ['Network Activities'] + } + ) + return to_push else: misperrors['error'] = 'Yeti Config Error' + return misperrors + def version(): From 7e1bf41d475d5598790d6cb95a13aabed62f26b1 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Mar 2021 15:08:32 +0100 Subject: [PATCH 017/101] Update yeti.py add logs --- misp_modules/modules/expansion/yeti.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index ccc614d..58ca63b 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -82,6 +82,7 @@ def handler(q=False): if yeti_client: obs = yeti_client.search(obs_value) + print(obs) values = [] types = [] to_push = {"results": []} From 9de5dd89eec3a9572e7151ab74a1d0b97311c16f Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Mar 2021 15:14:25 +0100 Subject: [PATCH 018/101] Update yeti.py add logs --- misp_modules/modules/expansion/yeti.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 58ca63b..828de47 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -86,9 +86,10 @@ def handler(q=False): values = [] types = [] to_push = {"results": []} - for obs in yeti_client.get_neighboors(obs['id']): - values.append(obs['value']) - types.append(yeti_client.dict[obs['type']]) + for obs_to_add in yeti_client.get_neighboors(obs['id']): + print(obs_to_add) + values.append(obs_to_add['value']) + types.append(yeti_client.dict[obs_to_add['type']]) to_push['results'].append( {'types': types, 'values': values, From bf617807df0c917dd699846393be52a4b1f161f6 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Mar 2021 15:19:30 +0100 Subject: [PATCH 019/101] Update yeti.py modify acess dict --- misp_modules/modules/expansion/yeti.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 828de47..f0f8099 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -31,7 +31,7 @@ class Yeti(): def get_neighboors(self, obs_id): neighboors = self.yeti_client.neighbors_observables(obs_id) if neighboors and 'objs' in neighboors: - for n in neighboors: + for n in neighboors['objs']: yield n def get_tags(self, value): @@ -69,7 +69,6 @@ def handler(q=False): yeti_client = None request = json.loads(q) - print(request) if 'config' in request and 'url' in request['config']: yeti_url = request['config']['url'] @@ -82,7 +81,6 @@ def handler(q=False): if yeti_client: obs = yeti_client.search(obs_value) - print(obs) values = [] types = [] to_push = {"results": []} From 33bba708bfd61ba40f05189fda04c53fc36fae88 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Mar 2021 16:53:49 +0100 Subject: [PATCH 020/101] Update yeti.py use format misp --- misp_modules/modules/expansion/yeti.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index f0f8099..db1b4c5 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -8,7 +8,8 @@ except ImportError: misperrors = {'error': 'Error'} mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'], - 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url']} + 'format': 'misp_standard' + } # possible module-types: 'expansion', 'hover' or both moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', 'description': 'Query on yeti', @@ -69,7 +70,7 @@ def handler(q=False): yeti_client = None request = json.loads(q) - + print(request) if 'config' in request and 'url' in request['config']: yeti_url = request['config']['url'] if 'config' in request and 'apikey' in request['config']: From 294bdee51afba5f4879c46fefd71eaa88554af30 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Mar 2021 16:57:55 +0100 Subject: [PATCH 021/101] Update yeti.py using attribute --- misp_modules/modules/expansion/yeti.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index db1b4c5..79a1800 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -77,11 +77,11 @@ def handler(q=False): apikey = request['config']['apikey'] if apikey and yeti_url: yeti_client = Yeti(yeti_url,apikey) - if request.get('ip-dst'): - obs_value = request['ip-dst'] + if request.get('attribute'): + attribute = request['attribute'] if yeti_client: - obs = yeti_client.search(obs_value) + obs = yeti_client.search(attribute['value']) values = [] types = [] to_push = {"results": []} From 6fc3b2a860f260bcfc052aec459d90dd475f526b Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 5 Mar 2021 19:01:25 +0100 Subject: [PATCH 022/101] Update yeti.py refactoring --- misp_modules/modules/expansion/yeti.py | 52 ++++++++++++++++---------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 79a1800..f92a3b0 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -5,6 +5,8 @@ try: except ImportError: print("pyeti module not installed.") +from pymisp import MISPEvent, MISPObject + misperrors = {'error': 'Error'} mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'], @@ -20,9 +22,12 @@ moduleconfig = ['apikey', 'url'] class Yeti(): - def __init__(self, url, key): + def __init__(self, url, key,attribute): self.dict = {'Ip': 'ip-dst', 'Domain': 'domain', 'Hostname': 'hostname', 'Url': 'url'} self.yeti_client = pyeti.YetiApi(url=url, api_key=key) + self.attribute = attribute + self.misp_event = MISPEvent() + self.misp_event.add_attribute(**attribute) def search(self, value): obs = self.yeti_client.observable_search(value=value) @@ -60,6 +65,28 @@ class Yeti(): for r in res: yield r['name'] + def parse_yeti_result(self): + obs = self.search(self.attribute['value']) + values = [] + types = [] + + for obs_to_add in self.get_neighboors(obs['id']): + object_misp = self.get_object(obs_to_add) + self.misp_event.add_object(object_misp) + + def get_result(self): + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object')} + return results + + def get_object(self,obj_to_add): + if (obj_to_add['type'] == 'Ip' and self.attribute in ['hostname','domain']) or\ + (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')): + domain_ip_object = MISPObject('domain-ip') + domain_ip_object.add_attribute() + domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') + return domain_ip_object + def handler(q=False): if q is False: return False @@ -70,32 +97,19 @@ def handler(q=False): yeti_client = None request = json.loads(q) + attribute = request['attribute'] + if attribute['type'] not in mispattributes['input']: + return {'error': 'Unsupported attributes type'} print(request) if 'config' in request and 'url' in request['config']: yeti_url = request['config']['url'] if 'config' in request and 'apikey' in request['config']: apikey = request['config']['apikey'] if apikey and yeti_url: - yeti_client = Yeti(yeti_url,apikey) - if request.get('attribute'): - attribute = request['attribute'] + yeti_client = Yeti(yeti_url, apikey, attribute) if yeti_client: - obs = yeti_client.search(attribute['value']) - values = [] - types = [] - to_push = {"results": []} - for obs_to_add in yeti_client.get_neighboors(obs['id']): - print(obs_to_add) - values.append(obs_to_add['value']) - types.append(yeti_client.dict[obs_to_add['type']]) - to_push['results'].append( - {'types': types, - 'values': values, - 'categories': ['Network Activities'] - } - ) - return to_push + else: misperrors['error'] = 'Yeti Config Error' return misperrors From 48f56b0690d2c4e77e3ef51660cf58a6067507b6 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 10:52:48 +0100 Subject: [PATCH 023/101] Update yeti.py add object --- misp_modules/modules/expansion/yeti.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index f92a3b0..0f474e2 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -23,7 +23,7 @@ moduleconfig = ['apikey', 'url'] class Yeti(): def __init__(self, url, key,attribute): - self.dict = {'Ip': 'ip-dst', 'Domain': 'domain', 'Hostname': 'hostname', 'Url': 'url'} + self.misp_mapping = {'Ip': 'ip-dst', 'Domain': 'domain', 'Hostname': 'hostname', 'Url': 'url'} self.yeti_client = pyeti.YetiApi(url=url, api_key=key) self.attribute = attribute self.misp_event = MISPEvent() @@ -83,10 +83,16 @@ class Yeti(): if (obj_to_add['type'] == 'Ip' and self.attribute in ['hostname','domain']) or\ (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')): domain_ip_object = MISPObject('domain-ip') - domain_ip_object.add_attribute() + domain_ip_object.add_attribute(**self.__get_attribute(obj_to_add)) domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') + domain_ip_object.add_attribute(**self.attribute) return domain_ip_object + def __get_attribute(self, obj_yeti): + typ_attribute = self.misp_mapping[obj_yeti['type']] + attr_misp = {'type':typ_attribute, 'value': obj_yeti['value']} + return attr_misp + def handler(q=False): if q is False: return False @@ -109,7 +115,8 @@ def handler(q=False): yeti_client = Yeti(yeti_url, apikey, attribute) if yeti_client: - + yeti_client.parse_yeti_result() + return yeti_client.get_result() else: misperrors['error'] = 'Yeti Config Error' return misperrors From 0618e288d3345880d91191f6710573f0abd9b199 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 11:01:02 +0100 Subject: [PATCH 024/101] Update yeti.py add relation object --- misp_modules/modules/expansion/yeti.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 0f474e2..676ffb2 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -90,14 +90,14 @@ class Yeti(): def __get_attribute(self, obj_yeti): typ_attribute = self.misp_mapping[obj_yeti['type']] - attr_misp = {'type':typ_attribute, 'value': obj_yeti['value']} + attr_misp = {'type':typ_attribute, 'value': obj_yeti['value'], + 'object_relation': 'pdns'} return attr_misp def handler(q=False): if q is False: return False - apikey = None yeti_url = None yeti_client = None @@ -106,7 +106,7 @@ def handler(q=False): attribute = request['attribute'] if attribute['type'] not in mispattributes['input']: return {'error': 'Unsupported attributes type'} - print(request) + if 'config' in request and 'url' in request['config']: yeti_url = request['config']['url'] if 'config' in request and 'apikey' in request['config']: From c9bc97c9f9ed99d0fb3756339d4d1767e2b26bc2 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 11:15:27 +0100 Subject: [PATCH 025/101] Update yeti.py change relation type and misp event init --- misp_modules/modules/expansion/yeti.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 676ffb2..38bd3f4 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -69,7 +69,7 @@ class Yeti(): obs = self.search(self.attribute['value']) values = [] types = [] - + self.misp_event.add_attribute(**self.attribute) for obs_to_add in self.get_neighboors(obs['id']): object_misp = self.get_object(obs_to_add) self.misp_event.add_object(object_misp) @@ -79,7 +79,7 @@ class Yeti(): results = {key: event[key] for key in ('Attribute', 'Object')} return results - def get_object(self,obj_to_add): + def get_object(self, obj_to_add): if (obj_to_add['type'] == 'Ip' and self.attribute in ['hostname','domain']) or\ (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')): domain_ip_object = MISPObject('domain-ip') @@ -90,8 +90,13 @@ class Yeti(): def __get_attribute(self, obj_yeti): typ_attribute = self.misp_mapping[obj_yeti['type']] - attr_misp = {'type':typ_attribute, 'value': obj_yeti['value'], - 'object_relation': 'pdns'} + attr_misp = {'type':typ_attribute, 'value': obj_yeti['value']} + if typ_attribute == 'ip-src' or typ_attribute =='ip-dst': + attr_misp['object_relation'] = 'ip' + elif 'domain' == typ_attribute: + attr_misp['object_relation'] = 'domain' + else: + attr_misp['object_relation'] = None return attr_misp def handler(q=False): From bd4a4b87fc9717b9fdf0dc3a5475b161015194d6 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 11:18:01 +0100 Subject: [PATCH 026/101] Update yeti.py add logs --- misp_modules/modules/expansion/yeti.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 38bd3f4..fa21676 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -73,6 +73,7 @@ class Yeti(): for obs_to_add in self.get_neighboors(obs['id']): object_misp = self.get_object(obs_to_add) self.misp_event.add_object(object_misp) + print(self.misp_event) def get_result(self): event = json.loads(self.misp_event.to_json()) @@ -91,7 +92,7 @@ class Yeti(): def __get_attribute(self, obj_yeti): typ_attribute = self.misp_mapping[obj_yeti['type']] attr_misp = {'type':typ_attribute, 'value': obj_yeti['value']} - if typ_attribute == 'ip-src' or typ_attribute =='ip-dst': + if typ_attribute == 'ip-src' or typ_attribute == 'ip-dst': attr_misp['object_relation'] = 'ip' elif 'domain' == typ_attribute: attr_misp['object_relation'] = 'domain' From d868373c5a1eb4bb7b07bf4ce18d0152b5ecdd48 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 11:24:10 +0100 Subject: [PATCH 027/101] Update yeti.py add logs --- misp_modules/modules/expansion/yeti.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index fa21676..aa0ead7 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -87,6 +87,7 @@ class Yeti(): domain_ip_object.add_attribute(**self.__get_attribute(obj_to_add)) domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') domain_ip_object.add_attribute(**self.attribute) + print(domain_ip_object) return domain_ip_object def __get_attribute(self, obj_yeti): From 347d12c78cf942078bef8a51f11c0a3f0c052775 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 11:27:23 +0100 Subject: [PATCH 028/101] Update yeti.py add logs --- misp_modules/modules/expansion/yeti.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index aa0ead7..ceafca0 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -99,6 +99,7 @@ class Yeti(): attr_misp['object_relation'] = 'domain' else: attr_misp['object_relation'] = None + print('Attribute %s' % attr_misp) return attr_misp def handler(q=False): From 1dfdb5a2a297da9d36ed4631e15a0b59f6c03208 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 11:29:57 +0100 Subject: [PATCH 029/101] Update yeti.py change type attr and relation --- misp_modules/modules/expansion/yeti.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index ceafca0..5792c8b 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -92,11 +92,14 @@ class Yeti(): def __get_attribute(self, obj_yeti): typ_attribute = self.misp_mapping[obj_yeti['type']] - attr_misp = {'type':typ_attribute, 'value': obj_yeti['value']} + attr_misp = {'type': typ_attribute, 'value': obj_yeti['value']} if typ_attribute == 'ip-src' or typ_attribute == 'ip-dst': attr_misp['object_relation'] = 'ip' elif 'domain' == typ_attribute: attr_misp['object_relation'] = 'domain' + elif 'hostname' == typ_attribute: + attr_misp['object_relation'] = 'domain' + attr_misp['type'] = 'domain' else: attr_misp['object_relation'] = None print('Attribute %s' % attr_misp) From bd5c1b0b535d58db72ca72c0ad071ef0c21a0aa2 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 11:40:23 +0100 Subject: [PATCH 030/101] Update yeti.py add logs --- misp_modules/modules/expansion/yeti.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 5792c8b..079697b 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -69,7 +69,6 @@ class Yeti(): obs = self.search(self.attribute['value']) values = [] types = [] - self.misp_event.add_attribute(**self.attribute) for obs_to_add in self.get_neighboors(obs['id']): object_misp = self.get_object(obs_to_add) self.misp_event.add_object(object_misp) @@ -87,7 +86,7 @@ class Yeti(): domain_ip_object.add_attribute(**self.__get_attribute(obj_to_add)) domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') domain_ip_object.add_attribute(**self.attribute) - print(domain_ip_object) + print(type(domain_ip_object)) return domain_ip_object def __get_attribute(self, obj_yeti): @@ -105,6 +104,7 @@ class Yeti(): print('Attribute %s' % attr_misp) return attr_misp + def handler(q=False): if q is False: return False From 633f5efd564ec23bc7fa800291fccc2586c733c0 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 11:48:55 +0100 Subject: [PATCH 031/101] Update yeti.py log object --- misp_modules/modules/expansion/yeti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 079697b..ffc0e34 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -86,7 +86,7 @@ class Yeti(): domain_ip_object.add_attribute(**self.__get_attribute(obj_to_add)) domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') domain_ip_object.add_attribute(**self.attribute) - print(type(domain_ip_object)) + print(domain_ip_object.to_json()) return domain_ip_object def __get_attribute(self, obj_yeti): From 65d8bb6b0735e92eb48267573f6bc93991997b96 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 11:51:55 +0100 Subject: [PATCH 032/101] Update yeti.py log json --- misp_modules/modules/expansion/yeti.py | 1 - 1 file changed, 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index ffc0e34..c06fcad 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -85,7 +85,6 @@ class Yeti(): domain_ip_object = MISPObject('domain-ip') domain_ip_object.add_attribute(**self.__get_attribute(obj_to_add)) domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') - domain_ip_object.add_attribute(**self.attribute) print(domain_ip_object.to_json()) return domain_ip_object From 7255a1eddc2b3f507be2263fd9ed64e667ffd116 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 12:09:54 +0100 Subject: [PATCH 033/101] Update yeti.py change relationship --- misp_modules/modules/expansion/yeti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index c06fcad..7e455e4 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -84,7 +84,7 @@ class Yeti(): (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')): domain_ip_object = MISPObject('domain-ip') domain_ip_object.add_attribute(**self.__get_attribute(obj_to_add)) - domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') + #domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') print(domain_ip_object.to_json()) return domain_ip_object From bc1bea0ec4d57e104e5487b051b7d30b91879fff Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 12:12:37 +0100 Subject: [PATCH 034/101] Update yeti.py change attribute add --- misp_modules/modules/expansion/yeti.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 7e455e4..fe4f4db 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -85,6 +85,7 @@ class Yeti(): domain_ip_object = MISPObject('domain-ip') domain_ip_object.add_attribute(**self.__get_attribute(obj_to_add)) #domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') + domain_ip_object.add_attribute(**self.attribute) print(domain_ip_object.to_json()) return domain_ip_object From 28b554d975818881135734790ed44d5f3f746474 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 12:24:15 +0100 Subject: [PATCH 035/101] Update yeti.py add test --- misp_modules/modules/expansion/yeti.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index fe4f4db..c76cc9d 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -71,7 +71,8 @@ class Yeti(): types = [] for obs_to_add in self.get_neighboors(obs['id']): object_misp = self.get_object(obs_to_add) - self.misp_event.add_object(object_misp) + if object_misp: + self.misp_event.add_object(object_misp) print(self.misp_event) def get_result(self): @@ -86,7 +87,6 @@ class Yeti(): domain_ip_object.add_attribute(**self.__get_attribute(obj_to_add)) #domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') domain_ip_object.add_attribute(**self.attribute) - print(domain_ip_object.to_json()) return domain_ip_object def __get_attribute(self, obj_yeti): From b9ce6d689c4eaba9c3538db655465271a92275ae Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 13:56:02 +0100 Subject: [PATCH 036/101] Update yeti.py add ref --- misp_modules/modules/expansion/yeti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index c76cc9d..cd78146 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -85,7 +85,7 @@ class Yeti(): (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')): domain_ip_object = MISPObject('domain-ip') domain_ip_object.add_attribute(**self.__get_attribute(obj_to_add)) - #domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') + domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') domain_ip_object.add_attribute(**self.attribute) return domain_ip_object From 0d035c02927737d590c08c71d66054644b1f06d3 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 14:22:51 +0100 Subject: [PATCH 037/101] Update yeti.py add relationship --- misp_modules/modules/expansion/yeti.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index cd78146..d566ef6 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -93,11 +93,14 @@ class Yeti(): typ_attribute = self.misp_mapping[obj_yeti['type']] attr_misp = {'type': typ_attribute, 'value': obj_yeti['value']} if typ_attribute == 'ip-src' or typ_attribute == 'ip-dst': - attr_misp['object_relation'] = 'ip' + attr_misp['object_relation'] = {'type': 'text', + 'object_relation': 'ip'} elif 'domain' == typ_attribute: - attr_misp['object_relation'] = 'domain' + attr_misp['object_relation'] = {'type': 'text', + 'object_relation': 'domain'} elif 'hostname' == typ_attribute: - attr_misp['object_relation'] = 'domain' + attr_misp['object_relation'] = {'type': 'text', + 'object_relation': 'domain'} attr_misp['type'] = 'domain' else: attr_misp['object_relation'] = None From 9eb41f4022a16fc87a7137386491c576f7ae2ce4 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 14:26:44 +0100 Subject: [PATCH 038/101] Update yeti.py change relation type --- misp_modules/modules/expansion/yeti.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index d566ef6..f44f2d6 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -93,14 +93,14 @@ class Yeti(): typ_attribute = self.misp_mapping[obj_yeti['type']] attr_misp = {'type': typ_attribute, 'value': obj_yeti['value']} if typ_attribute == 'ip-src' or typ_attribute == 'ip-dst': - attr_misp['object_relation'] = {'type': 'text', - 'object_relation': 'ip'} + attr_misp['object_relation'].update({'type': 'text', + 'object_relation': 'ip'}) elif 'domain' == typ_attribute: - attr_misp['object_relation'] = {'type': 'text', - 'object_relation': 'domain'} + attr_misp.update({'type': 'text', + 'object_relation': 'domain'}) elif 'hostname' == typ_attribute: - attr_misp['object_relation'] = {'type': 'text', - 'object_relation': 'domain'} + attr_misp.update({'type': 'text', + 'object_relation': 'domain'}) attr_misp['type'] = 'domain' else: attr_misp['object_relation'] = None From 0a364cf815e5f16fcf75dcb8246d68832010a2a3 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 14:32:00 +0100 Subject: [PATCH 039/101] Update yeti.py update relation --- misp_modules/modules/expansion/yeti.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index f44f2d6..199b5cf 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -96,12 +96,11 @@ class Yeti(): attr_misp['object_relation'].update({'type': 'text', 'object_relation': 'ip'}) elif 'domain' == typ_attribute: - attr_misp.update({'type': 'text', + attr_misp.update({'type': 'domain', 'object_relation': 'domain'}) elif 'hostname' == typ_attribute: - attr_misp.update({'type': 'text', + attr_misp.update({'type': 'domain', 'object_relation': 'domain'}) - attr_misp['type'] = 'domain' else: attr_misp['object_relation'] = None print('Attribute %s' % attr_misp) From 86275d7610d2f760297f9424bf14a4c2bb3e22ec Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 14:38:34 +0100 Subject: [PATCH 040/101] Update yeti.py change modification --- misp_modules/modules/expansion/yeti.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 199b5cf..f9ccec3 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -91,7 +91,7 @@ class Yeti(): def __get_attribute(self, obj_yeti): typ_attribute = self.misp_mapping[obj_yeti['type']] - attr_misp = {'type': typ_attribute, 'value': obj_yeti['value']} + attr_misp = {'value': obj_yeti['value']} if typ_attribute == 'ip-src' or typ_attribute == 'ip-dst': attr_misp['object_relation'].update({'type': 'text', 'object_relation': 'ip'}) @@ -101,8 +101,6 @@ class Yeti(): elif 'hostname' == typ_attribute: attr_misp.update({'type': 'domain', 'object_relation': 'domain'}) - else: - attr_misp['object_relation'] = None print('Attribute %s' % attr_misp) return attr_misp From 5176a36acfd0f3985598e21e62d68426dcb2fac4 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 15:16:00 +0100 Subject: [PATCH 041/101] Update yeti.py change relations --- misp_modules/modules/expansion/yeti.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index f9ccec3..4d78ffd 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -84,23 +84,23 @@ class Yeti(): if (obj_to_add['type'] == 'Ip' and self.attribute in ['hostname','domain']) or\ (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')): domain_ip_object = MISPObject('domain-ip') - domain_ip_object.add_attribute(**self.__get_attribute(obj_to_add)) - domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') + domain_ip_object.add_attribute(self.__get_relation(obj_to_add), + obj_to_add['value']) domain_ip_object.add_attribute(**self.attribute) + domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') + return domain_ip_object - def __get_attribute(self, obj_yeti): + def __get_relation(self, obj_yeti): typ_attribute = self.misp_mapping[obj_yeti['type']] attr_misp = {'value': obj_yeti['value']} if typ_attribute == 'ip-src' or typ_attribute == 'ip-dst': - attr_misp['object_relation'].update({'type': 'text', - 'object_relation': 'ip'}) + return 'ip' elif 'domain' == typ_attribute: - attr_misp.update({'type': 'domain', - 'object_relation': 'domain'}) + return 'domain' elif 'hostname' == typ_attribute: - attr_misp.update({'type': 'domain', - 'object_relation': 'domain'}) + return 'domain' + print('Attribute %s' % attr_misp) return attr_misp From 624f42326473d921cc402de091b2b5195894c355 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 15:19:37 +0100 Subject: [PATCH 042/101] Update yeti.py add logs --- misp_modules/modules/expansion/yeti.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 4d78ffd..298d086 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -94,6 +94,7 @@ class Yeti(): def __get_relation(self, obj_yeti): typ_attribute = self.misp_mapping[obj_yeti['type']] attr_misp = {'value': obj_yeti['value']} + print('att %s' % typ_attribute) if typ_attribute == 'ip-src' or typ_attribute == 'ip-dst': return 'ip' elif 'domain' == typ_attribute: From cd971867760371f6c7df70f4585c1e3d84e30492 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 15:20:58 +0100 Subject: [PATCH 043/101] Update yeti.py remove add --- misp_modules/modules/expansion/yeti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 298d086..3e2a094 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -86,7 +86,7 @@ class Yeti(): domain_ip_object = MISPObject('domain-ip') domain_ip_object.add_attribute(self.__get_relation(obj_to_add), obj_to_add['value']) - domain_ip_object.add_attribute(**self.attribute) + domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') return domain_ip_object From 83c4b2f4b0503b5c6f7f928578aabfa71f35b1db Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 15:22:53 +0100 Subject: [PATCH 044/101] Update yeti.py add relation --- misp_modules/modules/expansion/yeti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 3e2a094..3cba9c6 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -86,7 +86,7 @@ class Yeti(): domain_ip_object = MISPObject('domain-ip') domain_ip_object.add_attribute(self.__get_relation(obj_to_add), obj_to_add['value']) - + domain_ip_object.add_attribute('ip', self.attribute) domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') return domain_ip_object From 1be2c27131477a307e98795fd2769742348fcfcd Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 15:26:45 +0100 Subject: [PATCH 045/101] Update yeti.py add logs --- misp_modules/modules/expansion/yeti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 3cba9c6..c61b4d2 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -73,7 +73,7 @@ class Yeti(): object_misp = self.get_object(obs_to_add) if object_misp: self.misp_event.add_object(object_misp) - print(self.misp_event) + print(self.misp_event.to_json()) def get_result(self): event = json.loads(self.misp_event.to_json()) From ed3e0d56fde64567e67b74a81032f4db96d87efc Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 15:29:21 +0100 Subject: [PATCH 046/101] Update yeti.py change logs --- misp_modules/modules/expansion/yeti.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index c61b4d2..b201923 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -73,8 +73,7 @@ class Yeti(): object_misp = self.get_object(obs_to_add) if object_misp: self.misp_event.add_object(object_misp) - print(self.misp_event.to_json()) - + print('Event MISP %s' % self.misp_event.to_json()) def get_result(self): event = json.loads(self.misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object')} @@ -94,15 +93,12 @@ class Yeti(): def __get_relation(self, obj_yeti): typ_attribute = self.misp_mapping[obj_yeti['type']] attr_misp = {'value': obj_yeti['value']} - print('att %s' % typ_attribute) if typ_attribute == 'ip-src' or typ_attribute == 'ip-dst': return 'ip' elif 'domain' == typ_attribute: return 'domain' elif 'hostname' == typ_attribute: return 'domain' - - print('Attribute %s' % attr_misp) return attr_misp From 6b35a7ee4d2a54ff07e353de5af1efa79b31af4c Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 15:32:05 +0100 Subject: [PATCH 047/101] Update yeti.py value attribute --- misp_modules/modules/expansion/yeti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index b201923..12f0f17 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -85,7 +85,7 @@ class Yeti(): domain_ip_object = MISPObject('domain-ip') domain_ip_object.add_attribute(self.__get_relation(obj_to_add), obj_to_add['value']) - domain_ip_object.add_attribute('ip', self.attribute) + domain_ip_object.add_attribute('ip', self.attribute['value']) domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') return domain_ip_object From 76133ace8bcd48a3226cdcd15f8e7103f295f4b7 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 15:37:49 +0100 Subject: [PATCH 048/101] Update yeti.py change logs --- misp_modules/modules/expansion/yeti.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 12f0f17..9dd95a8 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -73,10 +73,12 @@ class Yeti(): object_misp = self.get_object(obs_to_add) if object_misp: self.misp_event.add_object(object_misp) - print('Event MISP %s' % self.misp_event.to_json()) + + def get_result(self): event = json.loads(self.misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object')} + print('results '% results) return results def get_object(self, obj_to_add): From ef2bf2962172f30dc3245d3ee4a9900646a65514 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 15:39:09 +0100 Subject: [PATCH 049/101] Update yeti.py correction format strings --- misp_modules/modules/expansion/yeti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 9dd95a8..261e403 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -78,7 +78,7 @@ class Yeti(): def get_result(self): event = json.loads(self.misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object')} - print('results '% results) + print('results %s'% results) return results def get_object(self, obj_to_add): From 240d043f912a19b8bd6c1020da39d2a6d0b95096 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 15:50:37 +0100 Subject: [PATCH 050/101] Update yeti.py delete attr --- misp_modules/modules/expansion/yeti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 261e403..a6318b5 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -27,7 +27,7 @@ class Yeti(): self.yeti_client = pyeti.YetiApi(url=url, api_key=key) self.attribute = attribute self.misp_event = MISPEvent() - self.misp_event.add_attribute(**attribute) + #self.misp_event.add_attribute(**attribute) def search(self, value): obs = self.yeti_client.observable_search(value=value) From b42da0435b7927668bea2a40a6790d4e14d316e6 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 19 Mar 2021 15:55:18 +0100 Subject: [PATCH 051/101] Update yeti.py add key results --- misp_modules/modules/expansion/yeti.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index a6318b5..1028f0c 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -27,7 +27,7 @@ class Yeti(): self.yeti_client = pyeti.YetiApi(url=url, api_key=key) self.attribute = attribute self.misp_event = MISPEvent() - #self.misp_event.add_attribute(**attribute) + self.misp_event.add_attribute(**attribute) def search(self, value): obs = self.yeti_client.observable_search(value=value) @@ -126,7 +126,7 @@ def handler(q=False): if yeti_client: yeti_client.parse_yeti_result() - return yeti_client.get_result() + return {'results': yeti_client.get_result()} else: misperrors['error'] = 'Yeti Config Error' return misperrors From a9f90d964c163be168b3a8907e01972f6a73c506 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Sun, 18 Apr 2021 18:11:37 +0200 Subject: [PATCH 052/101] fix: [tests] Back to the former ip address in the threatcrowd module test --- tests/test_expansions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_expansions.py b/tests/test_expansions.py index c5b5467..928ffe7 100644 --- a/tests/test_expansions.py +++ b/tests/test_expansions.py @@ -470,7 +470,7 @@ class TestExpansions(unittest.TestCase): def test_threatcrowd(self): query_types = ('domain', 'ip-src', 'md5', 'whois-registrant-email') - query_values = ('circl.lu', '185.194.93.14', '616eff3e9a7575ae73821b4668d2801c', 'hostmaster@eurodns.com') + query_values = ('circl.lu', '149.13.33.14', '616eff3e9a7575ae73821b4668d2801c', 'hostmaster@eurodns.com') results = ('149.13.33.4', 'cve.circl.lu', 'devilreturns.com', 'navabi.lu') for query_type, query_value, result in zip(query_types, query_values, results): query = {"module": "threatcrowd", query_type: query_value} From 6cd99c03e481d1db51bca8b57283d2c2439d1aee Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 10:46:07 +0200 Subject: [PATCH 053/101] Update yeti.py refactoring and add Url neighboors --- misp_modules/modules/expansion/yeti.py | 43 +++++++++++++++++--------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 1028f0c..05909d9 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -70,38 +70,53 @@ class Yeti(): values = [] types = [] for obs_to_add in self.get_neighboors(obs['id']): - object_misp = self.get_object(obs_to_add) - if object_misp: - self.misp_event.add_object(object_misp) - + object_misp_domain_ip = self.__get_object_domain_ip(obs_to_add) + if object_misp_domain_ip: + self.misp_event.add_object(object_misp_domain_ip) + object_misp_url = self.__get_object_url(obs_to_add) + if object_misp_url: + self.misp_event.add_object(object_misp_url) def get_result(self): event = json.loads(self.misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object')} - print('results %s'% results) + print('results %s' % results) return results - def get_object(self, obj_to_add): + def __get_object_domain_ip(self, obj_to_add): if (obj_to_add['type'] == 'Ip' and self.attribute in ['hostname','domain']) or\ (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')): domain_ip_object = MISPObject('domain-ip') domain_ip_object.add_attribute(self.__get_relation(obj_to_add), obj_to_add['value']) - domain_ip_object.add_attribute('ip', self.attribute['value']) + domain_ip_object.add_attribute(self.__get_relation(self.attribute, is_yeti_object=False), + self.attribute['value']) domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') return domain_ip_object - def __get_relation(self, obj_yeti): - typ_attribute = self.misp_mapping[obj_yeti['type']] - attr_misp = {'value': obj_yeti['value']} - if typ_attribute == 'ip-src' or typ_attribute == 'ip-dst': + def __get_object_url(self, obj_to_add): + if obj_to_add['type'] == 'Url': + url_object = MISPObject('Url') + url_object.add_attribute(self.__get_relation(obj_to_add), obj_to_add['value']) + url_object.add_attribute(self.__get_relation(self.attribute, is_yeti_object=False), + self.attribute['value']) + url_object.add_reference(self.attribute['uuid'], 'related_to') + return url_object + + def __get_relation(self, obj, is_yeti_object=True): + if is_yeti_object: + type_attribute = self.misp_mapping[obj['type']] + else: + type_attribute = obj['type'] + if type_attribute == 'ip-src' or type_attribute == 'ip-dst': return 'ip' - elif 'domain' == typ_attribute: + elif 'domain' == type_attribute: return 'domain' - elif 'hostname' == typ_attribute: + elif 'hostname' == type_attribute: return 'domain' - return attr_misp + elif type_attribute == 'url': + return type_attribute def handler(q=False): From 69a5584dfea5d0c912720445eaaad74afcb6d3a6 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 11:00:55 +0200 Subject: [PATCH 054/101] Update yeti.py add relation --- misp_modules/modules/expansion/yeti.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 05909d9..756464a 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -84,7 +84,7 @@ class Yeti(): return results def __get_object_domain_ip(self, obj_to_add): - if (obj_to_add['type'] == 'Ip' and self.attribute in ['hostname','domain']) or\ + if (obj_to_add['type'] == 'Ip' and self.attribute['type'] in ['hostname','domain']) or\ (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')): domain_ip_object = MISPObject('domain-ip') domain_ip_object.add_attribute(self.__get_relation(obj_to_add), @@ -96,10 +96,12 @@ class Yeti(): return domain_ip_object def __get_object_url(self, obj_to_add): - if obj_to_add['type'] == 'Url': + if (obj_to_add['type'] == 'Url' and self.attribute['type'] in ['hostname', 'domain', 'ip-src', 'ip-dest']) or ( + obj_to_add['type'] in ('Hostname', 'Domain', 'Ip') and self.attribute['type'] == 'url' + ): url_object = MISPObject('Url') url_object.add_attribute(self.__get_relation(obj_to_add), obj_to_add['value']) - url_object.add_attribute(self.__get_relation(self.attribute, is_yeti_object=False), + url_object.add_attribute(self.__get_relation(self.attribute), self.attribute['value']) url_object.add_reference(self.attribute['uuid'], 'related_to') return url_object @@ -116,7 +118,7 @@ class Yeti(): elif 'hostname' == type_attribute: return 'domain' elif type_attribute == 'url': - return type_attribute + return 'Url' def handler(q=False): From 07f54c1b8681451360749b64da5aefa90e89f132 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 11:03:39 +0200 Subject: [PATCH 055/101] Update yeti.py correct typo --- misp_modules/modules/expansion/yeti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 756464a..75af573 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -96,7 +96,7 @@ class Yeti(): return domain_ip_object def __get_object_url(self, obj_to_add): - if (obj_to_add['type'] == 'Url' and self.attribute['type'] in ['hostname', 'domain', 'ip-src', 'ip-dest']) or ( + if (obj_to_add['type'] == 'Url' and self.attribute['type'] in ['hostname', 'domain', 'ip-src', 'ip-dst']) or ( obj_to_add['type'] in ('Hostname', 'Domain', 'Ip') and self.attribute['type'] == 'url' ): url_object = MISPObject('Url') From af01db860ae27551f18ce70675c1f6e4407d519b Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 11:05:16 +0200 Subject: [PATCH 056/101] Update yeti.py add log --- misp_modules/modules/expansion/yeti.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 75af573..6a86e82 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -104,6 +104,7 @@ class Yeti(): url_object.add_attribute(self.__get_relation(self.attribute), self.attribute['value']) url_object.add_reference(self.attribute['uuid'], 'related_to') + print(url_object) return url_object def __get_relation(self, obj, is_yeti_object=True): From be212097a759cb6e39a2ca763921dc144b8f19a1 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 11:08:21 +0200 Subject: [PATCH 057/101] Update yeti.py add log --- misp_modules/modules/expansion/yeti.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 6a86e82..55c5fe0 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -112,6 +112,7 @@ class Yeti(): type_attribute = self.misp_mapping[obj['type']] else: type_attribute = obj['type'] + print(type_attribute) if type_attribute == 'ip-src' or type_attribute == 'ip-dst': return 'ip' elif 'domain' == type_attribute: From 4634567b23df811df185c37937674e890be50433 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 11:09:38 +0200 Subject: [PATCH 058/101] Update yeti.py correct bug --- misp_modules/modules/expansion/yeti.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 55c5fe0..82a20d5 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -112,7 +112,6 @@ class Yeti(): type_attribute = self.misp_mapping[obj['type']] else: type_attribute = obj['type'] - print(type_attribute) if type_attribute == 'ip-src' or type_attribute == 'ip-dst': return 'ip' elif 'domain' == type_attribute: @@ -120,7 +119,7 @@ class Yeti(): elif 'hostname' == type_attribute: return 'domain' elif type_attribute == 'url': - return 'Url' + return type_attribute def handler(q=False): From a29779eff6f758d2f22ecd3bc5395f85bd9f6b8e Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 11:24:01 +0200 Subject: [PATCH 059/101] Update yeti.py add check --- misp_modules/modules/expansion/yeti.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 82a20d5..ed4d02d 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -100,9 +100,16 @@ class Yeti(): obj_to_add['type'] in ('Hostname', 'Domain', 'Ip') and self.attribute['type'] == 'url' ): url_object = MISPObject('Url') - url_object.add_attribute(self.__get_relation(obj_to_add), obj_to_add['value']) - url_object.add_attribute(self.__get_relation(self.attribute), + obj_relation = self.__get_relation(obj_to_add) + if obj_relation: + print(obj_relation) + url_object.add_attribute(self.__get_relation(obj_to_add), obj_to_add['value']) + obj_relation = self.__get_relation(self.attribute) + if obj_relation: + print(obj_relation) + url_object.add_attribute(self.__get_relation(self.attribute), self.attribute['value']) + url_object.add_reference(self.attribute['uuid'], 'related_to') print(url_object) return url_object From 559533ea783e57b25015e3283f24f6a48a85e874 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 11:25:50 +0200 Subject: [PATCH 060/101] Update yeti.py try test --- misp_modules/modules/expansion/yeti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index ed4d02d..ab840b5 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -103,7 +103,7 @@ class Yeti(): obj_relation = self.__get_relation(obj_to_add) if obj_relation: print(obj_relation) - url_object.add_attribute(self.__get_relation(obj_to_add), obj_to_add['value']) + url_object.add_attribute('url', obj_to_add['value']) obj_relation = self.__get_relation(self.attribute) if obj_relation: print(obj_relation) From 8a24ed7fd6549ac8d7849d1337fa203db7014cdc Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 11:27:33 +0200 Subject: [PATCH 061/101] Update yeti.py add logs --- misp_modules/modules/expansion/yeti.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index ab840b5..f8ebe36 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -103,6 +103,7 @@ class Yeti(): obj_relation = self.__get_relation(obj_to_add) if obj_relation: print(obj_relation) + print(obj_to_add['value']) url_object.add_attribute('url', obj_to_add['value']) obj_relation = self.__get_relation(self.attribute) if obj_relation: From e3fc3a3f38609b53e0af3a91589e811ac89f286e Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 11:47:06 +0200 Subject: [PATCH 062/101] Update yeti.py test --- misp_modules/modules/expansion/yeti.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index f8ebe36..b53f653 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -73,9 +73,9 @@ class Yeti(): object_misp_domain_ip = self.__get_object_domain_ip(obs_to_add) if object_misp_domain_ip: self.misp_event.add_object(object_misp_domain_ip) - object_misp_url = self.__get_object_url(obs_to_add) - if object_misp_url: - self.misp_event.add_object(object_misp_url) + # object_misp_url = self.__get_object_url(obs_to_add) + # if object_misp_url: + # self.misp_event.add_object(object_misp_url) def get_result(self): event = json.loads(self.misp_event.to_json()) From ef6596637df513fd3c59b1d45916a41a4e82506f Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 11:49:24 +0200 Subject: [PATCH 063/101] Update yeti.py remove tests --- misp_modules/modules/expansion/yeti.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index b53f653..f044f76 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -73,9 +73,9 @@ class Yeti(): object_misp_domain_ip = self.__get_object_domain_ip(obs_to_add) if object_misp_domain_ip: self.misp_event.add_object(object_misp_domain_ip) - # object_misp_url = self.__get_object_url(obs_to_add) - # if object_misp_url: - # self.misp_event.add_object(object_misp_url) + object_misp_url = self.__get_object_url(obs_to_add) + if object_misp_url: + self.misp_event.add_object(object_misp_url) def get_result(self): event = json.loads(self.misp_event.to_json()) @@ -104,7 +104,7 @@ class Yeti(): if obj_relation: print(obj_relation) print(obj_to_add['value']) - url_object.add_attribute('url', obj_to_add['value']) + url_object.add_attribute(obj_relation, obj_to_add['value']) obj_relation = self.__get_relation(self.attribute) if obj_relation: print(obj_relation) From 53cc15adcda0986ad50f19cee58dce52d9910463 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 12:12:32 +0200 Subject: [PATCH 064/101] Update yeti.py remove print --- misp_modules/modules/expansion/yeti.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index f044f76..2f4afa6 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -102,12 +102,9 @@ class Yeti(): url_object = MISPObject('Url') obj_relation = self.__get_relation(obj_to_add) if obj_relation: - print(obj_relation) - print(obj_to_add['value']) url_object.add_attribute(obj_relation, obj_to_add['value']) obj_relation = self.__get_relation(self.attribute) if obj_relation: - print(obj_relation) url_object.add_attribute(self.__get_relation(self.attribute), self.attribute['value']) From 1e98f1d5752632bfdda81a4aae503dce3c0e27fb Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 12:20:25 +0200 Subject: [PATCH 065/101] Update yeti.py try typo --- misp_modules/modules/expansion/yeti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 2f4afa6..1d2c99e 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -99,7 +99,7 @@ class Yeti(): if (obj_to_add['type'] == 'Url' and self.attribute['type'] in ['hostname', 'domain', 'ip-src', 'ip-dst']) or ( obj_to_add['type'] in ('Hostname', 'Domain', 'Ip') and self.attribute['type'] == 'url' ): - url_object = MISPObject('Url') + url_object = MISPObject('url') obj_relation = self.__get_relation(obj_to_add) if obj_relation: url_object.add_attribute(obj_relation, obj_to_add['value']) From 0da40b34eeca5b63af0753dcc314ae2fed660fda Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 13:45:29 +0200 Subject: [PATCH 066/101] Update yeti.py add param --- misp_modules/modules/expansion/yeti.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 1d2c99e..96de599 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -105,8 +105,8 @@ class Yeti(): url_object.add_attribute(obj_relation, obj_to_add['value']) obj_relation = self.__get_relation(self.attribute) if obj_relation: - url_object.add_attribute(self.__get_relation(self.attribute), - self.attribute['value']) + url_object.add_attribute(self.__get_relation(self.attribute, is_yeti_object=False), + self.attribute['value']) url_object.add_reference(self.attribute['uuid'], 'related_to') print(url_object) From b46a3a8885169218498701ce0b1620e9a7ae307e Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 13:47:45 +0200 Subject: [PATCH 067/101] Update yeti.py fix bugs key error --- misp_modules/modules/expansion/yeti.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 96de599..c0004b0 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -103,11 +103,10 @@ class Yeti(): obj_relation = self.__get_relation(obj_to_add) if obj_relation: url_object.add_attribute(obj_relation, obj_to_add['value']) - obj_relation = self.__get_relation(self.attribute) + obj_relation = self.__get_relation(self.attribute, is_yeti_object=False) if obj_relation: - url_object.add_attribute(self.__get_relation(self.attribute, is_yeti_object=False), + url_object.add_attribute(obj_relation, self.attribute['value']) - url_object.add_reference(self.attribute['uuid'], 'related_to') print(url_object) return url_object From 5e6aec4162238ba909b97aa550aac0b7f6d4b37d Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 13:49:02 +0200 Subject: [PATCH 068/101] Update yeti.py remove print debug --- misp_modules/modules/expansion/yeti.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index c0004b0..c8ed0dd 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -80,7 +80,6 @@ class Yeti(): def get_result(self): event = json.loads(self.misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object')} - print('results %s' % results) return results def __get_object_domain_ip(self, obj_to_add): @@ -108,7 +107,7 @@ class Yeti(): url_object.add_attribute(obj_relation, self.attribute['value']) url_object.add_reference(self.attribute['uuid'], 'related_to') - print(url_object) + return url_object def __get_relation(self, obj, is_yeti_object=True): From 21b52dda155e7dd9a64e9d3e69b422699d350468 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 17:10:47 +0200 Subject: [PATCH 069/101] Update yeti.py add related observable and AS --- misp_modules/modules/expansion/yeti.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index c8ed0dd..bc2b107 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -1,11 +1,12 @@ import json +import logging try: import pyeti except ImportError: print("pyeti module not installed.") -from pymisp import MISPEvent, MISPObject +from pymisp import MISPEvent, MISPObject, MISPAttribute misperrors = {'error': 'Error'} @@ -23,7 +24,8 @@ moduleconfig = ['apikey', 'url'] class Yeti(): def __init__(self, url, key,attribute): - self.misp_mapping = {'Ip': 'ip-dst', 'Domain': 'domain', 'Hostname': 'hostname', 'Url': 'url'} + self.misp_mapping = {'Ip': 'ip-dst', 'Domain': 'domain', 'Hostname': 'hostname', 'Url': 'url', + 'AutonomousSystem': 'AS'} self.yeti_client = pyeti.YetiApi(url=url, api_key=key) self.attribute = attribute self.misp_event = MISPEvent() @@ -76,12 +78,27 @@ class Yeti(): object_misp_url = self.__get_object_url(obs_to_add) if object_misp_url: self.misp_event.add_object(object_misp_url) + if not object_misp_url and not object_misp_url: + attr = self.__get_attribute(obs_to_add) + if attr: + self.misp_event.add_attribute(attr.type, attr.value, tags=attr.tags) def get_result(self): event = json.loads(self.misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object')} return results + def __get_attribute(self, obs_to_add): + attr = MISPAttribute() + attr.value = obs_to_add['value'] + try: + attr.type = self.misp_mapping[obs_to_add['type']] + except KeyError: + logging.error('type not found %s' % obs_to_add['type']) + return + attr.tags.extend([t['name'] for t in obs_to_add['tags']]) + return attr + def __get_object_domain_ip(self, obj_to_add): if (obj_to_add['type'] == 'Ip' and self.attribute['type'] in ['hostname','domain']) or\ (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')): From ee7c06579551f89296d60a1929357d637d5c9389 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 17:16:59 +0200 Subject: [PATCH 070/101] Update yeti.py change tags method --- misp_modules/modules/expansion/yeti.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index bc2b107..a719270 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -96,7 +96,9 @@ class Yeti(): except KeyError: logging.error('type not found %s' % obs_to_add['type']) return - attr.tags.extend([t['name'] for t in obs_to_add['tags']]) + + for t in obs_to_add['tags']: + attr.tags.append(t['name']) return attr def __get_object_domain_ip(self, obj_to_add): From f7ca8bf140d3b43f7c284edfcb1c1b18ec5d36a9 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 17:19:23 +0200 Subject: [PATCH 071/101] Update yeti.py test tags --- misp_modules/modules/expansion/yeti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index a719270..18a900b 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -81,7 +81,7 @@ class Yeti(): if not object_misp_url and not object_misp_url: attr = self.__get_attribute(obs_to_add) if attr: - self.misp_event.add_attribute(attr.type, attr.value, tags=attr.tags) + self.misp_event.add_attribute(attr.type, attr.value, tags=['test','toto']) def get_result(self): event = json.loads(self.misp_event.to_json()) From 43672ee9a95a18fb72eb816e0b2ce5e386c6d7e6 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 17:20:13 +0200 Subject: [PATCH 072/101] Update yeti.py remove tag --- misp_modules/modules/expansion/yeti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 18a900b..841dcc0 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -81,7 +81,7 @@ class Yeti(): if not object_misp_url and not object_misp_url: attr = self.__get_attribute(obs_to_add) if attr: - self.misp_event.add_attribute(attr.type, attr.value, tags=['test','toto']) + self.misp_event.add_attribute(attr.type, attr.value) def get_result(self): event = json.loads(self.misp_event.to_json()) From 5d80b79bc499a6bbd407d7196751ee48ed3cacae Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Mon, 19 Apr 2021 17:55:29 +0200 Subject: [PATCH 073/101] Update yeti.py add tags for attribute --- misp_modules/modules/expansion/yeti.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 841dcc0..08ae8b9 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -79,27 +79,24 @@ class Yeti(): if object_misp_url: self.misp_event.add_object(object_misp_url) if not object_misp_url and not object_misp_url: - attr = self.__get_attribute(obs_to_add) - if attr: - self.misp_event.add_attribute(attr.type, attr.value) - + self.__get_attribute(obs_to_add) + def get_result(self): event = json.loads(self.misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object')} return results def __get_attribute(self, obs_to_add): - attr = MISPAttribute() - attr.value = obs_to_add['value'] + try: - attr.type = self.misp_mapping[obs_to_add['type']] + type_attr = self.misp_mapping[obs_to_add['type']] + attr = self.misp_event.add_attribute(value=obs_to_add['value'], type=type_attr) except KeyError: logging.error('type not found %s' % obs_to_add['type']) return for t in obs_to_add['tags']: - attr.tags.append(t['name']) - return attr + self.misp_event.add_attribute_tag(t['name'], attr['uuid']) def __get_object_domain_ip(self, obj_to_add): if (obj_to_add['type'] == 'Ip' and self.attribute['type'] in ['hostname','domain']) or\ From 8ea3d5c5c7a5e6e7c281347f01915fdfcfd4acdb Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 20 Apr 2021 10:41:44 +0200 Subject: [PATCH 074/101] Update yeti.py add file to add in attribute --- misp_modules/modules/expansion/yeti.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 08ae8b9..ecb647f 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -25,7 +25,7 @@ class Yeti(): def __init__(self, url, key,attribute): self.misp_mapping = {'Ip': 'ip-dst', 'Domain': 'domain', 'Hostname': 'hostname', 'Url': 'url', - 'AutonomousSystem': 'AS'} + 'AutonomousSystem': 'AS', 'File': 'sha256'} self.yeti_client = pyeti.YetiApi(url=url, api_key=key) self.attribute = attribute self.misp_event = MISPEvent() @@ -80,7 +80,7 @@ class Yeti(): self.misp_event.add_object(object_misp_url) if not object_misp_url and not object_misp_url: self.__get_attribute(obs_to_add) - + def get_result(self): event = json.loads(self.misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object')} @@ -90,7 +90,12 @@ class Yeti(): try: type_attr = self.misp_mapping[obs_to_add['type']] - attr = self.misp_event.add_attribute(value=obs_to_add['value'], type=type_attr) + value = None + if obs_to_add['type'] == 'File': + value = obs_to_add['value'].split(':')[1] + else: + value = obs_to_add['value'] + attr = self.misp_event.add_attribute(value=value, type=type_attr) except KeyError: logging.error('type not found %s' % obs_to_add['type']) return From 385af28a0ad62e5e945dc36f8db0f77349aa3ff5 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 20 Apr 2021 12:07:06 +0200 Subject: [PATCH 075/101] Update yeti.py add descripton --- misp_modules/modules/expansion/yeti.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index ecb647f..a27f5eb 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -39,8 +39,10 @@ class Yeti(): def get_neighboors(self, obs_id): neighboors = self.yeti_client.neighbors_observables(obs_id) if neighboors and 'objs' in neighboors: + links_by_id = {link['id']: link['description'] for link in neighboors['links']} + for n in neighboors['objs']: - yield n + yield n, links_by_id[n['id']] def get_tags(self, value): obs = self.search(value) @@ -71,7 +73,7 @@ class Yeti(): obs = self.search(self.attribute['value']) values = [] types = [] - for obs_to_add in self.get_neighboors(obs['id']): + for obs_to_add, link in self.get_neighboors(obs['id']): object_misp_domain_ip = self.__get_object_domain_ip(obs_to_add) if object_misp_domain_ip: self.misp_event.add_object(object_misp_domain_ip) @@ -79,14 +81,14 @@ class Yeti(): if object_misp_url: self.misp_event.add_object(object_misp_url) if not object_misp_url and not object_misp_url: - self.__get_attribute(obs_to_add) + self.__get_attribute(obs_to_add, link) def get_result(self): event = json.loads(self.misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object')} return results - def __get_attribute(self, obs_to_add): + def __get_attribute(self, obs_to_add, link): try: type_attr = self.misp_mapping[obs_to_add['type']] @@ -96,6 +98,7 @@ class Yeti(): else: value = obs_to_add['value'] attr = self.misp_event.add_attribute(value=value, type=type_attr) + attr.comment = '%s of %s' % (link, self.attribute['value']) except KeyError: logging.error('type not found %s' % obs_to_add['type']) return From 1a67f8ed96c971c9c9c2f21b7c3187718bcdf7f6 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 20 Apr 2021 12:08:59 +0200 Subject: [PATCH 076/101] Update yeti.py add log --- misp_modules/modules/expansion/yeti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index a27f5eb..9212470 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -40,7 +40,7 @@ class Yeti(): neighboors = self.yeti_client.neighbors_observables(obs_id) if neighboors and 'objs' in neighboors: links_by_id = {link['id']: link['description'] for link in neighboors['links']} - + print(links_by_id) for n in neighboors['objs']: yield n, links_by_id[n['id']] From abba63f32fcae805a674c0ff5a703f17f6568745 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 20 Apr 2021 12:17:17 +0200 Subject: [PATCH 077/101] Update yeti.py add test of id --- misp_modules/modules/expansion/yeti.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 9212470..093c30b 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -40,9 +40,8 @@ class Yeti(): neighboors = self.yeti_client.neighbors_observables(obs_id) if neighboors and 'objs' in neighboors: links_by_id = {link['id']: link['description'] for link in neighboors['links']} - print(links_by_id) for n in neighboors['objs']: - yield n, links_by_id[n['id']] + yield n, links_by_id def get_tags(self, value): obs = self.search(value) @@ -73,7 +72,7 @@ class Yeti(): obs = self.search(self.attribute['value']) values = [] types = [] - for obs_to_add, link in self.get_neighboors(obs['id']): + for obs_to_add, links in self.get_neighboors(obs['id']): object_misp_domain_ip = self.__get_object_domain_ip(obs_to_add) if object_misp_domain_ip: self.misp_event.add_object(object_misp_domain_ip) @@ -81,14 +80,14 @@ class Yeti(): if object_misp_url: self.misp_event.add_object(object_misp_url) if not object_misp_url and not object_misp_url: - self.__get_attribute(obs_to_add, link) + self.__get_attribute(obs_to_add, links) def get_result(self): event = json.loads(self.misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object')} return results - def __get_attribute(self, obs_to_add, link): + def __get_attribute(self, obs_to_add, links): try: type_attr = self.misp_mapping[obs_to_add['type']] @@ -98,7 +97,8 @@ class Yeti(): else: value = obs_to_add['value'] attr = self.misp_event.add_attribute(value=value, type=type_attr) - attr.comment = '%s of %s' % (link, self.attribute['value']) + if obs_to_add['id'] in links: + attr.comment = '%s of %s' % (links[obs_to_add['id']], self.attribute['value']) except KeyError: logging.error('type not found %s' % obs_to_add['type']) return From 507e56228f03efae5628bf15975f41faa36fb49c Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 20 Apr 2021 12:19:43 +0200 Subject: [PATCH 078/101] Update yeti.py add logs --- misp_modules/modules/expansion/yeti.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 093c30b..07a87bd 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -98,6 +98,7 @@ class Yeti(): value = obs_to_add['value'] attr = self.misp_event.add_attribute(value=value, type=type_attr) if obs_to_add['id'] in links: + print('%s of %s' % (links[obs_to_add['id']], self.attribute['value'])) attr.comment = '%s of %s' % (links[obs_to_add['id']], self.attribute['value']) except KeyError: logging.error('type not found %s' % obs_to_add['type']) From 37867f89eea61d6236aca0a9730cd57ff8e48d23 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 20 Apr 2021 12:21:56 +0200 Subject: [PATCH 079/101] Update yeti.py add logs --- misp_modules/modules/expansion/yeti.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 07a87bd..215456d 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -97,8 +97,9 @@ class Yeti(): else: value = obs_to_add['value'] attr = self.misp_event.add_attribute(value=value, type=type_attr) + print(links) + print(obs_to_add['id']) if obs_to_add['id'] in links: - print('%s of %s' % (links[obs_to_add['id']], self.attribute['value'])) attr.comment = '%s of %s' % (links[obs_to_add['id']], self.attribute['value']) except KeyError: logging.error('type not found %s' % obs_to_add['type']) From 9cb1a83e5432b158ed73964742392e936405c192 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 20 Apr 2021 12:24:34 +0200 Subject: [PATCH 080/101] Update yeti.py fix bug about id --- misp_modules/modules/expansion/yeti.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 215456d..f90d22a 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -41,7 +41,7 @@ class Yeti(): if neighboors and 'objs' in neighboors: links_by_id = {link['id']: link['description'] for link in neighboors['links']} for n in neighboors['objs']: - yield n, links_by_id + yield n, links_by_id[n['id']] def get_tags(self, value): obs = self.search(value) @@ -72,7 +72,7 @@ class Yeti(): obs = self.search(self.attribute['value']) values = [] types = [] - for obs_to_add, links in self.get_neighboors(obs['id']): + for obs_to_add, link in self.get_neighboors(obs['id']): object_misp_domain_ip = self.__get_object_domain_ip(obs_to_add) if object_misp_domain_ip: self.misp_event.add_object(object_misp_domain_ip) @@ -80,7 +80,7 @@ class Yeti(): if object_misp_url: self.misp_event.add_object(object_misp_url) if not object_misp_url and not object_misp_url: - self.__get_attribute(obs_to_add, links) + self.__get_attribute(obs_to_add, link) def get_result(self): event = json.loads(self.misp_event.to_json()) @@ -97,8 +97,6 @@ class Yeti(): else: value = obs_to_add['value'] attr = self.misp_event.add_attribute(value=value, type=type_attr) - print(links) - print(obs_to_add['id']) if obs_to_add['id'] in links: attr.comment = '%s of %s' % (links[obs_to_add['id']], self.attribute['value']) except KeyError: From a2741e8eb701c502fef4dac8e5adb67f1f8514ce Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 20 Apr 2021 12:30:22 +0200 Subject: [PATCH 081/101] Update yeti.py fix keyerror --- misp_modules/modules/expansion/yeti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index f90d22a..fb2aa88 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -39,7 +39,7 @@ class Yeti(): def get_neighboors(self, obs_id): neighboors = self.yeti_client.neighbors_observables(obs_id) if neighboors and 'objs' in neighboors: - links_by_id = {link['id']: link['description'] for link in neighboors['links']} + links_by_id = {link['dst']['id']: link['description'] for link in neighboors['links']} for n in neighboors['objs']: yield n, links_by_id[n['id']] From f7012560083ffc2975948d50be795f502094f092 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 20 Apr 2021 12:33:46 +0200 Subject: [PATCH 082/101] Update yeti.py add src --- misp_modules/modules/expansion/yeti.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index fb2aa88..672f76a 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -40,6 +40,7 @@ class Yeti(): neighboors = self.yeti_client.neighbors_observables(obs_id) if neighboors and 'objs' in neighboors: links_by_id = {link['dst']['id']: link['description'] for link in neighboors['links']} + links_by_id.update({link['src']['id']: link['description'] for link in neighboors['links']}) for n in neighboors['objs']: yield n, links_by_id[n['id']] From e0506ee31e3e6fbe013320ff6e35d165fed17cec Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 20 Apr 2021 12:40:01 +0200 Subject: [PATCH 083/101] Update yeti.py filter by id --- misp_modules/modules/expansion/yeti.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 672f76a..87fe1f8 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -39,8 +39,11 @@ class Yeti(): def get_neighboors(self, obs_id): neighboors = self.yeti_client.neighbors_observables(obs_id) if neighboors and 'objs' in neighboors: - links_by_id = {link['dst']['id']: link['description'] for link in neighboors['links']} - links_by_id.update({link['src']['id']: link['description'] for link in neighboors['links']}) + links_by_id = {link['dst']['id']: link['description'] for link in neighboors['links'] + if link['dst']['id'] != obs_id} + links_by_id.update({link['src']['id']: link['description'] for link in neighboors['links'] + if link['src']['id'] != obs_id}) + for n in neighboors['objs']: yield n, links_by_id[n['id']] From e037c4c767c6926e29382116e8cf3945dca8d955 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 20 Apr 2021 12:42:49 +0200 Subject: [PATCH 084/101] Update yeti.py remove tests --- misp_modules/modules/expansion/yeti.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 87fe1f8..fd9cb52 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -101,8 +101,7 @@ class Yeti(): else: value = obs_to_add['value'] attr = self.misp_event.add_attribute(value=value, type=type_attr) - if obs_to_add['id'] in links: - attr.comment = '%s of %s' % (links[obs_to_add['id']], self.attribute['value']) + attr.comment = '%s of %s' % (links[obs_to_add['id']], self.attribute['value']) except KeyError: logging.error('type not found %s' % obs_to_add['type']) return From bb1cd7c4de4b312ca426a155801ea39ac619d2f4 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 20 Apr 2021 12:43:43 +0200 Subject: [PATCH 085/101] Update yeti.py fix bug --- misp_modules/modules/expansion/yeti.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index fd9cb52..bd59bc5 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -91,7 +91,7 @@ class Yeti(): results = {key: event[key] for key in ('Attribute', 'Object')} return results - def __get_attribute(self, obs_to_add, links): + def __get_attribute(self, obs_to_add, link): try: type_attr = self.misp_mapping[obs_to_add['type']] @@ -101,7 +101,7 @@ class Yeti(): else: value = obs_to_add['value'] attr = self.misp_event.add_attribute(value=value, type=type_attr) - attr.comment = '%s of %s' % (links[obs_to_add['id']], self.attribute['value']) + attr.comment = '%s of %s' % (link, self.attribute['value']) except KeyError: logging.error('type not found %s' % obs_to_add['type']) return From cec06ed26d89bd11f2c961e2cf001f20eb3372f5 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 20 Apr 2021 13:38:45 +0200 Subject: [PATCH 086/101] Update yeti.py change loop --- misp_modules/modules/expansion/yeti.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index bd59bc5..c60c6a6 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -80,11 +80,13 @@ class Yeti(): object_misp_domain_ip = self.__get_object_domain_ip(obs_to_add) if object_misp_domain_ip: self.misp_event.add_object(object_misp_domain_ip) + continue object_misp_url = self.__get_object_url(obs_to_add) if object_misp_url: self.misp_event.add_object(object_misp_url) - if not object_misp_url and not object_misp_url: - self.__get_attribute(obs_to_add, link) + continue + + self.__get_attribute(obs_to_add, link) def get_result(self): event = json.loads(self.misp_event.to_json()) From baaaa81ec36177e006840709839b27078a2060b1 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 20 Apr 2021 13:53:06 +0200 Subject: [PATCH 087/101] Update yeti.py add ns_record object --- misp_modules/modules/expansion/yeti.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index c60c6a6..9ae29c7 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -23,7 +23,7 @@ moduleconfig = ['apikey', 'url'] class Yeti(): - def __init__(self, url, key,attribute): + def __init__(self, url, key, attribute): self.misp_mapping = {'Ip': 'ip-dst', 'Domain': 'domain', 'Hostname': 'hostname', 'Url': 'url', 'AutonomousSystem': 'AS', 'File': 'sha256'} self.yeti_client = pyeti.YetiApi(url=url, api_key=key) @@ -85,7 +85,10 @@ class Yeti(): if object_misp_url: self.misp_event.add_object(object_misp_url) continue - + if link == 'NS record': + object_ns_record = self.__get_object_ns_record(obs_to_add) + self.misp_event.add_object(object_ns_record) + continue self.__get_attribute(obs_to_add, link) def get_result(self): @@ -139,6 +142,15 @@ class Yeti(): return url_object + def __get_object_ns_record(self, obj_to_add): + object_dns_record = MISPObject('dns-record') + + object_dns_record.add_attribute(self.attribute['value'], 'queried_domain') + object_dns_record.add_attribute(obj_to_add['value', 'ns-record']) + object_dns_record.add_reference(self.attribute['uuid'], 'related_to') + + return object_dns_record + def __get_relation(self, obj, is_yeti_object=True): if is_yeti_object: type_attribute = self.misp_mapping[obj['type']] From dfa46b551a0c56c9788c45ee7dd460254a390391 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 20 Apr 2021 13:55:36 +0200 Subject: [PATCH 088/101] Update yeti.py change params --- misp_modules/modules/expansion/yeti.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 9ae29c7..7dddf2e 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -145,8 +145,8 @@ class Yeti(): def __get_object_ns_record(self, obj_to_add): object_dns_record = MISPObject('dns-record') - object_dns_record.add_attribute(self.attribute['value'], 'queried_domain') - object_dns_record.add_attribute(obj_to_add['value', 'ns-record']) + object_dns_record.add_attribute('queried_domain', self.attribute['value']) + object_dns_record.add_attribute('ns-record', obj_to_add['value']) object_dns_record.add_reference(self.attribute['uuid'], 'related_to') return object_dns_record From fd76e55093b0644e00e4cae215cdebfb99f72b66 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 20 Apr 2021 13:56:45 +0200 Subject: [PATCH 089/101] Update yeti.py fix typo --- misp_modules/modules/expansion/yeti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 7dddf2e..1288838 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -145,7 +145,7 @@ class Yeti(): def __get_object_ns_record(self, obj_to_add): object_dns_record = MISPObject('dns-record') - object_dns_record.add_attribute('queried_domain', self.attribute['value']) + object_dns_record.add_attribute('queried-domain', self.attribute['value']) object_dns_record.add_attribute('ns-record', obj_to_add['value']) object_dns_record.add_reference(self.attribute['uuid'], 'related_to') From 3426ad13c5de981bf50dd8c58277e44517ec875e Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 20 Apr 2021 14:05:51 +0200 Subject: [PATCH 090/101] Update yeti.py fix edges --- misp_modules/modules/expansion/yeti.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 1288838..e7ee859 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -39,9 +39,9 @@ class Yeti(): def get_neighboors(self, obs_id): neighboors = self.yeti_client.neighbors_observables(obs_id) if neighboors and 'objs' in neighboors: - links_by_id = {link['dst']['id']: link['description'] for link in neighboors['links'] + links_by_id = {link['dst']['id']: (link['description'],'dst') for link in neighboors['links'] if link['dst']['id'] != obs_id} - links_by_id.update({link['src']['id']: link['description'] for link in neighboors['links'] + links_by_id.update({link['src']['id']: (link['description'], 'src') for link in neighboors['links'] if link['src']['id'] != obs_id}) for n in neighboors['objs']: @@ -85,11 +85,11 @@ class Yeti(): if object_misp_url: self.misp_event.add_object(object_misp_url) continue - if link == 'NS record': + if link[0] == 'NS record' and link[1] == 'dst': object_ns_record = self.__get_object_ns_record(obs_to_add) self.misp_event.add_object(object_ns_record) continue - self.__get_attribute(obs_to_add, link) + self.__get_attribute(obs_to_add, link[0]) def get_result(self): event = json.loads(self.misp_event.to_json()) @@ -115,7 +115,7 @@ class Yeti(): self.misp_event.add_attribute_tag(t['name'], attr['uuid']) def __get_object_domain_ip(self, obj_to_add): - if (obj_to_add['type'] == 'Ip' and self.attribute['type'] in ['hostname','domain']) or\ + if (obj_to_add['type'] == 'Ip' and self.attribute['type'] in ['hostname', 'domain']) or\ (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')): domain_ip_object = MISPObject('domain-ip') domain_ip_object.add_attribute(self.__get_relation(obj_to_add), From 26bc02617f64c9c23483023ccfcef4010cae4420 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 20 Apr 2021 14:08:31 +0200 Subject: [PATCH 091/101] Update yeti.py add test to create result --- misp_modules/modules/expansion/yeti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index e7ee859..eea593e 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -93,7 +93,7 @@ class Yeti(): def get_result(self): event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object')} + results = {key: event[key] for key in ('Attribute', 'Object') if key in event} return results def __get_attribute(self, obs_to_add, link): From 8683c9e5cec8efe077e84e55c69957e539a270b5 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 20 Apr 2021 14:13:16 +0200 Subject: [PATCH 092/101] Update yeti.py add ns record dst and src link --- misp_modules/modules/expansion/yeti.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index eea593e..d048faf 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -85,8 +85,8 @@ class Yeti(): if object_misp_url: self.misp_event.add_object(object_misp_url) continue - if link[0] == 'NS record' and link[1] == 'dst': - object_ns_record = self.__get_object_ns_record(obs_to_add) + if link[0] == 'NS record': + object_ns_record = self.__get_object_ns_record(obs_to_add, link[1]) self.misp_event.add_object(object_ns_record) continue self.__get_attribute(obs_to_add, link[0]) @@ -142,11 +142,17 @@ class Yeti(): return url_object - def __get_object_ns_record(self, obj_to_add): + def __get_object_ns_record(self, obj_to_add, link): object_dns_record = MISPObject('dns-record') + if link == 'dst': + queried_domain = self.attribute['value'] + ns_domain = obj_to_add['value'] + elif link =='src': + queried_domain = obj_to_add['value'] + ns_domain = self.attribute['value'] - object_dns_record.add_attribute('queried-domain', self.attribute['value']) - object_dns_record.add_attribute('ns-record', obj_to_add['value']) + object_dns_record.add_attribute('queried-domain', queried_domain) + object_dns_record.add_attribute('ns-record', ns_domain) object_dns_record.add_reference(self.attribute['uuid'], 'related_to') return object_dns_record From 7e5238e8be3fcdbf4ea861d9e839100d968c2d7f Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 20 Apr 2021 14:35:18 +0200 Subject: [PATCH 093/101] Update yeti.py add tests --- misp_modules/modules/expansion/yeti.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index d048faf..758b560 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -87,8 +87,9 @@ class Yeti(): continue if link[0] == 'NS record': object_ns_record = self.__get_object_ns_record(obs_to_add, link[1]) - self.misp_event.add_object(object_ns_record) - continue + if object_ns_record: + self.misp_event.add_object(object_ns_record) + continue self.__get_attribute(obs_to_add, link[0]) def get_result(self): @@ -106,7 +107,7 @@ class Yeti(): else: value = obs_to_add['value'] attr = self.misp_event.add_attribute(value=value, type=type_attr) - attr.comment = '%s of %s' % (link, self.attribute['value']) + attr.comment = '%s: %s' % (link, self.attribute['value']) except KeyError: logging.error('type not found %s' % obs_to_add['type']) return @@ -143,6 +144,8 @@ class Yeti(): return url_object def __get_object_ns_record(self, obj_to_add, link): + queried_domain = None + ns_domain = None object_dns_record = MISPObject('dns-record') if link == 'dst': queried_domain = self.attribute['value'] @@ -150,12 +153,12 @@ class Yeti(): elif link =='src': queried_domain = obj_to_add['value'] ns_domain = self.attribute['value'] + if queried_domain and ns_domain: + object_dns_record.add_attribute('queried-domain', queried_domain) + object_dns_record.add_attribute('ns-record', ns_domain) + object_dns_record.add_reference(self.attribute['uuid'], 'related_to') - object_dns_record.add_attribute('queried-domain', queried_domain) - object_dns_record.add_attribute('ns-record', ns_domain) - object_dns_record.add_reference(self.attribute['uuid'], 'related_to') - - return object_dns_record + return object_dns_record def __get_relation(self, obj, is_yeti_object=True): if is_yeti_object: From a277cbb8bfa199830d4ddad5cf8865d585759203 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 21 Apr 2021 14:45:07 +0200 Subject: [PATCH 094/101] Update yeti.py add input --- misp_modules/modules/expansion/yeti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 758b560..4ec92d0 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -10,7 +10,7 @@ from pymisp import MISPEvent, MISPObject, MISPAttribute misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'], +mispattributes = {'input': ['AS', 'ip-src', 'ip-dst', 'hostname', 'domain', 'sha256', 'sha1', 'md5', 'url'], 'format': 'misp_standard' } # possible module-types: 'expansion', 'hover' or both From a76978d6c64c8024ccec410e5d2ba09efec5cb36 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 21 Apr 2021 15:40:46 +0200 Subject: [PATCH 095/101] Update yeti.py remove tags and entity --- misp_modules/modules/expansion/yeti.py | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 4ec92d0..6ef597e 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -47,31 +47,6 @@ class Yeti(): for n in neighboors['objs']: yield n, links_by_id[n['id']] - def get_tags(self, value): - obs = self.search(value) - if obs: - for t in obs['tags']: - yield t - - def get_entity(self, obs_id): - companies = self.yeti_client.observable_to_company(obs_id) - actors = self.yeti_client.observable_to_actor(obs_id) - campaigns = self.yeti_client.observable_to_campaign(obs_id) - exploit_kit = self.yeti_client.observable_to_exploitkit(obs_id) - exploit = self.yeti_client.observable_to_exploit(obs_id) - ind = self.yeti_client.observable_to_indicator(obs_id) - - res = [] - res.extend(companies) - res.extend(actors) - res.extend(campaigns) - res.extend(exploit) - res.extend(exploit_kit) - res.extend(ind) - - for r in res: - yield r['name'] - def parse_yeti_result(self): obs = self.search(self.attribute['value']) values = [] From 1b9d47dd3328222982040d848b6c4f70f1499f73 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 21 Apr 2021 15:41:20 +0200 Subject: [PATCH 096/101] Update yeti.py pep 8 compliant --- misp_modules/modules/expansion/yeti.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 6ef597e..efb781d 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -39,7 +39,7 @@ class Yeti(): def get_neighboors(self, obs_id): neighboors = self.yeti_client.neighbors_observables(obs_id) if neighboors and 'objs' in neighboors: - links_by_id = {link['dst']['id']: (link['description'],'dst') for link in neighboors['links'] + links_by_id = {link['dst']['id']: (link['description'], 'dst') for link in neighboors['links'] if link['dst']['id'] != obs_id} links_by_id.update({link['src']['id']: (link['description'], 'src') for link in neighboors['links'] if link['src']['id'] != obs_id}) @@ -91,7 +91,7 @@ class Yeti(): self.misp_event.add_attribute_tag(t['name'], attr['uuid']) def __get_object_domain_ip(self, obj_to_add): - if (obj_to_add['type'] == 'Ip' and self.attribute['type'] in ['hostname', 'domain']) or\ + if (obj_to_add['type'] == 'Ip' and self.attribute['type'] in ['hostname', 'domain']) or \ (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')): domain_ip_object = MISPObject('domain-ip') domain_ip_object.add_attribute(self.__get_relation(obj_to_add), @@ -104,7 +104,7 @@ class Yeti(): def __get_object_url(self, obj_to_add): if (obj_to_add['type'] == 'Url' and self.attribute['type'] in ['hostname', 'domain', 'ip-src', 'ip-dst']) or ( - obj_to_add['type'] in ('Hostname', 'Domain', 'Ip') and self.attribute['type'] == 'url' + obj_to_add['type'] in ('Hostname', 'Domain', 'Ip') and self.attribute['type'] == 'url' ): url_object = MISPObject('url') obj_relation = self.__get_relation(obj_to_add) @@ -125,7 +125,7 @@ class Yeti(): if link == 'dst': queried_domain = self.attribute['value'] ns_domain = obj_to_add['value'] - elif link =='src': + elif link == 'src': queried_domain = obj_to_add['value'] ns_domain = self.attribute['value'] if queried_domain and ns_domain: @@ -178,10 +178,10 @@ def handler(q=False): return misperrors - def version(): moduleinfo['config'] = moduleconfig return moduleinfo + def introspection(): - return mispattributes \ No newline at end of file + return mispattributes From da9d6a7dfd2f7acb623621311851ea8f154b8834 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 21 Apr 2021 17:34:40 +0200 Subject: [PATCH 097/101] Create yeti.json add doc --- doc/expansion/yeti.json | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/expansion/yeti.json diff --git a/doc/expansion/yeti.json b/doc/expansion/yeti.json new file mode 100644 index 0000000..3ec7789 --- /dev/null +++ b/doc/expansion/yeti.json @@ -0,0 +1,9 @@ +{ + "description": "Module to process a query on Yeti.", + "logo": "", + "requirements": ["pyeti", "API key "], + "input": "A domain, hostname,IP, sha256,sha1, md5, url of MISP attribute.", + "output": "MISP attributes and objects fetched from the Yeti instances.", + "references": ["https://github.com/yeti-platform/yeti", "https://github.com/sebdraven/pyeti"], + "features": "This module add context and links between observables using yeti" +} From abac4cfab76c722c1eb307225443d908fee3bb48 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 21 Apr 2021 17:51:22 +0200 Subject: [PATCH 098/101] remove import unused and add package in requirements --- REQUIREMENTS | 3 ++- misp_modules/modules/expansion/yeti.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/REQUIREMENTS b/REQUIREMENTS index 3abe64c..d76a7b4 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -44,7 +44,7 @@ importlib-metadata==1.6.0 ; python_version < '3.8' isodate==0.6.0 jbxapi==3.4.0 jsonschema==3.2.0 -lief==0.10.0 +lief==0.10.1 lxml==4.5.0 maclookup==1.0.3 maxminddb==1.5.2 @@ -107,6 +107,7 @@ websocket-client==0.57.0 wrapt==1.12.1 xlrd==1.2.0 xlsxwriter==1.2.8 +pyeti-python3=1.0 yara-python==3.8.1 yarl==1.4.2 zipp==3.1.0 diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index efb781d..9136b26 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -6,7 +6,7 @@ try: except ImportError: print("pyeti module not installed.") -from pymisp import MISPEvent, MISPObject, MISPAttribute +from pymisp import MISPEvent, MISPObject misperrors = {'error': 'Error'} From 9f5a4be9d7d98d3d11341e77f1dda002fbb7471c Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 21 Apr 2021 17:54:01 +0200 Subject: [PATCH 099/101] remove variable unused --- misp_modules/modules/expansion/yeti.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 9136b26..d16f355 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -49,8 +49,7 @@ class Yeti(): def parse_yeti_result(self): obs = self.search(self.attribute['value']) - values = [] - types = [] + for obs_to_add, link in self.get_neighboors(obs['id']): object_misp_domain_ip = self.__get_object_domain_ip(obs_to_add) if object_misp_domain_ip: From 7ab2e099f43ea077cc850a60a0792046a75db622 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 21 Apr 2021 18:15:16 +0200 Subject: [PATCH 100/101] fix typo --- REQUIREMENTS | 2 +- misp_modules/modules/expansion/yeti.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/REQUIREMENTS b/REQUIREMENTS index d76a7b4..7541f90 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -107,7 +107,7 @@ websocket-client==0.57.0 wrapt==1.12.1 xlrd==1.2.0 xlsxwriter==1.2.8 -pyeti-python3=1.0 +pyeti-python3==1.0 yara-python==3.8.1 yarl==1.4.2 zipp==3.1.0 diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index d16f355..3eeea95 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -49,7 +49,7 @@ class Yeti(): def parse_yeti_result(self): obs = self.search(self.attribute['value']) - + for obs_to_add, link in self.get_neighboors(obs['id']): object_misp_domain_ip = self.__get_object_domain_ip(obs_to_add) if object_misp_domain_ip: From cb091cdbdb4c6171dd45f04d27275e2bc492d7dd Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 22 Apr 2021 11:45:43 +0200 Subject: [PATCH 101/101] add pyeti package --- REQUIREMENTS | 1 + 1 file changed, 1 insertion(+) diff --git a/REQUIREMENTS b/REQUIREMENTS index ba5b5e5..34c15f3 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -106,6 +106,7 @@ python-socketio[client]==5.0.4 python-utils==2.5.2 pytz==2019.3 pyyaml==5.4.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' +pyeti-python3==1.0.0 pyzbar==0.1.8 pyzipper==0.3.4; python_version >= '3.5' rdflib==5.0.0