From 63ba7580d3d45cdc93495ce04286f906ed294cd8 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Fri, 27 Jul 2018 23:13:47 +0200
Subject: [PATCH 01/48] chg: Updated csvimport to support files from csv export
 + import MISP objects

---
 misp_modules/modules/import_mod/csvimport.py | 105 ++++++++++++++++---
 1 file changed, 88 insertions(+), 17 deletions(-)

diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py
index 5ccf287..9b19fc2 100644
--- a/misp_modules/modules/import_mod/csvimport.py
+++ b/misp_modules/modules/import_mod/csvimport.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import json, os, base64
-import pymisp
+from pymisp import __path__ as pymisp_path
+from collections import defaultdict
 
 misperrors = {'error': 'Error'}
 moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
@@ -13,20 +14,49 @@ userConfig = {'header': {
                 'message': 'Define the header of the csv file, with types (included in MISP attribute types or attribute fields) separated by commas.\nFor fields that do not match these types, please use space or simply nothing between commas.\nFor instance: ip-src,domain, ,timestamp'},
               'has_header':{
                 'type': 'Boolean',
-                'message': 'Tick this box ONLY if there is a header line, NOT COMMENTED, in the file (which will be skipped atm).'
+                'message': 'Tick this box ONLY if there is a header line, NOT COMMENTED, in the file.'
               }}
 
 duplicatedFields = {'mispType': {'mispComment': 'comment'},
                     'attrField': {'attrComment': 'comment'}}
 attributesFields = ['type', 'value', 'category', 'to_ids', 'comment', 'distribution']
+misp_standard_csv_header = ['uuid','event_id','category','type','value','comment','to_ids','date',
+                            'object_relation','object_uuid','object_name','object_meta_category']
 delimiters = [',', ';', '|', '/', '\t', '    ']
 
 class CsvParser():
-    def __init__(self, header, has_header):
-        self.header = header
-        self.fields_number = len(header)
-        self.has_header = has_header
-        self.attributes = []
+    def __init__(self, header, has_header, data):
+        if data[0].split(',') == misp_standard_csv_header:
+            self.header = misp_standard_csv_header
+            self.from_misp = True
+            self.data = data[1:]
+        else:
+            self.from_misp = False
+            self.has_header = has_header
+            if header:
+                self.header = header
+                self.fields_number = len(header)
+                self.parse_data(data)
+            else:
+                self.has_delimiter = True
+                self.fields_number, self.delimiter, self.header = self.get_delimiter_from_header(data[0])
+                self.data = data
+            self.result = []
+
+    def get_delimiter_from_header(self, data):
+        delimiters_count = {}
+        for d in delimiters:
+            length = data.count(d)
+            if length > 0:
+                delimiters_count[d] = data.count(d)
+        if len(delimiters_count) == 0:
+            length = 0
+            delimiter = None
+            header = [data]
+        else:
+            length, delimiter = max((n, v) for v, n in delimiters_count.items())
+            header = data.split(delimiter)
+        return length + 1, delimiter, header
 
     def parse_data(self, data):
         return_data = []
@@ -45,6 +75,7 @@ class CsvParser():
                     return_data.append(l)
             # find which delimiter is used
             self.delimiter = self.find_delimiter()
+            if self.fields_number == 0: self.header = return_data[0].split(self.delimiter)
         self.data = return_data[1:] if self.has_header else return_data
 
     def parse_delimiter(self, line):
@@ -56,6 +87,43 @@ class CsvParser():
         _, delimiter = max((n, v) for v, n in self.delimiter_count.items())
         return delimiter
 
+    def parse_csv(self):
+        if self.from_misp:
+            self.build_misp_event()
+        else:
+            self.buildAttributes()
+
+    def build_misp_event(self):
+        l_attributes = []
+        l_objects = []
+        objects = defaultdict(list)
+        attribute_fields = self.header[:1] + self.header[2:8]
+        relation_type = self.header[8]
+        object_fields = self.header[9:]
+        for line in self.data:
+            attribute = {}
+            try:
+                a_uuid,_,category,a_type,value,comment,to_ids,date,relation,o_uuid,o_name,o_meta_category = line.split(',')
+            except ValueError:
+                continue
+            for t, v in zip(attribute_fields, [a_uuid,category,a_type,value,comment,to_ids,date]):
+                attribute[t] = v.replace('"', '')
+            attribute['to_ids'] = True if to_ids == '1' else False
+            relation = relation.replace('"', '')
+            if relation:
+                attribute[relation_type] = relation
+                object_index = tuple(o.replace('"', '') for o in (o_uuid,o_name,o_meta_category))
+                objects[object_index].append(attribute)
+            else:
+                l_attributes.append(attribute)
+        for keys, attributes in objects.items():
+            misp_object = {}
+            for t, v in zip(['uuid','name','meta-category'], keys):
+                misp_object[t] = v
+            misp_object['Attribute'] = attributes
+            l_objects.append(misp_object)
+        self.result = {"Attribute": l_attributes, "Object": l_objects}
+
     def buildAttributes(self):
         # if there is only 1 field of data
         if self.delimiter is None:
@@ -63,7 +131,7 @@ class CsvParser():
             for data in self.data:
                 d = data.strip()
                 if d:
-                    self.attributes.append({'types': mispType, 'values': d})
+                    self.result.append({'types': mispType, 'values': d})
         else:
             # split fields that should be recognized as misp attribute types from the others
             list2pop, misp, head = self.findMispTypes()
@@ -83,10 +151,10 @@ class CsvParser():
                     for h, ds in zip(head, datasplit):
                         if h:
                             attribute[h] = ds.strip()
-                    self.attributes.append(attribute)
+                    self.result.append(attribute)
 
     def findMispTypes(self):
-        descFilename = os.path.join(pymisp.__path__[0], 'data/describeTypes.json')
+        descFilename = os.path.join(pymisp_path[0], 'data/describeTypes.json')
         with open(descFilename, 'r') as f:
             MispTypes = json.loads(f.read())['result'].get('types')
         list2pop = []
@@ -124,18 +192,21 @@ def handler(q=False):
     else:
         misperrors['error'] = "Unsupported attributes type"
         return misperrors
-    if not request.get('config') and not request['config'].get('header'):
-        misperrors['error'] = "Configuration error"
-        return misperrors
-    header = request['config'].get('header').split(',')
-    header = [c.strip() for c in header]
     has_header = request['config'].get('has_header')
     has_header = True if has_header == '1' else False
+    if not request.get('config') and not request['config'].get('header'):
+        if has_header:
+            header = []
+        else:
+            misperrors['error'] = "Configuration error"
+            return misperrors
+    else:
+        header = request['config'].get('header').split(',')
+        header = [c.strip() for c in header]
     csv_parser = CsvParser(header, has_header)
-    csv_parser.parse_data(data.split('\n'))
     # build the attributes
     csv_parser.buildAttributes()
-    r = {'results': csv_parser.attributes}
+    r = {'results': csv_parser.result}
     return r
 
 def introspection():

From 92fbcaeff60d82168351e4dbb49133ba26226308 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Sat, 28 Jul 2018 00:07:02 +0200
Subject: [PATCH 02/48] fix: Fixed changes omissions in handler function

---
 misp_modules/modules/import_mod/csvimport.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py
index 9b19fc2..d7be52a 100644
--- a/misp_modules/modules/import_mod/csvimport.py
+++ b/misp_modules/modules/import_mod/csvimport.py
@@ -203,9 +203,9 @@ def handler(q=False):
     else:
         header = request['config'].get('header').split(',')
         header = [c.strip() for c in header]
-    csv_parser = CsvParser(header, has_header)
+    csv_parser = CsvParser(header, has_header, data.split('\n'))
     # build the attributes
-    csv_parser.buildAttributes()
+    csv_parser.parse_csv()
     r = {'results': csv_parser.result}
     return r
 

From 7980aa045abaf4053bf2ad754eac7038c46edfd0 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Wed, 1 Aug 2018 17:59:00 +0200
Subject: [PATCH 03/48] fix: Handling the case of Context included in the csv
 file exported from MISP

---
 misp_modules/modules/import_mod/csvimport.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py
index d7be52a..90505b2 100644
--- a/misp_modules/modules/import_mod/csvimport.py
+++ b/misp_modules/modules/import_mod/csvimport.py
@@ -22,11 +22,14 @@ duplicatedFields = {'mispType': {'mispComment': 'comment'},
 attributesFields = ['type', 'value', 'category', 'to_ids', 'comment', 'distribution']
 misp_standard_csv_header = ['uuid','event_id','category','type','value','comment','to_ids','date',
                             'object_relation','object_uuid','object_name','object_meta_category']
+misp_context_additional_fields = ['event_info','event_member_org','event_source_org','event_distribution',
+                                  'event_threat_level_id','event_analysis','event_date','event_tag']
 delimiters = [',', ';', '|', '/', '\t', '    ']
 
 class CsvParser():
     def __init__(self, header, has_header, data):
-        if data[0].split(',') == misp_standard_csv_header:
+        data_header = data[0].split(',')
+        if data_header == misp_standard_csv_header or data_header == (misp_standard_csv_header + misp_context_additional_fields):
             self.header = misp_standard_csv_header
             self.from_misp = True
             self.data = data[1:]
@@ -100,10 +103,11 @@ class CsvParser():
         attribute_fields = self.header[:1] + self.header[2:8]
         relation_type = self.header[8]
         object_fields = self.header[9:]
+        header_length = len(self.header)
         for line in self.data:
             attribute = {}
             try:
-                a_uuid,_,category,a_type,value,comment,to_ids,date,relation,o_uuid,o_name,o_meta_category = line.split(',')
+                a_uuid,_,category,a_type,value,comment,to_ids,date,relation,o_uuid,o_name,o_meta_category = line.split(',')[:header_length]
             except ValueError:
                 continue
             for t, v in zip(attribute_fields, [a_uuid,category,a_type,value,comment,to_ids,date]):

From 8b4d24ba635d424c38936dc37223ffe8c1adb779 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Thu, 2 Aug 2018 15:42:59 +0200
Subject: [PATCH 04/48] fix: Fixed fields parsing to support files from csv
 export with additional context

---
 misp_modules/modules/import_mod/csvimport.py | 29 +++++++++++---------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py
index 90505b2..5b083a9 100644
--- a/misp_modules/modules/import_mod/csvimport.py
+++ b/misp_modules/modules/import_mod/csvimport.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-import json, os, base64
+import base64, csv, io, json, os
 from pymisp import __path__ as pymisp_path
 from collections import defaultdict
 
@@ -24,13 +24,14 @@ misp_standard_csv_header = ['uuid','event_id','category','type','value','comment
                             'object_relation','object_uuid','object_name','object_meta_category']
 misp_context_additional_fields = ['event_info','event_member_org','event_source_org','event_distribution',
                                   'event_threat_level_id','event_analysis','event_date','event_tag']
+misp_extended_csv_header = misp_standard_csv_header[:9] + ['attribute_tag'] + misp_standard_csv_header[9:] + misp_context_additional_fields
 delimiters = [',', ';', '|', '/', '\t', '    ']
 
 class CsvParser():
     def __init__(self, header, has_header, data):
-        data_header = data[0].split(',')
-        if data_header == misp_standard_csv_header or data_header == (misp_standard_csv_header + misp_context_additional_fields):
-            self.header = misp_standard_csv_header
+        data_header = data[0]
+        if data_header == misp_standard_csv_header or data_header == misp_extended_csv_header:
+            self.header = misp_standard_csv_header if data_header == misp_standard_csv_header else misp_extended_csv_header[:13]
             self.from_misp = True
             self.data = data[1:]
         else:
@@ -100,23 +101,24 @@ class CsvParser():
         l_attributes = []
         l_objects = []
         objects = defaultdict(list)
-        attribute_fields = self.header[:1] + self.header[2:8]
-        relation_type = self.header[8]
-        object_fields = self.header[9:]
         header_length = len(self.header)
+        attribute_fields = self.header[:1] + self.header[2:6]
         for line in self.data:
             attribute = {}
             try:
-                a_uuid,_,category,a_type,value,comment,to_ids,date,relation,o_uuid,o_name,o_meta_category = line.split(',')[:header_length]
+                try:
+                    a_uuid,_,a_category,a_type,value,comment,to_ids,_,relation,o_uuid,o_name,o_category = line[:header_length]
+                except ValueError:
+                    a_uuid,_,a_category,a_type,value,comment,to_ids,_,relation,tag,o_uuid,o_name,o_category = line[:header_length]
+                    if tag: attribute['tags'] = tag
             except ValueError:
                 continue
-            for t, v in zip(attribute_fields, [a_uuid,category,a_type,value,comment,to_ids,date]):
+            for t, v in zip(attribute_fields, [a_uuid, a_category, a_type, value, comment]):
                 attribute[t] = v.replace('"', '')
             attribute['to_ids'] = True if to_ids == '1' else False
-            relation = relation.replace('"', '')
             if relation:
-                attribute[relation_type] = relation
-                object_index = tuple(o.replace('"', '') for o in (o_uuid,o_name,o_meta_category))
+                attribute["object_relation"] = relation.replace('"', '')
+                object_index = tuple(o.replace('"', '') for o in (o_uuid,o_name,o_category))
                 objects[object_index].append(attribute)
             else:
                 l_attributes.append(attribute)
@@ -193,6 +195,7 @@ def handler(q=False):
     request = json.loads(q)
     if request.get('data'):
         data = base64.b64decode(request['data']).decode('utf-8')
+        data = [line for line in csv.reader(io.TextIOWrapper(io.BytesIO(data.encode()), encoding='utf-8'))]
     else:
         misperrors['error'] = "Unsupported attributes type"
         return misperrors
@@ -207,7 +210,7 @@ def handler(q=False):
     else:
         header = request['config'].get('header').split(',')
         header = [c.strip() for c in header]
-    csv_parser = CsvParser(header, has_header, data.split('\n'))
+    csv_parser = CsvParser(header, has_header, data)
     # build the attributes
     csv_parser.parse_csv()
     r = {'results': csv_parser.result}

From 9c8ee1f3d761987b263a9c68602e24b8f6385916 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Wed, 13 Mar 2019 09:57:28 +0100
Subject: [PATCH 05/48] new: Expansion module to query urlhaus API

- Using the next version of modules, taking a
  MISP attribute as input and able to return
  attributes and objects
- Work still in process in the core part
---
 misp_modules/modules/expansion/urlhaus.py | 136 ++++++++++++++++++++++
 1 file changed, 136 insertions(+)
 create mode 100644 misp_modules/modules/expansion/urlhaus.py

diff --git a/misp_modules/modules/expansion/urlhaus.py b/misp_modules/modules/expansion/urlhaus.py
new file mode 100644
index 0000000..1a2c80b
--- /dev/null
+++ b/misp_modules/modules/expansion/urlhaus.py
@@ -0,0 +1,136 @@
+from collections import defaultdict
+from pymisp import MISPAttribute, MISPObject
+import json
+import requests
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['domain', 'hostname', 'ip-src', 'ip-dst', 'md5', 'sha256', 'url'],
+                  'output': ['url', 'filename', 'md5', 'sha256'],
+                  'format': 'misp_standard'}
+moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
+              'description': 'Query of the URLhaus API to get additional information about some attributes.',
+              'module-type': ['expansion', 'hover']}
+moduleconfig = []
+
+
+def _create_file_object(file_attributes):
+    return _create_object(file_attributes, 'file')
+
+
+def _create_object(attributes, name):
+    misp_object = MISPObject(name)
+    for relation, attribute in attributes.items():
+        misp_object.add_attribute(relation, **attribute)
+    return [misp_object]
+
+
+def _create_objects_with_relationship(file_attributes, vt_attributes):
+    vt_object = _create_vt_object(vt_attributes)[0]
+    vt_uuid = vt_object.uuid
+    file_object = _create_file_object(file_attributes)[0]
+    file_object.add_reference(vt_uuid, 'analysed-with')
+    return [file_object, vt_object]
+
+
+def _create_url_attribute(value):
+    attribute = MISPAttribute()
+    attribute.from_dict(type='url', value=value)
+    return attribute
+
+
+def _create_vt_object(vt_attributes):
+    return _create_object(vt_attributes, 'virustotal_report')
+
+
+def _handle_payload_urls(response):
+    filenames = []
+    urls = []
+    if response:
+        for url in response:
+            urls.append(url['url'])
+            if url['filename']:
+                filenames.append(url['filename'])
+    return filenames, urls
+
+
+def _query_host_api(attribute):
+    response = requests.post('https://urlhaus-api.abuse.ch/v1/host/', data={'host': attribute['value']}).json()
+    attributes = []
+    if 'urls' in response and response['urls']:
+        for url in response['urls']:
+            attributes.append(_create_url_attribute(url['url']).to_json())
+    return {'results': {'Attribute': attributes}}
+
+
+def _query_payload_api(attribute):
+    hash_type = attribute['type']
+    response = requests.post('https://urlhaus-api.abuse.ch/v1/payload/', data={'{}_hash'.format(hash_type): attribute['value']}).json()
+    results = defaultdict(list)
+    filenames, urls = _handle_payload_urls(response['urls'])
+    other_hash_type = 'md5' if hash_type == 'sha256' else 'sha256'
+    file_object = MISPObject('file')
+    if attribute['object_id'] != '0':
+        file_object.id = attribute['object_id']
+    for key, relation in zip(('{}_hash'.format(other_hash_type), 'file_size'), (other_hash_type, 'size-in-bytes')):
+        if response[key]:
+            file_object.add_attribute(relation, **{'type': relation, 'value': response[key]})
+    for filename in filenames:
+        file_object.add_attribute('filename', **{'type': 'filename', 'value': filename})
+    for url in urls:
+        attribute = _create_url_attribute(url)
+        results['Attribute'].append(attribute.to_json())
+        file_object.add_reference(attribute.uuid, 'retrieved-from')
+    results['Object'].append(file_object.to_json())
+    return {'results': results}
+
+
+def _query_url_api(attribute):
+    response = requests.post('https://urlhaus-api.abuse.ch/v1/url/', data={'url': attribute['value']}).json()
+    results = defaultdict(list)
+    if 'payloads' in response and response['payloads']:
+        objects_mapping = {1: _create_file_object, 2: _create_vt_object, 3: _create_objects_with_relationship}
+        file_keys = ('filename', 'response_size', 'response_md5', 'response_sha256')
+        file_relations = ('filename', 'size-in-bytes', 'md5', 'sha256')
+        vt_keys = ('result', 'link')
+        vt_types = ('text', 'link')
+        vt_relations = ('detection-ratio', 'permalink')
+        for payload in response['payloads']:
+            args = []
+            object_score = 0
+            file_attributes = {relation: {'type': relation, 'value': payload[key]} for key, relation in zip(file_keys, file_relations) if payload[key]}
+            if file_attributes:
+                object_score += 1
+                args.append(file_attributes)
+            if payload['virustotal']:
+                virustotal = payload['virustotal']
+                vt_attributes = {relation: {'type': vt_type, 'value': virustotal[key]} for key, vt_type, relation in zip(vt_keys, vt_types, vt_relations)}
+                if vt_attributes:
+                    object_score += 2
+                    args.append(vt_attributes)
+            try:
+                results['Object'].extend([misp_object.to_json() for misp_object in objects_mapping[object_score](*args)])
+            except KeyError:
+                continue
+    return {'results': results}
+
+
+_misp_type_mapping = {'url': _query_url_api, 'md5': _query_payload_api, 'sha256': _query_payload_api,
+                      'domain': _query_host_api, 'hostname': _query_host_api,
+                      'ip-src': _query_host_api, 'ip-dst': _query_host_api}
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    attribute = request['attribute']
+    return _misp_type_mapping[attribute['type']](attribute)
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

From 62bc45e03a2c0e64c18a1125691b8873b986b21d Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Thu, 14 Mar 2019 14:31:38 +0100
Subject: [PATCH 06/48] fix: Using to_dict on attributes & objects instead of
 to_json to make json_decode happy in the core part

---
 misp_modules/modules/expansion/urlhaus.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/misp_modules/modules/expansion/urlhaus.py b/misp_modules/modules/expansion/urlhaus.py
index 1a2c80b..17ea1ea 100644
--- a/misp_modules/modules/expansion/urlhaus.py
+++ b/misp_modules/modules/expansion/urlhaus.py
@@ -58,7 +58,7 @@ def _query_host_api(attribute):
     attributes = []
     if 'urls' in response and response['urls']:
         for url in response['urls']:
-            attributes.append(_create_url_attribute(url['url']).to_json())
+            attributes.append(_create_url_attribute(url['url']).to_dict())
     return {'results': {'Attribute': attributes}}
 
 
@@ -78,9 +78,9 @@ def _query_payload_api(attribute):
         file_object.add_attribute('filename', **{'type': 'filename', 'value': filename})
     for url in urls:
         attribute = _create_url_attribute(url)
-        results['Attribute'].append(attribute.to_json())
+        results['Attribute'].append(attribute.to_dict())
         file_object.add_reference(attribute.uuid, 'retrieved-from')
-    results['Object'].append(file_object.to_json())
+    results['Object'].append(file_object.to_dict())
     return {'results': results}
 
 
@@ -108,7 +108,7 @@ def _query_url_api(attribute):
                     object_score += 2
                     args.append(vt_attributes)
             try:
-                results['Object'].extend([misp_object.to_json() for misp_object in objects_mapping[object_score](*args)])
+                results['Object'].extend([misp_object.to_dict() for misp_object in objects_mapping[object_score](*args)])
             except KeyError:
                 continue
     return {'results': results}

From 0b92fd5a5393d7be54878aa0494fedd3f2598ceb Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Thu, 14 Mar 2019 18:48:13 +0100
Subject: [PATCH 07/48] fix: Making json_decode even happier with full json
 format

- Using MISPEvent because it is cleaner & easier
- Also cleaner implementation globally
---
 misp_modules/modules/expansion/urlhaus.py | 188 +++++++++++-----------
 1 file changed, 93 insertions(+), 95 deletions(-)

diff --git a/misp_modules/modules/expansion/urlhaus.py b/misp_modules/modules/expansion/urlhaus.py
index 17ea1ea..b6abd92 100644
--- a/misp_modules/modules/expansion/urlhaus.py
+++ b/misp_modules/modules/expansion/urlhaus.py
@@ -1,5 +1,5 @@
 from collections import defaultdict
-from pymisp import MISPAttribute, MISPObject
+from pymisp import MISPAttribute, MISPEvent, MISPObject
 import json
 import requests
 
@@ -12,111 +12,107 @@ moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
               'module-type': ['expansion', 'hover']}
 moduleconfig = []
 
-
-def _create_file_object(file_attributes):
-    return _create_object(file_attributes, 'file')
+file_keys = ('filename', 'response_size', 'response_md5', 'response_sha256')
+file_relations = ('filename', 'size-in-bytes', 'md5', 'sha256')
+vt_keys = ('result', 'link')
+vt_types = ('text', 'link')
+vt_relations = ('detection-ratio', 'permalink')
 
 
-def _create_object(attributes, name):
-    misp_object = MISPObject(name)
-    for relation, attribute in attributes.items():
-        misp_object.add_attribute(relation, **attribute)
-    return [misp_object]
+class URLhaus():
+    def __init__(self):
+        super(URLhaus, self).__init__()
+        self.misp_event = MISPEvent()
+
+    @staticmethod
+    def _create_vt_object(virustotal):
+        vt_object = MISPObject('virustotal-report')
+        for key, vt_type, relation in zip(vt_keys, vt_types, vt_relations):
+            vt_object.add_attribute(relation, **{'type': vt_type, 'value': virustotal[key]})
+        return vt_object
+
+    def get_result(self):
+        event = json.loads(self.misp_event.to_json())['Event']
+        results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
+        return {'results': results}
 
 
-def _create_objects_with_relationship(file_attributes, vt_attributes):
-    vt_object = _create_vt_object(vt_attributes)[0]
-    vt_uuid = vt_object.uuid
-    file_object = _create_file_object(file_attributes)[0]
-    file_object.add_reference(vt_uuid, 'analysed-with')
-    return [file_object, vt_object]
+class HostQuery(URLhaus):
+    def __init__(self, attribute):
+        super(HostQuery, self).__init__()
+        self.attribute = MISPAttribute()
+        self.attribute.from_dict(**attribute)
+        self.url = 'https://urlhaus-api.abuse.ch/v1/host/'
+
+    def query_api(self):
+        response = requests.post(self.url, data={'host': self.attribute.value}).json()
+        if 'urls' in response and response['urls']:
+            for url in response['urls']:
+                self.misp_event.add_attribute(type='url', value=url['url'])
 
 
-def _create_url_attribute(value):
-    attribute = MISPAttribute()
-    attribute.from_dict(type='url', value=value)
-    return attribute
+class PayloadQuery(URLhaus):
+    def __init__(self, attribute):
+        super(PayloadQuery, self).__init__()
+        self.attribute = MISPAttribute()
+        self.attribute.from_dict(**attribute)
+        self.url = 'https://urlhaus-api.abuse.ch/v1/payload/'
 
-
-def _create_vt_object(vt_attributes):
-    return _create_object(vt_attributes, 'virustotal_report')
-
-
-def _handle_payload_urls(response):
-    filenames = []
-    urls = []
-    if response:
-        for url in response:
-            urls.append(url['url'])
-            if url['filename']:
-                filenames.append(url['filename'])
-    return filenames, urls
-
-
-def _query_host_api(attribute):
-    response = requests.post('https://urlhaus-api.abuse.ch/v1/host/', data={'host': attribute['value']}).json()
-    attributes = []
-    if 'urls' in response and response['urls']:
+    def query_api(self):
+        hash_type = self.attribute.type
+        file_object = MISPObject('file')
+        if self.attribute.event_id != '0':
+            file_object.id = self.attribute.event_id
+        response = requests.post(self.url, data={'{}_hash'.format(hash_type): self.attribute.value}).json()
+        other_hash_type = 'md5' if hash_type == 'sha256' else 'sha256'
+        for key, relation in zip(('{}_hash'.format(other_hash_type), 'file_size'), (other_hash_type, 'size-in-bytes')):
+            if response[key]:
+                file_object.add_attribute(relation, **{'type': relation, 'value': response[key]})
+        if response['virustotal']:
+            vt_object = self._create_vt_object(response['virustotal'])
+            file_object.add_reference(vt_object.uuid, 'analyzed-with')
+            self.misp_event.add_object(**vt_object)
+        _filename_ = 'filename'
         for url in response['urls']:
-            attributes.append(_create_url_attribute(url['url']).to_dict())
-    return {'results': {'Attribute': attributes}}
+            attribute = MISPAttribute()
+            attribute.from_dict(type='url', value=url['url'])
+            self.misp_event.add_attribute(**attribute)
+            file_object.add_reference(attribute.uuid, 'retrieved-from')
+            if url[_filename_]:
+                file_object.add_attribute(_filename_, **{'type': _filename_, 'value': url[_filename_]})
+        self.misp_event.add_object(**file_object)
 
 
-def _query_payload_api(attribute):
-    hash_type = attribute['type']
-    response = requests.post('https://urlhaus-api.abuse.ch/v1/payload/', data={'{}_hash'.format(hash_type): attribute['value']}).json()
-    results = defaultdict(list)
-    filenames, urls = _handle_payload_urls(response['urls'])
-    other_hash_type = 'md5' if hash_type == 'sha256' else 'sha256'
-    file_object = MISPObject('file')
-    if attribute['object_id'] != '0':
-        file_object.id = attribute['object_id']
-    for key, relation in zip(('{}_hash'.format(other_hash_type), 'file_size'), (other_hash_type, 'size-in-bytes')):
-        if response[key]:
-            file_object.add_attribute(relation, **{'type': relation, 'value': response[key]})
-    for filename in filenames:
-        file_object.add_attribute('filename', **{'type': 'filename', 'value': filename})
-    for url in urls:
-        attribute = _create_url_attribute(url)
-        results['Attribute'].append(attribute.to_dict())
-        file_object.add_reference(attribute.uuid, 'retrieved-from')
-    results['Object'].append(file_object.to_dict())
-    return {'results': results}
+class UrlQuery(URLhaus):
+    def __init__(self, attribute):
+        super(UrlQuery, self).__init__()
+        self.attribute = MISPAttribute()
+        self.attribute.from_dict(**attribute)
+        self.url = 'https://urlhaus-api.abuse.ch/v1/url/'
+
+    @staticmethod
+    def _create_file_object(payload):
+        file_object = MISPObject('file')
+        for key, relation in zip(file_keys, file_relations):
+            if payload[key]:
+                file_object.add_attribute(relation, **{'type': relation, 'value': payload[key]})
+        return file_object
+
+    def query_api(self):
+        response = requests.post(self.url, data={'url': self.attribute.value}).json()
+        if 'payloads' in response and response['payloads']:
+            for payload in response['payloads']:
+                file_object = self._create_file_object(payload)
+                if payload['virustotal']:
+                    vt_object = self._create_vt_object(payload['virustotal'])
+                    file_object.add_reference(vt_object.uuid, 'analyzed-with')
+                    self.misp_event.add_object(**vt_object)
+                self.misp_event.add_object(**file_object)
 
 
-def _query_url_api(attribute):
-    response = requests.post('https://urlhaus-api.abuse.ch/v1/url/', data={'url': attribute['value']}).json()
-    results = defaultdict(list)
-    if 'payloads' in response and response['payloads']:
-        objects_mapping = {1: _create_file_object, 2: _create_vt_object, 3: _create_objects_with_relationship}
-        file_keys = ('filename', 'response_size', 'response_md5', 'response_sha256')
-        file_relations = ('filename', 'size-in-bytes', 'md5', 'sha256')
-        vt_keys = ('result', 'link')
-        vt_types = ('text', 'link')
-        vt_relations = ('detection-ratio', 'permalink')
-        for payload in response['payloads']:
-            args = []
-            object_score = 0
-            file_attributes = {relation: {'type': relation, 'value': payload[key]} for key, relation in zip(file_keys, file_relations) if payload[key]}
-            if file_attributes:
-                object_score += 1
-                args.append(file_attributes)
-            if payload['virustotal']:
-                virustotal = payload['virustotal']
-                vt_attributes = {relation: {'type': vt_type, 'value': virustotal[key]} for key, vt_type, relation in zip(vt_keys, vt_types, vt_relations)}
-                if vt_attributes:
-                    object_score += 2
-                    args.append(vt_attributes)
-            try:
-                results['Object'].extend([misp_object.to_dict() for misp_object in objects_mapping[object_score](*args)])
-            except KeyError:
-                continue
-    return {'results': results}
-
-
-_misp_type_mapping = {'url': _query_url_api, 'md5': _query_payload_api, 'sha256': _query_payload_api,
-                      'domain': _query_host_api, 'hostname': _query_host_api,
-                      'ip-src': _query_host_api, 'ip-dst': _query_host_api}
+_misp_type_mapping = {'url': UrlQuery, 'md5': PayloadQuery, 'sha256': PayloadQuery,
+                      'domain': HostQuery, 'hostname': HostQuery,
+                      'ip-src': HostQuery, 'ip-dst': HostQuery}
 
 
 def handler(q=False):
@@ -124,7 +120,9 @@ def handler(q=False):
         return False
     request = json.loads(q)
     attribute = request['attribute']
-    return _misp_type_mapping[attribute['type']](attribute)
+    urlhaus_parser = _misp_type_mapping[attribute['type']](attribute)
+    urlhaus_parser.query_api()
+    return urlhaus_parser.get_result()
 
 
 def introspection():

From 2439d5f75ddeebdf8410586d13c8825449ec6cce Mon Sep 17 00:00:00 2001
From: root <chris.studer.68@gmail.com>
Date: Mon, 1 Apr 2019 16:28:19 +0200
Subject: [PATCH 08/48] fix: Fixed object_id variable name typo

---
 misp_modules/modules/expansion/urlhaus.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/misp_modules/modules/expansion/urlhaus.py b/misp_modules/modules/expansion/urlhaus.py
index b6abd92..dae6fd6 100644
--- a/misp_modules/modules/expansion/urlhaus.py
+++ b/misp_modules/modules/expansion/urlhaus.py
@@ -62,7 +62,7 @@ class PayloadQuery(URLhaus):
         hash_type = self.attribute.type
         file_object = MISPObject('file')
         if self.attribute.event_id != '0':
-            file_object.id = self.attribute.event_id
+            file_object.id = self.attribute.object_id
         response = requests.post(self.url, data={'{}_hash'.format(hash_type): self.attribute.value}).json()
         other_hash_type = 'md5' if hash_type == 'sha256' else 'sha256'
         for key, relation in zip(('{}_hash'.format(other_hash_type), 'file_size'), (other_hash_type, 'size-in-bytes')):

From 92351e66792c289f327abaace50b955e75bca569 Mon Sep 17 00:00:00 2001
From: root <chris.studer.68@gmail.com>
Date: Wed, 1 May 2019 22:22:10 +0200
Subject: [PATCH 09/48] add: Added urlhaus in the expansion modules init list

---
 misp_modules/modules/expansion/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py
index 70aca68..9c37970 100644
--- a/misp_modules/modules/expansion/__init__.py
+++ b/misp_modules/modules/expansion/__init__.py
@@ -10,4 +10,4 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c
            'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io',
            'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors',
            'qrcode', 'ocr-enrich', 'pdf-enrich', 'docx-enrich', 'xlsx-enrich', 'pptx-enrich',
-           'ods-enrich', 'odt-enrich']
+           'ods-enrich', 'odt-enrich', 'urlhaus']

From db74c5f49a1b5b4ed8200637dc32c9a32a75d1fd Mon Sep 17 00:00:00 2001
From: root <chris.studer.68@gmail.com>
Date: Wed, 1 May 2019 22:26:53 +0200
Subject: [PATCH 10/48] fix: Fixed libraries import that changed with the
 latest merge

---
 misp_modules/modules/import_mod/csvimport.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py
index 577229b..2eec1c6 100644
--- a/misp_modules/modules/import_mod/csvimport.py
+++ b/misp_modules/modules/import_mod/csvimport.py
@@ -1,4 +1,7 @@
 # -*- coding: utf-8 -*-
+from collections import defaultdict
+import csv
+import io
 import json
 import os
 import base64
@@ -162,7 +165,7 @@ class CsvParser():
                     self.result.append(attribute)
 
     def findMispTypes(self):
-        descFilename = os.path.join(pymisp_path[0], 'data/describeTypes.json')
+        descFilename = os.path.join(pymisp.__path__[0], 'data/describeTypes.json')
         with open(descFilename, 'r') as f:
             MispTypes = json.loads(f.read())['result'].get('types')
         list2pop = []

From f900cb7c68837204714f85e2e3e6c9e04042c794 Mon Sep 17 00:00:00 2001
From: root <chris.studer.68@gmail.com>
Date: Wed, 1 May 2019 22:28:19 +0200
Subject: [PATCH 11/48] fix: Fixed introspection fields for csvimport &
 goamlimport

- Added format field for goaml so the module is
  known as returning MISP attributes & objects
- Fixed introspection to make the format, user
  config and input source fields visible from
  MISP (format also added at the same time)
---
 misp_modules/modules/import_mod/csvimport.py   | 15 ++-------------
 misp_modules/modules/import_mod/goamlimport.py |  3 ++-
 2 files changed, 4 insertions(+), 14 deletions(-)

diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py
index 2eec1c6..bb86014 100644
--- a/misp_modules/modules/import_mod/csvimport.py
+++ b/misp_modules/modules/import_mod/csvimport.py
@@ -12,7 +12,6 @@ moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
               'description': 'Import Attributes from a csv file.',
               'module-type': ['import']}
 moduleconfig = []
-inputSource = ['file']
 userConfig = {'header': {
     'type': 'String',
     'message': 'Define the header of the csv file, with types (included in MISP attribute types or attribute fields) separated by commas.\nFor fields that do not match these types, please use space or simply nothing between commas.\nFor instance: ip-src,domain, ,timestamp'},
@@ -20,6 +19,7 @@ userConfig = {'header': {
         'type': 'Boolean',
         'message': 'Tick this box ONLY if there is a header line, NOT COMMENTED, in the file (which will be skipped atm).'
 }}
+mispattributes = {'userConfig': userConfig, 'inputSource': ['file'], 'format': 'misp_standard'}
 
 duplicatedFields = {'mispType': {'mispComment': 'comment'},
                     'attrField': {'attrComment': 'comment'}}
@@ -224,18 +224,7 @@ def handler(q=False):
 
 
 def introspection():
-    modulesetup = {}
-    try:
-        userConfig
-        modulesetup['userConfig'] = userConfig
-    except NameError:
-        pass
-    try:
-        inputSource
-        modulesetup['inputSource'] = inputSource
-    except NameError:
-        pass
-    return modulesetup
+    return mispattributes
 
 
 def version():
diff --git a/misp_modules/modules/import_mod/goamlimport.py b/misp_modules/modules/import_mod/goamlimport.py
index 7116b44..add6470 100644
--- a/misp_modules/modules/import_mod/goamlimport.py
+++ b/misp_modules/modules/import_mod/goamlimport.py
@@ -9,7 +9,8 @@ moduleinfo = {'version': 1, 'author': 'Christian Studer',
               'description': 'Import from GoAML',
               'module-type': ['import']}
 moduleconfig = []
-mispattributes = {'inputSource': ['file'], 'output': ['MISP objects']}
+mispattributes = {'inputSource': ['file'], 'output': ['MISP objects'],
+                  'format': 'misp_standard'}
 
 t_from_objects = {'nodes': ['from_person', 'from_account', 'from_entity'],
                   'leaves': ['from_funds_code', 'from_country']}

From c886247a649c3b6f70463a009d4d1960824378b8 Mon Sep 17 00:00:00 2001
From: root <chris.studer.68@gmail.com>
Date: Wed, 1 May 2019 22:32:06 +0200
Subject: [PATCH 12/48] fix: Fixed standard MISP csv format header

- The csv header we can find in data produced from
  MISP restSearch csv format is the one to use to
  recognize a csv file produced by MISP
---
 misp_modules/modules/import_mod/csvimport.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py
index bb86014..ec55a10 100644
--- a/misp_modules/modules/import_mod/csvimport.py
+++ b/misp_modules/modules/import_mod/csvimport.py
@@ -24,11 +24,11 @@ mispattributes = {'userConfig': userConfig, 'inputSource': ['file'], 'format': '
 duplicatedFields = {'mispType': {'mispComment': 'comment'},
                     'attrField': {'attrComment': 'comment'}}
 attributesFields = ['type', 'value', 'category', 'to_ids', 'comment', 'distribution']
-misp_standard_csv_header = ['uuid','event_id','category','type','value','comment','to_ids','date',
-                            'object_relation','object_uuid','object_name','object_meta_category']
+misp_standard_csv_header = ['uuid', 'event_id', 'category', 'type', 'value', 'comment', 'to_ids', 'date',
+                            'object_relation', 'attribute_tag', 'object_uuid', 'object_name', 'object_meta_category']
 misp_context_additional_fields = ['event_info','event_member_org','event_source_org','event_distribution',
                                   'event_threat_level_id','event_analysis','event_date','event_tag']
-misp_extended_csv_header = misp_standard_csv_header[:9] + ['attribute_tag'] + misp_standard_csv_header[9:] + misp_context_additional_fields
+misp_extended_csv_header = misp_standard_csv_header + misp_context_additional_fields
 delimiters = [',', ';', '|', '/', '\t', '    ']
 
 

From 6f4b88606b0d30b8f85e7a3a52d77599ab21eea1 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Thu, 2 May 2019 14:07:36 +0200
Subject: [PATCH 13/48] fix: Make pep8 happy

---
 misp_modules/modules/import_mod/csvimport.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py
index ec55a10..8e8bf1c 100644
--- a/misp_modules/modules/import_mod/csvimport.py
+++ b/misp_modules/modules/import_mod/csvimport.py
@@ -26,8 +26,8 @@ duplicatedFields = {'mispType': {'mispComment': 'comment'},
 attributesFields = ['type', 'value', 'category', 'to_ids', 'comment', 'distribution']
 misp_standard_csv_header = ['uuid', 'event_id', 'category', 'type', 'value', 'comment', 'to_ids', 'date',
                             'object_relation', 'attribute_tag', 'object_uuid', 'object_name', 'object_meta_category']
-misp_context_additional_fields = ['event_info','event_member_org','event_source_org','event_distribution',
-                                  'event_threat_level_id','event_analysis','event_date','event_tag']
+misp_context_additional_fields = ['event_info', 'event_member_org', 'event_source_org', 'event_distribution',
+                                  'event_threat_level_id', 'event_analysis', 'event_date', 'event_tag']
 misp_extended_csv_header = misp_standard_csv_header + misp_context_additional_fields
 delimiters = [',', ';', '|', '/', '\t', '    ']
 
@@ -84,7 +84,8 @@ class CsvParser():
                     return_data.append(line)
             # find which delimiter is used
             self.delimiter = self.find_delimiter()
-            if self.fields_number == 0: self.header = return_data[0].split(self.delimiter)
+            if self.fields_number == 0:
+                self.header = return_data[0].split(self.delimiter)
         self.data = return_data[1:] if self.has_header else return_data
 
     def parse_delimiter(self, line):
@@ -112,10 +113,11 @@ class CsvParser():
             attribute = {}
             try:
                 try:
-                    a_uuid,_,a_category,a_type,value,comment,to_ids,_,relation,o_uuid,o_name,o_category = line[:header_length]
+                    a_uuid, _, a_category, a_type, value, comment, to_ids, _, relation, o_uuid, o_name, o_category = line[:header_length]
                 except ValueError:
-                    a_uuid,_,a_category,a_type,value,comment,to_ids,_,relation,tag,o_uuid,o_name,o_category = line[:header_length]
-                    if tag: attribute['tags'] = tag
+                    a_uuid, _, a_category, a_type, value, comment, to_ids, _, relation, tag, o_uuid, o_name, o_category = line[:header_length]
+                    if tag:
+                        attribute['tags'] = tag
             except ValueError:
                 continue
             for t, v in zip(attribute_fields, [a_uuid, a_category, a_type, value, comment]):
@@ -123,13 +125,13 @@ class CsvParser():
             attribute['to_ids'] = True if to_ids == '1' else False
             if relation:
                 attribute["object_relation"] = relation.replace('"', '')
-                object_index = tuple(o.replace('"', '') for o in (o_uuid,o_name,o_category))
+                object_index = tuple(o.replace('"', '') for o in (o_uuid, o_name, o_category))
                 objects[object_index].append(attribute)
             else:
                 l_attributes.append(attribute)
         for keys, attributes in objects.items():
             misp_object = {}
-            for t, v in zip(['uuid','name','meta-category'], keys):
+            for t, v in zip(['uuid', 'name', 'meta-category'], keys):
                 misp_object[t] = v
             misp_object['Attribute'] = attributes
             l_objects.append(misp_object)

From d4bc85259db1a95c4d794cd804346a4174b15662 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Thu, 2 May 2019 14:15:12 +0200
Subject: [PATCH 14/48] fix: Removed unused library

---
 misp_modules/modules/expansion/urlhaus.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/misp_modules/modules/expansion/urlhaus.py b/misp_modules/modules/expansion/urlhaus.py
index dae6fd6..12893b9 100644
--- a/misp_modules/modules/expansion/urlhaus.py
+++ b/misp_modules/modules/expansion/urlhaus.py
@@ -1,4 +1,3 @@
-from collections import defaultdict
 from pymisp import MISPAttribute, MISPEvent, MISPObject
 import json
 import requests

From ae5bd8d06a07ddc67b581ca8e1483f4068a6333c Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Mon, 6 May 2019 22:15:14 +0200
Subject: [PATCH 15/48] fix: Clearer user config messages displayed in the
 import view

---
 misp_modules/modules/import_mod/csvimport.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py
index 8e8bf1c..ebd09d7 100644
--- a/misp_modules/modules/import_mod/csvimport.py
+++ b/misp_modules/modules/import_mod/csvimport.py
@@ -14,10 +14,10 @@ moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
 moduleconfig = []
 userConfig = {'header': {
     'type': 'String',
-    'message': 'Define the header of the csv file, with types (included in MISP attribute types or attribute fields) separated by commas.\nFor fields that do not match these types, please use space or simply nothing between commas.\nFor instance: ip-src,domain, ,timestamp'},
+    'message': 'Define the header of the csv file, with types (included in MISP attribute types or attribute fields) separated by commas.\nFor fields that do not match these types or that you want to skip, please use space or simply nothing between commas.\nFor instance: ip-src,domain, ,timestamp'},
     'has_header': {
         'type': 'Boolean',
-        'message': 'Tick this box ONLY if there is a header line, NOT COMMENTED, in the file (which will be skipped atm).'
+        'message': 'Tick this box ONLY if there is a header line, NOT COMMENTED, and all the fields of this header are respecting the recommendations above.'
 }}
 mispattributes = {'userConfig': userConfig, 'inputSource': ['file'], 'format': 'misp_standard'}
 

From 28eb92da53e9c53a3fe4b9fa0c2fe9ad02a37377 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Mon, 6 May 2019 22:16:14 +0200
Subject: [PATCH 16/48] fix: Using pymisp classes & methods to parse the module
 results

---
 misp_modules/modules/import_mod/csvimport.py | 58 ++++++++++----------
 1 file changed, 28 insertions(+), 30 deletions(-)

diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py
index ebd09d7..6701d3d 100644
--- a/misp_modules/modules/import_mod/csvimport.py
+++ b/misp_modules/modules/import_mod/csvimport.py
@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 from collections import defaultdict
+from pymisp import MISPEvent, MISPObject
+from pymisp import __path__ as pymisp_path
 import csv
 import io
 import json
@@ -35,6 +37,7 @@ delimiters = [',', ';', '|', '/', '\t', '    ']
 class CsvParser():
     def __init__(self, header, has_header, data):
         data_header = data[0]
+        self.misp_event = MISPEvent()
         if data_header == misp_standard_csv_header or data_header == misp_extended_csv_header:
             self.header = misp_standard_csv_header if data_header == misp_standard_csv_header else misp_extended_csv_header[:13]
             self.from_misp = True
@@ -50,7 +53,6 @@ class CsvParser():
                 self.has_delimiter = True
                 self.fields_number, self.delimiter, self.header = self.get_delimiter_from_header(data[0])
                 self.data = data
-            self.result = []
 
     def get_delimiter_from_header(self, data):
         delimiters_count = {}
@@ -104,38 +106,30 @@ class CsvParser():
             self.buildAttributes()
 
     def build_misp_event(self):
-        l_attributes = []
-        l_objects = []
-        objects = defaultdict(list)
+        objects = {}
         header_length = len(self.header)
-        attribute_fields = self.header[:1] + self.header[2:6]
+        attribute_fields = self.header[:1] + self.header[2:6] + self.header[7:8]
         for line in self.data:
             attribute = {}
             try:
-                try:
-                    a_uuid, _, a_category, a_type, value, comment, to_ids, _, relation, o_uuid, o_name, o_category = line[:header_length]
-                except ValueError:
-                    a_uuid, _, a_category, a_type, value, comment, to_ids, _, relation, tag, o_uuid, o_name, o_category = line[:header_length]
-                    if tag:
-                        attribute['tags'] = tag
+                a_uuid, _, a_category, a_type, value, comment, to_ids, timestamp, relation, tag, o_uuid, o_name, o_category = line[:header_length]
             except ValueError:
                 continue
-            for t, v in zip(attribute_fields, [a_uuid, a_category, a_type, value, comment]):
-                attribute[t] = v.replace('"', '')
+            for t, v in zip(attribute_fields, (a_uuid, a_category, a_type, value, comment, timestamp)):
+                attribute[t] = v.strip('"')
             attribute['to_ids'] = True if to_ids == '1' else False
+            if tag:
+                attribute['Tag'] = [{'name': t.strip()} for t in tag.split(',')]
             if relation:
-                attribute["object_relation"] = relation.replace('"', '')
-                object_index = tuple(o.replace('"', '') for o in (o_uuid, o_name, o_category))
-                objects[object_index].append(attribute)
+                if o_uuid not in objects:
+                    objects[o_uuid] = MISPObject(o_name)
+                objects[o_uuid].add_attribute(relation, **attribute)
             else:
-                l_attributes.append(attribute)
-        for keys, attributes in objects.items():
-            misp_object = {}
-            for t, v in zip(['uuid', 'name', 'meta-category'], keys):
-                misp_object[t] = v
-            misp_object['Attribute'] = attributes
-            l_objects.append(misp_object)
-        self.result = {"Attribute": l_attributes, "Object": l_objects}
+                self.misp_event.add_attribute(**attribute)
+        for uuid, misp_object in objects.items():
+            misp_object.uuid = uuid
+            self.misp_event.add_object(**misp_object)
+        self.finalize_results()
 
     def buildAttributes(self):
         # if there is only 1 field of data
@@ -144,7 +138,7 @@ class CsvParser():
             for data in self.data:
                 d = data.strip()
                 if d:
-                    self.result.append({'types': mispType, 'values': d})
+                    self.misp_event.add_attribute(**{'type': mispType, 'value': d})
         else:
             # split fields that should be recognized as misp attribute types from the others
             list2pop, misp, head = self.findMispTypes()
@@ -160,14 +154,15 @@ class CsvParser():
                     datamisp.append(datasplit.pop(l).strip())
                 # for each misp type, we create an attribute
                 for m, dm in zip(misp, datamisp):
-                    attribute = {'types': m, 'values': dm}
+                    attribute = {'type': m, 'value': dm}
                     for h, ds in zip(head, datasplit):
                         if h:
                             attribute[h] = ds.strip()
-                    self.result.append(attribute)
+                    self.misp_event.add_attribute(**attribute)
+        self.finalize_results()
 
     def findMispTypes(self):
-        descFilename = os.path.join(pymisp.__path__[0], 'data/describeTypes.json')
+        descFilename = os.path.join(pymisp_path[0], 'data/describeTypes.json')
         with open(descFilename, 'r') as f:
             MispTypes = json.loads(f.read())['result'].get('types')
         list2pop = []
@@ -196,6 +191,10 @@ class CsvParser():
         # return list of indexes of the misp types, list of the misp types, remaining fields that will be attribute fields
         return list2pop, misp, list(reversed(head))
 
+    def finalize_results(self):
+        event=json.loads(self.misp_event.to_json())['Event']
+        self.results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
+
 
 def handler(q=False):
     if q is False:
@@ -221,8 +220,7 @@ def handler(q=False):
     csv_parser = CsvParser(header, has_header, data)
     # build the attributes
     csv_parser.parse_csv()
-    r = {'results': csv_parser.result}
-    return r
+    return {'results': csv_parser.results}
 
 
 def introspection():

From f1b5f05bb33be21f2322dfdcbccf141b0a18b81b Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Tue, 7 May 2019 09:35:56 +0200
Subject: [PATCH 17/48] fix: Checking not MISP header fields

- Rejecting fields not recognizable by MISP
---
 misp_modules/modules/import_mod/csvimport.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py
index 6701d3d..fd2f27b 100644
--- a/misp_modules/modules/import_mod/csvimport.py
+++ b/misp_modules/modules/import_mod/csvimport.py
@@ -53,6 +53,13 @@ class CsvParser():
                 self.has_delimiter = True
                 self.fields_number, self.delimiter, self.header = self.get_delimiter_from_header(data[0])
                 self.data = data
+            descFilename = os.path.join(pymisp_path[0], 'data/describeTypes.json')
+            with open(descFilename, 'r') as f:
+                self.MispTypes = json.loads(f.read())['result'].get('types')
+            for h in self.header:
+                if not (h in self.MispTypes or h in misp_extended_csv_header):
+                    misperrors['error'] = 'Wrong header field: {}. Please use a header value that can be recognized by MISP (or alternatively skip it using a whitespace).'.format(h)
+                    return misperrors
 
     def get_delimiter_from_header(self, data):
         delimiters_count = {}
@@ -162,16 +169,13 @@ class CsvParser():
         self.finalize_results()
 
     def findMispTypes(self):
-        descFilename = os.path.join(pymisp_path[0], 'data/describeTypes.json')
-        with open(descFilename, 'r') as f:
-            MispTypes = json.loads(f.read())['result'].get('types')
         list2pop = []
         misp = []
         head = []
         for h in reversed(self.header):
             n = self.header.index(h)
             # fields that are misp attribute types
-            if h in MispTypes:
+            if h in self.MispTypes:
                 list2pop.append(n)
                 misp.append(h)
             # handle confusions between misp attribute types and attribute fields

From 77db21cf181034a7569b60a85c13771e1773cad0 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Tue, 7 May 2019 09:37:21 +0200
Subject: [PATCH 18/48] fix: Making pep8 happy

---
 misp_modules/modules/import_mod/csvimport.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py
index fd2f27b..5d7408c 100644
--- a/misp_modules/modules/import_mod/csvimport.py
+++ b/misp_modules/modules/import_mod/csvimport.py
@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-from collections import defaultdict
 from pymisp import MISPEvent, MISPObject
 from pymisp import __path__ as pymisp_path
 import csv
@@ -7,7 +6,6 @@ import io
 import json
 import os
 import base64
-import pymisp
 
 misperrors = {'error': 'Error'}
 moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
@@ -196,7 +194,7 @@ class CsvParser():
         return list2pop, misp, list(reversed(head))
 
     def finalize_results(self):
-        event=json.loads(self.misp_event.to_json())['Event']
+        event = json.loads(self.misp_event.to_json())['Event']
         self.results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
 
 

From 728386d8a0ff0ac32856ccf48a4b37ed53542d1a Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Wed, 8 May 2019 16:52:49 +0200
Subject: [PATCH 19/48] add: [new_module] Module to import data from Joe
 sandbox reports

- Parsing file, pe and pe-section objects from the
  report file info field
- Deeper file info parsing to come
- Other fields parsing to come as well
---
 misp_modules/modules/import_mod/joe_import.py | 128 ++++++++++++++++++
 1 file changed, 128 insertions(+)
 create mode 100644 misp_modules/modules/import_mod/joe_import.py

diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
new file mode 100644
index 0000000..bc14ba8
--- /dev/null
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+from pymisp import MISPEvent, MISPObject
+import json
+import base64
+
+misperrors = {'error': 'Error'}
+userConfig = {}
+inputSource = ['file']
+
+moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
+              'description': 'Import for Joe Sandbox JSON reports',
+              'module-type': ['import']}
+
+moduleconfig = []
+
+file_object_fields = ['filename', 'md5', 'sha1', 'sha256', 'sha512', 'ssdeep']
+file_object_mapping = {'entropy': ('float', 'entropy'),
+                       'filesize': ('size-in-bytes', 'size-in-bytes'),
+                       'filetype': ('mime-type', 'mimetype')}
+pe_object_fields = {'entrypoint': ('text', 'entrypoint-address'),
+                    'imphash': ('imphash', 'imphash')}
+pe_object_mapping = {'CompanyName': 'company-name', 'FileDescription': 'file-description',
+                     'FileVersion': 'file-version', 'InternalName': 'internal-filename',
+                     'LegalCopyright': 'legal-copyright', 'OriginalFilename': 'original-filename',
+                     'ProductName': 'product-filename', 'ProductVersion': 'product-version',
+                     'Translation': 'lang-id'}
+section_object_mapping = {'characteristics': ('text', 'characteristic'),
+                          'entropy': ('float', 'entropy'),
+                          'name': ('text', 'name'), 'rawaddr': ('hex', 'offset'),
+                          'rawsize': ('size-in-bytes', 'size-in-bytes'),
+                          'virtaddr': ('hex', 'virtual_address'),
+                          'virtsize': ('size-in-bytes', 'virtual_size')}
+signerinfo_object_mapping = {'sigissuer': ('text', 'issuer'),
+                             'version': ('text', 'version')}
+
+
+class JoeParser():
+    def __init__(self, data):
+        self.data = data
+        self.misp_event = MISPEvent()
+
+    def parse_joe(self):
+        self.parse_fileinfo()
+        self.finalize_results()
+
+    def parse_fileinfo(self):
+        fileinfo = self.data['fileinfo']
+        file_object = MISPObject('file')
+        for field in file_object_fields:
+            file_object.add_attribute(field, **{'type': field, 'value': fileinfo[field]})
+        for field, mapping in file_object_mapping.items():
+            attribute_type, object_relation = mapping
+            file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': fileinfo[field]})
+        pe_object = MISPObject('pe')
+        file_object.add_reference(pe_object.uuid, 'included-in')
+        self.misp_event.add_object(**file_object)
+        peinfo = fileinfo['pe']
+        for field, mapping in pe_object_fields.items():
+            attribute_type, object_relation = mapping
+            pe_object.add_attribute(object_relation, **{'type': attribute_type, 'value': peinfo[field]})
+        pe_object.add_attribute('compilation-timestamp', **{'type': 'datetime', 'value': int(peinfo['timestamp'].split()[0], 16)})
+        program_name = fileinfo['filename']
+        for feature in peinfo['versions']['version']:
+            name = feature['name']
+            if name == 'InternalName':
+                program_name = feature['value']
+            pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value']})
+        sections_number = len(peinfo['sections']['section'])
+        pe_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number})
+        for section in peinfo['sections']['section']:
+            section_object = self.parse_pe_section(section)
+            pe_object.add_reference(section_object.uuid, 'included-in')
+            self.misp_event.add_object(**section_object)
+        signerinfo_object = MISPObject('authenticode-signerinfo')
+        pe_object.add_reference(signerinfo_object.uuid, 'signed-by')
+        self.misp_event.add_object(**pe_object)
+        signerinfo_object.add_attribute('program-name', **{'type': 'text', 'value': program_name})
+        signatureinfo = peinfo['signature']
+        for feature, mapping in signerinfo_object_mapping.items():
+            attribute_type, object_relation = mapping
+            signerinfo_object.add_attribute(object_relation, **{'type': attribute_type, 'value': signatureinfo[feature]})
+        self.misp_event.add_object(**signerinfo_object)
+
+    def parse_pe_section(self, section):
+        section_object = MISPObject('pe-section')
+        for feature, mapping in section_object_mapping.items():
+            attribute_type, object_relation = mapping
+            section_object.add_attribute(object_relation, **{'type': attribute_type, 'value': section[feature]})
+        return section_object
+
+    def finalize_results(self):
+        event = json.loads(self.misp_event.to_json())['Event']
+        self.results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    q = json.loads(q)
+    data = base64.b64decode(q.get('data')).decode('utf-8')
+    if not data:
+        return json.dumps({'success': 0})
+    joe_data = json.loads(data)['analysis']
+    print(type(joe_data))
+    joe_parser = JoeParser(joe_data)
+    joe_parser.parse_joe()
+    return {'results': joe_parser.results}
+
+
+def introspection():
+    modulesetup = {}
+    try:
+        userConfig
+        modulesetup['userConfig'] = userConfig
+    except NameError:
+        pass
+    try:
+        inputSource
+        modulesetup['inputSource'] = inputSource
+    except NameError:
+        pass
+    modulesetup['format'] = 'misp_standard'
+    return modulesetup
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

From d39fb7da184f2335a2b860b384106c57c6a0bffc Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Mon, 13 May 2019 17:29:07 +0200
Subject: [PATCH 20/48] add: Parsing some object references at the end of the
 process

---
 misp_modules/modules/import_mod/joe_import.py | 20 +++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index bc14ba8..e378bc2 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+from collections import defaultdict
 from pymisp import MISPEvent, MISPObject
 import json
 import base64
@@ -38,11 +39,21 @@ class JoeParser():
     def __init__(self, data):
         self.data = data
         self.misp_event = MISPEvent()
+        self.references = defaultdict(list)
 
     def parse_joe(self):
         self.parse_fileinfo()
+        if self.references:
+            self.build_references()
         self.finalize_results()
 
+    def build_references(self):
+        for misp_object in self.misp_event.objects:
+            object_uuid = misp_object.uuid
+            if object_uuid in self.references:
+                for reference in self.references[object_uuid]:
+                    misp_object.add_reference(reference['idref'], reference['relationship'])
+
     def parse_fileinfo(self):
         fileinfo = self.data['fileinfo']
         file_object = MISPObject('file')
@@ -54,6 +65,7 @@ class JoeParser():
         pe_object = MISPObject('pe')
         file_object.add_reference(pe_object.uuid, 'included-in')
         self.misp_event.add_object(**file_object)
+        self.fileinfo_uuid = file_object.uuid
         peinfo = fileinfo['pe']
         for field, mapping in pe_object_fields.items():
             attribute_type, object_relation = mapping
@@ -67,10 +79,6 @@ class JoeParser():
             pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value']})
         sections_number = len(peinfo['sections']['section'])
         pe_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number})
-        for section in peinfo['sections']['section']:
-            section_object = self.parse_pe_section(section)
-            pe_object.add_reference(section_object.uuid, 'included-in')
-            self.misp_event.add_object(**section_object)
         signerinfo_object = MISPObject('authenticode-signerinfo')
         pe_object.add_reference(signerinfo_object.uuid, 'signed-by')
         self.misp_event.add_object(**pe_object)
@@ -80,6 +88,10 @@ class JoeParser():
             attribute_type, object_relation = mapping
             signerinfo_object.add_attribute(object_relation, **{'type': attribute_type, 'value': signatureinfo[feature]})
         self.misp_event.add_object(**signerinfo_object)
+        for section in peinfo['sections']['section']:
+            section_object = self.parse_pe_section(section)
+            self.references[pe_object.uuid].append({'idref': section_object.uuid, 'relationship': 'included-in'})
+            self.misp_event.add_object(**section_object)
 
     def parse_pe_section(self, section):
         section_object = MISPObject('pe-section')

From 29e681ef81ceea22ec3b91936fb89957ec4e5c83 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Mon, 13 May 2019 17:30:01 +0200
Subject: [PATCH 21/48] add: Parsing processes called by the file analyzed in
 the joe sandbox report

---
 misp_modules/modules/import_mod/joe_import.py | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index e378bc2..bae920f 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 from collections import defaultdict
+from datetime import datetime
 from pymisp import MISPEvent, MISPObject
 import json
 import base64
@@ -25,6 +26,9 @@ pe_object_mapping = {'CompanyName': 'company-name', 'FileDescription': 'file-des
                      'LegalCopyright': 'legal-copyright', 'OriginalFilename': 'original-filename',
                      'ProductName': 'product-filename', 'ProductVersion': 'product-version',
                      'Translation': 'lang-id'}
+process_object_fields = {'cmdline': 'command-line', 'name': 'name',
+                         'parentpid': 'parent-pid', 'pid': 'pid',
+                         'path': 'current-directory'}
 section_object_mapping = {'characteristics': ('text', 'characteristic'),
                           'entropy': ('float', 'entropy'),
                           'name': ('text', 'name'), 'rawaddr': ('hex', 'offset'),
@@ -43,6 +47,7 @@ class JoeParser():
 
     def parse_joe(self):
         self.parse_fileinfo()
+        self.parse_behavior()
         if self.references:
             self.build_references()
         self.finalize_results()
@@ -54,6 +59,24 @@ class JoeParser():
                 for reference in self.references[object_uuid]:
                     misp_object.add_reference(reference['idref'], reference['relationship'])
 
+    def parse_behavior(self):
+        self.parse_behavior_system()
+        self.parse_behavior_network()
+
+    def parse_behavior_network(self):
+        network = self.data['behavior']['network']
+
+    def parse_behavior_system(self):
+        processes = self.data['behavior']['system']['processes']['process'][0]
+        general = processes['general']
+        process_object = MISPObject('process')
+        for feature, relation in process_object_fields.items():
+            process_object.add_attribute(relation, **{'type': 'text', 'value': general[feature]})
+        start_time = datetime.strptime('{} {}'.format(general['date'], general['time']), '%d/%m/%Y %H:%M:%S')
+        process_object.add_attribute('start-time', **{'type': 'datetime', 'value': start_time})
+        self.misp_event.add_object(**process_object)
+        self.references[self.fileinfo_uuid].append({'idref': process_object.uuid, 'relationship': 'calls'})
+
     def parse_fileinfo(self):
         fileinfo = self.data['fileinfo']
         file_object = MISPObject('file')

From df7047dff0bd0f36441965e4d3a53b70f457590e Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Tue, 14 May 2019 10:50:11 +0200
Subject: [PATCH 22/48] fix: Fixed output format to match with the recent
 changes on modules

---
 misp_modules/modules/import_mod/goamlimport.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/misp_modules/modules/import_mod/goamlimport.py b/misp_modules/modules/import_mod/goamlimport.py
index add6470..79b4cfe 100644
--- a/misp_modules/modules/import_mod/goamlimport.py
+++ b/misp_modules/modules/import_mod/goamlimport.py
@@ -165,7 +165,7 @@ def handler(q=False):
         misperrors['error'] = "Impossible to read XML data"
         return misperrors
     aml_parser.parse_xml()
-    r = {'results': [obj.to_json() for obj in aml_parser.misp_event.objects]}
+    r = {'results': {'Object': [obj.to_json() for obj in aml_parser.misp_event.objects]}}
     return r
 
 

From fc8a56d1d9874a85afc9ee75b2835617c1d601fa Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Wed, 15 May 2019 15:49:29 +0200
Subject: [PATCH 23/48] fix: Removed test print

---
 misp_modules/modules/import_mod/joe_import.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index bae920f..206f108 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -136,7 +136,6 @@ def handler(q=False):
     if not data:
         return json.dumps({'success': 0})
     joe_data = json.loads(data)['analysis']
-    print(type(joe_data))
     joe_parser = JoeParser(joe_data)
     joe_parser.parse_joe()
     return {'results': joe_parser.results}

From d195b554a5165cd21d2ea1fbb46b3c24ff808365 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Wed, 15 May 2019 22:05:03 +0200
Subject: [PATCH 24/48] fix: Testing if some fields exist before trying to
 import them

- Testing for pe itself, pe versions and pe signature
---
 misp_modules/modules/import_mod/joe_import.py | 25 +++++++++++--------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index 206f108..9e12cfb 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -85,21 +85,25 @@ class JoeParser():
         for field, mapping in file_object_mapping.items():
             attribute_type, object_relation = mapping
             file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': fileinfo[field]})
+        self.fileinfo_uuid = file_object.uuid
+        if not fileinfo.get('pe'):
+            self.misp_event.add_object(**file_object)
+            return
+        peinfo = fileinfo['pe']
         pe_object = MISPObject('pe')
         file_object.add_reference(pe_object.uuid, 'included-in')
         self.misp_event.add_object(**file_object)
-        self.fileinfo_uuid = file_object.uuid
-        peinfo = fileinfo['pe']
         for field, mapping in pe_object_fields.items():
             attribute_type, object_relation = mapping
             pe_object.add_attribute(object_relation, **{'type': attribute_type, 'value': peinfo[field]})
         pe_object.add_attribute('compilation-timestamp', **{'type': 'datetime', 'value': int(peinfo['timestamp'].split()[0], 16)})
         program_name = fileinfo['filename']
-        for feature in peinfo['versions']['version']:
-            name = feature['name']
-            if name == 'InternalName':
-                program_name = feature['value']
-            pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value']})
+        if peinfo['versions']:
+            for feature in peinfo['versions']['version']:
+                name = feature['name']
+                if name == 'InternalName':
+                    program_name = feature['value']
+                pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value']})
         sections_number = len(peinfo['sections']['section'])
         pe_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number})
         signerinfo_object = MISPObject('authenticode-signerinfo')
@@ -107,9 +111,10 @@ class JoeParser():
         self.misp_event.add_object(**pe_object)
         signerinfo_object.add_attribute('program-name', **{'type': 'text', 'value': program_name})
         signatureinfo = peinfo['signature']
-        for feature, mapping in signerinfo_object_mapping.items():
-            attribute_type, object_relation = mapping
-            signerinfo_object.add_attribute(object_relation, **{'type': attribute_type, 'value': signatureinfo[feature]})
+        if signatureinfo['signed']:
+            for feature, mapping in signerinfo_object_mapping.items():
+                attribute_type, object_relation = mapping
+                signerinfo_object.add_attribute(object_relation, **{'type': attribute_type, 'value': signatureinfo[feature]})
         self.misp_event.add_object(**signerinfo_object)
         for section in peinfo['sections']['section']:
             section_object = self.parse_pe_section(section)

From 067b2292240be703de3a029ef4da5e16e0584a02 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Wed, 15 May 2019 22:06:55 +0200
Subject: [PATCH 25/48] fix: Handling case of multiple processes in behavior
 field

- Also starting parsing file activities
---
 misp_modules/modules/import_mod/joe_import.py | 29 ++++++++++++-------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index 9e12cfb..4ba53b4 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 from collections import defaultdict
 from datetime import datetime
-from pymisp import MISPEvent, MISPObject
+from pymisp import MISPAttribute, MISPEvent, MISPObject
 import json
 import base64
 
@@ -29,6 +29,8 @@ pe_object_mapping = {'CompanyName': 'company-name', 'FileDescription': 'file-des
 process_object_fields = {'cmdline': 'command-line', 'name': 'name',
                          'parentpid': 'parent-pid', 'pid': 'pid',
                          'path': 'current-directory'}
+process_references_mapping = {'fileCreated': 'creates', 'fileDeleted': 'deletes',
+                              'fileMoved': 'moves', 'fileRead': 'reads', 'fileWritten': 'writes'}
 section_object_mapping = {'characteristics': ('text', 'characteristic'),
                           'entropy': ('float', 'entropy'),
                           'name': ('text', 'name'), 'rawaddr': ('hex', 'offset'),
@@ -67,15 +69,22 @@ class JoeParser():
         network = self.data['behavior']['network']
 
     def parse_behavior_system(self):
-        processes = self.data['behavior']['system']['processes']['process'][0]
-        general = processes['general']
-        process_object = MISPObject('process')
-        for feature, relation in process_object_fields.items():
-            process_object.add_attribute(relation, **{'type': 'text', 'value': general[feature]})
-        start_time = datetime.strptime('{} {}'.format(general['date'], general['time']), '%d/%m/%Y %H:%M:%S')
-        process_object.add_attribute('start-time', **{'type': 'datetime', 'value': start_time})
-        self.misp_event.add_object(**process_object)
-        self.references[self.fileinfo_uuid].append({'idref': process_object.uuid, 'relationship': 'calls'})
+        for process in self.data['behavior']['system']['processes']['process']:
+            general = process['general']
+            process_object = MISPObject('process')
+            for feature, relation in process_object_fields.items():
+                process_object.add_attribute(relation, **{'type': 'text', 'value': general[feature]})
+            start_time = datetime.strptime('{} {}'.format(general['date'], general['time']), '%d/%m/%Y %H:%M:%S')
+            process_object.add_attribute('start-time', **{'type': 'datetime', 'value': start_time})
+            for feature, files in process['fileactivities'].items():
+                if files:
+                    for call in files['call']:
+                        file_attribute = MISPAttribute()
+                        file_attribute.from_dict(**{'type': 'filename', 'value': call['path']})
+                        process_object.add_reference(file_attribute.uuid, process_references_mapping[feature])
+                        self.misp_event.add_attribute(**file_attribute)
+            self.misp_event.add_object(**process_object)
+            self.references[self.fileinfo_uuid].append({'idref': process_object.uuid, 'relationship': 'calls'})
 
     def parse_fileinfo(self):
         fileinfo = self.data['fileinfo']

From 2246fc0d02b22ad3662b444a7743fd23f4e92584 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Thu, 16 May 2019 16:11:43 +0200
Subject: [PATCH 26/48] add: Parsing registry activities under processes

---
 misp_modules/modules/import_mod/joe_import.py | 63 ++++++++++++++-----
 1 file changed, 47 insertions(+), 16 deletions(-)

diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index 4ba53b4..8c0c514 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -37,6 +37,9 @@ section_object_mapping = {'characteristics': ('text', 'characteristic'),
                           'rawsize': ('size-in-bytes', 'size-in-bytes'),
                           'virtaddr': ('hex', 'virtual_address'),
                           'virtsize': ('size-in-bytes', 'virtual_size')}
+registry_references_mapping = {'keyValueCreated': 'creates', 'keyValueModified': 'modifies'}
+regkey_object_mapping = {'name': ('text', 'name'), 'newdata': ('text', 'data'),
+                         'path': ('regkey', 'key')}
 signerinfo_object_mapping = {'sigissuer': ('text', 'issuer'),
                              'version': ('text', 'version')}
 
@@ -69,22 +72,28 @@ class JoeParser():
         network = self.data['behavior']['network']
 
     def parse_behavior_system(self):
-        for process in self.data['behavior']['system']['processes']['process']:
-            general = process['general']
-            process_object = MISPObject('process')
-            for feature, relation in process_object_fields.items():
-                process_object.add_attribute(relation, **{'type': 'text', 'value': general[feature]})
-            start_time = datetime.strptime('{} {}'.format(general['date'], general['time']), '%d/%m/%Y %H:%M:%S')
-            process_object.add_attribute('start-time', **{'type': 'datetime', 'value': start_time})
-            for feature, files in process['fileactivities'].items():
-                if files:
-                    for call in files['call']:
-                        file_attribute = MISPAttribute()
-                        file_attribute.from_dict(**{'type': 'filename', 'value': call['path']})
-                        process_object.add_reference(file_attribute.uuid, process_references_mapping[feature])
-                        self.misp_event.add_attribute(**file_attribute)
-            self.misp_event.add_object(**process_object)
-            self.references[self.fileinfo_uuid].append({'idref': process_object.uuid, 'relationship': 'calls'})
+        system = self.data['behavior']['system']
+        if system.get('processes'):
+            process_activities = {'fileactivities': self.parse_fileactivities,
+                                  'registryactivities': self.parse_registryactivities}
+            for process in system['processes']['process']:
+                general = process['general']
+                process_object = MISPObject('process')
+                for feature, relation in process_object_fields.items():
+                    process_object.add_attribute(relation, **{'type': 'text', 'value': general[feature]})
+                start_time = datetime.strptime('{} {}'.format(general['date'], general['time']), '%d/%m/%Y %H:%M:%S')
+                process_object.add_attribute('start-time', **{'type': 'datetime', 'value': start_time})
+                self.misp_event.add_object(**process_object)
+                for field, to_call in process_activities.items():
+                    to_call(process_object.uuid, process[field])
+                self.references[self.fileinfo_uuid].append({'idref': process_object.uuid, 'relationship': 'calls'})
+
+    def parse_fileactivities(self, process_uuid, fileactivities):
+        for feature, files in fileactivities.items():
+            if files:
+                for call in files['call']:
+                    file_uuid = self.create_attribute(call, 'filename')
+                    self.references[process_uuid].append({'idref': file_uuid, 'relationship': process_references_mapping[feature]})
 
     def parse_fileinfo(self):
         fileinfo = self.data['fileinfo']
@@ -137,6 +146,28 @@ class JoeParser():
             section_object.add_attribute(object_relation, **{'type': attribute_type, 'value': section[feature]})
         return section_object
 
+    def parse_registryactivities(self, process_uuid, registryactivities):
+        if registryactivities['keyCreated']:
+            for call in registryactivities['keyCreated']['call']:
+                regkey_uuid = self.create_attribute(call, 'regkey')
+                self.references[process_uuid].append({'idref': regkey_uuid, 'relationship': 'creates'})
+        for feature, relationship_type in registry_references_mapping.items():
+            if registryactivities[feature]:
+                for call in registryactivities[feature]['call']:
+                    registry_key = MISPObject('registry-key')
+                    for field, mapping in regkey_object_mapping.items():
+                        attribute_type, object_relation = mapping
+                        registry_key.add_attribute(object_relation, **{'type': attribute_type, 'value': call[field]})
+                    registry_key.add_attribute('data-type', **{'type': 'text', 'value': 'REG_{}'.format(call['type'].upper())})
+                    self.misp_event.add_object(**registry_key)
+                    self.references[process_uuid].append({'idref': registry_key.uuid, 'relationship': relationship_type})
+
+    def create_attribute(self, field, attribute_type):
+        attribute = MISPAttribute()
+        attribute.from_dict(**{'type': attribute_type, 'value': field['path']})
+        self.misp_event.add_attribute(**attribute)
+        return attribute.uuid
+
     def finalize_results(self):
         event = json.loads(self.misp_event.to_json())['Event']
         self.results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}

From f9515c14d059599c6e54cbdbb6154d8adf3d14d9 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Thu, 16 May 2019 16:14:25 +0200
Subject: [PATCH 27/48] fix: Avoiding attribute & reference duplicates

---
 misp_modules/modules/import_mod/joe_import.py | 25 +++++++++++++------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index 8c0c514..6275c50 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -19,6 +19,8 @@ file_object_fields = ['filename', 'md5', 'sha1', 'sha256', 'sha512', 'ssdeep']
 file_object_mapping = {'entropy': ('float', 'entropy'),
                        'filesize': ('size-in-bytes', 'size-in-bytes'),
                        'filetype': ('mime-type', 'mimetype')}
+file_references_mapping = {'fileCreated': 'creates', 'fileDeleted': 'deletes',
+                           'fileMoved': 'moves', 'fileRead': 'reads', 'fileWritten': 'writes'}
 pe_object_fields = {'entrypoint': ('text', 'entrypoint-address'),
                     'imphash': ('imphash', 'imphash')}
 pe_object_mapping = {'CompanyName': 'company-name', 'FileDescription': 'file-description',
@@ -29,8 +31,6 @@ pe_object_mapping = {'CompanyName': 'company-name', 'FileDescription': 'file-des
 process_object_fields = {'cmdline': 'command-line', 'name': 'name',
                          'parentpid': 'parent-pid', 'pid': 'pid',
                          'path': 'current-directory'}
-process_references_mapping = {'fileCreated': 'creates', 'fileDeleted': 'deletes',
-                              'fileMoved': 'moves', 'fileRead': 'reads', 'fileWritten': 'writes'}
 section_object_mapping = {'characteristics': ('text', 'characteristic'),
                           'entropy': ('float', 'entropy'),
                           'name': ('text', 'name'), 'rawaddr': ('hex', 'offset'),
@@ -49,10 +49,13 @@ class JoeParser():
         self.data = data
         self.misp_event = MISPEvent()
         self.references = defaultdict(list)
+        self.attributes = defaultdict(lambda: defaultdict(set))
 
     def parse_joe(self):
         self.parse_fileinfo()
         self.parse_behavior()
+        if self.attributes:
+            self.handle_attributes()
         if self.references:
             self.build_references()
         self.finalize_results()
@@ -64,6 +67,14 @@ class JoeParser():
                 for reference in self.references[object_uuid]:
                     misp_object.add_reference(reference['idref'], reference['relationship'])
 
+    def handle_attributes(self):
+        for attribute_type, attribute in self.attributes.items():
+            for attribute_value, references in attribute.items():
+                attribute_uuid = self.create_attribute(attribute_type, attribute_value)
+                for reference in references:
+                    source_uuid, relationship = reference
+                    self.references[source_uuid].append({'idref': attribute_uuid, 'relationship': relationship})
+
     def parse_behavior(self):
         self.parse_behavior_system()
         self.parse_behavior_network()
@@ -92,8 +103,7 @@ class JoeParser():
         for feature, files in fileactivities.items():
             if files:
                 for call in files['call']:
-                    file_uuid = self.create_attribute(call, 'filename')
-                    self.references[process_uuid].append({'idref': file_uuid, 'relationship': process_references_mapping[feature]})
+                    self.attributes['filename'][call['path']].add((process_uuid, file_references_mapping[feature]))
 
     def parse_fileinfo(self):
         fileinfo = self.data['fileinfo']
@@ -149,8 +159,7 @@ class JoeParser():
     def parse_registryactivities(self, process_uuid, registryactivities):
         if registryactivities['keyCreated']:
             for call in registryactivities['keyCreated']['call']:
-                regkey_uuid = self.create_attribute(call, 'regkey')
-                self.references[process_uuid].append({'idref': regkey_uuid, 'relationship': 'creates'})
+                self.attributes['regkey'][call['path']].add((process_uuid, 'creates'))
         for feature, relationship_type in registry_references_mapping.items():
             if registryactivities[feature]:
                 for call in registryactivities[feature]['call']:
@@ -162,9 +171,9 @@ class JoeParser():
                     self.misp_event.add_object(**registry_key)
                     self.references[process_uuid].append({'idref': registry_key.uuid, 'relationship': relationship_type})
 
-    def create_attribute(self, field, attribute_type):
+    def create_attribute(self, attribute_type, attribute_value):
         attribute = MISPAttribute()
-        attribute.from_dict(**{'type': attribute_type, 'value': field['path']})
+        attribute.from_dict(**{'type': attribute_type, 'value': attribute_value})
         self.misp_event.add_attribute(**attribute)
         return attribute.uuid
 

From 0d5f86782518878ef752707eed8d833b25d051fb Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Fri, 17 May 2019 22:18:11 +0200
Subject: [PATCH 28/48] add: Starting parsing network behavior fields

---
 misp_modules/modules/import_mod/joe_import.py | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index 6275c50..0b854e3 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -21,6 +21,8 @@ file_object_mapping = {'entropy': ('float', 'entropy'),
                        'filetype': ('mime-type', 'mimetype')}
 file_references_mapping = {'fileCreated': 'creates', 'fileDeleted': 'deletes',
                            'fileMoved': 'moves', 'fileRead': 'reads', 'fileWritten': 'writes'}
+network_connection_object_mapping = {'srcip': ('ip-src', 'ip-src'), 'dstip': ('ip-dst', 'ip-dst'),
+                                     'srcport': ('port', 'src-port'), 'dstport': ('port', 'dst-port')}
 pe_object_fields = {'entrypoint': ('text', 'entrypoint-address'),
                     'imphash': ('imphash', 'imphash')}
 pe_object_mapping = {'CompanyName': 'company-name', 'FileDescription': 'file-description',
@@ -81,6 +83,25 @@ class JoeParser():
 
     def parse_behavior_network(self):
         network = self.data['behavior']['network']
+        protocols = {'tcp': 4, 'udp': 4, 'icmp': 3,
+                     'http': 7, 'https': 7, 'ftp': 7}
+        fields = ('srcip', 'dstip', 'srcport', 'dstport')
+        for protocol, layer in protocols.items():
+            if network.get(protocol):
+                connections = defaultdict(list)
+                for packet in network[protocol]['packet']:
+                    timestamp = self.parse_timestamp(packet['timestamp'])
+                    connections[(packet[field] for field in fields)].append(datetime.strptime(timestamp, '%B %d, %Y %H:%M:%S.%f'))
+                for connection, timestamps in connections.items():
+                    network_connection_object = MISPObject('network-connection')
+                    for field, value in zip(fields, connection):
+                        attribute_type, object_relation = network_connection_object_mapping[field]
+                        network_connection_object.add_attribute(object_relation, **{'type': attribute_type, 'value': value})
+                    network_connection_object.add_attribute('first-packet-seen', **{'type': 'datetime', 'value': min(timestamps)})
+                    network_connection_object.add_attribute('layer{}-protocol'.format(layer),
+                                                            **{'type': 'text', 'value': protocol})
+                    self.misp_event.add_object(**network_connection_object)
+                    self.references[self.fileinfo_uuid].append({'idref': network_connection_object.uuid, 'relationship': 'initiates'})
 
     def parse_behavior_system(self):
         system = self.data['behavior']['system']
@@ -181,6 +202,12 @@ class JoeParser():
         event = json.loads(self.misp_event.to_json())['Event']
         self.results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
 
+    @staticmethod
+    def parse_timestamp(timestamp):
+        timestamp = timestamp.split(':')
+        timestamp[-1] = str(round(float(timestamp[-1].split(' ')[0]), 6))
+        return ':'.join(timestamp)
+
 
 def handler(q=False):
     if q is False:

From 54f5fa6fa92b6410f343ab8f6430d23f6a419802 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Mon, 20 May 2019 09:19:38 +0200
Subject: [PATCH 29/48] fix: Avoiding dictionary indexes issues

- Using tuples as a dictionary indexes is better
  than using generators...
---
 misp_modules/modules/import_mod/joe_import.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index 0b854e3..614d430 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -91,7 +91,7 @@ class JoeParser():
                 connections = defaultdict(list)
                 for packet in network[protocol]['packet']:
                     timestamp = self.parse_timestamp(packet['timestamp'])
-                    connections[(packet[field] for field in fields)].append(datetime.strptime(timestamp, '%B %d, %Y %H:%M:%S.%f'))
+                    connections[tuple(packet[field] for field in fields)].append(datetime.strptime(timestamp, '%B %d, %Y %H:%M:%S.%f'))
                 for connection, timestamps in connections.items():
                     network_connection_object = MISPObject('network-connection')
                     for field, value in zip(fields, connection):

From 72e5f0099d322d90bf28d886a2276155ea6ec77c Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Mon, 20 May 2019 10:52:34 +0200
Subject: [PATCH 30/48] fix: Avoid creating a signer info object when the pe is
 not signed

---
 misp_modules/modules/import_mod/joe_import.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index 614d430..e8d9e90 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -155,16 +155,18 @@ class JoeParser():
                 pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value']})
         sections_number = len(peinfo['sections']['section'])
         pe_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number})
-        signerinfo_object = MISPObject('authenticode-signerinfo')
-        pe_object.add_reference(signerinfo_object.uuid, 'signed-by')
-        self.misp_event.add_object(**pe_object)
-        signerinfo_object.add_attribute('program-name', **{'type': 'text', 'value': program_name})
         signatureinfo = peinfo['signature']
         if signatureinfo['signed']:
+            signerinfo_object = MISPObject('authenticode-signerinfo')
+            pe_object.add_reference(signerinfo_object.uuid, 'signed-by')
+            self.misp_event.add_object(**pe_object)
+            signerinfo_object.add_attribute('program-name', **{'type': 'text', 'value': program_name})
             for feature, mapping in signerinfo_object_mapping.items():
                 attribute_type, object_relation = mapping
                 signerinfo_object.add_attribute(object_relation, **{'type': attribute_type, 'value': signatureinfo[feature]})
-        self.misp_event.add_object(**signerinfo_object)
+            self.misp_event.add_object(**signerinfo_object)
+        else:
+            self.misp_event.add_object(**pe_object)
         for section in peinfo['sections']['section']:
             section_object = self.parse_pe_section(section)
             self.references[pe_object.uuid].append({'idref': section_object.uuid, 'relationship': 'included-in'})

From 417c306ace6a49c0199a1ab2ebcf717ae94babfe Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Mon, 20 May 2019 15:59:18 +0200
Subject: [PATCH 31/48] fix: Avoiding network connection object duplicates

---
 misp_modules/modules/import_mod/joe_import.py | 43 +++++++++++++------
 1 file changed, 31 insertions(+), 12 deletions(-)

diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index e8d9e90..2721362 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -21,6 +21,7 @@ file_object_mapping = {'entropy': ('float', 'entropy'),
                        'filetype': ('mime-type', 'mimetype')}
 file_references_mapping = {'fileCreated': 'creates', 'fileDeleted': 'deletes',
                            'fileMoved': 'moves', 'fileRead': 'reads', 'fileWritten': 'writes'}
+network_behavior_fields = ('srcip', 'dstip', 'srcport', 'dstport')
 network_connection_object_mapping = {'srcip': ('ip-src', 'ip-src'), 'dstip': ('ip-dst', 'ip-dst'),
                                      'srcport': ('port', 'src-port'), 'dstport': ('port', 'dst-port')}
 pe_object_fields = {'entrypoint': ('text', 'entrypoint-address'),
@@ -33,6 +34,8 @@ pe_object_mapping = {'CompanyName': 'company-name', 'FileDescription': 'file-des
 process_object_fields = {'cmdline': 'command-line', 'name': 'name',
                          'parentpid': 'parent-pid', 'pid': 'pid',
                          'path': 'current-directory'}
+protocols = {'tcp': 4, 'udp': 4, 'icmp': 3,
+             'http': 7, 'https': 7, 'ftp': 7}
 section_object_mapping = {'characteristics': ('text', 'characteristic'),
                           'entropy': ('float', 'entropy'),
                           'name': ('text', 'name'), 'rawaddr': ('hex', 'offset'),
@@ -83,23 +86,31 @@ class JoeParser():
 
     def parse_behavior_network(self):
         network = self.data['behavior']['network']
-        protocols = {'tcp': 4, 'udp': 4, 'icmp': 3,
-                     'http': 7, 'https': 7, 'ftp': 7}
-        fields = ('srcip', 'dstip', 'srcport', 'dstport')
+        connections = defaultdict(lambda: defaultdict(set))
         for protocol, layer in protocols.items():
             if network.get(protocol):
-                connections = defaultdict(list)
                 for packet in network[protocol]['packet']:
-                    timestamp = self.parse_timestamp(packet['timestamp'])
-                    connections[tuple(packet[field] for field in fields)].append(datetime.strptime(timestamp, '%B %d, %Y %H:%M:%S.%f'))
-                for connection, timestamps in connections.items():
+                    timestamp = datetime.strptime(self.parse_timestamp(packet['timestamp']), '%B %d, %Y %H:%M:%S.%f')
+                    connections[tuple(packet[field] for field in network_behavior_fields)][protocol].add(timestamp)
+        for connection, data in connections.items():
+            attributes = self.prefetch_attributes_data(connection)
+            if len(data.keys()) == len(set(protocols[protocol] for protocol in data.keys())):
+                network_connection_object = MISPObject('network-connection')
+                for object_relation, attribute in attributes.items():
+                    network_connection_object.add_attribute(object_relation, **attribute)
+                network_connection_object.add_attribute('first-packet-seen',
+                                                        **{'type': 'datetime', 'value': min(tuple(min(timestamp) for timestamp in data.values()))})
+                for protocol in data.keys():
+                    network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol})
+                self.misp_event.add_object(**network_connection_object)
+                self.references[self.fileinfo_uuid].append({'idref': network_connection_object.uuid, 'relationship': 'initiates'})
+            else:
+                for protocol, timestamps in data.items():
                     network_connection_object = MISPObject('network-connection')
-                    for field, value in zip(fields, connection):
-                        attribute_type, object_relation = network_connection_object_mapping[field]
-                        network_connection_object.add_attribute(object_relation, **{'type': attribute_type, 'value': value})
+                    for object_relation, attribute in attributes.items():
+                        network_connection_object.add_attribute(object_relation, **attribute)
                     network_connection_object.add_attribute('first-packet-seen', **{'type': 'datetime', 'value': min(timestamps)})
-                    network_connection_object.add_attribute('layer{}-protocol'.format(layer),
-                                                            **{'type': 'text', 'value': protocol})
+                    network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol})
                     self.misp_event.add_object(**network_connection_object)
                     self.references[self.fileinfo_uuid].append({'idref': network_connection_object.uuid, 'relationship': 'initiates'})
 
@@ -210,6 +221,14 @@ class JoeParser():
         timestamp[-1] = str(round(float(timestamp[-1].split(' ')[0]), 6))
         return ':'.join(timestamp)
 
+    @staticmethod
+    def prefetch_attributes_data(connection):
+        attributes = {}
+        for field, value in zip(network_behavior_fields, connection):
+            attribute_type, object_relation = network_connection_object_mapping[field]
+            attributes[object_relation] = {'type': attribute_type, 'value': value}
+        return attributes
+
 
 def handler(q=False):
     if q is False:

From 191034d31111c447c7c0df31793279a910f36434 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Tue, 21 May 2019 23:37:53 +0200
Subject: [PATCH 32/48] add: Starting parsing dropped files

---
 misp_modules/modules/import_mod/joe_import.py | 30 +++++++++++++++----
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index 2721362..237218d 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -15,6 +15,11 @@ moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
 
 moduleconfig = []
 
+dropped_file_mapping = {'@entropy': ('float', 'entropy'),
+                        '@file': ('filename', 'filename'),
+                        '@size': ('size-in-bytes', 'size-in-bytes'),
+                        '@type': ('mime-type', 'mimetype')}
+dropped_hash_mapping = {'MD5': 'md5', 'SHA': 'sha1', 'SHA-256': 'sha256', 'SHA-512': 'sha512'}
 file_object_fields = ['filename', 'md5', 'sha1', 'sha256', 'sha512', 'ssdeep']
 file_object_mapping = {'entropy': ('float', 'entropy'),
                        'filesize': ('size-in-bytes', 'size-in-bytes'),
@@ -58,7 +63,9 @@ class JoeParser():
 
     def parse_joe(self):
         self.parse_fileinfo()
-        self.parse_behavior()
+        self.parse_system_behavior()
+        self.parse_network_behavior()
+        self.parse_dropped_files()
         if self.attributes:
             self.handle_attributes()
         if self.references:
@@ -80,11 +87,22 @@ class JoeParser():
                     source_uuid, relationship = reference
                     self.references[source_uuid].append({'idref': attribute_uuid, 'relationship': relationship})
 
-    def parse_behavior(self):
-        self.parse_behavior_system()
-        self.parse_behavior_network()
+    def parse_dropped_files(self):
+        droppedinfo = self.data['droppedinfo']
+        if droppedinfo:
+            for droppedfile in droppedinfo['hash']:
+                file_object = MISPObject('file')
+                for key, mapping in dropped_file_mapping.items():
+                    attribute_type, object_relation = mapping
+                    file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': droppedfile[key]})
+                if droppedfile['@malicious'] == 'true':
+                    file_object.add_attribute('state', **{'type': 'text', 'value': 'Malicious'})
+                for h in droppedfile['value']:
+                    hash_type = dropped_hash_mapping[h['@algo']]
+                    file_object.add_attribute(hash_type, **{'type': hash_type, 'value': h['$']})
+                self.misp_event.add_object(**file_object)
 
-    def parse_behavior_network(self):
+    def parse_network_behavior(self):
         network = self.data['behavior']['network']
         connections = defaultdict(lambda: defaultdict(set))
         for protocol, layer in protocols.items():
@@ -114,7 +132,7 @@ class JoeParser():
                     self.misp_event.add_object(**network_connection_object)
                     self.references[self.fileinfo_uuid].append({'idref': network_connection_object.uuid, 'relationship': 'initiates'})
 
-    def parse_behavior_system(self):
+    def parse_system_behavior(self):
         system = self.data['behavior']['system']
         if system.get('processes'):
             process_activities = {'fileactivities': self.parse_fileactivities,

From cfec9a6b1ca7aa811f3ffddef83124489b9050aa Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Wed, 22 May 2019 15:27:04 +0200
Subject: [PATCH 33/48] fix: Added references between processes and the files
 they drop

---
 misp_modules/modules/import_mod/joe_import.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index 237218d..c70531c 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -60,6 +60,7 @@ class JoeParser():
         self.misp_event = MISPEvent()
         self.references = defaultdict(list)
         self.attributes = defaultdict(lambda: defaultdict(set))
+        self.process_references = {}
 
     def parse_joe(self):
         self.parse_fileinfo()
@@ -101,6 +102,10 @@ class JoeParser():
                     hash_type = dropped_hash_mapping[h['@algo']]
                     file_object.add_attribute(hash_type, **{'type': hash_type, 'value': h['$']})
                 self.misp_event.add_object(**file_object)
+                self.references[self.process_references[(int(droppedfile['@targetid']), droppedfile['@process'])]].append({
+                    'idref': file_object.uuid,
+                    'relationship': 'drops'
+                })
 
     def parse_network_behavior(self):
         network = self.data['behavior']['network']
@@ -148,6 +153,7 @@ class JoeParser():
                 for field, to_call in process_activities.items():
                     to_call(process_object.uuid, process[field])
                 self.references[self.fileinfo_uuid].append({'idref': process_object.uuid, 'relationship': 'calls'})
+                self.process_references[(general['targetid'], general['path'])] = process_object.uuid
 
     def parse_fileactivities(self, process_uuid, fileactivities):
         for feature, files in fileactivities.items():

From e608107a09b489f2cfcee0d70bfe6a3ff28fa4e6 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Wed, 22 May 2019 17:12:49 +0200
Subject: [PATCH 34/48] add: Parsing domains, urls & ips contacted by processes

---
 misp_modules/modules/import_mod/joe_import.py | 43 +++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index c70531c..efefb3e 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -15,6 +15,7 @@ moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
 
 moduleconfig = []
 
+domain_object_mapping = {'@ip': ('ip-dst', 'ip'), '@name': ('domain', 'domain')}
 dropped_file_mapping = {'@entropy': ('float', 'entropy'),
                         '@file': ('filename', 'filename'),
                         '@size': ('size-in-bytes', 'size-in-bytes'),
@@ -66,6 +67,7 @@ class JoeParser():
         self.parse_fileinfo()
         self.parse_system_behavior()
         self.parse_network_behavior()
+        self.parse_network_interactions()
         self.parse_dropped_files()
         if self.attributes:
             self.handle_attributes()
@@ -207,6 +209,47 @@ class JoeParser():
             self.references[pe_object.uuid].append({'idref': section_object.uuid, 'relationship': 'included-in'})
             self.misp_event.add_object(**section_object)
 
+    def parse_network_interactions(self):
+        domaininfo = self.data['domaininfo']
+        if domaininfo:
+            for domain in domaininfo['domain']:
+                domain_object = MISPObject('domain-ip')
+                for key, mapping in domain_object_mapping.items():
+                    attribute_type, object_relation = mapping
+                    domain_object.add_attribute(object_relation, **{'type': attribute_type, 'value': domain[key]})
+                self.misp_event.add_object(**domain_object)
+                self.references[self.process_references[(int(domain['@targetid']), domain['@currentpath'])]].append({
+                    'idref': domain_object.uuid,
+                    'relationship': 'contacts'
+                })
+        ipinfo = self.data['ipinfo']
+        if ipinfo:
+            for ip in ipinfo['ip']:
+                attribute = MISPAttribute()
+                attribute.from_dict(**{'type': 'ip-dst', 'value': ip['@ip']})
+                self.misp_event.add_attribute(**attribute)
+                self.references[self.process_references[(int(ip['@targetid']), ip['@currentpath'])]].append({
+                    'idref': attribute.uuid,
+                    'relationship': 'contacts'
+                })
+        urlinfo = self.data['urlinfo']
+        if urlinfo:
+            for url in urlinfo['url']:
+                target_id = int(url['@targetid'])
+                current_path = url['@currentpath']
+                attribute = MISPAttribute()
+                attribute_dict = {'type': 'url', 'value': url['@name']}
+                if target_id != -1 and current_path != 'unknown':
+                    self.references[self.process_references[(target_id, current_path)]].append({
+                        'idref': attribute.uuid,
+                        'relationship': 'contacts'
+                    })
+                else:
+                    attribute_dict['comment'] = 'From Memory - Enriched via the joe_import module'
+                attribute.from_dict(**attribute_dict)
+                self.misp_event.add_attribute(**attribute)
+
+
     def parse_pe_section(self, section):
         section_object = MISPObject('pe-section')
         for feature, mapping in section_object_mapping.items():

From be05de62c008e57f4ac322e045ff6f52b8dac05c Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Thu, 23 May 2019 15:59:52 +0200
Subject: [PATCH 35/48] add: Parsing MITRE ATT&CK tactic matrix related to the
 Joe report

---
 misp_modules/modules/import_mod/joe_import.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index efefb3e..d20de60 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -73,6 +73,7 @@ class JoeParser():
             self.handle_attributes()
         if self.references:
             self.build_references()
+        self.parse_mitre_attack()
         self.finalize_results()
 
     def build_references(self):
@@ -109,6 +110,14 @@ class JoeParser():
                     'relationship': 'drops'
                 })
 
+    def parse_mitre_attack(self):
+        mitreattack = self.data['mitreattack']
+        if mitreattack:
+            for tactic in mitreattack['tactic']:
+                if tactic.get('technique'):
+                    for technique in tactic['technique']:
+                        self.misp_event.add_tag('misp-galaxy:mitre-attack-pattern="{} - {}"'.format(technique['name'], technique['id']))
+
     def parse_network_behavior(self):
         network = self.data['behavior']['network']
         connections = defaultdict(lambda: defaultdict(set))

From 8ac651562e215077ef7eb5333a409ac78e250223 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Thu, 23 May 2019 16:13:49 +0200
Subject: [PATCH 36/48] fix: Making pep8 & travis happy

---
 misp_modules/modules/import_mod/joe_import.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index d20de60..0d22074 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -258,7 +258,6 @@ class JoeParser():
                 attribute.from_dict(**attribute_dict)
                 self.misp_event.add_attribute(**attribute)
 
-
     def parse_pe_section(self, section):
         section_object = MISPObject('pe-section')
         for feature, mapping in section_object_mapping.items():

From 380b8d46ba6673a97b4a3043332e0285ec8a98c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Georg=20Sch=C3=B6lly?= <georg.schoelly@joesecurity.org>
Date: Tue, 28 May 2019 11:19:32 +0200
Subject: [PATCH 37/48] improve forwards-compatibility

---
 misp_modules/modules/import_mod/joe_import.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index 0d22074..8f258cd 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -168,6 +168,10 @@ class JoeParser():
 
     def parse_fileactivities(self, process_uuid, fileactivities):
         for feature, files in fileactivities.items():
+            # ignore unknown features
+            if feature not in file_references_mapping:
+                continue
+
             if files:
                 for call in files['call']:
                     self.attributes['filename'][call['path']].add((process_uuid, file_references_mapping[feature]))
@@ -198,7 +202,8 @@ class JoeParser():
                 name = feature['name']
                 if name == 'InternalName':
                     program_name = feature['value']
-                pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value']})
+                if name in pe_object_mapping:
+                    pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value']})
         sections_number = len(peinfo['sections']['section'])
         pe_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number})
         signatureinfo = peinfo['signature']

From 9377a892f4d6ed4d5540b0b41fc4f12dbbbf2991 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Georg=20Sch=C3=B6lly?= <georg.schoelly@joesecurity.org>
Date: Tue, 28 May 2019 11:20:00 +0200
Subject: [PATCH 38/48] support url analyses

---
 misp_modules/modules/import_mod/joe_import.py | 36 ++++++++++++++++---
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index 8f258cd..2dc8990 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -64,11 +64,16 @@ class JoeParser():
         self.process_references = {}
 
     def parse_joe(self):
-        self.parse_fileinfo()
+        if self.analysis_type() == "file":
+            self.parse_fileinfo()
+        else:
+            self.parse_url_analysis()
+
         self.parse_system_behavior()
         self.parse_network_behavior()
         self.parse_network_interactions()
         self.parse_dropped_files()
+
         if self.attributes:
             self.handle_attributes()
         if self.references:
@@ -137,7 +142,7 @@ class JoeParser():
                 for protocol in data.keys():
                     network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol})
                 self.misp_event.add_object(**network_connection_object)
-                self.references[self.fileinfo_uuid].append({'idref': network_connection_object.uuid, 'relationship': 'initiates'})
+                self.references[self.analysisinfo_uuid].append({'idref': network_connection_object.uuid, 'relationship': 'initiates'})
             else:
                 for protocol, timestamps in data.items():
                     network_connection_object = MISPObject('network-connection')
@@ -146,7 +151,7 @@ class JoeParser():
                     network_connection_object.add_attribute('first-packet-seen', **{'type': 'datetime', 'value': min(timestamps)})
                     network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol})
                     self.misp_event.add_object(**network_connection_object)
-                    self.references[self.fileinfo_uuid].append({'idref': network_connection_object.uuid, 'relationship': 'initiates'})
+                    self.references[self.analysisinfo_uuid].append({'idref': network_connection_object.uuid, 'relationship': 'initiates'})
 
     def parse_system_behavior(self):
         system = self.data['behavior']['system']
@@ -163,7 +168,7 @@ class JoeParser():
                 self.misp_event.add_object(**process_object)
                 for field, to_call in process_activities.items():
                     to_call(process_object.uuid, process[field])
-                self.references[self.fileinfo_uuid].append({'idref': process_object.uuid, 'relationship': 'calls'})
+                self.references[self.analysisinfo_uuid].append({'idref': process_object.uuid, 'relationship': 'calls'})
                 self.process_references[(general['targetid'], general['path'])] = process_object.uuid
 
     def parse_fileactivities(self, process_uuid, fileactivities):
@@ -176,15 +181,36 @@ class JoeParser():
                 for call in files['call']:
                     self.attributes['filename'][call['path']].add((process_uuid, file_references_mapping[feature]))
 
+    def analysis_type(self):
+        generalinfo = self.data['generalinfo']
+
+        if generalinfo['target']['sample']:
+            return "file"
+        elif generalinfo['target']['url']:
+            return "url"
+        else:
+            raise Exception("Unknown analysis type")
+
+    def parse_url_analysis(self):
+        generalinfo = self.data["generalinfo"]
+
+        url_object = MISPObject("url")
+        self.analysisinfo_uuid = url_object.uuid
+
+        url_object.add_attribute("url", generalinfo["target"]["url"])
+        self.misp_event.add_object(**url_object)
+
     def parse_fileinfo(self):
         fileinfo = self.data['fileinfo']
+
         file_object = MISPObject('file')
+        self.analysisinfo_uuid = file_object.uuid
+
         for field in file_object_fields:
             file_object.add_attribute(field, **{'type': field, 'value': fileinfo[field]})
         for field, mapping in file_object_mapping.items():
             attribute_type, object_relation = mapping
             file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': fileinfo[field]})
-        self.fileinfo_uuid = file_object.uuid
         if not fileinfo.get('pe'):
             self.misp_event.add_object(**file_object)
             return

From 74b73f93328b50836349e8d59b59133b11ecbfbe Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Wed, 29 May 2019 11:26:14 +1000
Subject: [PATCH 39/48] chg: Moved JoeParser class to make it reachable from
 expansion & import modules

---
 misp_modules/lib/__init__.py                  |   1 +
 misp_modules/lib/joe_parser.py                | 326 +++++++++++++++++
 misp_modules/modules/import_mod/joe_import.py | 329 +-----------------
 3 files changed, 332 insertions(+), 324 deletions(-)
 create mode 100644 misp_modules/lib/__init__.py
 create mode 100644 misp_modules/lib/joe_parser.py

diff --git a/misp_modules/lib/__init__.py b/misp_modules/lib/__init__.py
new file mode 100644
index 0000000..0dbceb8
--- /dev/null
+++ b/misp_modules/lib/__init__.py
@@ -0,0 +1 @@
+all = ['joe_parser']
diff --git a/misp_modules/lib/joe_parser.py b/misp_modules/lib/joe_parser.py
new file mode 100644
index 0000000..a3dc82c
--- /dev/null
+++ b/misp_modules/lib/joe_parser.py
@@ -0,0 +1,326 @@
+# -*- coding: utf-8 -*-
+from collections import defaultdict
+from datetime import datetime
+from pymisp import MISPAttribute, MISPEvent, MISPObject
+import json
+
+
+domain_object_mapping = {'@ip': ('ip-dst', 'ip'), '@name': ('domain', 'domain')}
+dropped_file_mapping = {'@entropy': ('float', 'entropy'),
+                        '@file': ('filename', 'filename'),
+                        '@size': ('size-in-bytes', 'size-in-bytes'),
+                        '@type': ('mime-type', 'mimetype')}
+dropped_hash_mapping = {'MD5': 'md5', 'SHA': 'sha1', 'SHA-256': 'sha256', 'SHA-512': 'sha512'}
+file_object_fields = ['filename', 'md5', 'sha1', 'sha256', 'sha512', 'ssdeep']
+file_object_mapping = {'entropy': ('float', 'entropy'),
+                       'filesize': ('size-in-bytes', 'size-in-bytes'),
+                       'filetype': ('mime-type', 'mimetype')}
+file_references_mapping = {'fileCreated': 'creates', 'fileDeleted': 'deletes',
+                           'fileMoved': 'moves', 'fileRead': 'reads', 'fileWritten': 'writes'}
+network_behavior_fields = ('srcip', 'dstip', 'srcport', 'dstport')
+network_connection_object_mapping = {'srcip': ('ip-src', 'ip-src'), 'dstip': ('ip-dst', 'ip-dst'),
+                                     'srcport': ('port', 'src-port'), 'dstport': ('port', 'dst-port')}
+pe_object_fields = {'entrypoint': ('text', 'entrypoint-address'),
+                    'imphash': ('imphash', 'imphash')}
+pe_object_mapping = {'CompanyName': 'company-name', 'FileDescription': 'file-description',
+                     'FileVersion': 'file-version', 'InternalName': 'internal-filename',
+                     'LegalCopyright': 'legal-copyright', 'OriginalFilename': 'original-filename',
+                     'ProductName': 'product-filename', 'ProductVersion': 'product-version',
+                     'Translation': 'lang-id'}
+process_object_fields = {'cmdline': 'command-line', 'name': 'name',
+                         'parentpid': 'parent-pid', 'pid': 'pid',
+                         'path': 'current-directory'}
+protocols = {'tcp': 4, 'udp': 4, 'icmp': 3,
+             'http': 7, 'https': 7, 'ftp': 7}
+section_object_mapping = {'characteristics': ('text', 'characteristic'),
+                          'entropy': ('float', 'entropy'),
+                          'name': ('text', 'name'), 'rawaddr': ('hex', 'offset'),
+                          'rawsize': ('size-in-bytes', 'size-in-bytes'),
+                          'virtaddr': ('hex', 'virtual_address'),
+                          'virtsize': ('size-in-bytes', 'virtual_size')}
+registry_references_mapping = {'keyValueCreated': 'creates', 'keyValueModified': 'modifies'}
+regkey_object_mapping = {'name': ('text', 'name'), 'newdata': ('text', 'data'),
+                         'path': ('regkey', 'key')}
+signerinfo_object_mapping = {'sigissuer': ('text', 'issuer'),
+                             'version': ('text', 'version')}
+
+
+class JoeParser():
+    def __init__(self, data):
+        self.data = data
+        self.misp_event = MISPEvent()
+        self.references = defaultdict(list)
+        self.attributes = defaultdict(lambda: defaultdict(set))
+        self.process_references = {}
+
+    def parse_joe(self):
+        if self.analysis_type() == "file":
+            self.parse_fileinfo()
+        else:
+            self.parse_url_analysis()
+
+        self.parse_system_behavior()
+        self.parse_network_behavior()
+        self.parse_network_interactions()
+        self.parse_dropped_files()
+
+        if self.attributes:
+            self.handle_attributes()
+        if self.references:
+            self.build_references()
+        self.parse_mitre_attack()
+        self.finalize_results()
+
+    def build_references(self):
+        for misp_object in self.misp_event.objects:
+            object_uuid = misp_object.uuid
+            if object_uuid in self.references:
+                for reference in self.references[object_uuid]:
+                    misp_object.add_reference(reference['idref'], reference['relationship'])
+
+    def handle_attributes(self):
+        for attribute_type, attribute in self.attributes.items():
+            for attribute_value, references in attribute.items():
+                attribute_uuid = self.create_attribute(attribute_type, attribute_value)
+                for reference in references:
+                    source_uuid, relationship = reference
+                    self.references[source_uuid].append({'idref': attribute_uuid, 'relationship': relationship})
+
+    def parse_dropped_files(self):
+        droppedinfo = self.data['droppedinfo']
+        if droppedinfo:
+            for droppedfile in droppedinfo['hash']:
+                file_object = MISPObject('file')
+                for key, mapping in dropped_file_mapping.items():
+                    attribute_type, object_relation = mapping
+                    file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': droppedfile[key]})
+                if droppedfile['@malicious'] == 'true':
+                    file_object.add_attribute('state', **{'type': 'text', 'value': 'Malicious'})
+                for h in droppedfile['value']:
+                    hash_type = dropped_hash_mapping[h['@algo']]
+                    file_object.add_attribute(hash_type, **{'type': hash_type, 'value': h['$']})
+                self.misp_event.add_object(**file_object)
+                self.references[self.process_references[(int(droppedfile['@targetid']), droppedfile['@process'])]].append({
+                    'idref': file_object.uuid,
+                    'relationship': 'drops'
+                })
+
+    def parse_mitre_attack(self):
+        mitreattack = self.data['mitreattack']
+        if mitreattack:
+            for tactic in mitreattack['tactic']:
+                if tactic.get('technique'):
+                    for technique in tactic['technique']:
+                        self.misp_event.add_tag('misp-galaxy:mitre-attack-pattern="{} - {}"'.format(technique['name'], technique['id']))
+
+    def parse_network_behavior(self):
+        network = self.data['behavior']['network']
+        connections = defaultdict(lambda: defaultdict(set))
+        for protocol, layer in protocols.items():
+            if network.get(protocol):
+                for packet in network[protocol]['packet']:
+                    timestamp = datetime.strptime(self.parse_timestamp(packet['timestamp']), '%B %d, %Y %H:%M:%S.%f')
+                    connections[tuple(packet[field] for field in network_behavior_fields)][protocol].add(timestamp)
+        for connection, data in connections.items():
+            attributes = self.prefetch_attributes_data(connection)
+            if len(data.keys()) == len(set(protocols[protocol] for protocol in data.keys())):
+                network_connection_object = MISPObject('network-connection')
+                for object_relation, attribute in attributes.items():
+                    network_connection_object.add_attribute(object_relation, **attribute)
+                network_connection_object.add_attribute('first-packet-seen',
+                                                        **{'type': 'datetime', 'value': min(tuple(min(timestamp) for timestamp in data.values()))})
+                for protocol in data.keys():
+                    network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol})
+                self.misp_event.add_object(**network_connection_object)
+                self.references[self.analysisinfo_uuid].append({'idref': network_connection_object.uuid, 'relationship': 'initiates'})
+            else:
+                for protocol, timestamps in data.items():
+                    network_connection_object = MISPObject('network-connection')
+                    for object_relation, attribute in attributes.items():
+                        network_connection_object.add_attribute(object_relation, **attribute)
+                    network_connection_object.add_attribute('first-packet-seen', **{'type': 'datetime', 'value': min(timestamps)})
+                    network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol})
+                    self.misp_event.add_object(**network_connection_object)
+                    self.references[self.analysisinfo_uuid].append({'idref': network_connection_object.uuid, 'relationship': 'initiates'})
+
+    def parse_system_behavior(self):
+        system = self.data['behavior']['system']
+        if system.get('processes'):
+            process_activities = {'fileactivities': self.parse_fileactivities,
+                                  'registryactivities': self.parse_registryactivities}
+            for process in system['processes']['process']:
+                general = process['general']
+                process_object = MISPObject('process')
+                for feature, relation in process_object_fields.items():
+                    process_object.add_attribute(relation, **{'type': 'text', 'value': general[feature]})
+                start_time = datetime.strptime('{} {}'.format(general['date'], general['time']), '%d/%m/%Y %H:%M:%S')
+                process_object.add_attribute('start-time', **{'type': 'datetime', 'value': start_time})
+                self.misp_event.add_object(**process_object)
+                for field, to_call in process_activities.items():
+                    to_call(process_object.uuid, process[field])
+                self.references[self.analysisinfo_uuid].append({'idref': process_object.uuid, 'relationship': 'calls'})
+                self.process_references[(general['targetid'], general['path'])] = process_object.uuid
+
+    def parse_fileactivities(self, process_uuid, fileactivities):
+        for feature, files in fileactivities.items():
+            # ignore unknown features
+            if feature not in file_references_mapping:
+                continue
+
+            if files:
+                for call in files['call']:
+                    self.attributes['filename'][call['path']].add((process_uuid, file_references_mapping[feature]))
+
+    def analysis_type(self):
+        generalinfo = self.data['generalinfo']
+
+        if generalinfo['target']['sample']:
+            return "file"
+        elif generalinfo['target']['url']:
+            return "url"
+        else:
+            raise Exception("Unknown analysis type")
+
+    def parse_url_analysis(self):
+        generalinfo = self.data["generalinfo"]
+
+        url_object = MISPObject("url")
+        self.analysisinfo_uuid = url_object.uuid
+
+        url_object.add_attribute("url", generalinfo["target"]["url"])
+        self.misp_event.add_object(**url_object)
+
+    def parse_fileinfo(self):
+        fileinfo = self.data['fileinfo']
+
+        file_object = MISPObject('file')
+        self.analysisinfo_uuid = file_object.uuid
+
+        for field in file_object_fields:
+            file_object.add_attribute(field, **{'type': field, 'value': fileinfo[field]})
+        for field, mapping in file_object_mapping.items():
+            attribute_type, object_relation = mapping
+            file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': fileinfo[field]})
+        if not fileinfo.get('pe'):
+            self.misp_event.add_object(**file_object)
+            return
+        peinfo = fileinfo['pe']
+        pe_object = MISPObject('pe')
+        file_object.add_reference(pe_object.uuid, 'included-in')
+        self.misp_event.add_object(**file_object)
+        for field, mapping in pe_object_fields.items():
+            attribute_type, object_relation = mapping
+            pe_object.add_attribute(object_relation, **{'type': attribute_type, 'value': peinfo[field]})
+        pe_object.add_attribute('compilation-timestamp', **{'type': 'datetime', 'value': int(peinfo['timestamp'].split()[0], 16)})
+        program_name = fileinfo['filename']
+        if peinfo['versions']:
+            for feature in peinfo['versions']['version']:
+                name = feature['name']
+                if name == 'InternalName':
+                    program_name = feature['value']
+                if name in pe_object_mapping:
+                    pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value']})
+        sections_number = len(peinfo['sections']['section'])
+        pe_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number})
+        signatureinfo = peinfo['signature']
+        if signatureinfo['signed']:
+            signerinfo_object = MISPObject('authenticode-signerinfo')
+            pe_object.add_reference(signerinfo_object.uuid, 'signed-by')
+            self.misp_event.add_object(**pe_object)
+            signerinfo_object.add_attribute('program-name', **{'type': 'text', 'value': program_name})
+            for feature, mapping in signerinfo_object_mapping.items():
+                attribute_type, object_relation = mapping
+                signerinfo_object.add_attribute(object_relation, **{'type': attribute_type, 'value': signatureinfo[feature]})
+            self.misp_event.add_object(**signerinfo_object)
+        else:
+            self.misp_event.add_object(**pe_object)
+        for section in peinfo['sections']['section']:
+            section_object = self.parse_pe_section(section)
+            self.references[pe_object.uuid].append({'idref': section_object.uuid, 'relationship': 'included-in'})
+            self.misp_event.add_object(**section_object)
+
+    def parse_network_interactions(self):
+        domaininfo = self.data['domaininfo']
+        if domaininfo:
+            for domain in domaininfo['domain']:
+                domain_object = MISPObject('domain-ip')
+                for key, mapping in domain_object_mapping.items():
+                    attribute_type, object_relation = mapping
+                    domain_object.add_attribute(object_relation, **{'type': attribute_type, 'value': domain[key]})
+                self.misp_event.add_object(**domain_object)
+                self.references[self.process_references[(int(domain['@targetid']), domain['@currentpath'])]].append({
+                    'idref': domain_object.uuid,
+                    'relationship': 'contacts'
+                })
+        ipinfo = self.data['ipinfo']
+        if ipinfo:
+            for ip in ipinfo['ip']:
+                attribute = MISPAttribute()
+                attribute.from_dict(**{'type': 'ip-dst', 'value': ip['@ip']})
+                self.misp_event.add_attribute(**attribute)
+                self.references[self.process_references[(int(ip['@targetid']), ip['@currentpath'])]].append({
+                    'idref': attribute.uuid,
+                    'relationship': 'contacts'
+                })
+        urlinfo = self.data['urlinfo']
+        if urlinfo:
+            for url in urlinfo['url']:
+                target_id = int(url['@targetid'])
+                current_path = url['@currentpath']
+                attribute = MISPAttribute()
+                attribute_dict = {'type': 'url', 'value': url['@name']}
+                if target_id != -1 and current_path != 'unknown':
+                    self.references[self.process_references[(target_id, current_path)]].append({
+                        'idref': attribute.uuid,
+                        'relationship': 'contacts'
+                    })
+                else:
+                    attribute_dict['comment'] = 'From Memory - Enriched via the joe_import module'
+                attribute.from_dict(**attribute_dict)
+                self.misp_event.add_attribute(**attribute)
+
+    def parse_pe_section(self, section):
+        section_object = MISPObject('pe-section')
+        for feature, mapping in section_object_mapping.items():
+            attribute_type, object_relation = mapping
+            section_object.add_attribute(object_relation, **{'type': attribute_type, 'value': section[feature]})
+        return section_object
+
+    def parse_registryactivities(self, process_uuid, registryactivities):
+        if registryactivities['keyCreated']:
+            for call in registryactivities['keyCreated']['call']:
+                self.attributes['regkey'][call['path']].add((process_uuid, 'creates'))
+        for feature, relationship_type in registry_references_mapping.items():
+            if registryactivities[feature]:
+                for call in registryactivities[feature]['call']:
+                    registry_key = MISPObject('registry-key')
+                    for field, mapping in regkey_object_mapping.items():
+                        attribute_type, object_relation = mapping
+                        registry_key.add_attribute(object_relation, **{'type': attribute_type, 'value': call[field]})
+                    registry_key.add_attribute('data-type', **{'type': 'text', 'value': 'REG_{}'.format(call['type'].upper())})
+                    self.misp_event.add_object(**registry_key)
+                    self.references[process_uuid].append({'idref': registry_key.uuid, 'relationship': relationship_type})
+
+    def create_attribute(self, attribute_type, attribute_value):
+        attribute = MISPAttribute()
+        attribute.from_dict(**{'type': attribute_type, 'value': attribute_value})
+        self.misp_event.add_attribute(**attribute)
+        return attribute.uuid
+
+    def finalize_results(self):
+        event = json.loads(self.misp_event.to_json())['Event']
+        self.results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
+
+    @staticmethod
+    def parse_timestamp(timestamp):
+        timestamp = timestamp.split(':')
+        timestamp[-1] = str(round(float(timestamp[-1].split(' ')[0]), 6))
+        return ':'.join(timestamp)
+
+    @staticmethod
+    def prefetch_attributes_data(connection):
+        attributes = {}
+        for field, value in zip(network_behavior_fields, connection):
+            attribute_type, object_relation = network_connection_object_mapping[field]
+            attributes[object_relation] = {'type': attribute_type, 'value': value}
+        return attributes
diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index 2dc8990..c1300c4 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
-from collections import defaultdict
-from datetime import datetime
-from pymisp import MISPAttribute, MISPEvent, MISPObject
-import json
 import base64
+import json
+import os
+import sys
+sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3])))
+from joe_parser import JoeParser
 
 misperrors = {'error': 'Error'}
 userConfig = {}
@@ -15,326 +16,6 @@ moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
 
 moduleconfig = []
 
-domain_object_mapping = {'@ip': ('ip-dst', 'ip'), '@name': ('domain', 'domain')}
-dropped_file_mapping = {'@entropy': ('float', 'entropy'),
-                        '@file': ('filename', 'filename'),
-                        '@size': ('size-in-bytes', 'size-in-bytes'),
-                        '@type': ('mime-type', 'mimetype')}
-dropped_hash_mapping = {'MD5': 'md5', 'SHA': 'sha1', 'SHA-256': 'sha256', 'SHA-512': 'sha512'}
-file_object_fields = ['filename', 'md5', 'sha1', 'sha256', 'sha512', 'ssdeep']
-file_object_mapping = {'entropy': ('float', 'entropy'),
-                       'filesize': ('size-in-bytes', 'size-in-bytes'),
-                       'filetype': ('mime-type', 'mimetype')}
-file_references_mapping = {'fileCreated': 'creates', 'fileDeleted': 'deletes',
-                           'fileMoved': 'moves', 'fileRead': 'reads', 'fileWritten': 'writes'}
-network_behavior_fields = ('srcip', 'dstip', 'srcport', 'dstport')
-network_connection_object_mapping = {'srcip': ('ip-src', 'ip-src'), 'dstip': ('ip-dst', 'ip-dst'),
-                                     'srcport': ('port', 'src-port'), 'dstport': ('port', 'dst-port')}
-pe_object_fields = {'entrypoint': ('text', 'entrypoint-address'),
-                    'imphash': ('imphash', 'imphash')}
-pe_object_mapping = {'CompanyName': 'company-name', 'FileDescription': 'file-description',
-                     'FileVersion': 'file-version', 'InternalName': 'internal-filename',
-                     'LegalCopyright': 'legal-copyright', 'OriginalFilename': 'original-filename',
-                     'ProductName': 'product-filename', 'ProductVersion': 'product-version',
-                     'Translation': 'lang-id'}
-process_object_fields = {'cmdline': 'command-line', 'name': 'name',
-                         'parentpid': 'parent-pid', 'pid': 'pid',
-                         'path': 'current-directory'}
-protocols = {'tcp': 4, 'udp': 4, 'icmp': 3,
-             'http': 7, 'https': 7, 'ftp': 7}
-section_object_mapping = {'characteristics': ('text', 'characteristic'),
-                          'entropy': ('float', 'entropy'),
-                          'name': ('text', 'name'), 'rawaddr': ('hex', 'offset'),
-                          'rawsize': ('size-in-bytes', 'size-in-bytes'),
-                          'virtaddr': ('hex', 'virtual_address'),
-                          'virtsize': ('size-in-bytes', 'virtual_size')}
-registry_references_mapping = {'keyValueCreated': 'creates', 'keyValueModified': 'modifies'}
-regkey_object_mapping = {'name': ('text', 'name'), 'newdata': ('text', 'data'),
-                         'path': ('regkey', 'key')}
-signerinfo_object_mapping = {'sigissuer': ('text', 'issuer'),
-                             'version': ('text', 'version')}
-
-
-class JoeParser():
-    def __init__(self, data):
-        self.data = data
-        self.misp_event = MISPEvent()
-        self.references = defaultdict(list)
-        self.attributes = defaultdict(lambda: defaultdict(set))
-        self.process_references = {}
-
-    def parse_joe(self):
-        if self.analysis_type() == "file":
-            self.parse_fileinfo()
-        else:
-            self.parse_url_analysis()
-
-        self.parse_system_behavior()
-        self.parse_network_behavior()
-        self.parse_network_interactions()
-        self.parse_dropped_files()
-
-        if self.attributes:
-            self.handle_attributes()
-        if self.references:
-            self.build_references()
-        self.parse_mitre_attack()
-        self.finalize_results()
-
-    def build_references(self):
-        for misp_object in self.misp_event.objects:
-            object_uuid = misp_object.uuid
-            if object_uuid in self.references:
-                for reference in self.references[object_uuid]:
-                    misp_object.add_reference(reference['idref'], reference['relationship'])
-
-    def handle_attributes(self):
-        for attribute_type, attribute in self.attributes.items():
-            for attribute_value, references in attribute.items():
-                attribute_uuid = self.create_attribute(attribute_type, attribute_value)
-                for reference in references:
-                    source_uuid, relationship = reference
-                    self.references[source_uuid].append({'idref': attribute_uuid, 'relationship': relationship})
-
-    def parse_dropped_files(self):
-        droppedinfo = self.data['droppedinfo']
-        if droppedinfo:
-            for droppedfile in droppedinfo['hash']:
-                file_object = MISPObject('file')
-                for key, mapping in dropped_file_mapping.items():
-                    attribute_type, object_relation = mapping
-                    file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': droppedfile[key]})
-                if droppedfile['@malicious'] == 'true':
-                    file_object.add_attribute('state', **{'type': 'text', 'value': 'Malicious'})
-                for h in droppedfile['value']:
-                    hash_type = dropped_hash_mapping[h['@algo']]
-                    file_object.add_attribute(hash_type, **{'type': hash_type, 'value': h['$']})
-                self.misp_event.add_object(**file_object)
-                self.references[self.process_references[(int(droppedfile['@targetid']), droppedfile['@process'])]].append({
-                    'idref': file_object.uuid,
-                    'relationship': 'drops'
-                })
-
-    def parse_mitre_attack(self):
-        mitreattack = self.data['mitreattack']
-        if mitreattack:
-            for tactic in mitreattack['tactic']:
-                if tactic.get('technique'):
-                    for technique in tactic['technique']:
-                        self.misp_event.add_tag('misp-galaxy:mitre-attack-pattern="{} - {}"'.format(technique['name'], technique['id']))
-
-    def parse_network_behavior(self):
-        network = self.data['behavior']['network']
-        connections = defaultdict(lambda: defaultdict(set))
-        for protocol, layer in protocols.items():
-            if network.get(protocol):
-                for packet in network[protocol]['packet']:
-                    timestamp = datetime.strptime(self.parse_timestamp(packet['timestamp']), '%B %d, %Y %H:%M:%S.%f')
-                    connections[tuple(packet[field] for field in network_behavior_fields)][protocol].add(timestamp)
-        for connection, data in connections.items():
-            attributes = self.prefetch_attributes_data(connection)
-            if len(data.keys()) == len(set(protocols[protocol] for protocol in data.keys())):
-                network_connection_object = MISPObject('network-connection')
-                for object_relation, attribute in attributes.items():
-                    network_connection_object.add_attribute(object_relation, **attribute)
-                network_connection_object.add_attribute('first-packet-seen',
-                                                        **{'type': 'datetime', 'value': min(tuple(min(timestamp) for timestamp in data.values()))})
-                for protocol in data.keys():
-                    network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol})
-                self.misp_event.add_object(**network_connection_object)
-                self.references[self.analysisinfo_uuid].append({'idref': network_connection_object.uuid, 'relationship': 'initiates'})
-            else:
-                for protocol, timestamps in data.items():
-                    network_connection_object = MISPObject('network-connection')
-                    for object_relation, attribute in attributes.items():
-                        network_connection_object.add_attribute(object_relation, **attribute)
-                    network_connection_object.add_attribute('first-packet-seen', **{'type': 'datetime', 'value': min(timestamps)})
-                    network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol})
-                    self.misp_event.add_object(**network_connection_object)
-                    self.references[self.analysisinfo_uuid].append({'idref': network_connection_object.uuid, 'relationship': 'initiates'})
-
-    def parse_system_behavior(self):
-        system = self.data['behavior']['system']
-        if system.get('processes'):
-            process_activities = {'fileactivities': self.parse_fileactivities,
-                                  'registryactivities': self.parse_registryactivities}
-            for process in system['processes']['process']:
-                general = process['general']
-                process_object = MISPObject('process')
-                for feature, relation in process_object_fields.items():
-                    process_object.add_attribute(relation, **{'type': 'text', 'value': general[feature]})
-                start_time = datetime.strptime('{} {}'.format(general['date'], general['time']), '%d/%m/%Y %H:%M:%S')
-                process_object.add_attribute('start-time', **{'type': 'datetime', 'value': start_time})
-                self.misp_event.add_object(**process_object)
-                for field, to_call in process_activities.items():
-                    to_call(process_object.uuid, process[field])
-                self.references[self.analysisinfo_uuid].append({'idref': process_object.uuid, 'relationship': 'calls'})
-                self.process_references[(general['targetid'], general['path'])] = process_object.uuid
-
-    def parse_fileactivities(self, process_uuid, fileactivities):
-        for feature, files in fileactivities.items():
-            # ignore unknown features
-            if feature not in file_references_mapping:
-                continue
-
-            if files:
-                for call in files['call']:
-                    self.attributes['filename'][call['path']].add((process_uuid, file_references_mapping[feature]))
-
-    def analysis_type(self):
-        generalinfo = self.data['generalinfo']
-
-        if generalinfo['target']['sample']:
-            return "file"
-        elif generalinfo['target']['url']:
-            return "url"
-        else:
-            raise Exception("Unknown analysis type")
-
-    def parse_url_analysis(self):
-        generalinfo = self.data["generalinfo"]
-
-        url_object = MISPObject("url")
-        self.analysisinfo_uuid = url_object.uuid
-
-        url_object.add_attribute("url", generalinfo["target"]["url"])
-        self.misp_event.add_object(**url_object)
-
-    def parse_fileinfo(self):
-        fileinfo = self.data['fileinfo']
-
-        file_object = MISPObject('file')
-        self.analysisinfo_uuid = file_object.uuid
-
-        for field in file_object_fields:
-            file_object.add_attribute(field, **{'type': field, 'value': fileinfo[field]})
-        for field, mapping in file_object_mapping.items():
-            attribute_type, object_relation = mapping
-            file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': fileinfo[field]})
-        if not fileinfo.get('pe'):
-            self.misp_event.add_object(**file_object)
-            return
-        peinfo = fileinfo['pe']
-        pe_object = MISPObject('pe')
-        file_object.add_reference(pe_object.uuid, 'included-in')
-        self.misp_event.add_object(**file_object)
-        for field, mapping in pe_object_fields.items():
-            attribute_type, object_relation = mapping
-            pe_object.add_attribute(object_relation, **{'type': attribute_type, 'value': peinfo[field]})
-        pe_object.add_attribute('compilation-timestamp', **{'type': 'datetime', 'value': int(peinfo['timestamp'].split()[0], 16)})
-        program_name = fileinfo['filename']
-        if peinfo['versions']:
-            for feature in peinfo['versions']['version']:
-                name = feature['name']
-                if name == 'InternalName':
-                    program_name = feature['value']
-                if name in pe_object_mapping:
-                    pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value']})
-        sections_number = len(peinfo['sections']['section'])
-        pe_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number})
-        signatureinfo = peinfo['signature']
-        if signatureinfo['signed']:
-            signerinfo_object = MISPObject('authenticode-signerinfo')
-            pe_object.add_reference(signerinfo_object.uuid, 'signed-by')
-            self.misp_event.add_object(**pe_object)
-            signerinfo_object.add_attribute('program-name', **{'type': 'text', 'value': program_name})
-            for feature, mapping in signerinfo_object_mapping.items():
-                attribute_type, object_relation = mapping
-                signerinfo_object.add_attribute(object_relation, **{'type': attribute_type, 'value': signatureinfo[feature]})
-            self.misp_event.add_object(**signerinfo_object)
-        else:
-            self.misp_event.add_object(**pe_object)
-        for section in peinfo['sections']['section']:
-            section_object = self.parse_pe_section(section)
-            self.references[pe_object.uuid].append({'idref': section_object.uuid, 'relationship': 'included-in'})
-            self.misp_event.add_object(**section_object)
-
-    def parse_network_interactions(self):
-        domaininfo = self.data['domaininfo']
-        if domaininfo:
-            for domain in domaininfo['domain']:
-                domain_object = MISPObject('domain-ip')
-                for key, mapping in domain_object_mapping.items():
-                    attribute_type, object_relation = mapping
-                    domain_object.add_attribute(object_relation, **{'type': attribute_type, 'value': domain[key]})
-                self.misp_event.add_object(**domain_object)
-                self.references[self.process_references[(int(domain['@targetid']), domain['@currentpath'])]].append({
-                    'idref': domain_object.uuid,
-                    'relationship': 'contacts'
-                })
-        ipinfo = self.data['ipinfo']
-        if ipinfo:
-            for ip in ipinfo['ip']:
-                attribute = MISPAttribute()
-                attribute.from_dict(**{'type': 'ip-dst', 'value': ip['@ip']})
-                self.misp_event.add_attribute(**attribute)
-                self.references[self.process_references[(int(ip['@targetid']), ip['@currentpath'])]].append({
-                    'idref': attribute.uuid,
-                    'relationship': 'contacts'
-                })
-        urlinfo = self.data['urlinfo']
-        if urlinfo:
-            for url in urlinfo['url']:
-                target_id = int(url['@targetid'])
-                current_path = url['@currentpath']
-                attribute = MISPAttribute()
-                attribute_dict = {'type': 'url', 'value': url['@name']}
-                if target_id != -1 and current_path != 'unknown':
-                    self.references[self.process_references[(target_id, current_path)]].append({
-                        'idref': attribute.uuid,
-                        'relationship': 'contacts'
-                    })
-                else:
-                    attribute_dict['comment'] = 'From Memory - Enriched via the joe_import module'
-                attribute.from_dict(**attribute_dict)
-                self.misp_event.add_attribute(**attribute)
-
-    def parse_pe_section(self, section):
-        section_object = MISPObject('pe-section')
-        for feature, mapping in section_object_mapping.items():
-            attribute_type, object_relation = mapping
-            section_object.add_attribute(object_relation, **{'type': attribute_type, 'value': section[feature]})
-        return section_object
-
-    def parse_registryactivities(self, process_uuid, registryactivities):
-        if registryactivities['keyCreated']:
-            for call in registryactivities['keyCreated']['call']:
-                self.attributes['regkey'][call['path']].add((process_uuid, 'creates'))
-        for feature, relationship_type in registry_references_mapping.items():
-            if registryactivities[feature]:
-                for call in registryactivities[feature]['call']:
-                    registry_key = MISPObject('registry-key')
-                    for field, mapping in regkey_object_mapping.items():
-                        attribute_type, object_relation = mapping
-                        registry_key.add_attribute(object_relation, **{'type': attribute_type, 'value': call[field]})
-                    registry_key.add_attribute('data-type', **{'type': 'text', 'value': 'REG_{}'.format(call['type'].upper())})
-                    self.misp_event.add_object(**registry_key)
-                    self.references[process_uuid].append({'idref': registry_key.uuid, 'relationship': relationship_type})
-
-    def create_attribute(self, attribute_type, attribute_value):
-        attribute = MISPAttribute()
-        attribute.from_dict(**{'type': attribute_type, 'value': attribute_value})
-        self.misp_event.add_attribute(**attribute)
-        return attribute.uuid
-
-    def finalize_results(self):
-        event = json.loads(self.misp_event.to_json())['Event']
-        self.results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
-
-    @staticmethod
-    def parse_timestamp(timestamp):
-        timestamp = timestamp.split(':')
-        timestamp[-1] = str(round(float(timestamp[-1].split(' ')[0]), 6))
-        return ':'.join(timestamp)
-
-    @staticmethod
-    def prefetch_attributes_data(connection):
-        attributes = {}
-        for field, value in zip(network_behavior_fields, connection):
-            attribute_type, object_relation = network_connection_object_mapping[field]
-            attributes[object_relation] = {'type': attribute_type, 'value': value}
-        return attributes
-
 
 def handler(q=False):
     if q is False:

From 0d40830a7f566dec9de1423edc24af079a205608 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Mon, 3 Jun 2019 18:35:58 +1000
Subject: [PATCH 40/48] fix: Some quick fixes

- Fixed strptime matching because months are
  expressed in abbreviated format
- Made data loaded while the parsing function is
  called, in case it has to be called multiple
  times at some point
---
 misp_modules/lib/joe_parser.py                | 12 ++++++------
 misp_modules/modules/import_mod/joe_import.py |  6 +++---
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/misp_modules/lib/joe_parser.py b/misp_modules/lib/joe_parser.py
index a3dc82c..5af78f2 100644
--- a/misp_modules/lib/joe_parser.py
+++ b/misp_modules/lib/joe_parser.py
@@ -46,14 +46,14 @@ signerinfo_object_mapping = {'sigissuer': ('text', 'issuer'),
 
 
 class JoeParser():
-    def __init__(self, data):
-        self.data = data
+    def __init__(self):
         self.misp_event = MISPEvent()
         self.references = defaultdict(list)
         self.attributes = defaultdict(lambda: defaultdict(set))
         self.process_references = {}
 
-    def parse_joe(self):
+    def parse_data(self, data):
+        self.data = data
         if self.analysis_type() == "file":
             self.parse_fileinfo()
         else:
@@ -66,8 +66,6 @@ class JoeParser():
 
         if self.attributes:
             self.handle_attributes()
-        if self.references:
-            self.build_references()
         self.parse_mitre_attack()
         self.finalize_results()
 
@@ -119,7 +117,7 @@ class JoeParser():
         for protocol, layer in protocols.items():
             if network.get(protocol):
                 for packet in network[protocol]['packet']:
-                    timestamp = datetime.strptime(self.parse_timestamp(packet['timestamp']), '%B %d, %Y %H:%M:%S.%f')
+                    timestamp = datetime.strptime(self.parse_timestamp(packet['timestamp']), '%b %d, %Y %H:%M:%S.%f')
                     connections[tuple(packet[field] for field in network_behavior_fields)][protocol].add(timestamp)
         for connection, data in connections.items():
             attributes = self.prefetch_attributes_data(connection)
@@ -308,6 +306,8 @@ class JoeParser():
         return attribute.uuid
 
     def finalize_results(self):
+        if self.references:
+            self.build_references()
         event = json.loads(self.misp_event.to_json())['Event']
         self.results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
 
diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index c1300c4..d4b9dfb 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -24,9 +24,9 @@ def handler(q=False):
     data = base64.b64decode(q.get('data')).decode('utf-8')
     if not data:
         return json.dumps({'success': 0})
-    joe_data = json.loads(data)['analysis']
-    joe_parser = JoeParser(joe_data)
-    joe_parser.parse_joe()
+    joe_parser = JoeParser()
+    joe_parser.parse_data(json.loads(data)['analysis'])
+    joe_parser.finalize_results()
     return {'results': joe_parser.results}
 
 

From 07698e5c72ef28c7d9b7eb91989e5c994b3092a4 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Mon, 3 Jun 2019 18:38:58 +1000
Subject: [PATCH 41/48] fix: Fixed references between domaininfo/ipinfo & their
 targets

- Fixed references when no target id is set
- Fixed domaininfo parsing when no ip is defined
---
 misp_modules/lib/joe_parser.py | 35 +++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/misp_modules/lib/joe_parser.py b/misp_modules/lib/joe_parser.py
index 5af78f2..bf5c974 100644
--- a/misp_modules/lib/joe_parser.py
+++ b/misp_modules/lib/joe_parser.py
@@ -241,25 +241,28 @@ class JoeParser():
         domaininfo = self.data['domaininfo']
         if domaininfo:
             for domain in domaininfo['domain']:
-                domain_object = MISPObject('domain-ip')
-                for key, mapping in domain_object_mapping.items():
-                    attribute_type, object_relation = mapping
-                    domain_object.add_attribute(object_relation, **{'type': attribute_type, 'value': domain[key]})
-                self.misp_event.add_object(**domain_object)
-                self.references[self.process_references[(int(domain['@targetid']), domain['@currentpath'])]].append({
-                    'idref': domain_object.uuid,
-                    'relationship': 'contacts'
-                })
+                if domain['@ip'] != 'unknown':
+                    domain_object = MISPObject('domain-ip')
+                    for key, mapping in domain_object_mapping.items():
+                        attribute_type, object_relation = mapping
+                        domain_object.add_attribute(object_relation,
+                                                    **{'type': attribute_type, 'value': domain[key]})
+                    self.misp_event.add_object(**domain_object)
+                    reference = {'idref': domain_object.uuid, 'relationship': 'contacts'}
+                    self.add_process_reference(domain['@targetid'], domain['@currentpath'], reference)
+                else:
+                    attribute = MISPAttribute()
+                    attribute.from_dict(**{'type': 'domain', 'value': domain['@name']})
+                    reference = {'idref': attribute.uuid, 'relationship': 'contacts'}
+                    self.add_process_reference(domain['@targetid'], domain['@currentpath'], reference)
         ipinfo = self.data['ipinfo']
         if ipinfo:
             for ip in ipinfo['ip']:
                 attribute = MISPAttribute()
                 attribute.from_dict(**{'type': 'ip-dst', 'value': ip['@ip']})
                 self.misp_event.add_attribute(**attribute)
-                self.references[self.process_references[(int(ip['@targetid']), ip['@currentpath'])]].append({
-                    'idref': attribute.uuid,
-                    'relationship': 'contacts'
-                })
+                reference = {'idref': attribute.uuid, 'relationship': 'contacts'}
+                self.add_process_reference(ip['@targetid'], ip['@currentpath'], reference)
         urlinfo = self.data['urlinfo']
         if urlinfo:
             for url in urlinfo['url']:
@@ -299,6 +302,12 @@ class JoeParser():
                     self.misp_event.add_object(**registry_key)
                     self.references[process_uuid].append({'idref': registry_key.uuid, 'relationship': relationship_type})
 
+    def add_process_reference(self, target, currentpath, reference):
+        try:
+            self.references[self.process_references[(int(target), currentpath)]].append(reference)
+        except KeyError:
+            self.references[self.analysisinfo_uuid].append(reference)
+
     def create_attribute(self, attribute_type, attribute_value):
         attribute = MISPAttribute()
         attribute.from_dict(**{'type': attribute_type, 'value': attribute_value})

From ee48d9984522da86c5a76fdc878f2fb0c1f7bccc Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Tue, 4 Jun 2019 09:48:50 +1000
Subject: [PATCH 42/48] add: New expansion module to query Joe Sandbox API with
 a report link

---
 misp_modules/modules/expansion/__init__.py    |  2 +-
 .../modules/expansion/joesandbox_query.py     | 63 +++++++++++++++++++
 2 files changed, 64 insertions(+), 1 deletion(-)
 create mode 100644 misp_modules/modules/expansion/joesandbox_query.py

diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py
index ba10b32..f64600a 100644
--- a/misp_modules/modules/expansion/__init__.py
+++ b/misp_modules/modules/expansion/__init__.py
@@ -10,4 +10,4 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c
            'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io',
            'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors',
            'qrcode', 'ocr-enrich', 'pdf-enrich', 'docx-enrich', 'xlsx-enrich', 'pptx-enrich',
-           'ods-enrich', 'odt-enrich', 'joesandbox_submit', 'urlhaus']
+           'ods-enrich', 'odt-enrich', 'joesandbox_submit', 'joesandbox_query', 'urlhaus']
diff --git a/misp_modules/modules/expansion/joesandbox_query.py b/misp_modules/modules/expansion/joesandbox_query.py
new file mode 100644
index 0000000..ea58cb6
--- /dev/null
+++ b/misp_modules/modules/expansion/joesandbox_query.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+import jbxapi
+import json
+import os
+import sys
+sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3])))
+from joe_parser import JoeParser
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['link'], 'format': 'misp_standard'}
+
+moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
+              'description': 'Query Joe Sandbox API with a report URL to get the parsed data.',
+              'module-type': ['expansion']}
+moduleconfig = ['apiurl', 'apikey', 'accept-tac']
+
+
+class _ParseError(Exception):
+    pass
+
+
+def _parse_bool(value, name="bool"):
+    if value is None or value == "":
+        return None
+    if value in ("true", "True"):
+        return True
+    if value in ("false", "False"):
+        return False
+    raise _ParseError("Cannot parse {}. Must be 'true' or 'false'".format(name))
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    apiurl = request['config'].get('apiurl') or 'https://jbxcloud.joesecurity.org/api'
+    apikey = request['config'].get('apikey')
+    if not apikey:
+        return {'error': 'No API key provided'}
+    try:
+        accept_tac = _parse_bool(request['config'].get('accept-tac'), 'accept-tac')
+    except _parseError as e:
+        return {'error': str(e)}
+    attribute = request['attribute']
+    joe = jbxapi.JoeSandbox(apiurl=apiurl, apikey=apikey, user_agent='MISP joesandbox_analysis', accept_tac=accept_tac)
+    joe_info = joe.submission_info(attribute['value'].split('/')[-1])
+    joe_parser = JoeParser()
+    most_relevant = joe_info['most_relevant_analysis']['webid']
+    for analyse in joe_info['analyses']:
+        if analyse['webid'] == most_relevant:
+            joe_data = json.loads(joe.analysis_download(most_relevant, 'jsonfixed')[1])
+            joe_parser.parse_data(joe_data['analysis'])
+            break
+    joe_parser.finalize_results()
+    return {'results': joe_parser.results}
+
+
+def introspection():
+    return mispattributes
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

From 42bc6f8d2b485b6aed77029b8cc2469ad151c0d4 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Tue, 4 Jun 2019 11:32:21 +1000
Subject: [PATCH 43/48] fix: Fixed variable name typo

---
 misp_modules/modules/expansion/joesandbox_query.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/misp_modules/modules/expansion/joesandbox_query.py b/misp_modules/modules/expansion/joesandbox_query.py
index ea58cb6..225cc16 100644
--- a/misp_modules/modules/expansion/joesandbox_query.py
+++ b/misp_modules/modules/expansion/joesandbox_query.py
@@ -39,7 +39,7 @@ def handler(q=False):
         return {'error': 'No API key provided'}
     try:
         accept_tac = _parse_bool(request['config'].get('accept-tac'), 'accept-tac')
-    except _parseError as e:
+    except _ParseError as e:
         return {'error': str(e)}
     attribute = request['attribute']
     joe = jbxapi.JoeSandbox(apiurl=apiurl, apikey=apikey, user_agent='MISP joesandbox_analysis', accept_tac=accept_tac)

From aa3e87384533da48eacae7e6a1ecef8a8e776e7f Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Tue, 4 Jun 2019 11:33:42 +1000
Subject: [PATCH 44/48] fix: Making pep8 happy + added joe_import module in the
 init list

---
 misp_modules/modules/expansion/__init__.py         | 3 +++
 misp_modules/modules/expansion/joesandbox_query.py | 4 +---
 misp_modules/modules/import_mod/__init__.py        | 5 ++++-
 misp_modules/modules/import_mod/joe_import.py      | 3 ---
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py
index f64600a..acf49f2 100644
--- a/misp_modules/modules/expansion/__init__.py
+++ b/misp_modules/modules/expansion/__init__.py
@@ -1,4 +1,7 @@
 from . import _vmray  # noqa
+import os
+import sys
+sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3])))
 
 __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl',
            'countrycode', 'cve', 'dns', 'btc_steroids', 'domaintools', 'eupi',
diff --git a/misp_modules/modules/expansion/joesandbox_query.py b/misp_modules/modules/expansion/joesandbox_query.py
index 225cc16..27ced1b 100644
--- a/misp_modules/modules/expansion/joesandbox_query.py
+++ b/misp_modules/modules/expansion/joesandbox_query.py
@@ -1,9 +1,6 @@
 # -*- coding: utf-8 -*-
 import jbxapi
 import json
-import os
-import sys
-sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3])))
 from joe_parser import JoeParser
 
 misperrors = {'error': 'Error'}
@@ -58,6 +55,7 @@ def handler(q=False):
 def introspection():
     return mispattributes
 
+
 def version():
     moduleinfo['config'] = moduleconfig
     return moduleinfo
diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py
index c3eea05..65a7069 100644
--- a/misp_modules/modules/import_mod/__init__.py
+++ b/misp_modules/modules/import_mod/__init__.py
@@ -1,3 +1,6 @@
 from . import _vmray  # noqa
+import os
+import sys
+sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3])))
 
-__all__ = ['vmray_import', 'ocr', 'cuckooimport', 'goamlimport', 'email_import', 'mispjson', 'openiocimport', 'threatanalyzer_import', 'csvimport']
+__all__ = ['vmray_import', 'ocr', 'cuckooimport', 'goamlimport', 'email_import', 'mispjson', 'openiocimport', 'threatanalyzer_import', 'csvimport', 'joe_import']
diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py
index d4b9dfb..d1c4d19 100644
--- a/misp_modules/modules/import_mod/joe_import.py
+++ b/misp_modules/modules/import_mod/joe_import.py
@@ -1,9 +1,6 @@
 # -*- coding: utf-8 -*-
 import base64
 import json
-import os
-import sys
-sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3])))
 from joe_parser import JoeParser
 
 misperrors = {'error': 'Error'}

From efb0a88eebd34d5320e7d06dbfa334183d776dbe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Georg=20Sch=C3=B6lly?= <georg.schoelly@joesecurity.org>
Date: Tue, 4 Jun 2019 11:29:40 +0200
Subject: [PATCH 45/48] joesandbox_query.py: improve behavior in unexpected
 circumstances

---
 .../modules/expansion/joesandbox_query.py     | 49 +++++++++----------
 1 file changed, 23 insertions(+), 26 deletions(-)

diff --git a/misp_modules/modules/expansion/joesandbox_query.py b/misp_modules/modules/expansion/joesandbox_query.py
index 27ced1b..dce63ea 100644
--- a/misp_modules/modules/expansion/joesandbox_query.py
+++ b/misp_modules/modules/expansion/joesandbox_query.py
@@ -9,21 +9,7 @@ mispattributes = {'input': ['link'], 'format': 'misp_standard'}
 moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
               'description': 'Query Joe Sandbox API with a report URL to get the parsed data.',
               'module-type': ['expansion']}
-moduleconfig = ['apiurl', 'apikey', 'accept-tac']
-
-
-class _ParseError(Exception):
-    pass
-
-
-def _parse_bool(value, name="bool"):
-    if value is None or value == "":
-        return None
-    if value in ("true", "True"):
-        return True
-    if value in ("false", "False"):
-        return False
-    raise _ParseError("Cannot parse {}. Must be 'true' or 'false'".format(name))
+moduleconfig = ['apiurl', 'apikey']
 
 
 def handler(q=False):
@@ -34,21 +20,32 @@ def handler(q=False):
     apikey = request['config'].get('apikey')
     if not apikey:
         return {'error': 'No API key provided'}
+
+    url = request['attribute']['value']
+    if "/submissions/" not in url:
+        return {'error': "The URL does not point to a Joe Sandbox analysis."}
+
+    submission_id = url.split('/')[-1]  # The URL has the format https://example.net/submissions/12345
+    joe = jbxapi.JoeSandbox(apiurl=apiurl, apikey=apikey, user_agent='MISP joesandbox_query')
+
     try:
-        accept_tac = _parse_bool(request['config'].get('accept-tac'), 'accept-tac')
-    except _ParseError as e:
+        joe_info = joe.submission_info(submission_id)
+    except jbxapi.ApiError as e:
         return {'error': str(e)}
-    attribute = request['attribute']
-    joe = jbxapi.JoeSandbox(apiurl=apiurl, apikey=apikey, user_agent='MISP joesandbox_analysis', accept_tac=accept_tac)
-    joe_info = joe.submission_info(attribute['value'].split('/')[-1])
+
+    if joe_info["status"] != "finished":
+        return {'error': "The analysis has not finished yet."}
+
+    if joe_info['most_relevant_analysis'] is None:
+        return {'error': "No analysis belongs to this submission."}
+
+    analysis_webid = joe_info['most_relevant_analysis']['webid']
+
     joe_parser = JoeParser()
-    most_relevant = joe_info['most_relevant_analysis']['webid']
-    for analyse in joe_info['analyses']:
-        if analyse['webid'] == most_relevant:
-            joe_data = json.loads(joe.analysis_download(most_relevant, 'jsonfixed')[1])
-            joe_parser.parse_data(joe_data['analysis'])
-            break
+    joe_data = json.loads(joe.analysis_download(analysis_webid, 'jsonfixed')[1])
+    joe_parser.parse_data(joe_data['analysis'])
     joe_parser.finalize_results()
+
     return {'results': joe_parser.results}
 
 

From b52e17fa8d670a0631d1f444b29864de28433a79 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Fri, 7 Jun 2019 11:38:50 +0200
Subject: [PATCH 46/48] fix: Removed duplicate finalize_results function call

---
 misp_modules/lib/joe_parser.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/misp_modules/lib/joe_parser.py b/misp_modules/lib/joe_parser.py
index bf5c974..7ee8a4b 100644
--- a/misp_modules/lib/joe_parser.py
+++ b/misp_modules/lib/joe_parser.py
@@ -67,7 +67,6 @@ class JoeParser():
         if self.attributes:
             self.handle_attributes()
         self.parse_mitre_attack()
-        self.finalize_results()
 
     def build_references(self):
         for misp_object in self.misp_event.objects:

From de966eac5157557cb20df6a3a44ba3017290ff32 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Fri, 7 Jun 2019 15:22:11 +0200
Subject: [PATCH 47/48] fix: Returning tags & galaxies with results

- Tags may exist with the current version of the
  parser
- Galaxies are not yet expected from the parser,
  nevertheless the principle is we want to return
  them as well if ever we have some galaxies from
  parsing a JoeSandbox report. Can be removed if
  we never galaxies at all
---
 misp_modules/lib/joe_parser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/misp_modules/lib/joe_parser.py b/misp_modules/lib/joe_parser.py
index 7ee8a4b..83f1fa0 100644
--- a/misp_modules/lib/joe_parser.py
+++ b/misp_modules/lib/joe_parser.py
@@ -317,7 +317,7 @@ class JoeParser():
         if self.references:
             self.build_references()
         event = json.loads(self.misp_event.to_json())['Event']
-        self.results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
+        self.results = {key: event[key] for key in ('Attribute', 'Object', 'Tag', 'Galaxy') if (key in event and event[key])}
 
     @staticmethod
     def parse_timestamp(timestamp):

From f885b6c5e197a084222594d4d6cdf8fb790cf8ea Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Wed, 12 Jun 2019 16:32:13 +0200
Subject: [PATCH 48/48] add: Added new modules to the list

---
 README.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index ecce406..bc2056a 100644
--- a/README.md
+++ b/README.md
@@ -40,7 +40,8 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
 * [intel471](misp_modules/modules/expansion/intel471.py) - an expansion module to get info from [Intel471](https://intel471.com).
 * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
 * [iprep](misp_modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net.
-* [Joe Sandbox](misp_modules/modules/expansion/joesandbox_submit.py) - Submit files and URLs to Joe Sandbox.
+* [Joe Sandbox submit](misp_modules/modules/expansion/joesandbox_submit.py) - Submit files and URLs to Joe Sandbox.
+* [Joe Sandbox query](misp_modules/modules/expansion/joesandbox_query.py) - Query Joe Sandbox with the link of an analysis and get the parsed data.
 * [macaddress.io](misp_modules/modules/expansion/macaddress_io.py) - a hover module to retrieve vendor details and other information regarding a given MAC address or an OUI from [MAC address Vendor Lookup](https://macaddress.io). See [integration tutorial here](https://macaddress.io/integrations/MISP-module).
 * [macvendors](misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information.
 * [ocr-enrich](misp_modules/modules/expansion/ocr-enrich.py) - an enrichment module to get OCRized data from images into MISP.
@@ -63,6 +64,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
 * [STIX2 pattern syntax validator](misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - a module to check a STIX2 pattern syntax.
 * [ThreatCrowd](misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/).
 * [threatminer](misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/).
+* [urlhaus](misp_modules/modules/expansion/urlhaus.py) - Query urlhaus to get additional data about a domain, hash, hostname, ip or url.
 * [urlscan](misp_modules/modules/expansion/urlscan.py) - an expansion module to query [urlscan.io](https://urlscan.io).
 * [virustotal](misp_modules/modules/expansion/virustotal.py) - an expansion module to pull known resolutions and malware samples related with an IP/Domain from virusTotal (this modules require a VirusTotal private API key)
 * [VMray](misp_modules/modules/expansion/vmray_submit.py) - a module to submit a sample to VMray.
@@ -92,7 +94,8 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
 * [CSV import](misp_modules/modules/import_mod/csvimport.py) Customizable CSV import module.
 * [Cuckoo JSON](misp_modules/modules/import_mod/cuckooimport.py) Cuckoo JSON import.
 * [Email Import](misp_modules/modules/import_mod/email_import.py) Email import module for MISP to import basic metadata.
-* [GoAML import](misp_modules/modules/import_mod/) Module to import [GoAML](http://goaml.unodc.org/goaml/en/index.html) XML format.
+* [GoAML import](misp_modules/modules/import_mod/goamlimport.py) Module to import [GoAML](http://goaml.unodc.org/goaml/en/index.html) XML format.
+* [Joe Sandbox import](misp_modules/modules/import_mod/joe_import.py) Parse data from a Joe Sandbox json report.
 * [OCR](misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes.
 * [OpenIOC](misp_modules/modules/import_mod/openiocimport.py) OpenIOC import based on PyMISP library.
 * [ThreatAnalyzer](misp_modules/modules/import_mod/threatanalyzer_import.py) - An import module to process ThreatAnalyzer archive.zip/analysis.json sandbox exports.