correction for https://travis-ci.org/MISP/misp-modules/jobs/231065469

code refactoring

add new composite

pymisp/tools/openioc.py

@@ -129,8 +129,10 @@ iocMispMapping = {
     'UrlHistoryItem/URL' : {'type': 'url','comment': 'UrlHistory. '},
     'UrlHistoryItem/HostName': {'type': 'hostname','comment': 'UrlHistory. '},
 
-    'Yara/Yara' : {'type': 'yara'},
+    'Yara/Yara' : {'type': 'yara'}
+}
 
+iocMispCompositeMapping = {
     # mapping for composite object
     # maybe later filename|sizeinbyte
     'FileItem/FileName|FileItem/Md5sum' : {'type': 'filename|md5'},
@@ -140,10 +142,11 @@ iocMispMapping = {
     'PortItem/remoteIP|PortItem/remotePort' : {'comment': 'ip-dst|port'},
     'RegistryItem/Path|RegistryItem/Value' : {'type': 'regkey|value'},
     'RegistryItem/KeyPath|RegistryItem/Value' : {'type': 'regkey|value'},
+    'RegistryItem/Path|RegistryItem/Text' : {'type': 'regkey|value'}
 }
 
-
 def extract_field(report, field_name):
+    if report:
         data = report.find(field_name.lower())
         if data and hasattr(data, 'text'):
             return data.text
@@ -186,43 +189,60 @@ def load_openioc(openioc):
     return misp_event
 
 
-def get_mapping(openioc_type):
+def get_mapping(openioc_type, mappingDict=iocMispMapping):
     t = openioc_type.lower()
-    for k, v in iocMispMapping.items():
+    for k, v in mappingDict.items():
         if k.lower() == t:
             return v
     return False
 
-def set_composite_values(value1, value2):
-    attribute_values = {'comment': ''}
 
+def set_values(value1, value2=None):
+    attribute_values = {}
+
+    if value2 is not None:
+        # construct attribut composite value
+        value = "{}|{}".format(
+                            extract_field(value1, 'Content'),
+                            extract_field(value2, 'Content')
+                        )
+    else:
+        value = extract_field(value1, 'Content')
+
+    if value:
+        attribute_values['value'] = value
+    else:
+        return None
+
+    if value2 is not None:
         # construct attribut composite type
         compositeMapping = value1.find('context')['search']+'|'+value2.find('context')['search']
-    mapping = get_mapping(compositeMapping)
+        mapping = get_mapping(compositeMapping, mappingDict=iocMispCompositeMapping)
+    else:
+        mapping = get_mapping(value1.find('context')['search'])
+
     if mapping:
         attribute_values.update(mapping)
     else:
-        # prevent some mistake error
+        # Unknown mapping, assign to default
-        attribute_values.update({'category': 'External analysis', 'type': 'other'})
+        attribute_values['category'] = 'External analysis'
+        attribute_values['type'] = 'other'
 
-    # construct attribut composite value
+    # change value to composite
-    compositeValue = value1.find('content').text + "|" +value2.find('content').text
+    # 127.0.0.1:80 ip-* to 127.0.0.1|80 ip-*|port
-    if compositeValue:
+    if attribute_values['type'] in ['ip-src', 'ip-dst'] and attribute_values['value'].count(':') == 1:
-        attribute_values['value'] = compositeValue
+        attribute_values['type'] = attribute_values['type'] + '|port'
+        attribute_values['value'] = attribute_values['value'].replace(':', '|')
 
-    # construct composite comment
+    attribute_values["comment"] = '{}{}'.format(
-    compositeComment = ""
+                        extract_field(value1, 'Comment'),
-    if value1.find('comment'):
+                        extract_field(value2, 'Comment')
-        compositeComment += value1.find('comment').text
+                    )
-    if value2.find('comment'):
-        compositeComment += value2.find('comment').text
-    attribute_values["comment"] = compositeComment
 
     return attribute_values
 
 def set_all_attributes(openioc, misp_event):
     processed = set()
-    hashName = ["FileItem/Md5sum","FileItem/Sha1sum","FileItem/Sha256sum"]
 
     # check for composite item
     for composite in openioc.find_all("indicator", operator="AND"):
@@ -233,95 +253,42 @@ def set_all_attributes(openioc, misp_event):
         if len(childs) == 2:
             childList = [child.find('context')['search'] for child in childs]
 
-            if ('FileItem/FileName' in childList) and\
+            def check_and_add(value1, value2):
-                (set(hashName) - set(childList) != set(hashName)):
+                if (value1 and value2) in childList:
-                if childs[0].find('context')['search'] == 'FileItem/FileName':
+                    if childs[0].find('context')['search'] == value1:
-                    value1, value2 = childs[0], childs[1]
+                        attribute_values = set_values(childs[0], childs[1])
                     else:
-                    value1, value2 = childs[1], childs[0]
+                        attribute_values = set_values(childs[1], childs[0])
 
-                attribute_values = set_composite_values(value1, value2)
                     misp_event.add_attribute(**attribute_values)
                     processed.add(childs[0]['id'])
                     processed.add(childs[1]['id'])
 
-
+            for k in iocMispCompositeMapping:
-            elif ("Network/DNS" and "PortItem/RemoteIP") in childList:
+                check_and_add(k.split('|')[0], k.split('|')[1])
-                if childs[0].find('context')['search'] == 'Network/DNS':
-                    value1, value2 = childs[0], childs[1]
-                else:
-                    value1, value2 = childs[1], childs[0]
-
-                attribute_values = set_composite_values(value1, value2)
-                misp_event.add_attribute(**attribute_values)
-                processed.add(childs[0]['id'])
-                processed.add(childs[1]['id'])
-
-
-            elif ("PortItem/RemoteIP" and "PortItem/RemotePort") in childList:
-                if childs[0].find('context')['search'] == 'PortItem/RemoteIP':
-                    value1, value2 = childs[0], childs[1]
-                else:
-                    value1, value2 = childs[1], childs[0]
-                
-                attribute_values = set_composite_values(value1, value2)
-                misp_event.add_attribute(**attribute_values)
-                processed.add(childs[0]['id'])
-                processed.add(childs[1]['id'])
-
-
-            elif ("RegistryItem/Path" and "RegistryItem/Value") in childList:
-                if childs[0].find('context')['search'] == 'RegistryItem/PathP':
-                    value1, value2 = childs[0], childs[1]
-                else:
-                    value1, value2 = childs[1], childs[0]
-
-                attribute_values = set_composite_values(value1, value2)
-                misp_event.add_attribute(**attribute_values)
-                processed.add(childs[0]['id'])
-                processed.add(childs[1]['id'])
 
     for item in openioc.find_all("indicatoritem"):
         # check if id in processed list
         if item['id'] in processed:
             continue
-        attribute_values = {'comment': ''}
+        attribute_values = set_values(item)
-        if item.find('context'):
+        
-            mapping = get_mapping(item.find('context')['search'])
+        if attribute_values is None:
-            if mapping:
+            continue
-                attribute_values.update(mapping)
-            else:
-                # Unknown mapping, assign to default
-                attribute_values.update({'category': 'External analysis', 'type': 'other'})
-                #continue
-        else:
-            continue
-        value = extract_field(item, 'Content')
-        if value:
-            attribute_values['value'] = value
-        else:
-            # No value, ignoring
-            continue
-        comment = extract_field(item, 'Comment')
-        if comment:
-            attribute_values["comment"] = '{} {}'.format(attribute_values["comment"], comment)
 
-        # change value to composite
-        # 127.0.0.1:80 ip-* to 127.0.0.1|80 ip-*|port
-        if mapping['type'] in ['ip-src', 'ip-dst'] and value.count(':') == 1:
-            attribute_values['type'] = mapping['type'] + '|port'
-            attribute_values['value'] = attribute_values['value'].replace(':', '|')
         misp_event.add_attribute(**attribute_values)
 
     return misp_event
 
 if __name__ == '__main__':
+    import requests
     # test file for composite
-    # https://github.com/fireeye/iocs/blob/master/BlogPosts/9cee306d-5441-4cd3-932d-f3119752634c.ioc
+    url = 'https://raw.githubusercontent.com/fireeye/iocs/master/BlogPosts/9cee306d-5441-4cd3-932d-f3119752634c.ioc'
-    x = open('test.ioc', 'r')
+    #~ url = 'https://raw.githubusercontent.com/MISP/misp-modules/master/tests/openioc.xml'
-    mispEvent = load_openioc(x.read())
+    x = requests.get(url)
-    #~ print(mispEvent._json_full())
+    mispEvent = load_openioc(x.text)
-    from pymisp import PyMISP
+    print(mispEvent)
-    misp = PyMISP('http://misp.local', 'xxxxx')
+    #~ from pymisp import PyMISP
-    r = misp.add_event(mispEvent)
+    #~ misp = PyMISP('http://misp.local', 'xxxxx')
-    print(r)
+    #~ r = misp.add_event(mispEvent)
+    #~ print(r)