Merge remote-tracking branch 'upstream/2.4' into distributionGraph

INSTALL/MYSQL.sql

@@ -1200,7 +1200,7 @@ INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modi
 VALUES (1, 'admin', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0);
 
 INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `default_role`)
-VALUES (2, 'Org Admin', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0);
+VALUES (2, 'Org Admin', NOW(), NOW(), 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0);
 
 INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `default_role`)
 VALUES (3, 'User', NOW(), NOW(), 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1);

VERSION.json

@@ -1 +1 @@
-{"major":2, "minor":4, "hotfix":89}
+{"major":2, "minor":4, "hotfix":90}

app/Controller/EventsController.php

@@ -628,15 +628,20 @@ class EventsController extends AppController {
 							)
 						)
 					));
-					foreach ($eventTags as $et) {
+					foreach ($eventTags as $ket => $et) {
-						$et['EventTag']['Tag'] = $et['Tag'];
+						if (empty($et['Tag']['id'])) {
-						unset($et['Tag']);
+							unset($eventTags[$ket]);
-						if (empty($event_tag_objects[$et['EventTag']['event_id']])) {
-							$event_tag_objects[$et['EventTag']['event_id']] = array($et['EventTag']);
 						} else {
-							$event_tag_objects[$et['EventTag']['event_id']][] = $et['EventTag'];
+							$et['EventTag']['Tag'] = $et['Tag'];
+							unset($et['Tag']);
+							if (empty($event_tag_objects[$et['EventTag']['event_id']])) {
+								$event_tag_objects[$et['EventTag']['event_id']] = array($et['EventTag']);
+							} else {
+								$event_tag_objects[$et['EventTag']['event_id']][] = $et['EventTag'];
+							}
 						}
 					}
+					$eventTags = array_values($eventTags);
 					for ($j = 0; $j < $elements; $j++) {
 						if (!empty($event_tag_objects[$events[($i*1000) + $j]['Event']['id']])) {
 							$events[($i*1000) + $j]['EventTag'] = $event_tag_objects[$events[($i*1000) + $j]['Event']['id']];

app/Controller/TagsController.php

@@ -171,7 +171,7 @@ class TagsController extends AppController {
 			if ($this->Tag->save($this->request->data)) {
 				if ($this->_isRest()) {
 					$tag = $this->Tag->find('first', array(
-						'contidions' => array(
+						'conditions' => array(
 							'Tag.id' => $this->Tag->id
 						),
 						'recursive' => -1

app/Model/Event.php

@@ -2700,6 +2700,7 @@ class Event extends AppModel {
 						}
 					}
 				}
+				throw new Exception();
 			}
 			if ($fromXml) $created_id = $this->id;
 			if (!empty($data['Event']['published']) && 1 == $data['Event']['published']) {
@@ -3608,7 +3609,8 @@ class Event extends AppModel {
 				$this->__fTool = new FinancialTool();
 		}
 		if ($object['type'] == 'attachment' && preg_match('/.*\.(jpg|png|jpeg|gif)$/i', $object['value'])) {
-			$object['image'] = $this->Attribute->base64EncodeAttachment($object);
+			if (!empty($object['data'])) $object['image'] = $object['data'];
+			else $object['image'] = $this->Attribute->base64EncodeAttachment($object);
 		}
 		if (isset($object['distribution']) && $object['distribution'] != 4) unset($object['SharingGroup']);
 		if ($object['objectType'] !== 'object') {

app/View/Attributes/ajax/toggle_correlation.ctp

@@ -10,7 +10,7 @@
 		if ($attribute['Attribute']['disable_correlation']) {
 			echo __('Re-enable the correlation for this attribute.');
 		} else {
-			echo __('This will remove all correlations that already exist for this attribute and prevents any attributes to be related as long as this setting is disabled. Make sure you understand the downasides of disabling correlations.');
+			echo __('This will remove all correlations that already exist for this attribute and prevents any attributes to be related as long as this setting is disabled. Make sure you understand the downsides of disabling correlations.');
 		}
 	?>
 	</p>

app/View/Users/admin_view.ctp

@@ -129,6 +129,8 @@ if (h($user['User']['change_pw']) == 1) {
 		</dd>
 	</dl>
 	<br />
+	<a href="<?php echo $baseurl . '/admin/users/view/' . h($user['User']['id']) . '.json'; ?>" class="btn btn-inverse" download>Download user profile for data portability</a>
+	<br />
 	<div id="userEvents"></div>
 </div>
 <?php

app/View/Users/view.ctp

@@ -86,6 +86,8 @@
 			</dd>
 		<?php endif; ?>
 	</dl>
+	<br />
+	<a href="<?php echo $baseurl . '/users/view/me.json'; ?>" class="btn btn-inverse" download>Download user profile for data portability</a>
 </div>
 <?php
 	echo $this->element('side_menu', array('menuList' => 'globalActions', 'menuItem' => 'view'));

app/files/scripts/stix2misp.py

@@ -43,6 +43,7 @@ class StixParser():
         self.misp_event = MISPEvent()
         self.misp_event['Galaxy'] = []
 
+    # Load data from STIX document, and other usefull data
     def load(self, args, pathname):
         try:
             filename = '{}/tmp/{}'.format(pathname, args[1])
@@ -62,6 +63,7 @@ class StixParser():
             print(json.dumps({'success': 0, 'message': 'The temporary STIX export file could not be read'}))
             sys.exit(0)
 
+    # Event loading function, recursively itterating as long as namespace errors appear
     def load_event(self, filename):
         try:
             return STIXPackage.from_xml(filename)
@@ -75,6 +77,7 @@ class StixParser():
             else:
                 return None
 
+    # Load the mapping dictionary for STIX object types
     def load_mapping(self):
         self.attribute_types_mapping = {
             'AddressObjectType': self.handle_address,
@@ -94,6 +97,8 @@ class StixParser():
             "WindowsExecutableFileObjectType": self.handle_pe
         }
 
+    # Define if the STIX document is from MISP or is an external one
+    # Call then the appropriate function to parse it
     def handler(self):
         self.outputname = '{}.json'.format(self.filename)
         if self.fromMISP:
@@ -103,12 +108,14 @@ class StixParser():
             # external STIX format file
             self.buildExternalDict()
 
+    # Build a MISP event, parsing STIX data following the structure used in our own exporter
     def buildMispDict(self):
         self.dictTimestampAndDate()
         self.eventInfo()
         for indicator in self.event.related_indicators.indicator:
             self.parse_misp_indicator(indicator)
 
+    # Try to parse data from external STIX documents
     def buildExternalDict(self):
         self.dictTimestampAndDate()
         self.eventInfo()
@@ -121,6 +128,7 @@ class StixParser():
         if self.event.courses_of_action:
             self.parse_coa(self.event.courses_of_action)
 
+    # Set timestamp & date values in the new MISP event
     def dictTimestampAndDate(self):
         if self.event.timestamp:
             stixTimestamp = self.event.timestamp
@@ -131,6 +139,7 @@ class StixParser():
             self.misp_event.date = date
             self.misp_event.timestamp = self.getTimestampfromDate(stixTimestamp)
 
+    # Translate date into timestamp
     @staticmethod
     def getTimestampfromDate(date):
         try:
@@ -144,6 +153,7 @@ class StixParser():
             d = int(time.mktime(date.timetuple()))
         return d
 
+    # Set info & title values in the new MISP event
     def eventInfo(self):
         info = "Imported from external STIX event"
         try:
@@ -157,6 +167,7 @@ class StixParser():
             pass
         self.misp_event.info = str(info)
 
+    # Parse indicators of a STIX document coming from our exporter
     def parse_misp_indicator(self, indicator):
         # define is an indicator will be imported as attribute or object
         if indicator.relationship in categories:
@@ -164,6 +175,7 @@ class StixParser():
         else:
             self.parse_misp_object(indicator)
 
+    # Parse STIX objects that we know will give MISP attributes
     def parse_misp_attribute(self, indicator):
         misp_attribute = {'category': str(indicator.relationship)}
         item = indicator.item
@@ -184,6 +196,7 @@ class StixParser():
                 attribute_type, attribute_value = self.composite_type(attribute_dict)
                 self.misp_event.add_attribute(attribute_type, attribute_value, **misp_attribute)
 
+    # Return type & value of a composite attribute in MISP
     @staticmethod
     def composite_type(attributes):
         if "port" in attributes:
@@ -200,6 +213,7 @@ class StixParser():
                 ip_value = attributes["ip-dst"]
             return "domain|ip", "{}|{}".format(attributes["domain"], ip_value)
 
+    # Define type & value of an attribute or object in MISP
     def handle_attribute_type(self, properties, is_object=False, title=None):
         xsi_type = properties._XSI_TYPE
         try:
@@ -214,6 +228,7 @@ class StixParser():
             print("Unparsed type: {}".format(xsi_type))
             sys.exit(1)
 
+    # Return type & value of an ip address attribute
     @staticmethod
     def handle_address(properties):
         if properties.is_source:
@@ -222,15 +237,18 @@ class StixParser():
             ip_type = "ip-dst"
         return ip_type, properties.address_value.value, "ip"
 
+    # Return type & value of an attachment attribute
     @staticmethod
     def handle_attachment(properties, title):
         return eventTypes[properties._XSI_TYPE]['type'], title, properties.raw_artifact.value
 
+    # Return type & value of a domain or url attribute
     @staticmethod
     def handle_domain_or_url(properties):
         event_types = eventTypes[properties._XSI_TYPE]
         return event_types['type'], properties.value.value, event_types['relation']
 
+    # Return type & value of an email attribute
     def handle_email_attribute(self, properties):
         try:
             if properties.from_:
@@ -257,11 +275,13 @@ class StixParser():
             print("Unsupported Email property")
             sys.exit(1)
 
+    # Return type & value of an email attachment
     @staticmethod
     def handle_email_attachment(indicator_object):
         properties = indicator_object.related_objects[0].properties
         return "email-attachment", properties.file_name.value, "attachment"
 
+    # Return type & attributes of a file object
     def handle_file(self, properties, is_object):
         b_hash, b_file = False, False
         attributes = []
@@ -296,6 +316,7 @@ class StixParser():
                 return self.handle_filename_object(attributes, is_object)
         return "file", self.return_attributes(attributes), ""
 
+    # Return the appropriate type & value when we have 1 filename & 1 hash value
     @staticmethod
     def handle_filename_object(attributes, is_object):
         for attribute in attributes:
@@ -313,6 +334,7 @@ class StixParser():
             # it could be malware-sample as well, but STIX is losing this information
             return "filename|{}".format(hash_type), value, ""
 
+    # Return type & value of a hash attribute
     @staticmethod
     def handle_hashes_attribute(properties):
         hash_type = properties.type_.value.lower()
@@ -322,11 +344,13 @@ class StixParser():
             hash_value = properties.fuzzy_hash_value.value
         return hash_type, hash_value, hash_type
 
+    # Return type & value of a hostname attribute
     @staticmethod
     def handle_hostname(properties):
         event_types = eventTypes[properties._XSI_TYPE]
         return event_types['type'], properties.hostname_value.value, event_types['relation']
 
+    # Return type & value of a http request attribute
     @staticmethod
     def handle_http(properties):
         client_request = properties.http_request_response[0].http_client_request
@@ -342,21 +366,25 @@ class StixParser():
             value = client_request.http_request_line.http_method.value
             return "http-method", value, "method"
 
+    # Return type & value of a mutex attribute
     @staticmethod
     def handle_mutex(properties):
         event_types = eventTypes[properties._XSI_TYPE]
         return event_types['type'], properties.name.value, event_types['relation']
 
+    # Return type & value of a port attribute
     @staticmethod
     def handle_port(properties):
         event_types = eventTypes[properties._XSI_TYPE]
         return event_types['type'], properties.port_value.value, event_types['relation']
 
+    # Return type & value of a regkey attribute
     @staticmethod
     def handle_regkey(properties):
         event_types = eventTypes[properties._XSI_TYPE]
         return event_types['type'], properties.key.value, event_types['relation']
 
+    # Return type & value of a composite attribute ip|port or hostname|port
     def handle_socket_address(self, properties):
         if properties.ip_address:
             type1, value1, _ = self.handle_address(properties.ip_address)
@@ -365,6 +393,9 @@ class StixParser():
             value1 = properties.hostname.hostname_value.value
         return "{}|port".format(type1), "{}|{}".format(value1, properties.port.port_value.value), ""
 
+    # Parse a whois object:
+    # Return type & attributes of a whois object if we have the required fields
+    # Otherwise create attributes and return type & value of the last attribute to avoid crashing the parent function
     def handle_whois(self, properties):
         required_one_of = False
         attributes = []
@@ -411,11 +442,14 @@ class StixParser():
                 self.misp_event.add_attribute(attribute_type, attribute_value, **misp_attributes)
             return last_attribute
 
+    # Return type & attributes of the file defining a portable executable object
     def handle_pe(self, properties):
         pe_uuid = self.parse_pe(properties)
         file_type, file_value, _ = self.handle_file(properties, False)
         return file_type, file_value, pe_uuid
 
+    # Parse attributes of a portable executable, create the corresponding object,
+    # and return its uuid to build the reference for the file object generated at the same time
     def parse_pe(self, properties):
         misp_object = MISPObject('pe')
         filename = properties.file_name.value
@@ -445,6 +479,8 @@ class StixParser():
         self.misp_event.add_object(**misp_object)
         return {"pe_uuid": misp_object.uuid}
 
+    # Parse attributes of a portable executable section, create the corresponding object,
+    # and return its uuid to build the reference for the pe object generated at the same time
     def parse_pe_section(self, section):
         section_object = MISPObject('pe-section')
         header_hashes = section.header_hashes
@@ -463,6 +499,7 @@ class StixParser():
         self.misp_event.add_object(**section_object)
         return section_object.uuid
 
+    # Parse STIX object that we know will give MISP objects
     def parse_misp_object(self, indicator):
         object_type = str(indicator.relationship)
         if object_type == 'file':
@@ -477,6 +514,7 @@ class StixParser():
             if object_type != "misc":
                 print("Unparsed Object type: {}".format(name))
 
+    # Create a MISP object, its attributes, and add it in the MISP event
     def fill_misp_object(self, item, name):
         misp_object = MISPObject(name)
         misp_object.timestamp = self.getTimestampfromDate(item.timestamp)
@@ -490,11 +528,13 @@ class StixParser():
             self.parse_observable(properties, misp_object)
         self.misp_event.add_object(**misp_object)
 
+    # Create a MISP attribute and add it in its MISP object
     def parse_observable(self, properties, misp_object):
         misp_attribute = MISPAttribute()
         misp_attribute.type, misp_attribute.value, misp_attribute.object_relation = self.handle_attribute_type(properties, is_object=True)
         misp_object.add_attribute(**misp_attribute)
 
+    # Parse indicators of an external STIX document
     def parse_external_indicator(self, indicators):
         for indicator in indicators:
             try:
@@ -514,6 +554,7 @@ class StixParser():
                     # otherwise, it is a dictionary of attributes, so we build an object
                     self.handle_object_case(attribute_type, attribute_value, compl_data)
 
+    # Parse observables of an external STIX document
     def parse_external_observable(self, observables):
         for observable in observables:
             title = observable.title
@@ -537,6 +578,7 @@ class StixParser():
                     if attribute_value:
                         self.handle_object_case(attribute_type, attribute_value, compl_data)
 
+    # Parse description of an external indicator or observable and add it in the MISP event as an attribute
     def parse_description(self, stix_object):
         if stix_object.description:
             misp_attribute = {}
@@ -544,11 +586,15 @@ class StixParser():
                 misp_attribute['timestamp'] = self.getTimestampfromDate(stix_object.timestamp)
             self.misp_event.add_attribute("text", stix_object.description.value, **misp_attribute)
 
+    # The value returned by the indicators or observables parser is of type str or int
+    # Thus we can add an attribute in the MISP event with the type & value
     def handle_attribute_case(self, attribute_type, attribute_value, data, attribute):
         if attribute_type == 'attachment':
             attribute['data'] = data
         self.misp_event.add_attribute(attribute_type, attribute_value, **attribute)
 
+    # The value returned by the indicators or observables parser is a list of dictionaries
+    # These dictionaries are the attributes we add in an object, itself added in the MISP event
     def handle_object_case(self, attribute_type, attribute_value, compl_data):
         misp_object = MISPObject(attribute_type)
         for attribute in attribute_value:
@@ -560,6 +606,7 @@ class StixParser():
             misp_object.add_reference(compl_data['pe_uuid'], 'pe')
         self.misp_event.add_object(**misp_object)
 
+    # Parse the ttps field of an external STIX document
     def parse_ttps(self, ttps):
         for ttp in ttps:
             if ttp.behavior and ttp.behavior.malware_instances:
@@ -580,6 +627,7 @@ class StixParser():
                     galaxy['GalaxyCluster'] = [cluster]
                     self.misp_event['Galaxy'].append(galaxy)
 
+    # Parse the courses of action field of an external STIX document
     def parse_coa(self, courses_of_action):
         for coa in courses_of_action:
             misp_object = MISPObject('course-of-action')
@@ -626,6 +674,7 @@ class StixParser():
                     misp_object.add_reference(referenced_uuid, 'observable', None, **attribute)
             self.misp_event.add_object(**misp_object)
 
+    # Return the attributes that will be added in a MISP object as a list of dictionaries
     @staticmethod
     def return_attributes(attributes):
         return_attributes = []
@@ -633,6 +682,8 @@ class StixParser():
             return_attributes.append(dict(zip(('type', 'value', 'object_relation'), attribute)))
         return return_attributes
 
+    # Convert the MISP event we create from the STIX document into json format
+    # and write it in the output file
     def saveFile(self):
         eventDict = self.misp_event.to_json()
         with open(self.outputname, 'w') as f: