From 44fef2903c2a8c0c140b0f63b64f450a23a59836 Mon Sep 17 00:00:00 2001 From: Patrizio Tufarolo Date: Thu, 27 Feb 2020 12:59:12 +0100 Subject: [PATCH 01/16] Ensure we only have the last line from the shell command when exporting STIX2 Same as e3b1e8c74a0b40cdb54be938bcea4d9b28a7f0b9 but for exporting STIX2 --- app/Lib/Export/Stix2Export.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/Lib/Export/Stix2Export.php b/app/Lib/Export/Stix2Export.php index ddf4f0b92..f471e204a 100644 --- a/app/Lib/Export/Stix2Export.php +++ b/app/Lib/Export/Stix2Export.php @@ -19,6 +19,8 @@ class Stix2Export extends StixExport $scriptFile = $this->__scripts_dir . $this->__script_name; $filename = $this->__scripts_dir . 'tmp/' . $filename; $my_server = ClassRegistry::init('Server'); - return shell_exec($my_server->getPythonVersion() . ' ' . $scriptFile . ' ' . $filename . $this->__end_of_cmd); + $result = shell_exec($my_server->getPythonVersion() . ' ' . $scriptFile . ' ' . $filename . $this->__end_of_cmd); + $result = end(preg_split("/\r\n|\n|\r/", trim($result))); + return $result; } } From cb572e50430cbd1c44915f2d82217e70586a87da Mon Sep 17 00:00:00 2001 From: Sebastien Tricaud Date: Wed, 25 Mar 2020 15:46:53 -0700 Subject: [PATCH 02/16] Various improvements: * Do not push a string for VERSION.json but use the file in the repository * If database already exist, move on --- debian/postinst | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/debian/postinst b/debian/postinst index 2839f413d..373e46524 100644 --- a/debian/postinst +++ b/debian/postinst @@ -29,7 +29,8 @@ if [ "$1" = "configure" ] ; then cd /usr/share/misp/app sudo -u www-data composer dump-autoload - + sudo -u www-data composer install --ignore-platform-reqs + phpenmod redis phpenmod gnupg @@ -55,9 +56,8 @@ if [ "$1" = "configure" ] ; then mysql -h$HOST -uroot -p$ROOTPWD -e "CREATE USER IF NOT EXISTS '$MISPDBUSER'@'localhost' IDENTIFIED BY '$MISPDBUSERPWD';" mysql -h$HOST -uroot -p$ROOTPWD -e "GRANT ALL PRIVILEGES ON misp.* TO '$MISPDBUSER'@'localhost';" mysql -h$HOST -uroot -p$ROOTPWD -e "FLUSH PRIVILEGES;" - mysql -h$HOST -uroot -p$ROOTPWD -e "CREATE DATABASE $MISPDB;" echo "Creating MISP Database..." - gunzip < /usr/share/doc/misp/MYSQL.sql.gz | mysql -h$HOST -u$MISPDBUSER -p$MISPDBUSERPWD $MISPDB + mysql -h$HOST -uroot -p$ROOTPWD -e "CREATE DATABASE $MISPDB;" && gunzip < /usr/share/doc/misp/MYSQL.sql.gz | mysql -h$HOST -u$MISPDBUSER -p$MISPDBUSERPWD $MISPDB || true # /usr/share/misp/app/Config/database.php echo "Updating salt..." @@ -69,11 +69,18 @@ if [ "$1" = "configure" ] ; then sed -i -E "s/'password'\s=>\s'db password'/'password' => '$MISPDBUSERPWD'/" /usr/share/misp/app/Config/database.php sed -i -E "s/'database'\s=>\s'misp'/'database' => '$MISPDB'/" /usr/share/misp/app/Config/database.php - composer require resque/php-resque || true + # composer require resque/php-resque || true # No composer.json in current directory, do you want to use the one at /usr/share/misp/app? [Y,n]? Y - sudo -u www-data /usr/share/misp/app/Console/cake admin setSetting MISP.baseurl "$BASEURL" - - echo "{\"major\":2, \"minor\":4, \"hotfix\":221}" > /usr/share/misp/VERSION.json + + # + # Starting MISP Workers at every boot + # + chmod +x /usr/share/misp/app/Console/worker/start.sh + cat /usr/share/doc/misp/misp-workers.service > /etc/systemd/system/misp-workers.service + sed -i -E "s/\/var\/www\/MISP/\/usr\/share\/misp/" /etc/systemd/system/misp-workers.service +# systemctl daemon-reload +# systemctl enable --now misp-workers fi + From 8b46a71cd79262b513ca94ea5cafcd1765f747cd Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 31 Mar 2020 09:58:36 +0200 Subject: [PATCH 03/16] chg: [warning-lists] updated to the latest version --- app/files/warninglists | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/warninglists b/app/files/warninglists index 10bcc1496..539c6bc8f 160000 --- a/app/files/warninglists +++ b/app/files/warninglists @@ -1 +1 @@ -Subproject commit 10bcc1496dce382665b04f760b7b0ce99b8ae3e7 +Subproject commit 539c6bc8fdf443d49fec58c6255649f2c1aa9eca From 36a4de95ebb187422b31f178867699e9058a48b2 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 31 Mar 2020 09:59:07 +0200 Subject: [PATCH 04/16] chg: [misp-galaxy] updated to the latest version --- app/files/misp-galaxy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy index 777c3188d..c7104e881 160000 --- a/app/files/misp-galaxy +++ b/app/files/misp-galaxy @@ -1 +1 @@ -Subproject commit 777c3188db6fd1f04fc81106a6c2eb293bb19d12 +Subproject commit c7104e8819d6b789b24a45655aa28625a8c4c346 From 832da852f8095033ee0cf0c5a4b8fbaed7775ef4 Mon Sep 17 00:00:00 2001 From: iglocska Date: Tue, 31 Mar 2020 15:51:59 +0200 Subject: [PATCH 05/16] chg: [settings] disabling background jobs now counts as a misconfiguration --- app/Model/Server.php | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/app/Model/Server.php b/app/Model/Server.php index 012ea1985..62f563c43 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -512,7 +512,7 @@ class Server extends AppModel 'description' => __('Enables the use of MISP\'s background processing.'), 'value' => '', 'errorMessage' => '', - 'test' => 'testBool', + 'test' => 'testBoolTrue', 'type' => 'boolean', ), 'attachments_dir' => array( @@ -3528,6 +3528,21 @@ class Server extends AppModel return true; } + public function testBoolTrue($value, $errorMessage = false) + { + if ($this->testBool($value, $errorMessage) !== true) { + return $this->testBool($value, $errorMessage); + } + if ($value === false) { + if ($errorMessage) { + return $errorMessage; + } + return 'It is highly recommended that this setting is enabled. Make sure you understand the impact of having this setting turned off.'; + } else { + return true; + } + } + public function testBoolFalse($value, $errorMessage = false) { if ($this->testBool($value, $errorMessage) !== true) { From b59b04a8fffd08449119d8a52c598260c1150f91 Mon Sep 17 00:00:00 2001 From: iglocska Date: Wed, 1 Apr 2020 16:28:11 +0200 Subject: [PATCH 06/16] chg: [warninglists] updated --- app/files/warninglists | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/warninglists b/app/files/warninglists index 539c6bc8f..c14634a03 160000 --- a/app/files/warninglists +++ b/app/files/warninglists @@ -1 +1 @@ -Subproject commit 539c6bc8fdf443d49fec58c6255649f2c1aa9eca +Subproject commit c14634a031a37b09e04e6b17ca5cd927de315979 From 150b5f7338cc4eac26763c30cf38e142464c4bcf Mon Sep 17 00:00:00 2001 From: iglocska Date: Wed, 1 Apr 2020 23:14:29 +0200 Subject: [PATCH 07/16] fix: [search] Fixed the UUID / ID searches on the attribute scope, fixes #5636 --- app/Controller/AttributesController.php | 25 +++++++++++++++++++++++++ app/Model/Attribute.php | 2 -- app/Model/Event.php | 18 ++++++++++++++---- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index a4b9b8188..80006607d 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -1678,6 +1678,31 @@ class AttributesController extends AppController ); $exception = false; $filters = $this->_harvestParameters($filterData, $exception); + if (!empty($filters['uuid'])) { + if (!is_array($filters['uuid'])) { + $filters['uuid'] = array($filters['uuid']); + } + $uuid = array(); + $ids = array(); + foreach ($filters['uuid'] as $k => $filter) { + if ($filter[0] === '!') { + $filter = substr($filter, 1); + } + if (Validation::uuid($filter)) { + $uuid[] = $filters['uuid'][$k]; + } else { + $ids[] = $filters['uuid'][$k]; + } + } + if (empty($uuid)) { + unset($filters['uuid']); + } else { + $filters['uuid'] = $uuid; + } + if (!empty($ids)) { + $filters['eventid'] = $ids; + } + } unset($filterData); if ($filters === false) { return $exception; diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index bffe14e5e..c9f529331 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -4426,7 +4426,6 @@ class Attribute extends AppModel 'event_timestamp' => array('function' => 'set_filter_timestamp', 'pop' => true), 'publish_timestamp' => array('function' => 'set_filter_timestamp'), 'org' => array('function' => 'set_filter_org'), - 'uuid' => array('function' => 'set_filter_uuid'), 'published' => array('function' => 'set_filter_published') ), 'Object' => array( @@ -4487,7 +4486,6 @@ class Attribute extends AppModel $subqueryElements = $this->Event->harvestSubqueryElements($filters); $filters = $this->Event->addFiltersFromSubqueryElements($filters, $subqueryElements); - $conditions = $this->buildFilterConditions($user, $filters); $params = array( 'conditions' => $conditions, diff --git a/app/Model/Event.php b/app/Model/Event.php index e4a428da1..91dc6d304 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -2587,11 +2587,21 @@ class Event extends AppModel { if (!empty($params['uuid'])) { $params['uuid'] = $this->convert_filters($params['uuid']); - if (!empty($options['scope']) && $options['scope'] === 'Event') { - $conditions = $this->generic_add_filter($conditions, $params['uuid'], 'Event.uuid'); + if (!empty($params['uuid']['OR'])) { + $conditions['AND'][] = array( + 'OR' => array( + 'Event.uuid' => $params['uuid']['OR'], + 'Attribute.uuid' => $params['uuid']['OR'] + ) + ); } - if (!empty($options['scope']) && $options['scope'] === 'Attribute') { - $conditions = $this->generic_add_filter($conditions, $params['uuid'], 'Attribute.uuid'); + if (!empty($params['uuid']['NOT'])) { + $conditions['AND'][] = array( + 'NOT' => array( + 'Event.uuid' => $params['uuid']['NOT'], + 'Attribute.uuid' => $params['uuid']['NOT'] + ) + ); } } return $conditions; From c6739b7659320784ee3b556a5eb0b1d4e971e40d Mon Sep 17 00:00:00 2001 From: iglocska Date: Wed, 1 Apr 2020 23:53:08 +0200 Subject: [PATCH 08/16] chg: [syslog] added title of log entry --- app/Model/Log.php | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/app/Model/Log.php b/app/Model/Log.php index 4df47f92e..e041fce97 100644 --- a/app/Model/Log.php +++ b/app/Model/Log.php @@ -323,7 +323,6 @@ class Log extends AppModel $elasticSearchClient = $this->getElasticSearchTool(); $elasticSearchClient->pushDocument($logIndex, "log", $data); } - if (Configure::read('Security.syslog')) { // write to syslogd as well $syslog = new SysLog(); @@ -338,8 +337,17 @@ class Log extends AppModel } $entry = $data['Log']['action']; + if (!empty($data['Log']['title'])) { + $entry .= sprintf( + ' -- %s', + $data['Log']['title'] + ); + } if (!empty($data['Log']['description'])) { - $entry .= sprintf(' -- %s', $data['Log']['description']); + $entry .= sprintf( + ' -- %s', + $data['Log']['description'] + ); } $syslog->write($action, $entry); } From 20e1c62d1142f3f61cd0de7f176a2c44b1476b41 Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 2 Apr 2020 07:02:24 +0200 Subject: [PATCH 09/16] fix: [API] fixes to set_filter_uuid --- app/Model/Attribute.php | 24 ++++++++++ app/Model/Event.php | 98 +++++++++++++++++++++++++++++++++-------- 2 files changed, 103 insertions(+), 19 deletions(-) diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index c9f529331..6f6b5e982 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -4615,4 +4615,28 @@ class Attribute extends AppModel } return true; } + + public function set_filter_uuid(&$params, $conditions, $options) + { + if (!empty($params['uuid'])) { + $params['uuid'] = $this->convert_filters($params['uuid']); + if (!empty($params['uuid']['OR'])) { + $conditions['AND'][] = array( + 'OR' => array( + 'Event.uuid' => $params['uuid']['OR'], + 'Attribute.uuid' => $params['uuid']['OR'] + ) + ); + } + if (!empty($params['uuid']['NOT'])) { + $conditions['AND'][] = array( + 'NOT' => array( + 'Event.uuid' => $params['uuid']['NOT'], + 'Attribute.uuid' => $params['uuid']['NOT'] + ) + ); + } + } + return $conditions; + } } diff --git a/app/Model/Event.php b/app/Model/Event.php index 91dc6d304..248a3466f 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -1667,7 +1667,6 @@ class Event extends AppModel 'object_relation' => array('function' => 'set_filter_simple_attribute'), 'tags' => array('function' => 'set_filter_tags', 'pop' => true), 'ignore' => array('function' => 'set_filter_ignore'), - 'uuid' => array('function' => 'set_filter_uuid'), 'deleted' => array('function' => 'set_filter_deleted'), 'to_ids' => array('function' => 'set_filter_to_ids'), 'comment' => array('function' => 'set_filter_comment') @@ -1707,7 +1706,6 @@ class Event extends AppModel } } } - $fields = array('Event.id'); if (!empty($params['include_attribute_count'])) { $fields[] = 'Event.attribute_count'; @@ -2583,26 +2581,88 @@ class Event extends AppModel return $conditions; } + /* + * Add parameters to the already existing condition branch. + * Alternatively just add it. + * + * Example: + * { + * "OR" => array( + * "Event.uuid" => "5e850711-69c8-4d50-8e16-0eea011fb688" + * ) + * } + * + * We would like to add an Attribute.uuid condition to the same branch if it + * exists, passing the existing conditions, the new conditions, the boolean + * branch for the lookup and the pattern of the key (regex) to look for + * + * adding "Attribute.uuid" => "5e850711-69c8-4d50-8e16-0eea011fb688" + * to the OR branch that contains Event.uuid would result in: + * + * Example: + * { + * "OR" => array( + * "Event.uuid" => "5e850711-69c8-4d50-8e16-0eea011fb688", + * "Attribute.uuid" => "5e850711-69c8-4d50-8e16-0eea011fb688" + * ) + * } + */ + public function update_condition_branch($conditions, $data, $booleanOperator, $key) + { + $found = false; + if (!empty($conditions['AND'])) { + foreach ($conditions['AND'] as $k => $conditionData) { + if (!empty($conditionData[$booleanOperator])) { + foreach ($conditionData[$booleanOperator] as $existingKey => $subConditions) { + if (preg_match($key, $existingKey) !== false) { + $conditions['AND'][$k][$booleanOperator] = $conditionData[$booleanOperator] + $data; + continue 2; + } + } + } + } + } else { + $conditions['AND'][] = array( + $booleanOperator = $data + ); + } + return $conditions; + } + public function set_filter_uuid(&$params, $conditions, $options) { - if (!empty($params['uuid'])) { - $params['uuid'] = $this->convert_filters($params['uuid']); - if (!empty($params['uuid']['OR'])) { - $conditions['AND'][] = array( - 'OR' => array( - 'Event.uuid' => $params['uuid']['OR'], - 'Attribute.uuid' => $params['uuid']['OR'] - ) - ); - } - if (!empty($params['uuid']['NOT'])) { - $conditions['AND'][] = array( - 'NOT' => array( - 'Event.uuid' => $params['uuid']['NOT'], - 'Attribute.uuid' => $params['uuid']['NOT'] - ) - ); + if ($options['scope'] === 'Event') { + if (!empty($params['uuid'])) { + $params['uuid'] = $this->convert_filters($params['uuid']); + if (!empty($params['uuid']['OR'])) { + $subQueryOptions = array( + 'conditions' => array('Attribute.uuid' => $params['uuid']['OR']), + 'fields' => array('event_id') + ); + $attributeSubquery = $this->subQueryGenerator($this->Attribute, $subQueryOptions, 'Event.id'); + $conditions['AND'][] = array( + 'OR' => array( + 'Event.uuid' => $params['uuid']['OR'], + $attributeSubquery + ) + ); + } + if (!empty($params['uuid']['NOT'])) { + $subQueryOptions = array( + 'conditions' => array('Attribute.uuid' => $params['uuid']['NOT']), + 'fields' => array('event_id') + ); + $attributeSubquery = $this->subQueryGenerator($this->Attribute, $subQueryOptions, 'Event.id'); + $conditions['AND'][] = array( + 'NOT' => array( + 'Event.uuid' => $params['uuid']['NOT'], + $attributeSubquery + ) + ); + } } + } else { + $conditions = $this->{$options['scope']}->set_filter_uuid($params, $conditions, $options); } return $conditions; } From 94bbb037120102bf85d888fa4997a1952ba3af89 Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 2 Apr 2020 07:04:06 +0200 Subject: [PATCH 10/16] chg: [cleanup] removed bad idea that got barfed into the codebase --- app/Model/Event.php | 50 +-------------------------------------------- 1 file changed, 1 insertion(+), 49 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index 248a3466f..c3267a0a3 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -2580,55 +2580,7 @@ class Event extends AppModel } return $conditions; } - - /* - * Add parameters to the already existing condition branch. - * Alternatively just add it. - * - * Example: - * { - * "OR" => array( - * "Event.uuid" => "5e850711-69c8-4d50-8e16-0eea011fb688" - * ) - * } - * - * We would like to add an Attribute.uuid condition to the same branch if it - * exists, passing the existing conditions, the new conditions, the boolean - * branch for the lookup and the pattern of the key (regex) to look for - * - * adding "Attribute.uuid" => "5e850711-69c8-4d50-8e16-0eea011fb688" - * to the OR branch that contains Event.uuid would result in: - * - * Example: - * { - * "OR" => array( - * "Event.uuid" => "5e850711-69c8-4d50-8e16-0eea011fb688", - * "Attribute.uuid" => "5e850711-69c8-4d50-8e16-0eea011fb688" - * ) - * } - */ - public function update_condition_branch($conditions, $data, $booleanOperator, $key) - { - $found = false; - if (!empty($conditions['AND'])) { - foreach ($conditions['AND'] as $k => $conditionData) { - if (!empty($conditionData[$booleanOperator])) { - foreach ($conditionData[$booleanOperator] as $existingKey => $subConditions) { - if (preg_match($key, $existingKey) !== false) { - $conditions['AND'][$k][$booleanOperator] = $conditionData[$booleanOperator] + $data; - continue 2; - } - } - } - } - } else { - $conditions['AND'][] = array( - $booleanOperator = $data - ); - } - return $conditions; - } - + public function set_filter_uuid(&$params, $conditions, $options) { if ($options['scope'] === 'Event') { From e504d993065c35eb7bccf2112d80238b4afe3118 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 2 Apr 2020 18:16:44 +0200 Subject: [PATCH 11/16] chg: [stix2 export] Setting datetime fields - Instead of letting the created and modified fields set by default, we set them with the timestamp value (or date in case of an event) - The first_seen & last_seen values (or equivalent like valid_from, valid_until, depending on the STIX object type) are set to the first_seen / last_seen if possible, otherwise timestamp --- app/files/scripts/stix2/misp2stix2.py | 76 ++++++++++++++++----------- 1 file changed, 45 insertions(+), 31 deletions(-) diff --git a/app/files/scripts/stix2/misp2stix2.py b/app/files/scripts/stix2/misp2stix2.py index 1e34319ab..5a332561a 100644 --- a/app/files/scripts/stix2/misp2stix2.py +++ b/app/files/scripts/stix2/misp2stix2.py @@ -39,6 +39,8 @@ threat_actor_galaxies_list = ('threat-actor', 'microsoft-activity-group') tool_galaxies_list = ('botnet', 'rat', 'exploit-kit', 'tds', 'tool', 'mitre-tool', 'mitre-enterprise-attack-tool', 'mitre-mobile-attack-tool') _MISP_event_tags = ['Threat-Report', 'misp:tool="misp2stix2"'] +_time_fields = {'indicator': ('valid_from', 'valid_until'), + 'observed-data': ('first_observed', 'last_observed')} class StixBuilder(): def __init__(self): @@ -73,6 +75,7 @@ class StixBuilder(): report_args = {'type': 'report', 'id': self.report_id, 'name': self.misp_event['info'], 'created_by_ref': self.identity_id, 'created': self.misp_event['date'], 'published': self.get_datetime_from_timestamp(self.misp_event['publish_timestamp']), + 'modified': self.get_datetime_from_timestamp(self.misp_event['timestamp']), 'interoperability': True} labels = [tag for tag in _MISP_event_tags] if self.misp_event.get('Tag'): @@ -377,8 +380,7 @@ class StixBuilder(): self.galaxies.append(galaxy_uuid) self.relationships['defined'][source_id].append("{}--{}".format(stix_type, galaxy_uuid)) - @staticmethod - def generate_galaxy_args(galaxy, b_killchain, b_alias, sdo_type): + def generate_galaxy_args(self, galaxy, b_killchain, b_alias, sdo_type): cluster = galaxy['GalaxyCluster'][0] try: cluster_uuid = cluster['collection_uuid'] @@ -387,8 +389,9 @@ class StixBuilder(): sdo_id = "{}--{}".format(sdo_type, cluster_uuid) description = "{} | {}".format(galaxy['description'], cluster['description']) labels = ['misp:name=\"{}\"'.format(galaxy['name'])] - sdo_args = {'id': sdo_id, 'type': sdo_type, 'name': cluster['value'], - 'description': description, 'interoperability': True} + sdo_args = {'id': sdo_id, 'type': sdo_type, 'created': self.misp_event['date'], + 'modified': self.get_datetime_from_timestamp(self.misp_event['timestamp']), + 'name': cluster['value'], 'description': description, 'interoperability': True} if b_killchain: killchain = [{'kill_chain_name': 'misp-category', 'phase_name': galaxy['type']}] @@ -452,8 +455,9 @@ class StixBuilder(): custom_object_id = "x-misp-object-{}--{}".format(attribute_type, attribute['uuid']) custom_object_type = "x-misp-object-{}".format(attribute_type) labels, markings = self.create_labels(attribute) - custom_object_args = {'id': custom_object_id, 'x_misp_category': attribute['category'], 'labels': labels, - 'x_misp_timestamp': self.get_datetime_from_timestamp(attribute['timestamp']), + timestamp = self.get_datetime_from_timestamp(attribute['timestamp']) + custom_object_args = {'id': custom_object_id, 'x_misp_category': attribute['category'], + 'created': timestamp, 'modified': timestamp, 'labels': labels, 'x_misp_value': attribute['value'], 'created_by_ref': self.identity_id} if attribute.get('comment'): custom_object_args['x_misp_comment'] = attribute['comment'] @@ -461,14 +465,15 @@ class StixBuilder(): markings = self.handle_tags(markings) custom_object_args['object_marking_refs'] = markings @CustomObject(custom_object_type, [('id', properties.StringProperty(required=True)), - ('x_misp_timestamp', properties.StringProperty(required=True)), - ('labels', properties.ListProperty(labels, required=True)), - ('x_misp_value', properties.StringProperty(required=True)), - ('created_by_ref', properties.StringProperty(required=True)), - ('object_marking_refs', properties.ListProperty(markings)), - ('x_misp_comment', properties.StringProperty()), - ('x_misp_category', properties.StringProperty()) - ]) + ('labels', properties.ListProperty(labels, required=True)), + ('x_misp_value', properties.StringProperty(required=True)), + ('created', properties.TimestampProperty(required=True)), + ('modified', properties.TimestampProperty(required=True)), + ('created_by_ref', properties.StringProperty(required=True)), + ('object_marking_refs', properties.ListProperty(markings)), + ('x_misp_comment', properties.StringProperty()), + ('x_misp_category', properties.StringProperty()) + ]) class Custom(object): def __init__(self, **kwargs): return @@ -498,14 +503,11 @@ class StixBuilder(): labels, markings = self.create_labels(attribute) attribute_value = attribute['value'] if attribute_type != "AS" else self.define_attribute_value(attribute['value'], attribute['comment']) pattern = mispTypesMapping[attribute_type]['pattern'](attribute_type, attribute_value, attribute['data']) if attribute.get('data') else self.define_pattern(attribute_type, attribute_value) - indicator_args = {'id': indicator_id, 'type': 'indicator', 'labels': labels, 'kill_chain_phases': killchain, - 'valid_from': self.misp_event['date'], 'created_by_ref': self.identity_id, - 'pattern': pattern, 'interoperability': True} - if hasattr(attribute, 'Sighting'): - for sighting in attribute['Sighting']: - if sighting['Organisation']['name'] == self.misp_event['Orgc']['name'] and sighting['type'] == "2": - indicator_args['valid_until'] = self.get_datetime_from_timestamp(sighting['date_sighting']) - break + timestamp = self.get_datetime_from_timestamp(attribute['timestamp']) + indicator_args = {'id': indicator_id, 'type': 'indicator', 'labels': labels, + 'kill_chain_phases': killchain, 'created_by_ref': self.identity_id, + 'pattern': pattern, 'interoperability': True} + indicator_args.update(self.handle_time_fields(attribute, timestamp, 'indicator')) if attribute.get('comment'): indicator_args['description'] = attribute['comment'] if markings: @@ -534,8 +536,9 @@ class StixBuilder(): attribute_value = attribute['value'] if attribute_type != "AS" else self.define_attribute_value(attribute['value'], attribute['comment']) observable = mispTypesMapping[attribute_type]['observable'](attribute_type, attribute_value, attribute['data']) if attribute.get('data') else self.define_observable(attribute_type, attribute_value) observed_data_args = {'id': observed_data_id, 'type': 'observed-data', 'number_observed': 1, - 'first_observed': timestamp, 'last_observed': timestamp, 'labels': labels, - 'created_by_ref': self.identity_id, 'objects': observable, 'interoperability': True} + 'objects': observable, 'created_by_ref': self.identity_id, + 'labels': labels, 'interoperability': True} + observed_data_args.update(self.handle_time_fields(attribute, timestamp, 'observed-data')) if markings: observed_data_args['object_marking_refs'] = self.handle_tags(markings) observed_data = ObservedData(**observed_data_args) @@ -593,15 +596,17 @@ class StixBuilder(): category = misp_object.get('meta-category') labels = self.create_object_labels(name, category, to_ids) values = self.fetch_custom_values(misp_object['Attribute'], custom_object_id) - custom_object_args = {'id': custom_object_id, 'x_misp_values': values, 'labels': labels, - 'x_misp_category': category, 'created_by_ref': self.identity_id, - 'x_misp_timestamp': self.get_datetime_from_timestamp(misp_object['timestamp'])} + timestamp = self.get_datetime_from_timestamp(misp_object['timestamp']) + custom_object_args = {'id': custom_object_id, 'x_misp_values': values, + 'created': timestamp, 'modified': timestamp, 'labels': labels, + 'x_misp_category': category, 'created_by_ref': self.identity_id} if hasattr(misp_object, 'comment') and misp_object.get('comment'): custom_object_args['x_misp_comment'] = misp_object['comment'] @CustomObject(custom_object_type, [('id', properties.StringProperty(required=True)), - ('x_misp_timestamp', properties.StringProperty(required=True)), ('labels', properties.ListProperty(labels, required=True)), ('x_misp_values', properties.DictionaryProperty(required=True)), + ('created', properties.TimestampProperty(required=True)), + ('modified', properties.TimestampProperty(required=True)), ('created_by_ref', properties.StringProperty(required=True)), ('x_misp_comment', properties.StringProperty()), ('x_misp_category', properties.StringProperty()) @@ -623,11 +628,13 @@ class StixBuilder(): category = misp_object.get('meta-category') killchain = self.create_killchain(category) labels = self.create_object_labels(name, category, True) - indicator_args = {'id': indicator_id, 'valid_from': self.misp_event['date'], - 'type': 'indicator', 'labels': labels, 'pattern': pattern, + timestamp = self.get_datetime_from_timestamp(misp_object['timestamp']) + indicator_args = {'id': indicator_id, 'type': 'indicator', + 'labels': labels, 'pattern': pattern, 'description': misp_object['description'], 'allow_custom': True, 'kill_chain_phases': killchain, 'interoperability': True, 'created_by_ref': self.identity_id} + indicator_args.update(self.handle_time_fields(misp_object, timestamp, 'indicator')) indicator = Indicator(**indicator_args) self.append_object(indicator) @@ -644,8 +651,8 @@ class StixBuilder(): timestamp = self.get_datetime_from_timestamp(misp_object['timestamp']) observed_data_args = {'id': observed_data_id, 'type': 'observed-data', 'labels': labels, 'number_observed': 1, 'objects': observable_objects, 'allow_custom': True, - 'first_observed': timestamp, 'last_observed': timestamp, 'created_by_ref': self.identity_id, 'interoperability': True} + observed_data_args.update(self.handle_time_fields(misp_object, timestamp, 'observed-data')) try: observed_data = ObservedData(**observed_data_args) except exceptions.InvalidValueError: @@ -1417,6 +1424,13 @@ class StixBuilder(): def get_datetime_from_timestamp(timestamp): return datetime.datetime.utcfromtimestamp(int(timestamp)) + @staticmethod + def handle_time_fields(attribute, timestamp, stix_type): + to_return = {'created': timestamp, 'modified': timestamp} + for misp_field, stix_field in zip(('first_seen', 'last_seen'), _time_fields[stix_type]): + to_return[stix_field] = attribute[misp_field] if attribute[misp_field] else timestamp + return to_return + def main(args): stix_builder = StixBuilder() stix_builder.loadEvent(args) From f32c50fe6de18a4374f603b1dee8b5fec79870d4 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 2 Apr 2020 18:35:33 +0200 Subject: [PATCH 12/16] chg: [stix2 libray] Bumped latest python library version --- cti-python-stix2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cti-python-stix2 b/cti-python-stix2 index 77ca5ae2f..65a943d89 160000 --- a/cti-python-stix2 +++ b/cti-python-stix2 @@ -1 +1 @@ -Subproject commit 77ca5ae2f9742652cd90a53f1b83308a8343f2cc +Subproject commit 65a943d8929c578041f789665b05810ea68986cb From ffc67e0b10a1fbe91466ef723c80433d61540dcb Mon Sep 17 00:00:00 2001 From: iglocska Date: Fri, 3 Apr 2020 06:15:14 +0200 Subject: [PATCH 13/16] chg: [warninglists] bump --- app/files/warninglists | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/warninglists b/app/files/warninglists index c14634a03..bad8b17ff 160000 --- a/app/files/warninglists +++ b/app/files/warninglists @@ -1 +1 @@ -Subproject commit c14634a031a37b09e04e6b17ca5cd927de315979 +Subproject commit bad8b17fffd059f8fd739ce16fc885d79e749022 From d9f4a25f62f1d3423bbc390a045dc859d9e0f0ba Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Fri, 3 Apr 2020 11:11:48 +0200 Subject: [PATCH 14/16] chg: [taxonomies] updated to the latest version --- app/files/taxonomies | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/taxonomies b/app/files/taxonomies index d577ad875..84b5a5306 160000 --- a/app/files/taxonomies +++ b/app/files/taxonomies @@ -1 +1 @@ -Subproject commit d577ad8758713e4d7c0523bbe2bead64c941ebdb +Subproject commit 84b5a5306c3186497f83e264b21006d8468a1b85 From 42728ef609ac56e3c3d061a0221cc019995cdc05 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Fri, 3 Apr 2020 12:20:12 +0200 Subject: [PATCH 15/16] chg: [taxonomies] revert --- app/files/taxonomies | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/taxonomies b/app/files/taxonomies index 84b5a5306..d577ad875 160000 --- a/app/files/taxonomies +++ b/app/files/taxonomies @@ -1 +1 @@ -Subproject commit 84b5a5306c3186497f83e264b21006d8468a1b85 +Subproject commit d577ad8758713e4d7c0523bbe2bead64c941ebdb From 2670e2b443297cf6431bc2a9163ce2963a103694 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 3 Apr 2020 18:30:14 +0200 Subject: [PATCH 16/16] fix: [stix2 export] Fixed datetime fields format in custom objects --- app/files/scripts/stix2/misp2stix2.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/files/scripts/stix2/misp2stix2.py b/app/files/scripts/stix2/misp2stix2.py index 5a332561a..1540e113f 100644 --- a/app/files/scripts/stix2/misp2stix2.py +++ b/app/files/scripts/stix2/misp2stix2.py @@ -467,8 +467,8 @@ class StixBuilder(): @CustomObject(custom_object_type, [('id', properties.StringProperty(required=True)), ('labels', properties.ListProperty(labels, required=True)), ('x_misp_value', properties.StringProperty(required=True)), - ('created', properties.TimestampProperty(required=True)), - ('modified', properties.TimestampProperty(required=True)), + ('created', properties.TimestampProperty(required=True, precision='millisecond')), + ('modified', properties.TimestampProperty(required=True, precision='millisecond')), ('created_by_ref', properties.StringProperty(required=True)), ('object_marking_refs', properties.ListProperty(markings)), ('x_misp_comment', properties.StringProperty()), @@ -605,8 +605,8 @@ class StixBuilder(): @CustomObject(custom_object_type, [('id', properties.StringProperty(required=True)), ('labels', properties.ListProperty(labels, required=True)), ('x_misp_values', properties.DictionaryProperty(required=True)), - ('created', properties.TimestampProperty(required=True)), - ('modified', properties.TimestampProperty(required=True)), + ('created', properties.TimestampProperty(required=True, precision='millisecond')), + ('modified', properties.TimestampProperty(required=True, precision='millisecond')), ('created_by_ref', properties.StringProperty(required=True)), ('x_misp_comment', properties.StringProperty()), ('x_misp_category', properties.StringProperty())