From 08c9337e6c69b27737fd609bcfc47453ff756cf3 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 23 Sep 2019 18:31:44 +0200 Subject: [PATCH 01/33] fix: [internal] Just site admin can force when saving freetext --- app/Controller/EventsController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 628000092..5e0912d83 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -4120,7 +4120,7 @@ class EventsController extends AppController $this->Event->insertLock($this->Auth->user(), $id); $attributes = json_decode($this->request->data['Attribute']['JsonObject'], true); $default_comment = $this->request->data['Attribute']['default_comment']; - $force = $this->request->data['Attribute']['force']; + $force = $this->_isSiteAdmin() && $this->request->data['Attribute']['force']; $flashMessage = $this->Event->processFreeTextDataRouter($this->Auth->user(), $attributes, $id, $default_comment, $force); $this->Flash->info($flashMessage); $this->redirect(array('controller' => 'events', 'action' => 'view', $id)); From 7cd21755dd512c15575c1a1ce824b896fe2bed69 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Fri, 17 Apr 2020 11:22:15 +0200 Subject: [PATCH 02/33] fix: [event:fetchEvent] Block viewing the event if user does not belong to the sharing_group Even if the event belongs to the user. This scenario can happen if a remote sync is badly configured where the remote sync user have site_admin right, thus allowing the user to see the event even though he is not part of the SG --- app/Model/Event.php | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/Model/Event.php b/app/Model/Event.php index c1cddda5a..c962886d3 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -2150,6 +2150,22 @@ class Event extends AppModel 'Object' => array('name', 'meta-category') ); foreach ($results as $eventKey => &$event) { + if ($event['Event']['distribution'] == 4 && !in_array($event['Event']['sharing_group_id'], $sgids)) { + $this->Log = ClassRegistry::init('Log'); + $this->Log->create(); + $this->Log->save(array( + 'org' => $user['Organisation']['name'], + 'model' => 'Event', + 'model_id' => $event['Event']['id'], + 'email' => $user['email'], + 'action' => 'fetchEvent', + 'user_id' => $user['id'], + 'title' => 'User was able to fetch the event but not the sharing_group it belongs to', + 'change' => '' + )); + unset($results[$eventKey]); // Current user cannot access sharing_group associated to this event + continue; + } $this->__attachReferences($user, $event, $sgids, $fields); $event = $this->Orgc->attachOrgsToEvent($event, $fieldsOrg); if (!$options['sgReferenceOnly'] && $event['Event']['sharing_group_id']) { From c9481b23140d5f2ef3460e02574dae233493aacb Mon Sep 17 00:00:00 2001 From: mokaddem Date: Fri, 17 Apr 2020 11:26:22 +0200 Subject: [PATCH 03/33] fix: [event:fetchEvent] Block viewing Objects/Attributes if the user does not belong to the sharing_group Even if these elements belong to the user. Similar explanation than for 7cd2175 --- app/Model/Event.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index c962886d3..93e008f8d 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -2464,7 +2464,11 @@ class Event extends AppModel } foreach ($data as $k => $v) { if ($v['distribution'] == 4) { - $data[$k]['SharingGroup'] = $sharingGroupData[$v['sharing_group_id']]['SharingGroup']; + if (isset($sharingGroupData[$v['sharing_group_id']])) { + $data[$k]['SharingGroup'] = $sharingGroupData[$v['sharing_group_id']]['SharingGroup']; + } else { + unset($data[$k]); // current user could not fetch the sharing_group + } } } return $data; From 3547a8a888c899f43c8ce7c603119787db511f03 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Fri, 17 Apr 2020 11:29:09 +0200 Subject: [PATCH 04/33] fix: [correlations] Update correlations on Attribute or Event `distribution` change --- app/Model/Attribute.php | 6 ++++-- app/Model/Event.php | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 6f6b5e982..74972365c 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -698,7 +698,7 @@ class Attribute extends AppModel * Only recorrelate if: * - We are dealing with a new attribute OR * - The existing attribute's previous state is known AND - * value, type or disable correlation have changed + * value, type, disable correlation or distribution have changed * This will avoid recorrelations when it's not really needed, such as adding a tag */ if (!$created) { @@ -706,7 +706,9 @@ class Attribute extends AppModel empty($this->old) || $this->data['Attribute']['value'] != $this->old['Attribute']['value'] || $this->data['Attribute']['disable_correlation'] != $this->old['Attribute']['disable_correlation'] || - $this->data['Attribute']['type'] != $this->old['Attribute']['type'] + $this->data['Attribute']['type'] != $this->old['Attribute']['type'] || + $this->data['Attribute']['distribution'] != $this->old['Attribute']['distribution'] || + $this->data['Attribute']['sharing_group_id'] != $this->old['Attribute']['sharing_group_id'] ) { $this->__beforeSaveCorrelation($this->data['Attribute']); $this->__afterSaveCorrelation($this->data['Attribute'], false, $passedEvent); diff --git a/app/Model/Event.php b/app/Model/Event.php index 93e008f8d..8fe3d4cfe 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -641,6 +641,12 @@ class Event extends AppModel if (isset($this->data['Event']['info'])) { $this->Correlation->updateAll(array('Correlation.info' => $db->value($this->data['Event']['info'])), array('Correlation.event_id' => intval($this->data['Event']['id']))); } + if (isset($this->data['Event']['distribution'])) { + $this->Correlation->updateAll(array('Correlation.distribution' => $db->value($this->data['Event']['distribution'])), array('Correlation.event_id' => intval($this->data['Event']['id']))); + } + if (isset($this->data['Event']['sharing_group_id'])) { + $this->Correlation->updateAll(array('Correlation.sharing_group_id' => $db->value($this->data['Event']['sharing_group_id'])), array('Correlation.event_id' => intval($this->data['Event']['id']))); + } } if (empty($this->data['Event']['unpublishAction']) && empty($this->data['Event']['skip_zmq']) && Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) { $pubSubTool = $this->getPubSubTool(); From 549028c7af7fcacf1f6d5113583155ab02a45eed Mon Sep 17 00:00:00 2001 From: mokaddem Date: Fri, 17 Apr 2020 14:59:25 +0200 Subject: [PATCH 05/33] fix: [event:view] Restored disabled_correlation toggle --- app/View/Events/view.ctp | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/app/View/Events/view.ctp b/app/View/Events/view.ctp index baee82ee6..cc35bd3f0 100644 --- a/app/View/Events/view.ctp +++ b/app/View/Events/view.ctp @@ -301,27 +301,27 @@ ) ) ); - if (!Configure::read('MISP.completely_disable_correlation') && Configure::read('MISP.allow_disabling_correlation')) { - $table_data[] = array( - 'key' => __('Correlation'), - 'class' => $event['Event']['disable_correlation'] ? 'background-red bold' : '', - 'html' => sprintf( - '%s%s', - $event['Event']['disable_correlation'] ? __('Disabled') : __('Enabled'), - (!$mayModify && !$isSiteAdmin) ? '' : sprintf( + } + if (!Configure::read('MISP.completely_disable_correlation') && Configure::read('MISP.allow_disabling_correlation')) { + $table_data[] = array( + 'key' => __('Correlation'), + 'class' => $event['Event']['disable_correlation'] ? 'background-red bold' : '', + 'html' => sprintf( + '%s%s', + $event['Event']['disable_correlation'] ? __('Disabled') : __('Enabled'), + (!$mayModify && !$isSiteAdmin) ? '' : sprintf( + sprintf( + ' (%s)', sprintf( - ' (%s)', - sprintf( - "'%s', 'events', 'toggleCorrelation', '', '#confirmation_box'", - h($event['Event']['id']) - ), - $event['Event']['disable_correlation'] ? 'color:white;' : '', - $event['Event']['disable_correlation'] ? __('enable') : __('disable') - ) + "'%s', 'events', 'toggleCorrelation', '', '#confirmation_box'", + h($event['Event']['id']) + ), + $event['Event']['disable_correlation'] ? 'color:white;' : '', + $event['Event']['disable_correlation'] ? __('enable') : __('disable') ) ) - ); - } + ) + ); } ?> From e9dc28fda7292fd49113e6b027f2fd3b88f81222 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 20 Apr 2020 08:51:01 +0200 Subject: [PATCH 06/33] chg: [sharingGroup:capture] Prevent capture of SG in some specific cases - Need more testing Should fix #5784 --- app/Model/Event.php | 81 +++++++++++++++++++++----------------- app/Model/SharingGroup.php | 21 +++++++++- 2 files changed, 65 insertions(+), 37 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index 8fe3d4cfe..d5ad62957 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -3263,10 +3263,10 @@ class Event extends AppModel return array($bodyevent, $body); } - private function __captureSGForElement($element, $user) + private function __captureSGForElement($element, $user, $syncLocal=false) { if (isset($element['SharingGroup'])) { - $sg = $this->SharingGroup->captureSG($element['SharingGroup'], $user); + $sg = $this->SharingGroup->captureSG($element['SharingGroup'], $user, $syncLocal); unset($element['SharingGroup']); } elseif (isset($element['sharing_group_id'])) { $sg = $this->SharingGroup->checkIfAuthorised($user, $element['sharing_group_id']) ? $element['sharing_group_id'] : false; @@ -3283,17 +3283,17 @@ class Event extends AppModel // When we receive an event via REST, we might end up with organisations, sharing groups, tags that we do not know // or which we need to update. All of that is controlled in this method. - private function __captureObjects($data, $user) + private function __captureObjects($data, $user, $syncLocal=false) { // First we need to check whether the event or any attributes are tied to a sharing group and whether the user is even allowed to create the sharing group / is part of it if (isset($data['Event']['distribution']) && $data['Event']['distribution'] == 4) { - $data['Event'] = $this->__captureSGForElement($data['Event'], $user); + $data['Event'] = $this->__captureSGForElement($data['Event'], $user, $syncLocal); } if (!empty($data['Event']['Attribute'])) { foreach ($data['Event']['Attribute'] as $k => $a) { unset($data['Event']['Attribute']['id']); if (isset($a['distribution']) && $a['distribution'] == 4) { - $data['Event']['Attribute'][$k] = $this->__captureSGForElement($a, $user); + $data['Event']['Attribute'][$k] = $this->__captureSGForElement($a, $user, $syncLocal); if ($data['Event']['Attribute'][$k] === false) { unset($data['Event']['Attribute']); } @@ -3303,7 +3303,7 @@ class Event extends AppModel if (!empty($data['Event']['Object'])) { foreach ($data['Event']['Object'] as $k => $o) { if (isset($o['distribution']) && $o['distribution'] == 4) { - $data['Event']['Object'][$k] = $this->__captureSGForElement($o, $user); + $data['Event']['Object'][$k] = $this->__captureSGForElement($o, $user, $syncLocal); if ($data['Event']['Object'][$k] === false) { unset($data['Event']['Object'][$k]); continue; @@ -3311,7 +3311,7 @@ class Event extends AppModel } foreach ($o['Attribute'] as $k2 => $a) { if (isset($a['distribution']) && $a['distribution'] == 4) { - $data['Event']['Object'][$k]['Attribute'][$k2] = $this->__captureSGForElement($a, $user); + $data['Event']['Object'][$k]['Attribute'][$k2] = $this->__captureSGForElement($a, $user, $syncLocal); if ($data['Event']['Object'][$k]['Attribute'][$k2] === false) { unset($data['Event']['Object'][$k]['Attribute'][$k2]); } @@ -3479,6 +3479,24 @@ class Event extends AppModel return 'blocked'; } } + if ($passAlong) { + $this->Server = ClassRegistry::init('Server'); + $server = $this->Server->find('first', array( + 'conditions' => array( + 'Server.id' => $passAlong + ), + 'recursive' => -1, + 'fields' => array( + 'Server.name', + 'Server.id', + 'Server.unpublish_event', + 'Server.publish_without_email', + 'Server.internal' + ) + )); + } else { + $server['Server']['internal'] = false; + } if ($fromXml) { // Workaround for different structure in XML/array than what CakePHP expects $data = $this->cleanupEventArrayFromXML($data); @@ -3505,7 +3523,7 @@ class Event extends AppModel return $existingEvent['Event']['id']; } else { if ($fromXml) { - $data = $this->__captureObjects($data, $user); + $data = $this->__captureObjects($data, $user, $server['Server']['internal']); } if ($data === false) { $failedCapture = true; @@ -3513,7 +3531,7 @@ class Event extends AppModel } } else { if ($fromXml) { - $data = $this->__captureObjects($data, $user); + $data = $this->__captureObjects($data, $user, $server['Server']['internal']); } if ($data === false) { $failedCapture = true; @@ -3574,19 +3592,6 @@ class Event extends AppModel $this->Log = ClassRegistry::init('Log'); if ($saveResult) { if ($passAlong) { - $this->Server = ClassRegistry::init('Server'); - $server = $this->Server->find('first', array( - 'conditions' => array( - 'Server.id' => $passAlong - ), - 'recursive' => -1, - 'fields' => array( - 'Server.name', - 'Server.id', - 'Server.unpublish_event', - 'Server.publish_without_email' - ) - )); if ($server['Server']['publish_without_email'] == 0) { $st = "enabled"; } else { @@ -3729,6 +3734,23 @@ class Event extends AppModel } else { $existingEvent = $this->findById($id); } + if ($passAlong) { + $this->Server = ClassRegistry::init('Server'); + $server = $this->Server->find('first', array( + 'conditions' => array( + 'Server.id' => $passAlong + ), + 'recursive' => -1, + 'fields' => array( + 'Server.name', + 'Server.id', + 'Server.unpublish_event', + 'Server.publish_without_email' + ) + )); + } else { + $server['Server']['internal'] = false; + } // If the event exists... $dateObj = new DateTime(); $date = $dateObj->getTimestamp(); @@ -3751,7 +3773,7 @@ class Event extends AppModel return(array('error' => 'Event could not be saved: Invalid sharing group or you don\'t have access to that sharing group.')); } } else { - $data['Event']['sharing_group_id'] = $this->SharingGroup->captureSG($data['Event']['SharingGroup'], $user); + $data['Event']['sharing_group_id'] = $this->SharingGroup->captureSG($data['Event']['SharingGroup'], $user, $server['Server']['internal']); unset($data['Event']['SharingGroup']); if ($data['Event']['sharing_group_id'] === false) { return (array('error' => 'Event could not be saved: User not authorised to create the associated sharing group.')); @@ -3872,19 +3894,6 @@ class Event extends AppModel if ((!empty($data['Event']['published']) && 1 == $data['Event']['published'])) { // The edited event is from a remote server ? if ($passAlong) { - $this->Server = ClassRegistry::init('Server'); - $server = $this->Server->find('first', array( - 'conditions' => array( - 'Server.id' => $passAlong - ), - 'recursive' => -1, - 'fields' => array( - 'Server.name', - 'Server.id', - 'Server.unpublish_event', - 'Server.publish_without_email' - ) - )); if ($server['Server']['publish_without_email'] == 0) { $st = "enabled"; } else { diff --git a/app/Model/SharingGroup.php b/app/Model/SharingGroup.php index 89c51a1aa..2e15f5182 100644 --- a/app/Model/SharingGroup.php +++ b/app/Model/SharingGroup.php @@ -485,7 +485,7 @@ class SharingGroup extends AppModel return $results; } - public function captureSG($sg, $user) + public function captureSG($sg, $user, $syncLocal=false) { $existingSG = !isset($sg['uuid']) ? null : $this->find('first', array( 'recursive' => -1, @@ -501,6 +501,25 @@ class SharingGroup extends AppModel if (!$user['Role']['perm_sharing_group']) { return false; } + // check if current user is contained in the SG and we are in a local sync setup + $authorizedToSave = $this->checkIfAuthorisedToSave($user, $sg); + if (!$user['Role']['perm_site_admin'] && + !($user['Role']['perm_sync'] && $syncLocal ) && + !$authorizedToSave + ) { + $this->Log->create(); + $entry = array( + 'org' => $user['Organisation']['name'], + 'model' => 'SharingGroup', + 'model_id' => $sg['SharingGroup']['uuid'], + 'email' => $user['email'], + 'action' => 'error', + 'user_id' => $user['id'], + 'title' => 'Tried to save a sharing group but the user does not belong to it.' + ); + $this->Log->save($entry); + return false; + } $this->create(); $newSG = array(); $attributes = array( From a99c96adcfdfbadc652a14cab392dc61638073b1 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 20 Apr 2020 09:43:53 +0200 Subject: [PATCH 07/33] fix: [attribute:add] Prevent save for invalid sharing_groups ids --- app/Controller/AttributesController.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 80006607d..13d89201e 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -148,6 +148,12 @@ class AttributesController extends AppController if (!isset($this->request->data['Attribute'])) { $this->request->data = array('Attribute' => $this->request->data); } + if ($this->request->data['Attribute']['distribution'] == 4) { + $sg = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1, $this->request->data['Attribute']['sharing_group_id']); + if (empty($sg)) { + throw new MethodNotAllowedException(__('Invalid Sharing Group or not authorised.')); + } + } // // multiple attributes in batch import // From f29474325d6e2d94302373474662dab0d9ce444c Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 20 Apr 2020 09:49:12 +0200 Subject: [PATCH 08/33] fix: [attribute:edit] Prevent save for invalid sharing_groups ids --- app/Controller/AttributesController.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 13d89201e..95766179a 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -858,6 +858,12 @@ class AttributesController extends AppController if (!isset($this->request->data['Attribute'])) { $this->request->data = array('Attribute' => $this->request->data); } + if ($this->request->data['Attribute']['distribution'] == 4) { + $sg = $this->Attribute->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1, $this->request->data['Attribute']['sharing_group_id']); + if (empty($sg)) { + throw new MethodNotAllowedException(__('Invalid Sharing Group or not authorised.')); + } + } $existingAttribute = $this->Attribute->findByUuid($this->Attribute->data['Attribute']['uuid']); // check if the attribute has a timestamp already set (from a previous instance that is trying to edit via synchronisation) // check which attribute is newer From 93bd5eddba6b53b34e3aefad064ee6a849203056 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 23 Apr 2020 10:08:34 +0200 Subject: [PATCH 09/33] chg: [event:timeline] Added Sightings visualisation --- app/Controller/EventsController.php | 8 +- app/Lib/Tools/EventTimelineTool.php | 141 ++++++++++++++++++++++++++++ app/webroot/css/event-timeline.css | 9 ++ app/webroot/js/event-timeline.js | 57 +++++++++-- 4 files changed, 207 insertions(+), 8 deletions(-) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 220f1989c..f13107bc3 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -4514,16 +4514,20 @@ class EventsController extends AppController if (!in_array($type, $validTools)) { throw new MethodNotAllowedException('Invalid type.'); } - App::uses('EventTimelineTool', 'Tools'); $grapher = new EventTimelineTool(); $data = $this->request->is('post') ? $this->request->data : array(); $dataFiltering = array_key_exists('filtering', $data) ? $data['filtering'] : array(); + $scope = isset($data['scope']) ? $data['scope'] : 'seen'; $extended = isset($this->params['named']['extended']) ? 1 : 0; $grapher->construct($this->Event, $this->Auth->user(), $dataFiltering, $extended); - $json = $grapher->get_timeline($id); + if ($scope == 'seen') { + $json = $grapher->get_timeline($id); + } elseif ($scope == 'sightings') { + $json = $grapher->get_sighting_timeline($id); + } array_walk_recursive($json, function (&$item, $key) { if (!mb_detect_encoding($item, 'utf-8', true)) { diff --git a/app/Lib/Tools/EventTimelineTool.php b/app/Lib/Tools/EventTimelineTool.php index d2eb44943..46fef4604 100644 --- a/app/Lib/Tools/EventTimelineTool.php +++ b/app/Lib/Tools/EventTimelineTool.php @@ -135,4 +135,145 @@ return $this->__json; } + + /* + * Extrapolation strategy: + * - If only positive sightings: Will be from first to last sighting + * - If both positive and false positive: False positive get priority. It will be marked as false positive until next positive sighting + */ + public function get_sighting_timeline($id) + { + $event = $this->__eventModel->fetchEvent($this->__user, array( + 'eventid' => $id, + 'flatten' => 1, + 'includeTagRelations' => 1, + 'extended' => $this->__extended_view + )); + $this->__json['items'] = array(); + + if (empty($event)) { + return $this->__json; + } else { + $event = $event[0]; + } + + $lookupAttribute = array(); + foreach ($event['Attribute'] as $k => $attribute) { + $lookupAttribute[$attribute['id']] = &$event['Attribute'][$k]; + } + + // regroup sightings per attribute + $regroupedSightings = array(); + foreach ($event['Sighting'] as $k => $sighting) { + $event['Sighting'][$k]['date_sighting'] *= 1000; // adapt to use micro + $regroupedSightings[$sighting['attribute_id']][] = &$event['Sighting'][$k]; + } + // make sure sightings are ordered + uksort($regroupedSightings, function ($a, $b) { + return $a['date_sighting'] > $b['date_sighting']; + }); + // generate extrapolation + $now = time()*1000; + foreach ($regroupedSightings as $attributeId => $sightings) { + $i = 0; + while ($i < count($sightings)) { + $sighting = $sightings[$i]; + $attribute = $lookupAttribute[$attributeId]; + $fpSightingIndex = $this->getNextFalsePositiveSightingIndex($sightings, $i+1); + if ($fpSightingIndex === false) { // No next FP, extrapolate to now + $this->__json['items'][] = array( + 'attribute_id' => $attributeId, + 'id' => sprintf('%s-%s', $attributeId, $sighting['id']), + 'uuid' => $sighting['uuid'], + 'content' => $attribute['value'], + 'event_id' => $attribute['event_id'], + 'group' => 'sighting_positive', + 'timestamp' => $attribute['timestamp'], + 'first_seen' => $sighting['date_sighting'], + 'last_seen' => $now, + ); + break; + } else { + // set up until last positive + $pSightingIndex = $fpSightingIndex - 1; + $halfTime = 0; + if ($pSightingIndex == $i) { + // we have only one positive sighting, thus the UP time should be take from a pooling frequence + // for now, consider it UP only for half the time until the next FP + $halfTime = ($sightings[$i+1]['date_sighting'] - $sighting['date_sighting'])/2; + } + $pSighting = $sightings[$pSightingIndex]; + $this->__json['items'][] = array( + 'attribute_id' => $attributeId, + 'id' => sprintf('%s-%s', $attributeId, $sighting['id']), + 'uuid' => $sighting['uuid'], + 'content' => $attribute['value'], + 'event_id' => $attribute['event_id'], + 'group' => 'sighting_positive', + 'timestamp' => $attribute['timestamp'], + 'first_seen' => $sighting['date_sighting'], + 'last_seen' => $pSighting['date_sighting'] + $halfTime, + ); + // No next FP, extrapolate to now + $fpSighting = $sightings[$fpSightingIndex]; + $secondNextPSightingIndex = $this->getNextPositiveSightingIndex($sightings, $fpSightingIndex+1); + if ($secondNextPSightingIndex === false) { // No next P, extrapolate to now + $this->__json['items'][] = array( + 'attribute_id' => $attributeId, + 'id' => sprintf('%s-%s', $attributeId, $sighting['id']), + 'uuid' => $sighting['uuid'], + 'content' => $attribute['value'], + 'event_id' => $attribute['event_id'], + 'group' => 'sighting_negative', + 'timestamp' => $attribute['timestamp'], + 'first_seen' => $pSighting['date_sighting'] - $halfTime, + 'last_seen' => $now, + ); + break; + } else { + if ($halfTime > 0) { // We need to fake a previous P + $pSightingIndex = $pSightingIndex+1; + $pSighting = $sightings[$pSightingIndex]; + } + // set down until next postive + $secondNextPSighting = $sightings[$secondNextPSightingIndex]; + $this->__json['items'][] = array( + 'attribute_id' => $attributeId, + 'id' => sprintf('%s-%s', $attributeId, $sighting['id']), + 'uuid' => $pSighting['uuid'], + 'content' => $attribute['value'], + 'event_id' => $attribute['event_id'], + 'group' => 'sighting_negative', + 'timestamp' => $attribute['timestamp'], + 'first_seen' => $pSighting['date_sighting'] - $halfTime, + 'last_seen' => $secondNextPSighting['date_sighting'], + ); + $i = $secondNextPSightingIndex; + } + } + } + } + return $this->__json; + } + + private function getNextFalsePositiveSightingIndex($sightings, $startIndex) + { + for ($i=$startIndex; $i < count($sightings) ; $i++) { + $sighting = $sightings[$i]; + if ($sighting['type'] == 1) { // is false positive + return $i; + } + } + return false; + } + private function getNextPositiveSightingIndex($sightings, $startIndex) + { + for ($i=$startIndex; $i < count($sightings) ; $i++) { + $sighting = $sightings[$i]; + if ($sighting['type'] == 0) { // is false positive + return $i; + } + } + return false; + } } diff --git a/app/webroot/css/event-timeline.css b/app/webroot/css/event-timeline.css index d5653e776..b82318aa7 100644 --- a/app/webroot/css/event-timeline.css +++ b/app/webroot/css/event-timeline.css @@ -59,6 +59,15 @@ box-shadow: 0 0 20px rgba(82, 168, 236, 1);` } +.vis-item.sighting_positive { + background-color: green; + border-color: white; +} +.vis-item.sighting_negative { + background-color: red; + border-color: white; +} + .vis-item.object { background-color: #3465a4; border-color: black; diff --git a/app/webroot/js/event-timeline.js b/app/webroot/js/event-timeline.js index 3d810143b..2e00638a1 100644 --- a/app/webroot/js/event-timeline.js +++ b/app/webroot/js/event-timeline.js @@ -29,10 +29,16 @@ var options = { return build_object_template(item); case "object_attribute": - console.log('Error'); + console.log('Error: Group not valid'); break; default: + if (item.className == "sighting_positive" || item.className == "sighting_negative") { + return build_sighting_template(item); + } else { + console.log(item) + console.log('Error: Unkown group'); + } break; } }, @@ -199,6 +205,13 @@ function build_object_template(obj) { return html; } +function build_sighting_template(attr){ + var span = $(''); + span.text(attr.content); + var html = span[0].outerHTML; + return html; +} + function contain_seen_attribute(obj) { if (obj['Attribute'] === undefined) { return false; @@ -372,6 +385,8 @@ function map_scope(val) { return 'seen'; case 'Object relationship': return 'relationship'; + case 'Sightings': + return 'sightings'; default: return 'seen'; } @@ -400,7 +415,8 @@ function update_badge() { function reload_timeline() { update_badge(); - var payload = {scope: map_scope($('#select_timeline_scope').val())}; + var selectedScope = map_scope($('#select_timeline_scope').val()); + var payload = {scope: selectedScope}; $.ajax({ url: "/events/"+"getEventTimeline"+"/"+scope_id+"/"+extended_text+"event.json", dataType: 'json', @@ -413,6 +429,8 @@ function reload_timeline() { }, success: function( data, textStatus, jQxhr ){ items_timeline.clear(); + mapping_text_to_id = new Map(); + var itemIds = {}; for (var item of data.items) { item.className = item.group; item.orig_id = item.id; @@ -420,16 +438,43 @@ function reload_timeline() { set_spanned_time(item); if (item.group == 'object') { for (var attr of item.Attribute) { - mapping_text_to_id.set(attr.contentType+': '+attr.content+' ('+item.orig_id+')', item.id); + if (selectedScope == 'sightings') { + var k = attr.contentType+': '+attr.content+' ('+item.orig_id.split('-')[0]+')' + if (!mapping_text_to_id.get(k)) { + mapping_text_to_id.set(k, item.id); + } + } else { + mapping_text_to_id.set(attr.contentType+': '+attr.content+' ('+item.orig_id+')', item.id); + } adjust_text_length(attr); } } else { - mapping_text_to_id.set(item.content+' ('+item.orig_id+')', item.id); + if (selectedScope == 'sightings') { + var k = item.content+' ('+item.orig_id.split('-')[0]+')' + if (!mapping_text_to_id.get(k)) { + mapping_text_to_id.set(k, item.id); + } + } else { + mapping_text_to_id.set(item.content+' ('+item.orig_id+')', item.id); + } adjust_text_length(item); } + itemIds[item.attribute_id] = item.content; + if (selectedScope == 'sightings') { + item.group = item.attribute_id; + item.content = ''; + } } items_timeline.add(data.items); handle_not_seen_enabled($('#checkbox_timeline_display_hide_not_seen_enabled').prop('checked'), false) + if (selectedScope == 'sightings') { + var groups = Object.keys(itemIds).map(function(id) { + return {id: id, content: itemIds[id]} + }) + eventTimeline.setGroups(groups); + } else { + eventTimeline.setGroups([]); + } }, error: function( jqXhr, textStatus, errorThrown ){ console.log( errorThrown ); @@ -610,11 +655,11 @@ function init_popover() { label: "Scope", tooltip: "The time scope represented by the timeline", event: function(value) { - if (value == "First seen/Last seen") { + if (value == "First seen/Last seen" || value == "Sightings") { reload_timeline(); } }, - options: ["First seen/Last seen"], + options: ["First seen/Last seen", "Sightings"], default: "First seen/Last seen" }); From 5034c1c798f22fc2054f46569f550609dffe443a Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 23 Apr 2020 10:18:34 +0200 Subject: [PATCH 10/33] chg: [event:timeline] Prevent item selection while in the sighting context --- app/webroot/js/event-timeline.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/webroot/js/event-timeline.js b/app/webroot/js/event-timeline.js index 2e00638a1..787b056aa 100644 --- a/app/webroot/js/event-timeline.js +++ b/app/webroot/js/event-timeline.js @@ -472,7 +472,9 @@ function reload_timeline() { return {id: id, content: itemIds[id]} }) eventTimeline.setGroups(groups); + eventTimeline.setOptions({selectable: false}); } else { + eventTimeline.setOptions({selectable: true}); eventTimeline.setGroups([]); } }, From 6950db8eb6f1cafa3ed25c8e39156f58ef4b2c60 Mon Sep 17 00:00:00 2001 From: kscheetz Date: Fri, 24 Apr 2020 13:10:30 -0400 Subject: [PATCH 11/33] Stix2 importer naming change. --- app/files/scripts/stix2/stix2misp.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/app/files/scripts/stix2/stix2misp.py b/app/files/scripts/stix2/stix2misp.py index f3ab86084..322e08290 100644 --- a/app/files/scripts/stix2/stix2misp.py +++ b/app/files/scripts/stix2/stix2misp.py @@ -130,12 +130,17 @@ class StixParser(): def build_from_STIX_with_report(self): report_attributes = defaultdict(set) + report_attributes['name'] = None + for ruuid, report in self.report.items(): try: report_attributes['orgs'].add(report.created_by_ref.split('--')[1]) except AttributeError: pass - report_attributes['name'].add(report.name) + + if report_attributes['name'] is None: + report_attributes['name'] = report.name + if report.get('published'): report_attributes['published'].add(report.published) if 'labels' in report: @@ -155,10 +160,14 @@ class StixParser(): self.misp_event['Org'] = {'name': identity['name']} if len(report_attributes['published']) == 1: self.misp_event.publish_timestamp = self.getTimestampfromDate(report_attributes['published'].pop()) - if len(report_attributes['name']) == 1: - self.misp_event.info = report_attributes['name'].pop() + + if report_attributes['name'] is None: + self.misp_event.info = "Imported with MISP import script for {} from {}.".format(self.stix_version, + os.path.basename( + self.filename)) else: - self.misp_event.info = "Imported with MISP import script for {}.".format(self.stix_version) + self.misp_event.info = report_attributes['name'] + for l in report_attributes['labels']: self.misp_event.add_tag(l) From 84bdfbc8d6a3a320f9b25f9393063a2043d483e5 Mon Sep 17 00:00:00 2001 From: kscheetz Date: Fri, 24 Apr 2020 13:13:55 -0400 Subject: [PATCH 12/33] Preserve report order. --- app/files/scripts/stix2/stix2misp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/files/scripts/stix2/stix2misp.py b/app/files/scripts/stix2/stix2misp.py index 322e08290..980b59ad3 100644 --- a/app/files/scripts/stix2/stix2misp.py +++ b/app/files/scripts/stix2/stix2misp.py @@ -24,7 +24,7 @@ import io import re import stix2 from stix2misp_mapping import * -from collections import defaultdict +from collections import defaultdict, OrderedDict _MISP_dir = "/".join([p for p in os.path.dirname(os.path.realpath(__file__)).split('/')[:-4]]) _PyMISP_dir = '{_MISP_dir}/PyMISP'.format(_MISP_dir=_MISP_dir) @@ -107,7 +107,7 @@ class StixParser(): try: self.report[parsed_object['id'].split('--')[1]] = parsed_object except AttributeError: - self.report = {parsed_object['id'].split('--')[1]: parsed_object} + self.report = OrderedDict({parsed_object['id'].split('--')[1]: parsed_object}) def _load_usual_object(self, parsed_object): self.event[parsed_object._type][parsed_object['id'].split('--')[1]] = parsed_object From 40077e9ea75cb07c8fe820dcb67fb97a414c9d68 Mon Sep 17 00:00:00 2001 From: kscheetz Date: Tue, 28 Apr 2020 10:09:25 -0400 Subject: [PATCH 13/33] Simplification for code complexity reqs. --- PyMISP | 2 +- app/Lib/cakephp | 2 +- app/files/misp-galaxy | 2 +- app/files/misp-objects | 2 +- app/files/scripts/stix2/stix2misp.py | 8 +++++--- app/files/taxonomies | 2 +- app/files/warninglists | 2 +- cti-python-stix2 | 2 +- 8 files changed, 12 insertions(+), 10 deletions(-) diff --git a/PyMISP b/PyMISP index 0faa75824..64d7c9a24 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit 0faa75824f4dbac2b14919bb17e9d0fef79026d7 +Subproject commit 64d7c9a24ad9d3a7ccc1b96fb643c235d2b9e02e diff --git a/app/Lib/cakephp b/app/Lib/cakephp index 5ccb12354..d2e1681eb 160000 --- a/app/Lib/cakephp +++ b/app/Lib/cakephp @@ -1 +1 @@ -Subproject commit 5ccb12354dfc08ca1b3e0a430e8668bf1610b5d3 +Subproject commit d2e1681eb8ec75e6c2819fa113834843fed6995a diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy index c7104e881..e37f320df 160000 --- a/app/files/misp-galaxy +++ b/app/files/misp-galaxy @@ -1 +1 @@ -Subproject commit c7104e8819d6b789b24a45655aa28625a8c4c346 +Subproject commit e37f320df5a6ba4d9c67662a3670b160e9941bcf diff --git a/app/files/misp-objects b/app/files/misp-objects index 3b5451c32..7ef9a2ba5 160000 --- a/app/files/misp-objects +++ b/app/files/misp-objects @@ -1 +1 @@ -Subproject commit 3b5451c32518da3e29c575e868d245f27c18dcf4 +Subproject commit 7ef9a2ba56efc6553a720d6df27c9ee547e24242 diff --git a/app/files/scripts/stix2/stix2misp.py b/app/files/scripts/stix2/stix2misp.py index 980b59ad3..b2f1a7908 100644 --- a/app/files/scripts/stix2/stix2misp.py +++ b/app/files/scripts/stix2/stix2misp.py @@ -128,6 +128,10 @@ class StixParser(): except PyMISPInvalidFormat: continue + @property + def basename(self): + return os.path.basename(self.filename) + def build_from_STIX_with_report(self): report_attributes = defaultdict(set) report_attributes['name'] = None @@ -162,9 +166,7 @@ class StixParser(): self.misp_event.publish_timestamp = self.getTimestampfromDate(report_attributes['published'].pop()) if report_attributes['name'] is None: - self.misp_event.info = "Imported with MISP import script for {} from {}.".format(self.stix_version, - os.path.basename( - self.filename)) + self.misp_event.info = "Imported with MISP import script for {} from {}.".format(self.stix_version, self.basename) else: self.misp_event.info = report_attributes['name'] diff --git a/app/files/taxonomies b/app/files/taxonomies index 8c4e2a8e8..0edcb08fa 160000 --- a/app/files/taxonomies +++ b/app/files/taxonomies @@ -1 +1 @@ -Subproject commit 8c4e2a8e8b4d3ec287c43dcae129123fcde8cb2f +Subproject commit 0edcb08fa618f15b51a2bbf0eae6b89c5f603ded diff --git a/app/files/warninglists b/app/files/warninglists index 28687d90d..192d11272 160000 --- a/app/files/warninglists +++ b/app/files/warninglists @@ -1 +1 @@ -Subproject commit 28687d90d575332776480cd5d683361e7485033c +Subproject commit 192d112728f3cab83570a0ad4f6f296e073cf738 diff --git a/cti-python-stix2 b/cti-python-stix2 index e4f08557e..77ca5ae2f 160000 --- a/cti-python-stix2 +++ b/cti-python-stix2 @@ -1 +1 @@ -Subproject commit e4f08557ec93c589a71a6e4060134661f1c4b2c0 +Subproject commit 77ca5ae2f9742652cd90a53f1b83308a8343f2cc From 238a6adeffd1e05e502137e8181e05da11d68034 Mon Sep 17 00:00:00 2001 From: kscheetz Date: Tue, 28 Apr 2020 10:14:29 -0400 Subject: [PATCH 14/33] Moved info assignment to method. --- app/files/scripts/stix2/stix2misp.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/app/files/scripts/stix2/stix2misp.py b/app/files/scripts/stix2/stix2misp.py index b2f1a7908..8e7cd9609 100644 --- a/app/files/scripts/stix2/stix2misp.py +++ b/app/files/scripts/stix2/stix2misp.py @@ -128,9 +128,11 @@ class StixParser(): except PyMISPInvalidFormat: continue - @property - def basename(self): - return os.path.basename(self.filename) + def _set_info_from_report(report_attributes): + if report_attributes['name'] is None: + self.misp_event.info = "Imported with MISP import script for {} from {}.".format(self.stix_version, os.path.basename(self.filename)) + else: + self.misp_event.info = report_attributes['name'] def build_from_STIX_with_report(self): report_attributes = defaultdict(set) @@ -141,10 +143,8 @@ class StixParser(): report_attributes['orgs'].add(report.created_by_ref.split('--')[1]) except AttributeError: pass - if report_attributes['name'] is None: report_attributes['name'] = report.name - if report.get('published'): report_attributes['published'].add(report.published) if 'labels' in report: @@ -164,12 +164,7 @@ class StixParser(): self.misp_event['Org'] = {'name': identity['name']} if len(report_attributes['published']) == 1: self.misp_event.publish_timestamp = self.getTimestampfromDate(report_attributes['published'].pop()) - - if report_attributes['name'] is None: - self.misp_event.info = "Imported with MISP import script for {} from {}.".format(self.stix_version, self.basename) - else: - self.misp_event.info = report_attributes['name'] - + self._set_info_from_report(report_attributes) for l in report_attributes['labels']: self.misp_event.add_tag(l) From 4dec19dedfe3e5c48c44947133fe3e78ba41dc03 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Tue, 28 Apr 2020 15:31:27 +0200 Subject: [PATCH 15/33] new: [feed] Support for compressed feeds --- app/Model/AppModel.php | 17 +++++++ app/Model/Feed.php | 110 ++++++++++++++++++++++++++++++++++------- 2 files changed, 108 insertions(+), 19 deletions(-) diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index f25cc18bd..a62223a1a 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -2847,4 +2847,21 @@ class AppModel extends Model $message .= "\nStack Trace:\n" . $exception->getTraceAsString(); return $this->log($message, $type); } + + /** + * Generates random file name in tmp dir. + * @return string + */ + protected function tempFileName() + { + return $this->tempDir() . DS . $this->generateRandomFileName(); + } + + /** + * @return string + */ + protected function tempDir() + { + return Configure::read('MISP.tmpdir') ?: sys_get_temp_dir(); + } } diff --git a/app/Model/Feed.php b/app/Model/Feed.php index ceefb22fa..b96a016cd 100644 --- a/app/Model/Feed.php +++ b/app/Model/Feed.php @@ -247,10 +247,7 @@ class Feed extends AppModel $data = $this->feedGetUri($feed, $feedUrl, $HttpSocket, true); if (!$isLocal) { - $redis = $this->setupRedis(); - if ($redis === false) { - throw new Exception('Could not reach Redis.'); - } + $redis = $this->setupRedisWithException(); $redis->del('misp:feed_cache:' . $feed['Feed']['id']); file_put_contents($feedCache, $data); } @@ -502,12 +499,17 @@ class Feed extends AppModel $result = array( 'header' => array( - 'Accept' => array('application/json', 'text/plain'), - 'Content-Type' => 'application/json', - 'MISP-version' => $version, - 'MISP-uuid' => Configure::read('MISP.uuid') + 'Accept' => array('application/json', 'text/plain'), + 'MISP-version' => $version, + 'MISP-uuid' => Configure::read('MISP.uuid'), ) ); + + // Enable gzipped responses if PHP has 'gzdecode' method + if (function_exists('gzdecode')) { + $result['header']['Accept-Encoding'] = 'gzip'; + } + if ($commit) { $result['header']['commit'] = $commit; } @@ -1590,24 +1592,51 @@ class Feed extends AppModel if ($data === false) { throw new Exception("Could not read local file '$uri'."); } + return $data; } else { throw new Exception("Local file '$uri' doesn't exists."); } + } + + $request = $this->__createFeedRequest($feed['Feed']['headers']); + + if ($followRedirect) { + $response = $this->getFollowRedirect($HttpSocket, $uri, $request); } else { - $request = $this->__createFeedRequest($feed['Feed']['headers']); + $response = $HttpSocket->get($uri, array(), $request); + } - if ($followRedirect) { - $response = $this->getFollowRedirect($HttpSocket, $uri, $request); - } else { - $response = $HttpSocket->get($uri, array(), $request); - } + if ($response === false) { + throw new Exception("Could not reach '$uri'."); + } else if ($response->code != 200) { // intentionally != + throw new Exception("Fetching the '$uri' failed with HTTP error {$response->code}: {$response->reasonPhrase}"); + } - if ($response === false) { - throw new Exception("Could not reach '$uri'."); - } else if ($response->code != 200) { // intentionally != - throw new Exception("Fetching the '$uri' failed with HTTP error {$response->code}: {$response->reasonPhrase}"); + $data = $response->body; + + $contentEncoding = $response->getHeader('Content-Encoding'); + if ($contentEncoding === 'gzip') { + $data = gzdecode($data); + if ($data === false) { + throw new Exception("Fetching the '$uri' failed, response should be gzip encoded, but gzip decoding failed."); + } + } else if ($contentEncoding) { + throw new Exception("Fetching the '$uri' failed, because remote server returns unsupported content encoding '$contentEncoding'"); + } + + $contentType = $response->getHeader('Content-Type'); + if ($contentType === 'application/zip') { + $zipFile = new File($this->tempFileName()); + $zipFile->write($data); + $zipFile->close(); + + try { + $data = $this->unzipFirstFile($zipFile); + } catch (Exception $e) { + throw new Exception("Fetching the '$uri' failed: {$e->getMessage()}"); + } finally { + $zipFile->delete(); } - $data = $response->body; } return $data; @@ -1720,4 +1749,47 @@ class Feed extends AppModel $this->save($feed); return $count; } + + /** + * @param File $zipFile + * @return string Uncompressed data + * @throws Exception + */ + private function unzipFirstFile(File $zipFile) + { + if (!class_exists('ZipArchive')) { + throw new Exception("ZIP archive decompressing is not supported."); + } + + $zip = new ZipArchive(); + $result = $zip->open($zipFile->pwd()); + if ($result !== true) { + throw new Exception("Remote server returns ZIP file, that cannot be open (error $result)"); + } + + if ($zip->numFiles !== 1) { + throw new Exception("Remote server returns ZIP file, that contains multiple files."); + } + + $filename = $zip->getNameIndex(0); + if ($filename === false) { + throw new Exception("Remote server returns ZIP file, but there is a problem with reading filename."); + } + + $zip->close(); + + $destinationFile = $this->tempFileName(); + $result = copy("zip://{$zipFile->pwd()}#$filename", $destinationFile); + if ($result === false) { + throw new Exception("Remote server returns ZIP file, that contains '$filename' file, that cannot be extracted."); + } + + $unzipped = new File($destinationFile); + $data = $unzipped->read(); + if ($data === false) { + throw new Exception("Couldn't read extracted file content."); + } + $unzipped->delete(); + return $data; + } } From 80d80162b4a77b56723a6abb3660a1719f2152cd Mon Sep 17 00:00:00 2001 From: kscheetz Date: Tue, 28 Apr 2020 10:28:38 -0400 Subject: [PATCH 16/33] Fixes missing self argument bug. --- app/files/scripts/stix2/stix2misp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/scripts/stix2/stix2misp.py b/app/files/scripts/stix2/stix2misp.py index 8e7cd9609..41dfa72ad 100644 --- a/app/files/scripts/stix2/stix2misp.py +++ b/app/files/scripts/stix2/stix2misp.py @@ -128,7 +128,7 @@ class StixParser(): except PyMISPInvalidFormat: continue - def _set_info_from_report(report_attributes): + def _set_info_from_report(self, report_attributes): if report_attributes['name'] is None: self.misp_event.info = "Imported with MISP import script for {} from {}.".format(self.stix_version, os.path.basename(self.filename)) else: From 10cf9e296db10f461ed3fd1a6879ba3d357b6d34 Mon Sep 17 00:00:00 2001 From: kscheetz Date: Fri, 24 Apr 2020 13:10:30 -0400 Subject: [PATCH 17/33] Stix2 importer naming change. --- app/files/scripts/stix2/stix2misp.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/app/files/scripts/stix2/stix2misp.py b/app/files/scripts/stix2/stix2misp.py index f3ab86084..322e08290 100644 --- a/app/files/scripts/stix2/stix2misp.py +++ b/app/files/scripts/stix2/stix2misp.py @@ -130,12 +130,17 @@ class StixParser(): def build_from_STIX_with_report(self): report_attributes = defaultdict(set) + report_attributes['name'] = None + for ruuid, report in self.report.items(): try: report_attributes['orgs'].add(report.created_by_ref.split('--')[1]) except AttributeError: pass - report_attributes['name'].add(report.name) + + if report_attributes['name'] is None: + report_attributes['name'] = report.name + if report.get('published'): report_attributes['published'].add(report.published) if 'labels' in report: @@ -155,10 +160,14 @@ class StixParser(): self.misp_event['Org'] = {'name': identity['name']} if len(report_attributes['published']) == 1: self.misp_event.publish_timestamp = self.getTimestampfromDate(report_attributes['published'].pop()) - if len(report_attributes['name']) == 1: - self.misp_event.info = report_attributes['name'].pop() + + if report_attributes['name'] is None: + self.misp_event.info = "Imported with MISP import script for {} from {}.".format(self.stix_version, + os.path.basename( + self.filename)) else: - self.misp_event.info = "Imported with MISP import script for {}.".format(self.stix_version) + self.misp_event.info = report_attributes['name'] + for l in report_attributes['labels']: self.misp_event.add_tag(l) From 9fcc9f91bb751f6cc785b23fa7946190812b6c07 Mon Sep 17 00:00:00 2001 From: kscheetz Date: Fri, 24 Apr 2020 13:13:55 -0400 Subject: [PATCH 18/33] Preserve report order. --- app/files/scripts/stix2/stix2misp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/files/scripts/stix2/stix2misp.py b/app/files/scripts/stix2/stix2misp.py index 322e08290..980b59ad3 100644 --- a/app/files/scripts/stix2/stix2misp.py +++ b/app/files/scripts/stix2/stix2misp.py @@ -24,7 +24,7 @@ import io import re import stix2 from stix2misp_mapping import * -from collections import defaultdict +from collections import defaultdict, OrderedDict _MISP_dir = "/".join([p for p in os.path.dirname(os.path.realpath(__file__)).split('/')[:-4]]) _PyMISP_dir = '{_MISP_dir}/PyMISP'.format(_MISP_dir=_MISP_dir) @@ -107,7 +107,7 @@ class StixParser(): try: self.report[parsed_object['id'].split('--')[1]] = parsed_object except AttributeError: - self.report = {parsed_object['id'].split('--')[1]: parsed_object} + self.report = OrderedDict({parsed_object['id'].split('--')[1]: parsed_object}) def _load_usual_object(self, parsed_object): self.event[parsed_object._type][parsed_object['id'].split('--')[1]] = parsed_object From 631800b80e44c85dc6d5fce710346720294e399a Mon Sep 17 00:00:00 2001 From: kscheetz Date: Tue, 28 Apr 2020 10:09:25 -0400 Subject: [PATCH 19/33] Simplification for code complexity reqs. --- PyMISP | 2 +- app/Lib/cakephp | 2 +- app/files/misp-galaxy | 2 +- app/files/misp-objects | 2 +- app/files/scripts/stix2/stix2misp.py | 8 +++++--- app/files/taxonomies | 2 +- app/files/warninglists | 2 +- cti-python-stix2 | 2 +- 8 files changed, 12 insertions(+), 10 deletions(-) diff --git a/PyMISP b/PyMISP index 0faa75824..64d7c9a24 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit 0faa75824f4dbac2b14919bb17e9d0fef79026d7 +Subproject commit 64d7c9a24ad9d3a7ccc1b96fb643c235d2b9e02e diff --git a/app/Lib/cakephp b/app/Lib/cakephp index 5ccb12354..d2e1681eb 160000 --- a/app/Lib/cakephp +++ b/app/Lib/cakephp @@ -1 +1 @@ -Subproject commit 5ccb12354dfc08ca1b3e0a430e8668bf1610b5d3 +Subproject commit d2e1681eb8ec75e6c2819fa113834843fed6995a diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy index c7104e881..e37f320df 160000 --- a/app/files/misp-galaxy +++ b/app/files/misp-galaxy @@ -1 +1 @@ -Subproject commit c7104e8819d6b789b24a45655aa28625a8c4c346 +Subproject commit e37f320df5a6ba4d9c67662a3670b160e9941bcf diff --git a/app/files/misp-objects b/app/files/misp-objects index 3b5451c32..7ef9a2ba5 160000 --- a/app/files/misp-objects +++ b/app/files/misp-objects @@ -1 +1 @@ -Subproject commit 3b5451c32518da3e29c575e868d245f27c18dcf4 +Subproject commit 7ef9a2ba56efc6553a720d6df27c9ee547e24242 diff --git a/app/files/scripts/stix2/stix2misp.py b/app/files/scripts/stix2/stix2misp.py index 980b59ad3..b2f1a7908 100644 --- a/app/files/scripts/stix2/stix2misp.py +++ b/app/files/scripts/stix2/stix2misp.py @@ -128,6 +128,10 @@ class StixParser(): except PyMISPInvalidFormat: continue + @property + def basename(self): + return os.path.basename(self.filename) + def build_from_STIX_with_report(self): report_attributes = defaultdict(set) report_attributes['name'] = None @@ -162,9 +166,7 @@ class StixParser(): self.misp_event.publish_timestamp = self.getTimestampfromDate(report_attributes['published'].pop()) if report_attributes['name'] is None: - self.misp_event.info = "Imported with MISP import script for {} from {}.".format(self.stix_version, - os.path.basename( - self.filename)) + self.misp_event.info = "Imported with MISP import script for {} from {}.".format(self.stix_version, self.basename) else: self.misp_event.info = report_attributes['name'] diff --git a/app/files/taxonomies b/app/files/taxonomies index 8c4e2a8e8..0edcb08fa 160000 --- a/app/files/taxonomies +++ b/app/files/taxonomies @@ -1 +1 @@ -Subproject commit 8c4e2a8e8b4d3ec287c43dcae129123fcde8cb2f +Subproject commit 0edcb08fa618f15b51a2bbf0eae6b89c5f603ded diff --git a/app/files/warninglists b/app/files/warninglists index 28687d90d..192d11272 160000 --- a/app/files/warninglists +++ b/app/files/warninglists @@ -1 +1 @@ -Subproject commit 28687d90d575332776480cd5d683361e7485033c +Subproject commit 192d112728f3cab83570a0ad4f6f296e073cf738 diff --git a/cti-python-stix2 b/cti-python-stix2 index e4f08557e..77ca5ae2f 160000 --- a/cti-python-stix2 +++ b/cti-python-stix2 @@ -1 +1 @@ -Subproject commit e4f08557ec93c589a71a6e4060134661f1c4b2c0 +Subproject commit 77ca5ae2f9742652cd90a53f1b83308a8343f2cc From b1124f23d19a539dab0bdddbc076d91a8ffce17e Mon Sep 17 00:00:00 2001 From: kscheetz Date: Tue, 28 Apr 2020 10:14:29 -0400 Subject: [PATCH 20/33] Moved info assignment to method. --- app/files/scripts/stix2/stix2misp.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/app/files/scripts/stix2/stix2misp.py b/app/files/scripts/stix2/stix2misp.py index b2f1a7908..8e7cd9609 100644 --- a/app/files/scripts/stix2/stix2misp.py +++ b/app/files/scripts/stix2/stix2misp.py @@ -128,9 +128,11 @@ class StixParser(): except PyMISPInvalidFormat: continue - @property - def basename(self): - return os.path.basename(self.filename) + def _set_info_from_report(report_attributes): + if report_attributes['name'] is None: + self.misp_event.info = "Imported with MISP import script for {} from {}.".format(self.stix_version, os.path.basename(self.filename)) + else: + self.misp_event.info = report_attributes['name'] def build_from_STIX_with_report(self): report_attributes = defaultdict(set) @@ -141,10 +143,8 @@ class StixParser(): report_attributes['orgs'].add(report.created_by_ref.split('--')[1]) except AttributeError: pass - if report_attributes['name'] is None: report_attributes['name'] = report.name - if report.get('published'): report_attributes['published'].add(report.published) if 'labels' in report: @@ -164,12 +164,7 @@ class StixParser(): self.misp_event['Org'] = {'name': identity['name']} if len(report_attributes['published']) == 1: self.misp_event.publish_timestamp = self.getTimestampfromDate(report_attributes['published'].pop()) - - if report_attributes['name'] is None: - self.misp_event.info = "Imported with MISP import script for {} from {}.".format(self.stix_version, self.basename) - else: - self.misp_event.info = report_attributes['name'] - + self._set_info_from_report(report_attributes) for l in report_attributes['labels']: self.misp_event.add_tag(l) From e381621a39f7b7817d06f75e56e4c4379862c07c Mon Sep 17 00:00:00 2001 From: kscheetz Date: Tue, 28 Apr 2020 10:28:38 -0400 Subject: [PATCH 21/33] Fixes missing self argument bug. --- app/files/scripts/stix2/stix2misp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/scripts/stix2/stix2misp.py b/app/files/scripts/stix2/stix2misp.py index 8e7cd9609..41dfa72ad 100644 --- a/app/files/scripts/stix2/stix2misp.py +++ b/app/files/scripts/stix2/stix2misp.py @@ -128,7 +128,7 @@ class StixParser(): except PyMISPInvalidFormat: continue - def _set_info_from_report(report_attributes): + def _set_info_from_report(self, report_attributes): if report_attributes['name'] is None: self.misp_event.info = "Imported with MISP import script for {} from {}.".format(self.stix_version, os.path.basename(self.filename)) else: From 76ed9a1eeb00528932d9776a7228598ec1f9635e Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 30 Apr 2020 13:42:28 +0900 Subject: [PATCH 22/33] chg: [installer] Update installer to latest --- INSTALL/INSTALL.sh | 27 ++++++++++++++++++--------- INSTALL/INSTALL.sh.sfv | 6 +++--- INSTALL/INSTALL.sh.sha1 | 2 +- INSTALL/INSTALL.sh.sha256 | 2 +- INSTALL/INSTALL.sh.sha384 | 2 +- INSTALL/INSTALL.sh.sha512 | 2 +- 6 files changed, 25 insertions(+), 16 deletions(-) diff --git a/INSTALL/INSTALL.sh b/INSTALL/INSTALL.sh index 7b18f5544..13e26bd45 100755 --- a/INSTALL/INSTALL.sh +++ b/INSTALL/INSTALL.sh @@ -377,18 +377,19 @@ EOF checkInstaller () { # Workaround: shasum is not available on RHEL, only checking sha512 if [[ $FLAVOUR == "rhel" ]] || [[ $FLAVOUR == "centos" ]]; then - INSTsum=$(sha512sum ${0} | cut -f1 -d\ ) - /usr/bin/wget --no-cache -q -O /tmp/INSTALL.sh.sha512 https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/INSTALL.sh.sha512 + INSTsum=$(sha512sum ${0} | cut -f1 -d\ ) + /usr/bin/wget --no-cache -q -O /tmp/INSTALL.sh.sha512 https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/INSTALL.sh.sha512 chsum=$(cat /tmp/INSTALL.sh.sha512) - if [[ "${chsum}" == "${INSTsum}" ]]; then - echo "SHA512 matches" - else - echo "SHA512: ${chsum} does not match the installer sum of: ${INSTsum}" - # exit 1 # uncomment when/if PR is merged - fi + if [[ "${chsum}" == "${INSTsum}" ]]; then + echo "SHA512 matches" + else + echo "SHA512: ${chsum} does not match the installer sum of: ${INSTsum}" + # exit 1 # uncomment when/if PR is merged + fi else # TODO: Implement $FLAVOUR checks and install depending on the platform we are on if [[ $(which shasum > /dev/null 2>&1 ; echo $?) != 0 ]]; then + sudo apt update sudo apt install libdigest-sha-perl -qyy fi # SHAsums to be computed, not the -- notatiation is for ease of use with rhash @@ -3173,6 +3174,7 @@ x86_64-fedora-30 x86_64-debian-stretch x86_64-debian-buster x86_64-ubuntu-bionic +x86_64-ubuntu-focal x86_64-kali-2019.1 x86_64-kali-2019.2 x86_64-kali-2019.3 @@ -3187,6 +3189,7 @@ armv7l-debian-jessie armv7l-debian-stretch armv7l-debian-buster armv7l-ubuntu-bionic +armv7l-ubuntu-focal " # Check if we actually support this configuration @@ -3208,12 +3211,18 @@ if [ "${FLAVOUR}" == "ubuntu" ]; then echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues" installSupported && exit || exit fi + if [ "${RELEASE}" == "20.04" ]; then + echo "Install on Ubuntu 20.04 LTS fully supported." + echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues" + installSupported PHP="7.4" && exit || exit + fi if [ "${RELEASE}" == "18.10" ]; then echo "Install on Ubuntu 18.10 partially supported, bye." + echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues" installSupported && exit || exit fi if [ "${RELEASE}" == "19.04" ]; then - echo "Install on Ubuntu 19.04 under development." + echo "Install on Ubuntu 19.04 partially supported bye." echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues" installSupported && exit || exit exit 1 diff --git a/INSTALL/INSTALL.sh.sfv b/INSTALL/INSTALL.sh.sfv index e98414656..c5212a214 100644 --- a/INSTALL/INSTALL.sh.sfv +++ b/INSTALL/INSTALL.sh.sfv @@ -1,5 +1,5 @@ -; Generated by RHash v1.3.8 on 2020-04-27 at 19:39.56 +; Generated by RHash v1.3.8 on 2020-04-30 at 13:42.00 ; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/ ; -; 130366 19:39.56 2020-04-27 INSTALL.sh -INSTALL.sh 8AFDDFA23C1154790947FA1C09DBC7599614F48D 1DC92AFF146065ECB85D5C5C211E252D5C54D6F86A61C29DDED37A9ECA4540E4 3F2C936AEE2773A29DD02477E463402945576DEC32C3003BFF3132B783E2C93D14EA99E8FADFEEE655C746D4C634BE08 DBBF519D97372DDD5337C1F58252BFE9A86B0C8ABB95A375420D8D543C5239D71C24287A75FAFC389DBFDC657B1B4080530D0D3CB44D5F94152F2ABBB9B23174 +; 130750 13:42.00 2020-04-30 INSTALL.sh +INSTALL.sh 5B66DBA7E71771AA95A12413E661E00688C03610 18F6997170191327CD2B9E40C1C0120E48EC425D533C4624EB050594B99833CE 6C1DB0EEEE105E73F1125905FD48D057C15DC05794C8ABD2EC24DD31270423B8580992B5EFBBF0E1CDD54228A6899F39 BD545DB6BC3B746A99E2CB738D178C1812EB5AFB9E8D53A172AF282E225CC2644A4AD1A84277FDCCD2C53C3F1FC0EF04758E25226AC1960470D0A4565E21CAB1 diff --git a/INSTALL/INSTALL.sh.sha1 b/INSTALL/INSTALL.sh.sha1 index fdbabddc8..6ed16a3ad 100644 --- a/INSTALL/INSTALL.sh.sha1 +++ b/INSTALL/INSTALL.sh.sha1 @@ -1 +1 @@ -8afddfa23c1154790947fa1c09dbc7599614f48d INSTALL.sh +5b66dba7e71771aa95a12413e661e00688c03610 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha256 b/INSTALL/INSTALL.sh.sha256 index ea4bbcc5b..bad463300 100644 --- a/INSTALL/INSTALL.sh.sha256 +++ b/INSTALL/INSTALL.sh.sha256 @@ -1 +1 @@ -1dc92aff146065ecb85d5c5c211e252d5c54d6f86a61c29dded37a9eca4540e4 INSTALL.sh +18f6997170191327cd2b9e40c1c0120e48ec425d533c4624eb050594b99833ce INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha384 b/INSTALL/INSTALL.sh.sha384 index 50b08f292..ac1e9d9ad 100644 --- a/INSTALL/INSTALL.sh.sha384 +++ b/INSTALL/INSTALL.sh.sha384 @@ -1 +1 @@ -3f2c936aee2773a29dd02477e463402945576dec32c3003bff3132b783e2c93d14ea99e8fadfeee655c746d4c634be08 INSTALL.sh +6c1db0eeee105e73f1125905fd48d057c15dc05794c8abd2ec24dd31270423b8580992b5efbbf0e1cdd54228a6899f39 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha512 b/INSTALL/INSTALL.sh.sha512 index d3a517d82..540ccf94a 100644 --- a/INSTALL/INSTALL.sh.sha512 +++ b/INSTALL/INSTALL.sh.sha512 @@ -1 +1 @@ -dbbf519d97372ddd5337c1f58252bfe9a86b0c8abb95a375420d8d543c5239d71c24287a75fafc389dbfdc657b1b4080530d0d3cb44d5f94152f2abbb9b23174 INSTALL.sh +bd545db6bc3b746a99e2cb738d178c1812eb5afb9e8d53a172af282e225cc2644a4ad1a84277fdccd2c53c3f1fc0ef04758e25226ac1960470d0a4565e21cab1 INSTALL.sh From 2de54d59e5b772024d4e7814a42f36f3c7835caa Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 30 Apr 2020 06:46:46 +0200 Subject: [PATCH 23/33] fix: [user settings] corrected field name --- app/Model/AppModel.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 971e5d82e..f868b9e4f 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -78,7 +78,7 @@ class AppModel extends Model 33 => false, 34 => false, 35 => false, 36 => false, 37 => false, 38 => false, 39 => false, 40 => false, 41 => false, 42 => false, 43 => false, 44 => false, 45 => false, 46 => false, 47 => false, 48 => false, 49 => false, 50 => false, - 51 => false, 52 => false + 51 => false, 52 => false, 53 => false ); public $advanced_updates_description = array( @@ -1383,6 +1383,12 @@ class AppModel extends Model $this->__addIndex('admin_settings', 'setting'); } break; + case 53: + if (!empty($this->query("SHOW COLUMNS FROM `user_settings` LIKE 'key';"))) { + $sqlArray[] = "ALTER TABLE user_settings CHANGE `key` `setting` varchar(255) COLLATE utf8_bin NOT NULL;"; + $this->__addIndex('user_settings', 'setting'); + } + break; case 'fixNonEmptySharingGroupID': $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; From 9a22aa1f3c1295ab4715e7043e09fa3797b592cb Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 30 Apr 2020 06:55:21 +0200 Subject: [PATCH 24/33] chg: [db_schema] bumped --- db_schema.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db_schema.json b/db_schema.json index 8d2a6da54..ab40567c3 100644 --- a/db_schema.json +++ b/db_schema.json @@ -6567,5 +6567,5 @@ "id" ] }, - "db_version": "51" -} \ No newline at end of file + "db_version": "53" +} From 01b552d7e9917171cdd84ce43dc67f627a7e87ac Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 30 Apr 2020 07:09:56 +0200 Subject: [PATCH 25/33] Revert "Merge branch '5835' into 2.4" This reverts commit 48132af1796b13e888ecdc77fa0e25787d517242, reversing changes made to 9a22aa1f3c1295ab4715e7043e09fa3797b592cb. --- PyMISP | 2 +- app/Lib/cakephp | 2 +- app/files/misp-galaxy | 2 +- app/files/misp-objects | 2 +- app/files/scripts/stix2/stix2misp.py | 20 +++++++------------- app/files/taxonomies | 2 +- app/files/warninglists | 2 +- cti-python-stix2 | 2 +- 8 files changed, 14 insertions(+), 20 deletions(-) diff --git a/PyMISP b/PyMISP index 64d7c9a24..0faa75824 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit 64d7c9a24ad9d3a7ccc1b96fb643c235d2b9e02e +Subproject commit 0faa75824f4dbac2b14919bb17e9d0fef79026d7 diff --git a/app/Lib/cakephp b/app/Lib/cakephp index d2e1681eb..5ccb12354 160000 --- a/app/Lib/cakephp +++ b/app/Lib/cakephp @@ -1 +1 @@ -Subproject commit d2e1681eb8ec75e6c2819fa113834843fed6995a +Subproject commit 5ccb12354dfc08ca1b3e0a430e8668bf1610b5d3 diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy index e37f320df..c7104e881 160000 --- a/app/files/misp-galaxy +++ b/app/files/misp-galaxy @@ -1 +1 @@ -Subproject commit e37f320df5a6ba4d9c67662a3670b160e9941bcf +Subproject commit c7104e8819d6b789b24a45655aa28625a8c4c346 diff --git a/app/files/misp-objects b/app/files/misp-objects index 7ef9a2ba5..3b5451c32 160000 --- a/app/files/misp-objects +++ b/app/files/misp-objects @@ -1 +1 @@ -Subproject commit 7ef9a2ba56efc6553a720d6df27c9ee547e24242 +Subproject commit 3b5451c32518da3e29c575e868d245f27c18dcf4 diff --git a/app/files/scripts/stix2/stix2misp.py b/app/files/scripts/stix2/stix2misp.py index 41dfa72ad..f3ab86084 100644 --- a/app/files/scripts/stix2/stix2misp.py +++ b/app/files/scripts/stix2/stix2misp.py @@ -24,7 +24,7 @@ import io import re import stix2 from stix2misp_mapping import * -from collections import defaultdict, OrderedDict +from collections import defaultdict _MISP_dir = "/".join([p for p in os.path.dirname(os.path.realpath(__file__)).split('/')[:-4]]) _PyMISP_dir = '{_MISP_dir}/PyMISP'.format(_MISP_dir=_MISP_dir) @@ -107,7 +107,7 @@ class StixParser(): try: self.report[parsed_object['id'].split('--')[1]] = parsed_object except AttributeError: - self.report = OrderedDict({parsed_object['id'].split('--')[1]: parsed_object}) + self.report = {parsed_object['id'].split('--')[1]: parsed_object} def _load_usual_object(self, parsed_object): self.event[parsed_object._type][parsed_object['id'].split('--')[1]] = parsed_object @@ -128,23 +128,14 @@ class StixParser(): except PyMISPInvalidFormat: continue - def _set_info_from_report(self, report_attributes): - if report_attributes['name'] is None: - self.misp_event.info = "Imported with MISP import script for {} from {}.".format(self.stix_version, os.path.basename(self.filename)) - else: - self.misp_event.info = report_attributes['name'] - def build_from_STIX_with_report(self): report_attributes = defaultdict(set) - report_attributes['name'] = None - for ruuid, report in self.report.items(): try: report_attributes['orgs'].add(report.created_by_ref.split('--')[1]) except AttributeError: pass - if report_attributes['name'] is None: - report_attributes['name'] = report.name + report_attributes['name'].add(report.name) if report.get('published'): report_attributes['published'].add(report.published) if 'labels' in report: @@ -164,7 +155,10 @@ class StixParser(): self.misp_event['Org'] = {'name': identity['name']} if len(report_attributes['published']) == 1: self.misp_event.publish_timestamp = self.getTimestampfromDate(report_attributes['published'].pop()) - self._set_info_from_report(report_attributes) + if len(report_attributes['name']) == 1: + self.misp_event.info = report_attributes['name'].pop() + else: + self.misp_event.info = "Imported with MISP import script for {}.".format(self.stix_version) for l in report_attributes['labels']: self.misp_event.add_tag(l) diff --git a/app/files/taxonomies b/app/files/taxonomies index 0edcb08fa..8c4e2a8e8 160000 --- a/app/files/taxonomies +++ b/app/files/taxonomies @@ -1 +1 @@ -Subproject commit 0edcb08fa618f15b51a2bbf0eae6b89c5f603ded +Subproject commit 8c4e2a8e8b4d3ec287c43dcae129123fcde8cb2f diff --git a/app/files/warninglists b/app/files/warninglists index 192d11272..28687d90d 160000 --- a/app/files/warninglists +++ b/app/files/warninglists @@ -1 +1 @@ -Subproject commit 192d112728f3cab83570a0ad4f6f296e073cf738 +Subproject commit 28687d90d575332776480cd5d683361e7485033c diff --git a/cti-python-stix2 b/cti-python-stix2 index 77ca5ae2f..e4f08557e 160000 --- a/cti-python-stix2 +++ b/cti-python-stix2 @@ -1 +1 @@ -Subproject commit 77ca5ae2f9742652cd90a53f1b83308a8343f2cc +Subproject commit e4f08557ec93c589a71a6e4060134661f1c4b2c0 From 4ac89bec8650dfbbabe85f32c0094b33943ce602 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 30 Apr 2020 14:56:06 +0900 Subject: [PATCH 26/33] fix: [installer] Bug where the wrong php deps would get installed --- INSTALL/INSTALL.tpl.sh | 8 ++++---- docs/generic/supportFunctions.md | 10 +++++++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/INSTALL/INSTALL.tpl.sh b/INSTALL/INSTALL.tpl.sh index d10970406..3c5132e49 100755 --- a/INSTALL/INSTALL.tpl.sh +++ b/INSTALL/INSTALL.tpl.sh @@ -278,16 +278,16 @@ installSupported () { if [[ "$1" =~ ^PHP= ]]; then PHP_VER=$(echo $1 |cut -f2 -d=) - if [[ "$PHP_VER" == "7.2" ]]; then + if [[ "$PHP_VER" == 7.2 ]]; then # Install PHP 7.2 Dependencies - functionLocation('INSTALL.ubuntu1804.md') [[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp72 - elif [[ "$PHP_VER" == "7.3" ]]; then + elif [[ "$PHP_VER" == 7.3 ]]; then # Install PHP 7.4 Dependencies - functionLocation('INSTALL.ubuntu2004.md') [[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp74 - elif [[ "$PHP_VER" == "7.4" ]]; then + elif [[ "$PHP_VER" == 7.4 ]]; then # Install PHP 7.3 Dependencies - functionLocation('generic/supportFunctions.md') [[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp73 - elif [[ "$PHP_VER" == "7.0" ]]; then + elif [[ "$PHP_VER" == 7.0 ]]; then # Install PHP 7.0 Dependencies - functionLocation('generic/supportFunctions.md') [[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp70 fi diff --git a/docs/generic/supportFunctions.md b/docs/generic/supportFunctions.md index 2d8766fe9..a88afd4a2 100644 --- a/docs/generic/supportFunctions.md +++ b/docs/generic/supportFunctions.md @@ -541,12 +541,20 @@ setBaseURL () { MISP_BASEURL="https://misp.local" # Webserver configuration FQDN='misp.local' - else + elif [[ "$(checkManufacturer)" == "innotek GmbH" ]]; then MISP_BASEURL='https://localhost:8443' IP=$(ip addr show | awk '$1 == "inet" {gsub(/\/.*$/, "", $2); print $2}' |grep -v "127.0.0.1" |tail -1) sudo iptables -t nat -A OUTPUT -p tcp --dport 8443 -j DNAT --to ${IP}:443 # Webserver configuration FQDN='localhost.localdomain' + elif [[ "$(checkManufacturer)" == "VMware, Inc." ]]; then + MISP_BASEURL='""' + # Webserver configuration + FQDN='misp.local' + else + MISP_BASEURL='""' + # Webserver configuration + FQDN='misp.local' fi } From adadaba0646a7425035601cb744d7572d95c58aa Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 30 Apr 2020 15:20:30 +0900 Subject: [PATCH 27/33] chg: [installer] Version bump --- INSTALL/INSTALL.sh | 18 +++++++++++++----- INSTALL/INSTALL.sh.sfv | 6 +++--- INSTALL/INSTALL.sh.sha1 | 2 +- INSTALL/INSTALL.sh.sha256 | 2 +- INSTALL/INSTALL.sh.sha384 | 2 +- INSTALL/INSTALL.sh.sha512 | 2 +- 6 files changed, 20 insertions(+), 12 deletions(-) diff --git a/INSTALL/INSTALL.sh b/INSTALL/INSTALL.sh index 13e26bd45..d497e26cf 100755 --- a/INSTALL/INSTALL.sh +++ b/INSTALL/INSTALL.sh @@ -702,12 +702,20 @@ setBaseURL () { MISP_BASEURL="https://misp.local" # Webserver configuration FQDN='misp.local' - else + elif [[ "$(checkManufacturer)" == "innotek GmbH" ]]; then MISP_BASEURL='https://localhost:8443' IP=$(ip addr show | awk '$1 == "inet" {gsub(/\/.*$/, "", $2); print $2}' |grep -v "127.0.0.1" |tail -1) sudo iptables -t nat -A OUTPUT -p tcp --dport 8443 -j DNAT --to ${IP}:443 # Webserver configuration FQDN='localhost.localdomain' + elif [[ "$(checkManufacturer)" == "VMware, Inc." ]]; then + MISP_BASEURL='""' + # Webserver configuration + FQDN='misp.local' + else + MISP_BASEURL='""' + # Webserver configuration + FQDN='misp.local' fi } @@ -2652,16 +2660,16 @@ installSupported () { if [[ "$1" =~ ^PHP= ]]; then PHP_VER=$(echo $1 |cut -f2 -d=) - if [[ "$PHP_VER" == "7.2" ]]; then + if [[ "$PHP_VER" == 7.2 ]]; then # Install PHP 7.2 Dependencies - functionLocation('INSTALL.ubuntu1804.md') [[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp72 - elif [[ "$PHP_VER" == "7.3" ]]; then + elif [[ "$PHP_VER" == 7.3 ]]; then # Install PHP 7.4 Dependencies - functionLocation('INSTALL.ubuntu2004.md') [[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp74 - elif [[ "$PHP_VER" == "7.4" ]]; then + elif [[ "$PHP_VER" == 7.4 ]]; then # Install PHP 7.3 Dependencies - functionLocation('generic/supportFunctions.md') [[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp73 - elif [[ "$PHP_VER" == "7.0" ]]; then + elif [[ "$PHP_VER" == 7.0 ]]; then # Install PHP 7.0 Dependencies - functionLocation('generic/supportFunctions.md') [[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp70 fi diff --git a/INSTALL/INSTALL.sh.sfv b/INSTALL/INSTALL.sh.sfv index c5212a214..a8bdaeace 100644 --- a/INSTALL/INSTALL.sh.sfv +++ b/INSTALL/INSTALL.sh.sfv @@ -1,5 +1,5 @@ -; Generated by RHash v1.3.8 on 2020-04-30 at 13:42.00 +; Generated by RHash v1.3.8 on 2020-04-30 at 15:20.13 ; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/ ; -; 130750 13:42.00 2020-04-30 INSTALL.sh -INSTALL.sh 5B66DBA7E71771AA95A12413E661E00688C03610 18F6997170191327CD2B9E40C1C0120E48EC425D533C4624EB050594B99833CE 6C1DB0EEEE105E73F1125905FD48D057C15DC05794C8ABD2EC24DD31270423B8580992B5EFBBF0E1CDD54228A6899F39 BD545DB6BC3B746A99E2CB738D178C1812EB5AFB9E8D53A172AF282E225CC2644A4AD1A84277FDCCD2C53C3F1FC0EF04758E25226AC1960470D0A4565E21CAB1 +; 131010 15:20.13 2020-04-30 INSTALL.sh +INSTALL.sh 660E0D51D88B57CE5BE725117482207E39371038 DCF69118CD37B43C308FD25E6BADAF03549BAF0FFA2AC11A1E919005D700F4AC 74E03A8054AF2E4BCB90426A3B813F57BF032734AB7B4E9D4F6F96961D7371FB051180BEE8357642EB9CC58603C13DA3 C4D1D02980808A92E8E11C72A49AA354DDEFA71C6E85FAC739645CEDEB4B36415243F7FB4B8BC75B6AE7B5D9660E0F88A35E884EBD51EA107128B0D7FB20C946 diff --git a/INSTALL/INSTALL.sh.sha1 b/INSTALL/INSTALL.sh.sha1 index 6ed16a3ad..46b782d89 100644 --- a/INSTALL/INSTALL.sh.sha1 +++ b/INSTALL/INSTALL.sh.sha1 @@ -1 +1 @@ -5b66dba7e71771aa95a12413e661e00688c03610 INSTALL.sh +660e0d51d88b57ce5be725117482207e39371038 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha256 b/INSTALL/INSTALL.sh.sha256 index bad463300..8da70f5ba 100644 --- a/INSTALL/INSTALL.sh.sha256 +++ b/INSTALL/INSTALL.sh.sha256 @@ -1 +1 @@ -18f6997170191327cd2b9e40c1c0120e48ec425d533c4624eb050594b99833ce INSTALL.sh +dcf69118cd37b43c308fd25e6badaf03549baf0ffa2ac11a1e919005d700f4ac INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha384 b/INSTALL/INSTALL.sh.sha384 index ac1e9d9ad..7fb05a8a5 100644 --- a/INSTALL/INSTALL.sh.sha384 +++ b/INSTALL/INSTALL.sh.sha384 @@ -1 +1 @@ -6c1db0eeee105e73f1125905fd48d057c15dc05794c8abd2ec24dd31270423b8580992b5efbbf0e1cdd54228a6899f39 INSTALL.sh +74e03a8054af2e4bcb90426a3b813f57bf032734ab7b4e9d4f6f96961d7371fb051180bee8357642eb9cc58603c13da3 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha512 b/INSTALL/INSTALL.sh.sha512 index 540ccf94a..15701f93f 100644 --- a/INSTALL/INSTALL.sh.sha512 +++ b/INSTALL/INSTALL.sh.sha512 @@ -1 +1 @@ -bd545db6bc3b746a99e2cb738d178c1812eb5afb9e8d53a172af282e225cc2644a4ad1a84277fdccd2c53c3f1fc0ef04758e25226ac1960470d0a4565e21cab1 INSTALL.sh +c4d1d02980808a92e8e11c72a49aa354ddefa71c6e85fac739645cedeb4b36415243f7fb4b8bc75b6ae7b5d9660e0f88a35e884ebd51ea107128b0d7fb20c946 INSTALL.sh From 4e0ef6f28ea4db42bc57be1afc9bf62bba8f5356 Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 30 Apr 2020 08:29:17 +0200 Subject: [PATCH 28/33] new: [internal] cache the sharing group access lookups - should reduce the number of queries drastically for events heavy on object/attribute level sharing groups --- app/Model/SharingGroup.php | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/app/Model/SharingGroup.php b/app/Model/SharingGroup.php index 2e15f5182..ba9605411 100644 --- a/app/Model/SharingGroup.php +++ b/app/Model/SharingGroup.php @@ -54,6 +54,10 @@ class SharingGroup extends AppModel ); private $__sgoCache = array(); + private $__sgAuthorisationCache = array( + 'save' => array(), + 'access' => array() + ); public function beforeValidate($options = array()) @@ -353,6 +357,9 @@ class SharingGroup extends AppModel // returns true if the SG exists and the user is allowed to see it public function checkIfAuthorised($user, $id, $adminCheck = true) { + if (isset($this->__sgAuthorisationCache['access'][boolval($adminCheck)][$id])) { + return $this->__sgAuthorisationCache['access'][boolval($adminCheck)][$id]; + } if (Validation::uuid($id)) { $sgid = $this->SharingGroup->find('first', array( 'conditions' => array('SharingGroup.uuid' => $id), @@ -372,8 +379,10 @@ class SharingGroup extends AppModel return false; } if (($adminCheck && $user['Role']['perm_site_admin']) || $this->SharingGroupServer->checkIfAuthorised($id) || $this->SharingGroupOrg->checkIfAuthorised($id, $user['org_id'])) { + $this->__sgAuthorisationCache['access'][boolval($adminCheck)][$id] = true; return true; } + $this->__sgAuthorisationCache['access'][boolval($adminCheck)][$id] = false; return false; } @@ -502,10 +511,19 @@ class SharingGroup extends AppModel return false; } // check if current user is contained in the SG and we are in a local sync setup - $authorizedToSave = $this->checkIfAuthorisedToSave($user, $sg); + if (!empty($sg['uuid'])) { + if (isset($this->__sgAuthorisationCache['save'][boolval($syncLocal)][$sg['uuid']])) { + $authorisedToSave = $this->__sgAuthorisationCache['save'][boolval($syncLocal)][$sg['uuid']]; + } else { + $authorisedToSave = $this->checkIfAuthorisedToSave($user, $sg); + $this->__sgAuthorisationCache['save'][boolval($syncLocal)][$sg['uuid']] = $authorisedToSave; + } + } else { + $authorisedToSave = $this->checkIfAuthorisedToSave($user, $sg); + } if (!$user['Role']['perm_site_admin'] && !($user['Role']['perm_sync'] && $syncLocal ) && - !$authorizedToSave + !$authorisedToSave ) { $this->Log->create(); $entry = array( From 799506dcc97998a8ae0caa0a65392eceab0e2df7 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Thu, 30 Apr 2020 09:12:55 +0200 Subject: [PATCH 29/33] chg: [misp-galaxy] updated to the latest version --- app/files/misp-galaxy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy index c7104e881..cb9bff18e 160000 --- a/app/files/misp-galaxy +++ b/app/files/misp-galaxy @@ -1 +1 @@ -Subproject commit c7104e8819d6b789b24a45655aa28625a8c4c346 +Subproject commit cb9bff18e8c94584588ec1333affb4959cdc08d0 From b19a72493f71a3ffc3cdbade3cf49561326d602d Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Thu, 30 Apr 2020 09:14:06 +0200 Subject: [PATCH 30/33] chg: [misp-objects] updated to the latest version --- app/files/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/misp-objects b/app/files/misp-objects index 3b5451c32..84a7bb07a 160000 --- a/app/files/misp-objects +++ b/app/files/misp-objects @@ -1 +1 @@ -Subproject commit 3b5451c32518da3e29c575e868d245f27c18dcf4 +Subproject commit 84a7bb07a4f1807546cf5c2e03b35dbc0773699d From 2431cda0571dc3fc95e18f773a7a4aa1cde247a6 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Thu, 30 Apr 2020 09:15:10 +0200 Subject: [PATCH 31/33] chg: [warning-lists] updated to the latest version --- app/files/warninglists | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/warninglists b/app/files/warninglists index 28687d90d..2c116cbd1 160000 --- a/app/files/warninglists +++ b/app/files/warninglists @@ -1 +1 @@ -Subproject commit 28687d90d575332776480cd5d683361e7485033c +Subproject commit 2c116cbd1ff50b2ed677ea6af08c8a6bf511a6fd From 9b46c6242887073d70bd05a8e66f633f442d0e6e Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 30 Apr 2020 11:49:42 +0200 Subject: [PATCH 32/33] chg: [pymisp] bump --- PyMISP | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyMISP b/PyMISP index 0faa75824..5cc7a1ad5 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit 0faa75824f4dbac2b14919bb17e9d0fef79026d7 +Subproject commit 5cc7a1ad57c2e5a90092644e61c3de1e3e449a36 From f278407e915ff7f6e2e40bc1035206c8be2a1581 Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 30 Apr 2020 11:50:22 +0200 Subject: [PATCH 33/33] chg: [VERSION] bump --- VERSION.json | 2 +- app/Controller/AppController.php | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/VERSION.json b/VERSION.json index 4ebf2a526..1071f10ac 100644 --- a/VERSION.json +++ b/VERSION.json @@ -1 +1 @@ -{"major":2, "minor":4, "hotfix":124} +{"major":2, "minor":4, "hotfix":125} diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index e63f4264e..c838e9d7d 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -46,8 +46,8 @@ class AppController extends Controller public $helpers = array('Utility', 'OrgImg', 'FontAwesome', 'UserName', 'DataPathCollector'); - private $__queryVersion = '104'; - public $pyMispVersion = '2.4.123'; + private $__queryVersion = '105'; + public $pyMispVersion = '2.4.125'; public $phpmin = '7.2'; public $phprec = '7.4'; public $pythonmin = '3.6';