From 49271cd8d4fffbc1d1140e5c93a5c4435eff5e9c Mon Sep 17 00:00:00 2001 From: Dan Nelson <34968876+nandelson@users.noreply.github.com> Date: Tue, 16 Aug 2022 16:53:39 -0400 Subject: [PATCH 01/54] Update INSTALL.rhel7.md --- docs/INSTALL.rhel7.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index 238628f43..6f15fe7c8 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -659,4 +659,3 @@ systemctl restart misp-workers.service via this guide and will need additional investigation. {!generic/hardening.md!} - From 6703d82ad0ddec3153c8c27bca6fdd423b96bcc5 Mon Sep 17 00:00:00 2001 From: iglocska Date: Sat, 24 Dec 2022 06:25:43 +0100 Subject: [PATCH 02/54] chg: [indexTable] added option for the generic Field to set a default value, if the referenced value is empty - via the key 'empty' --- .../genericElements/IndexTable/Fields/generic_field.ctp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/View/Elements/genericElements/IndexTable/Fields/generic_field.ctp b/app/View/Elements/genericElements/IndexTable/Fields/generic_field.ctp index 0d63cdc3f..2698be658 100644 --- a/app/View/Elements/genericElements/IndexTable/Fields/generic_field.ctp +++ b/app/View/Elements/genericElements/IndexTable/Fields/generic_field.ctp @@ -1,5 +1,8 @@ 1) { $implodeGlue = isset($field['array_implode_glue']) ? $field['array_implode_glue'] : ', '; From f551639a96605a1e6ce94a6903ba0aed0b53ccdb Mon Sep 17 00:00:00 2001 From: iglocska Date: Sat, 24 Dec 2022 06:27:01 +0100 Subject: [PATCH 03/54] new: [indexTable] added 3 new simple elements - custom_element -> loop the data through an element set via element_path - model -> for the various log indeces, format the log entry's model entry as MODEL #MODEL_ID - time -> loop the data through the time helper's time() function --- .../genericElements/IndexTable/Fields/custom_element.ctp | 1 + app/View/Elements/genericElements/IndexTable/Fields/model.ctp | 3 +++ app/View/Elements/genericElements/IndexTable/Fields/time.ctp | 3 +++ 3 files changed, 7 insertions(+) create mode 100644 app/View/Elements/genericElements/IndexTable/Fields/custom_element.ctp create mode 100644 app/View/Elements/genericElements/IndexTable/Fields/model.ctp create mode 100644 app/View/Elements/genericElements/IndexTable/Fields/time.ctp diff --git a/app/View/Elements/genericElements/IndexTable/Fields/custom_element.ctp b/app/View/Elements/genericElements/IndexTable/Fields/custom_element.ctp new file mode 100644 index 000000000..b661e6bda --- /dev/null +++ b/app/View/Elements/genericElements/IndexTable/Fields/custom_element.ctp @@ -0,0 +1 @@ +element($field['element_path'], ['item' => $row]) ?> \ No newline at end of file diff --git a/app/View/Elements/genericElements/IndexTable/Fields/model.ctp b/app/View/Elements/genericElements/IndexTable/Fields/model.ctp new file mode 100644 index 000000000..52d0eaeda --- /dev/null +++ b/app/View/Elements/genericElements/IndexTable/Fields/model.ctp @@ -0,0 +1,3 @@ +Time->time($time); \ No newline at end of file From 5667a3dd84aac6aa85206f06a1dc8bfca2e5c912 Mon Sep 17 00:00:00 2001 From: iglocska Date: Sat, 24 Dec 2022 06:28:42 +0100 Subject: [PATCH 04/54] chg: [event index] changed to the indextable generator --- app/Controller/AuditLogsController.php | 7 +- app/View/AuditLogs/event_index.ctp | 107 ++++++++++++++----------- 2 files changed, 64 insertions(+), 50 deletions(-) diff --git a/app/Controller/AuditLogsController.php b/app/Controller/AuditLogsController.php index 24d4f709d..4199c748f 100644 --- a/app/Controller/AuditLogsController.php +++ b/app/Controller/AuditLogsController.php @@ -192,10 +192,13 @@ class AuditLogsController extends AppController $list[$k]['AuditLog']['action_human'] = $this->actions[$item['AuditLog']['action']]; } - $this->set('list', $list); + $this->set('data', $list); $this->set('event', $event); $this->set('mayModify', $this->__canModifyEvent($event)); - $this->set('title_for_layout', __('Audit logs for event #%s', $event['Event']['id'])); + $this->set('menuData', [ + 'menuList' => 'event', + 'menuItem' => 'eventLog' + ]); } public function fullChange($id) diff --git a/app/View/AuditLogs/event_index.ctp b/app/View/AuditLogs/event_index.ctp index 9fd0aa7d4..b78afc18b 100644 --- a/app/View/AuditLogs/event_index.ctp +++ b/app/View/AuditLogs/event_index.ctp @@ -1,48 +1,59 @@ -
-

- - - - - - - - - - - - - - - - - - - - - - -
LightPaginator->sort('created') ?>LightPaginator->sort('user_id', __('User')) ?>LightPaginator->sort('org_id', __('Org')) ?>LightPaginator->sort('action') ?>
Time->time($item['AuditLog']['created']); ?>OrgImg->getOrgLogo($item, 24) : '' ?>element('AuditLog/change', ['item' => $item]) ?>
- -
-element('/genericElements/SideMenu/side_menu', ['menuList' => 'event', 'menuItem' => 'eventLog']); - +', empty($ajax) ? ' class="index"' : ''); +echo $this->element('genericElements/IndexTable/index_table', [ + 'data' => [ + 'light_paginator' => 1, + 'data' => $data, + 'fields' => [ + [ + 'name' => __('Created'), + 'data_path' => 'AuditLog.created', + 'sort' => 'AuditLog.created', + 'class' => 'short', + 'element' => 'time' + ], + [ + 'name' => __('User'), + 'data_path' => 'User.email', + 'sort' => 'User.email', + 'class' => 'short', + 'empty' => 'SYSTEM' + ], + [ + 'name' => __('Organisation'), + 'data_path' => 'Organisation', + 'sort' => 'Organisation.name', + 'element' => 'org', + 'class' => 'short' + ], + [ + 'name' => __('Action'), + 'data_path' => 'AuditLog.action_human', + 'sort' => 'AuditLog.action_human', + 'class' => 'short' + ], + [ + 'name' => __('Model'), + 'data_path' => 'AuditLog', + 'element' => 'model', + 'class' => 'short' + ], + [ + 'name' => __('Title'), + 'data_path' => 'AuditLog.title', + 'class' => 'limitedWidth' + ], + [ + 'name' => __('Change'), + 'data_path' => 'AuditLog', + 'element' => 'custom_element', + 'element_path' => 'AuditLog/change' + ] + ], + 'title' => __('Audit logs for event #%s', intval($event['Event']['id'])) + ] +]); +echo ''; +if (empty($ajax)) { + echo $this->element('/genericElements/SideMenu/side_menu', $menuData); +} From e3d716b9a748cdd45848e199be8db3a4781bf574 Mon Sep 17 00:00:00 2001 From: iglocska Date: Mon, 2 Jan 2023 12:29:33 +0100 Subject: [PATCH 05/54] fix: [postTest] speculative fix for case sensitivity of headers - as reported by @DavoDirty --- app/Controller/ServersController.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index 284c6e945..2123522ec 100644 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -1691,8 +1691,9 @@ class ServersController extends AppController if (!function_exists('getallheaders')) { $headers = []; foreach ($_SERVER as $name => $value) { - if (substr($name, 0, 5) === 'HTTP_') { - $headers[strtolower(str_replace('_', '-', substr($name, 5)))] = $value; + $name = strtolower($name); + if (substr($name, 0, 5) === 'http_') { + $headers[str_replace('_', '-', substr($name, 5))] = $value; } } } else { From 672b33ca20aaa5d4ebd3b15431aee5e356ccd97b Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 2 Jan 2023 16:16:48 +0100 Subject: [PATCH 06/54] chg: [warning-lists] updated --- app/files/warninglists | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/warninglists b/app/files/warninglists index ab2938e00..3a059f8bd 160000 --- a/app/files/warninglists +++ b/app/files/warninglists @@ -1 +1 @@ -Subproject commit ab2938e008eb5ec085d9e9c8e7c5c4d6fd466a39 +Subproject commit 3a059f8bd95360837ab3fe96830a4af6c2869372 From faa1fc5300fc8e1e6e98c72112801625e583a3fb Mon Sep 17 00:00:00 2001 From: Andras Iklody Date: Tue, 3 Jan 2023 12:20:35 +0100 Subject: [PATCH 07/54] fix: [querystring] bumped --- app/Controller/AppController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index f969cbe95..4b0d81383 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -33,7 +33,7 @@ class AppController extends Controller public $helpers = array('OrgImg', 'FontAwesome', 'UserName'); - private $__queryVersion = '146'; + private $__queryVersion = '147'; public $pyMispVersion = '2.4.167'; public $phpmin = '7.2'; public $phprec = '7.4'; From 9115a3042336e2938a6e3a3c4effcb3c125f413e Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Fri, 6 Jan 2023 09:19:36 +0100 Subject: [PATCH 08/54] minor code cosmetic fix --- app/Controller/UsersController.php | 45 ++++++++++++++++-------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php index 51ca9e6d2..08ab97cb0 100644 --- a/app/Controller/UsersController.php +++ b/app/Controller/UsersController.php @@ -1230,6 +1230,11 @@ class UsersController extends AppController $this->Bruteforce->insert($this->request->data['User']['email']); } } + + // + // Actions needed for the first access, when the database is not populated yet. + // + // populate the DB with the first role (site admin) if it's empty if (!$this->User->Role->hasAny()) { $siteAdmin = array('Role' => array( @@ -1279,7 +1284,6 @@ class UsersController extends AppController } $org_id = $this->User->Organisation->id; } - // populate the DB with the first user if it's empty if (!$this->User->hasAny()) { if (!isset($org_id)) { @@ -1291,7 +1295,6 @@ class UsersController extends AppController $org_id = $firstOrg['Organisation']['id']; } } - $this->User->runUpdates(); $this->User->createInitialUser($org_id); } @@ -1300,25 +1303,25 @@ class UsersController extends AppController private function _postlogin() { - $this->User->extralog($this->Auth->user(), "login"); - $this->User->Behaviors->disable('SysLogLogable.SysLogLogable'); - $this->User->id = $this->Auth->user('id'); - $user = $this->User->find('first', array( - 'conditions' => array( - 'User.id' => $this->Auth->user('id') - ), - 'recursive' => -1 - )); - unset($user['User']['password']); - $this->User->updateLoginTimes($user['User']); - $lastUserLogin = $user['User']['last_login']; - $this->User->Behaviors->enable('SysLogLogable.SysLogLogable'); - if ($lastUserLogin) { - $readableDatetime = (new DateTime())->setTimestamp($lastUserLogin)->format('D, d M y H:i:s O'); // RFC822 - $this->Flash->info(__('Welcome! Last login was on %s', $readableDatetime)); - } - // no state changes are ever done via GET requests, so it is safe to return to the original page: - $this->redirect($this->Auth->redirectUrl()); + $this->User->extralog($this->Auth->user(), "login"); + $this->User->Behaviors->disable('SysLogLogable.SysLogLogable'); + $this->User->id = $this->Auth->user('id'); + $user = $this->User->find('first', array( + 'conditions' => array( + 'User.id' => $this->Auth->user('id') + ), + 'recursive' => -1 + )); + unset($user['User']['password']); + $this->User->updateLoginTimes($user['User']); + $lastUserLogin = $user['User']['last_login']; + $this->User->Behaviors->enable('SysLogLogable.SysLogLogable'); + if ($lastUserLogin) { + $readableDatetime = (new DateTime())->setTimestamp($lastUserLogin)->format('D, d M y H:i:s O'); // RFC822 + $this->Flash->info(__('Welcome! Last login was on %s', $readableDatetime)); + } + // no state changes are ever done via GET requests, so it is safe to return to the original page: + $this->redirect($this->Auth->redirectUrl()); } public function routeafterlogin() From 56fe9a312956745414493ea484cc36fecd9201a2 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 9 Jan 2023 08:47:55 +0100 Subject: [PATCH 09/54] fix: [doc] New year - copyrights updated --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8e6dc245a..d74b8594a 100644 --- a/README.md +++ b/README.md @@ -108,16 +108,16 @@ License This software is licensed under [GNU Affero General Public License version 3](http://www.gnu.org/licenses/agpl-3.0.html) -* Copyright (C) 2012-2022 Christophe Vandeplas +* Copyright (C) 2012-2023 Christophe Vandeplas * Copyright (C) 2012 Belgian Defence * Copyright (C) 2012 NATO / NCIRC -* Copyright (C) 2013-2022 Andras Iklody -* Copyright (C) 2015-2022 CIRCL - Computer Incident Response Center Luxembourg +* Copyright (C) 2013-2023 Andras Iklody +* Copyright (C) 2015-2023 CIRCL - Computer Incident Response Center Luxembourg * Copyright (C) 2016 Andreas Ziegler -* Copyright (C) 2018-2022 Sami Mokaddem -* Copyright (C) 2018-2022 Christian Studer -* Copyright (C) 2015-2022 Alexandre Dulaunoy +* Copyright (C) 2018-2023 Sami Mokaddem +* Copyright (C) 2018-2023 Christian Studer +* Copyright (C) 2015-2023 Alexandre Dulaunoy * Copyright (C) 2018-2022 Steve Clement -* Copyright (C) 2020-2022 Jakub Onderka +* Copyright (C) 2020-2023 Jakub Onderka For more information, [the list of authors and contributors](AUTHORS) is available. From f7238fe5e71ac065daa43c8607d02f8ac682f18f Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 10 Jan 2023 10:05:22 +0100 Subject: [PATCH 10/54] fix: [security] XSS in authkey add - as reported by Dawid Czarnecki from Zigrin Security --- app/Controller/Component/CRUDComponent.php | 2 +- app/View/AuthKeys/authkey_display.ctp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/Controller/Component/CRUDComponent.php b/app/Controller/Component/CRUDComponent.php index fb276e34d..81ad2bfb1 100644 --- a/app/Controller/Component/CRUDComponent.php +++ b/app/Controller/Component/CRUDComponent.php @@ -114,7 +114,7 @@ class CRUDComponent extends Component $this->Controller->Flash->success($message); if (!empty($params['displayOnSuccess'])) { $this->Controller->set('entity', $data); - $this->Controller->set('referer', $this->Controller->referer()); + $this->Controller->set('referer', $this->Controller->referer(['action' => 'view', $model->id], true)); $this->Controller->render($params['displayOnSuccess']); return; } diff --git a/app/View/AuthKeys/authkey_display.ctp b/app/View/AuthKeys/authkey_display.ctp index 5cd4161d3..f64321798 100644 --- a/app/View/AuthKeys/authkey_display.ctp +++ b/app/View/AuthKeys/authkey_display.ctp @@ -12,7 +12,7 @@
- +

- + From a44b681aca32ff3df352f73070a1bb58256d5e7e Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 10 Jan 2023 11:17:07 +0100 Subject: [PATCH 11/54] chg: [logs:event_index] Added notice about displayed data and usage of LightPaginator --- app/View/Logs/event_index.ctp | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/app/View/Logs/event_index.ctp b/app/View/Logs/event_index.ctp index 7f38ecdeb..d53dec313 100644 --- a/app/View/Logs/event_index.ctp +++ b/app/View/Logs/event_index.ctp @@ -1,22 +1,24 @@

+

- - - - - - + + + + + + @@ -39,7 +41,7 @@
Paginator->sort('org');?>Paginator->sort('email');?>Paginator->sort('action');?>Paginator->sort('model');?>Paginator->sort('title');?>Paginator->sort('created');?>LightPaginator->sort('org');?>LightPaginator->sort('email');?>LightPaginator->sort('action');?>LightPaginator->sort('model');?>LightPaginator->sort('title');?>LightPaginator->sort('created');?>

Paginator->counter(array( + echo $this->LightPaginator->counter(array( 'format' => __('Page {:page} of {:pages}, showing {:current} records out of {:count} total, starting on record {:start}, ending on {:end}') )); ?> @@ -47,9 +49,9 @@

From 683a90199cabe3f2234568bafe8e039d711eb09e Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 10 Jan 2023 12:04:56 +0100 Subject: [PATCH 12/54] fix: [feed:edit] Make sure to keep orgc_id to its saved value --- app/View/Feeds/add.ctp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/View/Feeds/add.ctp b/app/View/Feeds/add.ctp index 47ccdefbd..dc49d38a2 100755 --- a/app/View/Feeds/add.ctp +++ b/app/View/Feeds/add.ctp @@ -73,7 +73,7 @@ echo $this->element('genericElements/Form/genericForm', [ 'field' => 'orgc_id', 'label' => __('Creator organisation'), 'options' => $dropdownData['orgs'], - 'value' => $this->request->params['action'] === 'add' ? $me['org_id'] : '', + 'value' => $this->request->params['action'] === 'add' ? $me['org_id'] : null, 'type' => 'dropdown', 'div' => ['id' => 'OrgcDiv', 'style' => 'display:none', 'class' => 'optionalField'], 'class' => 'form-control span6' From f2bb822d391f6ecf2bbbd6d7cd8ccd558d73bcea Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 11 Jan 2023 16:19:41 +0100 Subject: [PATCH 13/54] chg: [taxonomies] updated --- app/files/taxonomies | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/taxonomies b/app/files/taxonomies index a33dd623d..e4d0c5807 160000 --- a/app/files/taxonomies +++ b/app/files/taxonomies @@ -1 +1 @@ -Subproject commit a33dd623d1936dc015a3ba286a1b83eea1f71459 +Subproject commit e4d0c58076c2c1af20491b1d9be56967a16ac515 From 102b625042815bf79b0cbed5cc7001d8ebb9bb54 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 11 Jan 2023 16:20:29 +0100 Subject: [PATCH 14/54] chg: [taxonomies] updated to the latest version --- app/files/taxonomies | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/taxonomies b/app/files/taxonomies index a33dd623d..e4d0c5807 160000 --- a/app/files/taxonomies +++ b/app/files/taxonomies @@ -1 +1 @@ -Subproject commit a33dd623d1936dc015a3ba286a1b83eea1f71459 +Subproject commit e4d0c58076c2c1af20491b1d9be56967a16ac515 From 97cd2604d63dadb322afcce94c9f783a044c758b Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 13 Jan 2023 11:09:56 +0100 Subject: [PATCH 15/54] fix: [tags:relationship] Fixed synchronisation of relationship_type --- app/Model/Event.php | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index 98e6166f8..5affaad15 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -3373,7 +3373,11 @@ class Event extends AppModel foreach ($event['EventTag'] as $tag) { $tagId = $this->captureTagWithCache($tag['Tag'], $user, $capturedTags); if ($tagId && !in_array($tagId, $event_tag_ids)) { - $eventTags[] = array('tag_id' => $tagId); + $eventTags[] = array( + 'tag_id' => $tagId, + 'local' => isset($tag['local']) ? $tag['local'] : 0, + 'relationship_type' => isset($tag['relationship_type']) ? $tag['relationship_type'] : '', + ); $event_tag_ids[] = $tagId; } } @@ -3388,6 +3392,7 @@ class Event extends AppModel $eventTags[] = [ 'tag_id' => $tag_id, 'local' => isset($tag['local']) ? $tag['local'] : 0, + 'relationship_type' => isset($tag['relationship_type']) ? $tag['relationship_type'] : '', ]; $event_tag_ids[] = $tag_id; } @@ -3448,7 +3453,11 @@ class Event extends AppModel $a['AttributeTag'] = array($a['AttributeTag']); } foreach ($a['AttributeTag'] as $tag) { - $attributeTags[] = array('tag_id' => $this->captureTagWithCache($tag['Tag'], $user, $capturedTags)); + $attributeTags[] = array( + 'tag_id' => $this->captureTagWithCache($tag['Tag'], $user, $capturedTags), + 'local' => isset($tag['local']) ? $tag['local'] : 0, + 'relationship_type' => isset($tag['relationship_type']) ? $tag['relationship_type'] : '', + ); } } if (isset($a['Tag'])) { @@ -3461,6 +3470,7 @@ class Event extends AppModel $attributeTags[] = [ 'tag_id' => $tagId, 'local' => isset($tag['local']) ? $tag['local'] : 0, + 'relationship_type' => isset($tag['relationship_type']) ? $tag['relationship_type'] : '', ]; } } From 2b738d474551e3f93602bf4fedac30a7d5a028a5 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 13 Jan 2023 11:09:56 +0100 Subject: [PATCH 16/54] fix: [tags:relationship] Fixed synchronisation of relationship_type --- app/Model/Event.php | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index 98e6166f8..5affaad15 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -3373,7 +3373,11 @@ class Event extends AppModel foreach ($event['EventTag'] as $tag) { $tagId = $this->captureTagWithCache($tag['Tag'], $user, $capturedTags); if ($tagId && !in_array($tagId, $event_tag_ids)) { - $eventTags[] = array('tag_id' => $tagId); + $eventTags[] = array( + 'tag_id' => $tagId, + 'local' => isset($tag['local']) ? $tag['local'] : 0, + 'relationship_type' => isset($tag['relationship_type']) ? $tag['relationship_type'] : '', + ); $event_tag_ids[] = $tagId; } } @@ -3388,6 +3392,7 @@ class Event extends AppModel $eventTags[] = [ 'tag_id' => $tag_id, 'local' => isset($tag['local']) ? $tag['local'] : 0, + 'relationship_type' => isset($tag['relationship_type']) ? $tag['relationship_type'] : '', ]; $event_tag_ids[] = $tag_id; } @@ -3448,7 +3453,11 @@ class Event extends AppModel $a['AttributeTag'] = array($a['AttributeTag']); } foreach ($a['AttributeTag'] as $tag) { - $attributeTags[] = array('tag_id' => $this->captureTagWithCache($tag['Tag'], $user, $capturedTags)); + $attributeTags[] = array( + 'tag_id' => $this->captureTagWithCache($tag['Tag'], $user, $capturedTags), + 'local' => isset($tag['local']) ? $tag['local'] : 0, + 'relationship_type' => isset($tag['relationship_type']) ? $tag['relationship_type'] : '', + ); } } if (isset($a['Tag'])) { @@ -3461,6 +3470,7 @@ class Event extends AppModel $attributeTags[] = [ 'tag_id' => $tagId, 'local' => isset($tag['local']) ? $tag['local'] : 0, + 'relationship_type' => isset($tag['relationship_type']) ? $tag['relationship_type'] : '', ]; } } From 72c5424034c378583d128fc1e769aae33fb1c8b9 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Wed, 18 Jan 2023 15:00:50 +0100 Subject: [PATCH 17/54] fix: [security] XSS through network history name - as reported by Cyber Controls from SIX Group --- app/webroot/js/action_table.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/webroot/js/action_table.js b/app/webroot/js/action_table.js index 2bd54af97..103e36f4d 100644 --- a/app/webroot/js/action_table.js +++ b/app/webroot/js/action_table.js @@ -162,7 +162,7 @@ class ActionTable { tr.id = "tr_" + this.__uuidv4(); for (var col of row) { var td = document.createElement('td'); - td.innerHTML = col; + td.textContent = col; tr.appendChild(td); } this.__add_action_button(tr); From a46f794a136001101cbec84fccf3cc824e983493 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Wed, 18 Jan 2023 15:04:45 +0100 Subject: [PATCH 18/54] fix: [security] XSS in eventgraph preview payload - as reported by Cyber Controls from SIX Group --- app/webroot/js/event-graph.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/webroot/js/event-graph.js b/app/webroot/js/event-graph.js index 97cf672e8..150e9b662 100644 --- a/app/webroot/js/event-graph.js +++ b/app/webroot/js/event-graph.js @@ -636,7 +636,7 @@ class EventGraph { btn_plot.data('network-preview', preview); btn_plot.popover({ container: 'body', - content: function() { return ''; }, + content: function() { return ''; }, placement: 'right', trigger: 'hover', template: '', @@ -2002,7 +2002,7 @@ function reset_graph_history() { btn_plot.data('network-preview', preview); btn_plot.popover({ container: 'body', - content: function() { return ''; }, + content: function() { return ''; }, placement: 'right', trigger: 'hover', template: '', From 93bf15d3bd703a32ebfe86cb6c1c9b735cf23e30 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Wed, 18 Jan 2023 15:05:46 +0100 Subject: [PATCH 19/54] fix: [security] Prevent unauthorized access to decaying import function - as reported by Cyber Controls from SIX Group --- app/Controller/Component/ACLComponent.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php index c78f98c15..99390e34d 100644 --- a/app/Controller/Component/ACLComponent.php +++ b/app/Controller/Component/ACLComponent.php @@ -125,7 +125,7 @@ class ACLComponent extends Component 'decayingModel' => array( "update" => array(), "export" => array('*'), - "import" => array('*'), + "import" => array('OR' => array('perm_admin', 'perm_decaying')), "view" => array('*'), "index" => array('*'), "add" => array( 'OR' => array('perm_admin', 'perm_decaying')), From 5e4f9e692b4b0341f1d89bc0f20688078de2fb37 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Thu, 19 Jan 2023 11:44:21 +0100 Subject: [PATCH 20/54] chg: [misp-objects] updated --- app/files/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/misp-objects b/app/files/misp-objects index 4e19aa30b..fd603be32 160000 --- a/app/files/misp-objects +++ b/app/files/misp-objects @@ -1 +1 @@ -Subproject commit 4e19aa30ba94940e38bf50164248a572139ca520 +Subproject commit fd603be3283953b68ed48ede7afd2e19f43577ac From 3100254716356890a8eb27a54cc156e495f47c7a Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Thu, 19 Jan 2023 11:44:59 +0100 Subject: [PATCH 21/54] chg: [PyMISP] updated --- PyMISP | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyMISP b/PyMISP index cca5287b2..b4f305a07 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit cca5287b2b11a1951789680c75e10636dde98add +Subproject commit b4f305a077920e969f84db5c8c3dc42b6515a0d4 From 56b9aefb49dd869afa4d3ee863a86559e029042b Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 20 Jan 2023 11:02:54 +0100 Subject: [PATCH 22/54] chg: [servers:testConnection] Prematurely close the session allowing concurrent requests --- app/Controller/ServersController.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index 284c6e945..48c84f243 100644 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -1725,6 +1725,7 @@ class ServersController extends AppController if (!$server) { throw new NotFoundException(__('Invalid server')); } + @session_write_close(); // close session to allow concurrent requests $result = $this->Server->runConnectionTest($server); if ($result['status'] == 1) { if (isset($result['info']['version']) && preg_match('/^[0-9]+\.+[0-9]+\.[0-9]+$/', $result['info']['version'])) { From 5edcd9083f2104e4eeab4be75109b797d756c14b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 23 Jan 2023 10:09:01 +0100 Subject: [PATCH 23/54] chg: [PyMISP] Bump --- PyMISP | 2 +- app/Controller/AppController.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PyMISP b/PyMISP index b4f305a07..9a7adb2e0 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit b4f305a077920e969f84db5c8c3dc42b6515a0d4 +Subproject commit 9a7adb2e0d60d2edee9f541db808652875bae20e diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index f969cbe95..cfc9907ab 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -34,7 +34,7 @@ class AppController extends Controller public $helpers = array('OrgImg', 'FontAwesome', 'UserName'); private $__queryVersion = '146'; - public $pyMispVersion = '2.4.167'; + public $pyMispVersion = '2.4.168'; public $phpmin = '7.2'; public $phprec = '7.4'; public $phptoonew = '8.0'; From 6da894dc795af420554fe6eeed631b4980bb1986 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 23 Jan 2023 10:23:54 +0100 Subject: [PATCH 24/54] chg: [taxonomies] updated to the latest version --- app/files/taxonomies | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/taxonomies b/app/files/taxonomies index e4d0c5807..14f1349fa 160000 --- a/app/files/taxonomies +++ b/app/files/taxonomies @@ -1 +1 @@ -Subproject commit e4d0c58076c2c1af20491b1d9be56967a16ac515 +Subproject commit 14f1349fad189091960311018a4cabcb1018adb9 From c213ede209d84213c11bf9640b5e192188e86acb Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 23 Jan 2023 10:24:15 +0100 Subject: [PATCH 25/54] chg: [misp-galaxy] updated to the latest version --- app/files/misp-galaxy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy index ac1242a40..2f0dfc765 160000 --- a/app/files/misp-galaxy +++ b/app/files/misp-galaxy @@ -1 +1 @@ -Subproject commit ac1242a40e3a165ea664f3a8640dc346a5877c3a +Subproject commit 2f0dfc765667f1b7ef2dd21eb2aaa3c587e13c26 From a988cba4a965a053b38a461d1a803dcd6d923391 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 23 Jan 2023 10:25:17 +0100 Subject: [PATCH 26/54] chg: [misp-warninglists] updated to the latest version --- app/files/warninglists | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/warninglists b/app/files/warninglists index 3a059f8bd..a51a9adc6 160000 --- a/app/files/warninglists +++ b/app/files/warninglists @@ -1 +1 @@ -Subproject commit 3a059f8bd95360837ab3fe96830a4af6c2869372 +Subproject commit a51a9adc6c693bd5d2bf83f49920df7247ced298 From 9da3991a4a97cc5e1fa2079f6fd365774d42632b Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 24 Jan 2023 01:28:33 +0100 Subject: [PATCH 27/54] chg: [misp-stix] Bumped latest version --- app/files/scripts/misp-stix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/scripts/misp-stix b/app/files/scripts/misp-stix index c60a6e76c..93da9ae3a 160000 --- a/app/files/scripts/misp-stix +++ b/app/files/scripts/misp-stix @@ -1 +1 @@ -Subproject commit c60a6e76ccd735d460b382effee9af6aae7fa4ab +Subproject commit 93da9ae3a468f1beacc7edbfe241e6df89b94250 From 4a3964880d1efde9399a2efcd165b144ad301191 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 24 Jan 2023 01:29:35 +0100 Subject: [PATCH 28/54] fix: [misp-galaxy] Bumped latest version --- app/files/misp-galaxy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy index 2f0dfc765..f605f041d 160000 --- a/app/files/misp-galaxy +++ b/app/files/misp-galaxy @@ -1 +1 @@ -Subproject commit 2f0dfc765667f1b7ef2dd21eb2aaa3c587e13c26 +Subproject commit f605f041d99057b767231831d7f5970be6c1fcb9 From 6aa5f322150c1d53e5eac1d69379a8a21dde5c44 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Wed, 25 Jan 2023 11:51:41 +0100 Subject: [PATCH 29/54] fix: [shadowAttribute:accept] Restored accepting functionality Replace cake's magic finder by the standard way to fetch data --- app/Controller/ShadowAttributesController.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/Controller/ShadowAttributesController.php b/app/Controller/ShadowAttributesController.php index 48b448091..6ba08cbd1 100644 --- a/app/Controller/ShadowAttributesController.php +++ b/app/Controller/ShadowAttributesController.php @@ -62,8 +62,10 @@ class ShadowAttributesController extends AppController // If the old_id is set to anything but 0 then we're dealing with a proposed edit to an existing attribute if ($shadow['old_id'] != 0) { // Find the live attribute by the shadow attribute's uuid, so we can begin editing it - $this->Attribute->contain = 'Event'; - $activeAttribute = $this->Attribute->findByUuid($shadow['uuid']); + $activeAttribute = $this->Attribute->find('first', [ + 'conditions' => ['Attribute.uuid' => $shadow['uuid']], + 'contain' => ['Event'], + ]); // Send those away that shouldn't be able to edit this if (!$this->__canModifyEvent($activeAttribute)) { From 2211d5e00b3b0bc700cb6577befc18452329f303 Mon Sep 17 00:00:00 2001 From: goodlandsecurity Date: Thu, 26 Jan 2023 10:23:20 -0600 Subject: [PATCH 30/54] chg: [internal] allow site admins ability to view event_creator_email for all events in export --- app/Model/Event.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index 5affaad15..19264aec3 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -2097,7 +2097,7 @@ class Event extends AppModel // Include information about event creator user email. This information is included for: // - users from event creator org // - site admins - // In export, this information will be included in `event_creator_email` field just for auditors of event creator org. + // In export, this information will be included in `event_creator_email` field for auditors of event creator org and site admins. $sameOrg = $event['Event']['orgc_id'] === $user['org_id']; if ($sameOrg || $user['Role']['perm_site_admin']) { if (!isset($userEmails[$event['Event']['user_id']])) { @@ -2105,7 +2105,7 @@ class Event extends AppModel } $userEmail = $userEmails[$event['Event']['user_id']]; - if ($sameOrg && $user['Role']['perm_audit']) { + if ($sameOrg && $user['Role']['perm_audit'] || $user['Role']['perm_site_admin']) { $event['Event']['event_creator_email'] = $userEmail; } $event['User']['email'] = $userEmail; From 827bf50f3c00a18a2c93727c834becfb3cab178b Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Fri, 27 Jan 2023 02:42:40 +0100 Subject: [PATCH 31/54] chg: [auth] group authentication code --- app/Controller/AppController.php | 64 +++++++++++++++++--------------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index cfc9907ab..d332f6689 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -152,20 +152,25 @@ class AppController extends Controller } $this->User = ClassRegistry::init('User'); - if ($this->Auth->user()) { - if ($this->User->checkForSessionDestruction($this->Auth->user('id'))) { - $this->Auth->logout(); - $this->Session->destroy(); - $message = __('User deauthenticated on administrator request. Please reauthenticate.'); - if ($this->_isRest()) { - throw new ForbiddenException($message); - } else { - $this->Flash->warning($message); - $this->_redirectToLogin(); - } - } + + if (!empty($this->request->params['named']['disable_background_processing'])) { + Configure::write('MISP.background_jobs', 0); } + Configure::write('CurrentController', $controller); + Configure::write('CurrentAction', $action); + $versionArray = $this->User->checkMISPVersion(); + $this->mispVersion = implode('.', $versionArray); + $this->Security->blackHoleCallback = 'blackHole'; + + // send users away that are using ancient versions of IE + // Make sure to update this if IE 20 comes out :) + if (isset($_SERVER['HTTP_USER_AGENT'])) { + if (preg_match('/(?i)msie [2-8]/', $_SERVER['HTTP_USER_AGENT']) && !strpos($_SERVER['HTTP_USER_AGENT'], 'Opera')) { + throw new MethodNotAllowedException('You are using an unsecure and outdated version of IE, please download Google Chrome, Mozilla Firefox or update to a newer version of IE. If you are running IE9 or newer and still receive this error message, please make sure that you are not running your browser in compatibility mode. If you still have issues accessing the site, get in touch with your administration team at ' . Configure::read('MISP.contact')); + } + } + // For fresh installation (salt empty) generate a new salt if (!Configure::read('Security.salt')) { $this->User->Server->serverSettingsSaveValue('Security.salt', $this->User->generateRandomPassword(32)); @@ -176,6 +181,10 @@ class AppController extends Controller $this->User->Server->serverSettingsSaveValue('MISP.uuid', CakeText::uuid()); } + /** + * Authentication related activities + */ + // Check if Apache provides kerberos authentication data $authUserFields = $this->User->describeAuthFields(); $envvar = Configure::read('ApacheSecureAuth.apacheEnv'); @@ -191,22 +200,7 @@ class AppController extends Controller } else { $this->Auth->authenticate[AuthComponent::ALL]['userFields'] = $authUserFields; } - if (!empty($this->request->params['named']['disable_background_processing'])) { - Configure::write('MISP.background_jobs', 0); - } - Configure::write('CurrentController', $controller); - Configure::write('CurrentAction', $action); - $versionArray = $this->User->checkMISPVersion(); - $this->mispVersion = implode('.', $versionArray); - $this->Security->blackHoleCallback = 'blackHole'; - - // send users away that are using ancient versions of IE - // Make sure to update this if IE 20 comes out :) - if (isset($_SERVER['HTTP_USER_AGENT'])) { - if (preg_match('/(?i)msie [2-8]/', $_SERVER['HTTP_USER_AGENT']) && !strpos($_SERVER['HTTP_USER_AGENT'], 'Opera')) { - throw new MethodNotAllowedException('You are using an unsecure and outdated version of IE, please download Google Chrome, Mozilla Firefox or update to a newer version of IE. If you are running IE9 or newer and still receive this error message, please make sure that you are not running your browser in compatibility mode. If you still have issues accessing the site, get in touch with your administration team at ' . Configure::read('MISP.contact')); - } - } + $userLoggedIn = false; if (Configure::read('Plugin.CustomAuth_enable')) { $userLoggedIn = $this->__customAuthentication($_SERVER); @@ -528,12 +522,24 @@ class AppController extends Controller } $this->Flash->info($message); $this->Auth->logout(); - throw new MethodNotAllowedException($message);//todo this should pb be removed? + $this->_redirectToLogin(); + return false; } else { $this->Flash->error(__('Warning: MISP is currently disabled for all users. Enable it in Server Settings (Administration -> Server Settings -> MISP tab -> live). An update might also be in progress, you can see the progress in ') , array('params' => array('url' => $this->baseurl . '/servers/updateProgress/', 'urlName' => __('Update Progress')), 'clear' => 1)); } } + // kill existing sessions for a user if the admin/instance decides so + // exclude API authentication as it doesn't make sense + if (!$this->isApiAuthed && $this->User->checkForSessionDestruction($user['id'])) { + $this->Auth->logout(); + $this->Session->destroy(); + $message = __('User deauthenticated on administrator request. Please reauthenticate.'); + $this->Flash->warning($message); + $this->_redirectToLogin(); + return false; + } + // Force logout doesn't make sense for API key authentication if (!$this->isApiAuthed && $user['force_logout']) { $this->User->id = $user['id']; From 52e3abd73e47fc42764702d90ebc62600766d0a6 Mon Sep 17 00:00:00 2001 From: Andras Iklody Date: Mon, 30 Jan 2023 11:16:43 +0100 Subject: [PATCH 32/54] chg: [README.md] added a missing comma Just testing mail filters.... :) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8e6dc245a..cbd03aa13 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ The objective of MISP is to foster the sharing of structured information within MISP, Malware Information Sharing Platform and Threat Sharing, core functionalities are: -- An **efficient IOC and indicators** database allowing to store technical and non-technical information about malware samples, incidents, attackers and intelligence. +- An **efficient IOC and indicators** database, allowing to store technical and non-technical information about malware samples, incidents, attackers and intelligence. - Automatic **correlation** finding relationships between attributes and indicators from malware, attack campaigns or analysis. The correlation engine includes correlation between attributes and more advanced correlations like Fuzzy hashing correlation (e.g. ssdeep) or CIDR block matching. Correlation can also be enabled or event disabled per attribute. - A **flexible data model** where complex [objects](https://www.misp-project.org/objects.html) can be expressed and **linked together to express threat intelligence, incidents or connected elements**. - Built-in **sharing functionality** to ease data sharing using different model of distributions. MISP can automatically synchronize events and attributes among different MISP instances. Advanced filtering functionalities can be used to meet each organization's sharing policy including a **flexible sharing group** capacity and an attribute level distribution mechanisms. From 918f18ee912871537f3b619dff184176c936aaa3 Mon Sep 17 00:00:00 2001 From: Andras Iklody Date: Mon, 30 Jan 2023 11:21:41 +0100 Subject: [PATCH 33/54] fix: [README.md] typo fixed testing mail filters further --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cbd03aa13..193fd8f91 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ MISP, Malware Information Sharing Platform and Threat Sharing, core functionalit - Flexible **free text import** tool to ease the integration of unstructured reports into MISP. - A gentle system to **collaborate** on events and attributes allowing MISP users to propose changes or updates to attributes/indicators. - **data-sharing**: automatically exchange and synchronize with other parties and trust-groups using MISP. -- **delegating of sharing**: allows a simple pseudo-anonymous mechanism to delegate publication of event/indicators to another organization. +- **delegating of sharing**: allows for a simple, pseudo-anonymous mechanism to delegate publication of event/indicators to another organization. - Flexible **API** to integrate MISP with your own solutions. MISP is bundled with [PyMISP](https://github.com/MISP/PyMISP) which is a flexible Python Library to fetch, add or update events attributes, handle malware samples or search for attributes. An exhaustive restSearch API to easily search for indicators in MISP and exports those in all the format supported by MISP. - **Adjustable taxonomy** to classify and tag events following your own classification schemes or [existing classification](https://github.com/MISP/misp-taxonomies). The taxonomy can be local to your MISP but also shareable among MISP instances. - **Intelligence vocabularies** called MISP galaxy and bundled with existing [threat actors, malware, RAT, ransomware or MITRE ATT&CK](https://www.misp-project.org/galaxy.html) which can be easily linked with events and attributes in MISP. From 32711df31264010392412e91314f1615c9ad05a7 Mon Sep 17 00:00:00 2001 From: Andras Iklody Date: Mon, 30 Jan 2023 11:31:30 +0100 Subject: [PATCH 34/54] chg: [README.md] phrasing --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 193fd8f91..f7516f8bb 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ MISP, Malware Information Sharing Platform and Threat Sharing, core functionalit - **export**: generating IDS, OpenIOC, plain text, CSV, MISP XML or JSON output to integrate with other systems (network IDS, host IDS, custom tools), Cache format (used for forensic tools), STIX (XML and JSON) 1 and 2, NIDS export (Suricata, Snort and Bro/Zeek) or RPZ zone. Many other formats can be easily added via the [misp-modules](https://github.com/MISP/misp-modules). - **import**: bulk-import, batch-import, import from OpenIOC, GFI sandbox, ThreatConnect CSV, MISP standard format or STIX 1.1/2.0. Many other formats easily added via the [misp-modules](https://github.com/MISP/misp-modules). - Flexible **free text import** tool to ease the integration of unstructured reports into MISP. -- A gentle system to **collaborate** on events and attributes allowing MISP users to propose changes or updates to attributes/indicators. +- A user-friendly system to **collaborate** on events and attributes allowing MISP users to propose changes or updates to attributes/indicators. - **data-sharing**: automatically exchange and synchronize with other parties and trust-groups using MISP. - **delegating of sharing**: allows for a simple, pseudo-anonymous mechanism to delegate publication of event/indicators to another organization. - Flexible **API** to integrate MISP with your own solutions. MISP is bundled with [PyMISP](https://github.com/MISP/PyMISP) which is a flexible Python Library to fetch, add or update events attributes, handle malware samples or search for attributes. An exhaustive restSearch API to easily search for indicators in MISP and exports those in all the format supported by MISP. From cd6bff6e201bf330d15716fcfceee4245d5ee72b Mon Sep 17 00:00:00 2001 From: iglocska Date: Wed, 1 Feb 2023 14:34:53 +0100 Subject: [PATCH 35/54] fix: [VERSION] bump --- VERSION.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.json b/VERSION.json index 7b36fa445..6e1915b14 100644 --- a/VERSION.json +++ b/VERSION.json @@ -1 +1 @@ -{"major":2, "minor":4, "hotfix":167} +{"major":2, "minor":4, "hotfix":168} From 44dac20bf0e301cc44019b93d7761df9f8602afc Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 9 Feb 2023 10:06:11 +0100 Subject: [PATCH 36/54] chg: [dashboard-widget:TrendingTags] Added filtering and over time functionalities --- app/Lib/Dashboard/TrendingTagsWidget.php | 97 +++++++++++++++++++----- 1 file changed, 76 insertions(+), 21 deletions(-) diff --git a/app/Lib/Dashboard/TrendingTagsWidget.php b/app/Lib/Dashboard/TrendingTagsWidget.php index b9bdc9b7f..ae49c8c94 100644 --- a/app/Lib/Dashboard/TrendingTagsWidget.php +++ b/app/Lib/Dashboard/TrendingTagsWidget.php @@ -10,14 +10,17 @@ class TrendingTagsWidget 'time_window' => 'The time window, going back in seconds, that should be included.', 'exclude' => 'List of substrings to exclude tags by - for example "sofacy" would exclude any tag containing sofacy.', 'include' => 'List of substrings to include tags by - for example "sofacy" would include any tag containing sofacy.', - 'threshold' => 'Limits the number of displayed tags. Default: 10' + 'threshold' => 'Limits the number of displayed tags. Default: 10', + 'filter_event_tags' => 'Filters to be applied on event tags', + 'over_time' => 'Toggle the trending to be over time', ); public $placeholder = -'{ + '{ "time_window": "86400", "threshold": 15, "exclude": ["tlp:", "pap:"], - "include": ["misp-galaxy:", "my-internal-taxonomy"] + "include": ["misp-galaxy:", "my-internal-taxonomy"], + "filter_event_tags": ["misp-galaxy:threat-actor="APT 29"], }'; public $description = 'Widget showing the trending tags over the past x seconds, along with the possibility to include/exclude tags.'; public $cacheLifetime = 600; @@ -30,33 +33,85 @@ class TrendingTagsWidget $params = [ 'timestamp' => time() - (empty($options['time_window']) ? 8640000 : $options['time_window']), ]; + if (!empty($options['filter_event_tags'])) { + $params['event_tags'] = $options['filter_event_tags']; + } $eventIds = $eventModel->filterEventIds($user, $params); - $tags = []; $tagColours = []; - if (!empty($eventIds)) { - $eventTags = $eventModel->EventTag->find('all', [ - 'conditions' => ['EventTag.event_id' => $eventIds], - 'contain' => ['Tag' => ['fields' => ['name', 'colour']]], - 'recursive' => -1, - 'fields' => ['id'], - ]); + $allTags = []; + if (!empty($options['over_time'])) { + $this->render = 'MultiLineChart'; - foreach ($eventTags as $eventTag) { - $tagName = $eventTag['Tag']['name']; - if (isset($tags[$tagName])) { - $tags[$tagName]++; - } else if ($this->checkTag($options, $tagName)) { - $tags[$tagName] = 1; - $tagColours[$tagName] = $eventTag['Tag']['colour']; + $tagOvertime = []; + if (!empty($eventIds)) { + $events = $eventModel->fetchEvent($user, [ + 'eventid' => $eventIds, + 'order' => 'Event.timestamp', + ]); + + foreach ($events as $event) { + $timestamp = $event['Event']['timestamp']; + $timestamp = strftime('%Y-%m-%d', $timestamp); + foreach ($event['EventTag'] as $tag) { + $tagName = $tag['Tag']['name']; + if (isset($tagOvertime[$timestamp][$tagName])) { + $tagOvertime[$timestamp][$tagName]++; + } else if ($this->checkTag($options, $tagName)) { + $tagOvertime[$timestamp][$tagName] = 1; + $tagColours[$tagName] = $tag['Tag']['colour']; + $allTags[$tagName] = $tagName; + } + } } } - arsort($tags); + $data = []; + $data['data'] = []; + foreach($tagOvertime as $date => $tagCount) { + $item = []; + $item['date'] = $date; + foreach ($allTags as $tagName) { + if (!empty($tagCount[$tagName])) { + $item[$tagName] = $tagCount[$tagName]; + } else { + $item[$tagName] = 0; + } + } + $data['data'][] = $item; + } + uasort($data['data'], function ($a, $b) { + return ($a['date'] < $b['date']) ? -1 : 1; + }); + $data['data'] = array_values($data['data']); + return $data; + } else { + $tags = []; + if (!empty($eventIds)) { + $eventTags = $eventModel->EventTag->find('all', [ + 'conditions' => ['EventTag.event_id' => $eventIds], + 'contain' => ['Tag' => ['fields' => ['name', 'colour']]], + 'recursive' => -1, + 'fields' => ['id'], + ]); + + foreach ($eventTags as $eventTag) { + $tagName = $eventTag['Tag']['name']; + if (isset($tags[$tagName])) { + $tags[$tagName]++; + } else if ($this->checkTag($options, $tagName)) { + $tags[$tagName] = 1; + $tagColours[$tagName] = $eventTag['Tag']['colour']; + } + } + + arsort($tags); + $data['data'] = array_slice($tags, 0, $threshold); + $data['colours'] = $tagColours; + } + } - $data['data'] = array_slice($tags, 0, $threshold); - $data['colours'] = $tagColours; return $data; } From 92d88f6f9d9b3930f5e66ecafaf435640ed6ede3 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 9 Feb 2023 11:42:21 +0100 Subject: [PATCH 37/54] fix: [dashboard:widget_render] Use the correct render when using cache --- app/Controller/DashboardsController.php | 4 ++-- app/Lib/Dashboard/TrendingTagsWidget.php | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/app/Controller/DashboardsController.php b/app/Controller/DashboardsController.php index 9b87e8afd..a974be4c4 100644 --- a/app/Controller/DashboardsController.php +++ b/app/Controller/DashboardsController.php @@ -183,9 +183,9 @@ class DashboardsController extends AppController } else { $data = $dashboardWidget->handler($user, $valueConfig); } - + $renderer = method_exists($dashboardWidget, 'getRenderer') ? $dashboardWidget->getRenderer($valueConfig) : $dashboardWidget->render; $config = array( - 'render' => $dashboardWidget->render, + 'render' => $renderer, 'autoRefreshDelay' => empty($dashboardWidget->autoRefreshDelay) ? false : $dashboardWidget->autoRefreshDelay, 'widget_config' => empty($valueConfig['widget_config']) ? array() : $valueConfig['widget_config'] ); diff --git a/app/Lib/Dashboard/TrendingTagsWidget.php b/app/Lib/Dashboard/TrendingTagsWidget.php index ae49c8c94..cf50cf707 100644 --- a/app/Lib/Dashboard/TrendingTagsWidget.php +++ b/app/Lib/Dashboard/TrendingTagsWidget.php @@ -40,8 +40,8 @@ class TrendingTagsWidget $tagColours = []; $allTags = []; + $this->render = $this->getRenderer($options); if (!empty($options['over_time'])) { - $this->render = 'MultiLineChart'; $tagOvertime = []; if (!empty($eventIds)) { @@ -135,4 +135,9 @@ class TrendingTagsWidget return true; } } + + public function getRenderer(array $options) + { + return !empty($options['over_time']) ? 'MultiLineChart' : 'BarChart'; + } } From c92667ce1f85df7151ae67726bc299afb0475e11 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 13 Feb 2023 10:53:57 +0100 Subject: [PATCH 38/54] chg: [view:ajaxTags] Added placeholder for highlighted taxonomies without a tag --- app/View/Elements/ajaxTags.ctp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/View/Elements/ajaxTags.ctp b/app/View/Elements/ajaxTags.ctp index 216b0a925..7053e813e 100644 --- a/app/View/Elements/ajaxTags.ctp +++ b/app/View/Elements/ajaxTags.ctp @@ -75,6 +75,9 @@ 'tag_display_style' => 2 ]); } + if (empty($hTags)) { + $hTags = sprintf('-%s-', __('none')); + } $highlightedTagsString .= sprintf( '%s%s%s', From dab1abb2af9d7b7c27146ab237102a17221fc9c6 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 14 Feb 2023 08:34:47 +0100 Subject: [PATCH 39/54] chg: [events:populate] Improved support of MISP core format --- app/Controller/EventsController.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 45f9a22cf..9b053f8d6 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -2584,6 +2584,9 @@ class EventsController extends AppController if (isset($this->request->data['json'])) { $this->request->data = $this->_jsonDecode($this->request->data['json']); } + if (isset($this->request->data['Event'])) { + $this->request->data = $this->request->data['Event']; + } $eventToSave = $event; $capturedObjects = ['Attribute', 'Object', 'Tag', 'Galaxy', 'EventReport']; foreach ($capturedObjects as $objectType) { From d51bebcbf46ec4abe631743499e757b4db07dc82 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 14 Feb 2023 08:59:06 +0100 Subject: [PATCH 40/54] chg: [events:populate] Added support of regeneration of UUIDs --- app/Controller/EventsController.php | 9 ++++++++- app/View/Events/populate.ctp | 8 +++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 9b053f8d6..29d2eda31 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -2557,7 +2557,7 @@ class EventsController extends AppController } } - public function populate($id) + public function populate($id, $regenerateUUIDs=false) { if ($this->request->is('get') && $this->_isRest()) { return $this->RestResponse->describe('Events', 'populate', false, $this->response->type()); @@ -2579,6 +2579,7 @@ class EventsController extends AppController } if ($this->request->is('post') || $this->request->is('put')) { if (isset($this->request->data['Event'])) { + $regenerateUUIDs = $this->request->data['Event']['regenerate_uuids'] ?? false; $this->request->data = $this->request->data['Event']; } if (isset($this->request->data['json'])) { @@ -2591,6 +2592,12 @@ class EventsController extends AppController $capturedObjects = ['Attribute', 'Object', 'Tag', 'Galaxy', 'EventReport']; foreach ($capturedObjects as $objectType) { if (!empty($this->request->data[$objectType])) { + if (!empty($regenerateUUIDs)) { + foreach ($this->request->data[$objectType] as $i => $obj) { + unset($this->request->data[$objectType][$i]['id']); + unset($this->request->data[$objectType][$i]['uuid']); + } + } $eventToSave['Event'][$objectType] = $this->request->data[$objectType]; } } diff --git a/app/View/Events/populate.ctp b/app/View/Events/populate.ctp index c837e13e6..15fb1a764 100644 --- a/app/View/Events/populate.ctp +++ b/app/View/Events/populate.ctp @@ -10,7 +10,13 @@ [ 'field' => 'json', 'class' => 'input-big-chungus', - 'type' => 'textarea' + 'type' => 'textarea', + ], + [ + 'field' => 'regenerate_uuids', + 'type' => 'checkbox', + 'label' => __('Regenerate UUIDs'), + 'title' => __('test'), ], ], 'submit' => [ From 63430374563e72ff3be2a2f20034f6b6027346b1 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Thu, 16 Feb 2023 15:05:19 +0100 Subject: [PATCH 41/54] chg: [misp-stix] updated to the latest version --- app/files/scripts/misp-stix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/scripts/misp-stix b/app/files/scripts/misp-stix index 93da9ae3a..a794fdf0d 160000 --- a/app/files/scripts/misp-stix +++ b/app/files/scripts/misp-stix @@ -1 +1 @@ -Subproject commit 93da9ae3a468f1beacc7edbfe241e6df89b94250 +Subproject commit a794fdf0d55aa9bdb15f778c826935ada52c6b39 From edfdc5bfa2efd3865d56d69e889957405452548c Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 17 Feb 2023 15:59:34 +0100 Subject: [PATCH 42/54] chg: [events:add_misp_export] Handle case of missing Event key --- app/Controller/EventsController.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 29d2eda31..f2c223c96 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -2324,6 +2324,9 @@ class EventsController extends AppController if ($this->request->is('post')) { $results = array(); if (!empty($this->request->data)) { + if (empty($this->request->data['Event'])) { + $this->request->data['Event'] = $this->request->data; + } if (!empty($this->request->data['Event']['filecontent'])) { $data = $this->request->data['Event']['filecontent']; $isXml = $data[0] === '<'; @@ -4317,8 +4320,8 @@ class EventsController extends AppController 'checkbox' => false, ), 'bro' => array( - 'url' => $this->baseurl . '/attributes/bro/download/all/false/' . $id, - // 'url' => '/attributes/restSearch/returnFormat:bro/published:1||0/eventid:' . $id, + // 'url' => $this->baseurl . '/attributes/bro/download/all/false/' . $id, + 'url' => $this->baseurl . '/attributes/restSearch/returnFormat:bro/published:1||0/eventid:' . $id, 'text' => __('Bro rules'), 'requiresPublished' => false, 'checkbox' => false, From 2c30e9af7eff9ce772d01fc74f6da626a1d4abf4 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 17 Feb 2023 16:14:00 +0100 Subject: [PATCH 43/54] fix: [attribute:bro] Restored bro export The broExport should probably be rewritten to sue the standard restSearch export later on --- app/Controller/EventsController.php | 4 +- app/Model/Attribute.php | 94 +++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index f2c223c96..2e65f2b68 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -4320,8 +4320,8 @@ class EventsController extends AppController 'checkbox' => false, ), 'bro' => array( - // 'url' => $this->baseurl . '/attributes/bro/download/all/false/' . $id, - 'url' => $this->baseurl . '/attributes/restSearch/returnFormat:bro/published:1||0/eventid:' . $id, + 'url' => $this->baseurl . '/attributes/bro/download/all/false/' . $id, + // 'url' => $this->baseurl . '/attributes/restSearch/returnFormat:bro/published:1||0/eventid:' . $id, 'text' => __('Bro rules'), 'requiresPublished' => false, 'checkbox' => false, diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index b5dbb3713..ab2e09db2 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -3023,6 +3023,100 @@ class Attribute extends AppModel return $totalCount + $incrementTotalBy; } + public function bro($user, $type, $tags = false, $eventId = false, $from = false, $to = false, $last = false, $enforceWarninglist = false, $skipHeader = false) + { + App::uses('BroExport', 'Export'); + $export = new BroExport(); + if ($type == 'all') { + $types = array_keys($export->mispTypes); + } else { + $types = array($type); + } + $intel = array(); + foreach ($types as $type) { + //restricting to non-private or same org if the user is not a site-admin. + $conditions['AND'] = array('Attribute.to_ids' => 1, 'Event.published' => 1); + if ($from) { + $conditions['AND']['Event.date >='] = $from; + } + if ($to) { + $conditions['AND']['Event.date <='] = $to; + } + if ($last) { + $conditions['AND']['Event.publish_timestamp >='] = $last; + } + if ($eventId !== false) { + $temp = array(); + $args = $this->dissectArgs($eventId); + foreach ($args[0] as $accepted) { + $temp['OR'][] = array('Event.id' => $accepted); + } + $conditions['AND'][] = $temp; + $temp = array(); + foreach ($args[1] as $rejected) { + $temp['AND'][] = array('Event.id !=' => $rejected); + } + $conditions['AND'][] = $temp; + } + if ($tags !== false) { + // If we sent any tags along, load the associated tag names for each attribute + $tag = ClassRegistry::init('Tag'); + $args = $this->dissectArgs($tags); + $tagArray = $tag->fetchEventTagIds($args[0], $args[1]); + $temp = array(); + foreach ($tagArray[0] as $accepted) { + $temp['OR'][] = array('Event.id' => $accepted); + } + $conditions['AND'][] = $temp; + $temp = array(); + foreach ($tagArray[1] as $rejected) { + $temp['AND'][] = array('Event.id !=' => $rejected); + } + $conditions['AND'][] = $temp; + } + $this->Allowedlist = ClassRegistry::init('Allowedlist'); + $this->allowedlist = $this->Allowedlist->getBlockedValues(); + $instanceString = 'MISP'; + if (Configure::read('MISP.host_org_id') && Configure::read('MISP.host_org_id') > 0) { + $this->Event->Orgc->id = Configure::read('MISP.host_org_id'); + if ($this->Event->Orgc->exists()) { + $instanceString = $this->Event->Orgc->field('name') . ' MISP'; + } + } + $mispTypes = $export->getMispTypes($type); + foreach ($mispTypes as $mispType) { + $conditions['AND']['Attribute.type'] = $mispType[0]; + $intel = array_merge($intel, $this->__bro($user, $conditions, $mispType[1], $export, $this->allowedlist, $instanceString, $enforceWarninglist)); + } + } + natsort($intel); + $intel = array_unique($intel); + if (empty($skipHeader)) { + array_unshift($intel, $export->header); + } + return $intel; + } + + private function __bro($user, $conditions, $valueField, $export, $allowedlist, $instanceString, $enforceWarninglist) + { + $attributes = $this->fetchAttributes( + $user, + array( + 'conditions' => $conditions, // array of conditions + 'order' => 'Attribute.value' . $valueField . ' ASC', + 'recursive' => -1, // int + 'fields' => array('Attribute.id', 'Attribute.event_id', 'Attribute.type', 'Attribute.category', 'Attribute.comment', 'Attribute.to_ids', 'Attribute.value', 'Attribute.value' . $valueField), + 'contain' => array('Event' => array('fields' => array('Event.id', 'Event.threat_level_id', 'Event.orgc_id', 'Event.uuid'))), + 'enforceWarninglist' => $enforceWarninglist, + 'flatten' => 1 + ) + ); + $orgs = $this->Event->Orgc->find('list', array( + 'fields' => array('Orgc.id', 'Orgc.name') + )); + return $export->export($attributes, $orgs, $valueField, $allowedlist, $instanceString); + } + public function set_filter_uuid(&$params, $conditions, $options) { if (!empty($params['uuid'])) { From 6df6924ca3d361084a44b96f1314523dd39d4d81 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 17 Feb 2023 16:14:00 +0100 Subject: [PATCH 44/54] fix: [attribute:bro] Restored bro export The broExport should probably be rewritten to sue the standard restSearch export later on --- app/Controller/EventsController.php | 2 +- app/Model/Attribute.php | 94 +++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 45f9a22cf..6f00e9f20 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -4308,7 +4308,7 @@ class EventsController extends AppController ), 'bro' => array( 'url' => $this->baseurl . '/attributes/bro/download/all/false/' . $id, - // 'url' => '/attributes/restSearch/returnFormat:bro/published:1||0/eventid:' . $id, + // 'url' => $this->baseurl . '/attributes/restSearch/returnFormat:bro/published:1||0/eventid:' . $id, 'text' => __('Bro rules'), 'requiresPublished' => false, 'checkbox' => false, diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index b5dbb3713..ab2e09db2 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -3023,6 +3023,100 @@ class Attribute extends AppModel return $totalCount + $incrementTotalBy; } + public function bro($user, $type, $tags = false, $eventId = false, $from = false, $to = false, $last = false, $enforceWarninglist = false, $skipHeader = false) + { + App::uses('BroExport', 'Export'); + $export = new BroExport(); + if ($type == 'all') { + $types = array_keys($export->mispTypes); + } else { + $types = array($type); + } + $intel = array(); + foreach ($types as $type) { + //restricting to non-private or same org if the user is not a site-admin. + $conditions['AND'] = array('Attribute.to_ids' => 1, 'Event.published' => 1); + if ($from) { + $conditions['AND']['Event.date >='] = $from; + } + if ($to) { + $conditions['AND']['Event.date <='] = $to; + } + if ($last) { + $conditions['AND']['Event.publish_timestamp >='] = $last; + } + if ($eventId !== false) { + $temp = array(); + $args = $this->dissectArgs($eventId); + foreach ($args[0] as $accepted) { + $temp['OR'][] = array('Event.id' => $accepted); + } + $conditions['AND'][] = $temp; + $temp = array(); + foreach ($args[1] as $rejected) { + $temp['AND'][] = array('Event.id !=' => $rejected); + } + $conditions['AND'][] = $temp; + } + if ($tags !== false) { + // If we sent any tags along, load the associated tag names for each attribute + $tag = ClassRegistry::init('Tag'); + $args = $this->dissectArgs($tags); + $tagArray = $tag->fetchEventTagIds($args[0], $args[1]); + $temp = array(); + foreach ($tagArray[0] as $accepted) { + $temp['OR'][] = array('Event.id' => $accepted); + } + $conditions['AND'][] = $temp; + $temp = array(); + foreach ($tagArray[1] as $rejected) { + $temp['AND'][] = array('Event.id !=' => $rejected); + } + $conditions['AND'][] = $temp; + } + $this->Allowedlist = ClassRegistry::init('Allowedlist'); + $this->allowedlist = $this->Allowedlist->getBlockedValues(); + $instanceString = 'MISP'; + if (Configure::read('MISP.host_org_id') && Configure::read('MISP.host_org_id') > 0) { + $this->Event->Orgc->id = Configure::read('MISP.host_org_id'); + if ($this->Event->Orgc->exists()) { + $instanceString = $this->Event->Orgc->field('name') . ' MISP'; + } + } + $mispTypes = $export->getMispTypes($type); + foreach ($mispTypes as $mispType) { + $conditions['AND']['Attribute.type'] = $mispType[0]; + $intel = array_merge($intel, $this->__bro($user, $conditions, $mispType[1], $export, $this->allowedlist, $instanceString, $enforceWarninglist)); + } + } + natsort($intel); + $intel = array_unique($intel); + if (empty($skipHeader)) { + array_unshift($intel, $export->header); + } + return $intel; + } + + private function __bro($user, $conditions, $valueField, $export, $allowedlist, $instanceString, $enforceWarninglist) + { + $attributes = $this->fetchAttributes( + $user, + array( + 'conditions' => $conditions, // array of conditions + 'order' => 'Attribute.value' . $valueField . ' ASC', + 'recursive' => -1, // int + 'fields' => array('Attribute.id', 'Attribute.event_id', 'Attribute.type', 'Attribute.category', 'Attribute.comment', 'Attribute.to_ids', 'Attribute.value', 'Attribute.value' . $valueField), + 'contain' => array('Event' => array('fields' => array('Event.id', 'Event.threat_level_id', 'Event.orgc_id', 'Event.uuid'))), + 'enforceWarninglist' => $enforceWarninglist, + 'flatten' => 1 + ) + ); + $orgs = $this->Event->Orgc->find('list', array( + 'fields' => array('Orgc.id', 'Orgc.name') + )); + return $export->export($attributes, $orgs, $valueField, $allowedlist, $instanceString); + } + public function set_filter_uuid(&$params, $conditions, $options) { if (!empty($params['uuid'])) { From 3bb6f9f43f5cf9662fee43ea865b12947939ded9 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 20 Feb 2023 15:04:31 +0100 Subject: [PATCH 45/54] fix: [objects:group_attributes_into_object] Typo in find options --- app/Controller/ObjectsController.php | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/app/Controller/ObjectsController.php b/app/Controller/ObjectsController.php index e626b2b00..c58aca671 100644 --- a/app/Controller/ObjectsController.php +++ b/app/Controller/ObjectsController.php @@ -1218,8 +1218,18 @@ class ObjectsController extends AppController )); foreach ($object_references as $i => $object_reference) { - $temp_object = $this->MispObject->find('first', array('id' => $object_reference['ObjectReference']['object_id'], 'recursive' => -1)); - $temp_attribute = $this->MispObject->Attribute->find('first', array('id' => $object_reference['ObjectReference']['referenced_id'], 'recursive' => -1)); + $temp_object = $this->MispObject->find('first', [ + 'conditions' => [ + 'id' => $object_reference['ObjectReference']['object_id'] + ], + 'recursive' => -1 + ]); + $temp_attribute = $this->MispObject->Attribute->find('first', [ + 'conditions' => [ + 'id' => $object_reference['ObjectReference']['referenced_id'], + ], + 'recursive' => -1 + ]); if (!empty($temp_object) && !empty($temp_attribute)) { $temp_object = $temp_object['Object']; $temp_attribute = $temp_attribute['Attribute']; From df80a5c369236f42c1adf98ddcabe73ea5ded6bf Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 21 Feb 2023 11:48:42 +0100 Subject: [PATCH 46/54] chg: [misp-galaxy] updated --- app/files/misp-galaxy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy index f605f041d..aad2e33b8 160000 --- a/app/files/misp-galaxy +++ b/app/files/misp-galaxy @@ -1 +1 @@ -Subproject commit f605f041d99057b767231831d7f5970be6c1fcb9 +Subproject commit aad2e33b80e3c6221cc63529b58ee837d65e651b From 39408794efdac0d82f8004432424be8823f15854 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 23 Feb 2023 10:45:00 +0100 Subject: [PATCH 47/54] fix: [workflow:baseModule] Removed unused line --- app/Model/WorkflowModules/WorkflowBaseModule.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/Model/WorkflowModules/WorkflowBaseModule.php b/app/Model/WorkflowModules/WorkflowBaseModule.php index c61edc64a..911c35626 100644 --- a/app/Model/WorkflowModules/WorkflowBaseModule.php +++ b/app/Model/WorkflowModules/WorkflowBaseModule.php @@ -36,7 +36,6 @@ class WorkflowBaseModule { $fullIndexedParams = []; foreach ($this->params as $param) { - $param['value'] = $nodeParamByID[$param['id']]['value'] ?? null; $param['value'] = $node['data']['indexed_params'][$param['id']] ?? null; $fullIndexedParams[$param['id']] = $param; } From d418f33835d1e69957850aa0a867ab8bf807a4b2 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 23 Feb 2023 10:45:28 +0100 Subject: [PATCH 48/54] fix: [workflow:standalone_module_execute] Clear error output on sucess --- app/View/Elements/Workflows/execute_module.ctp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/View/Elements/Workflows/execute_module.ctp b/app/View/Elements/Workflows/execute_module.ctp index c8ffd6e71..67e5fdbbb 100644 --- a/app/View/Elements/Workflows/execute_module.ctp +++ b/app/View/Elements/Workflows/execute_module.ctp @@ -159,9 +159,9 @@ $type_mapper = [ .addClass(jqXHR.status == 200 ? 'label-success' : 'label-important') if (typeof result === 'object') { $executionResultText.text(JSON.stringify(result, '', 4)); + $('#executionResultHtml').empty(); } else { $('#executionResultHtml').html(result); - // $executionResultText.text(result); } } From 1994f35e9591b92a048ed4ec821cd57de422e89e Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 23 Feb 2023 10:46:52 +0100 Subject: [PATCH 49/54] chg: [workflow:module_webhook] Added support of more parameter to perform a request --- .../action/Module_ms_teams_webhook.php | 3 ++- app/Model/WorkflowModules/action/Module_webhook.php | 11 ++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/app/Model/WorkflowModules/action/Module_ms_teams_webhook.php b/app/Model/WorkflowModules/action/Module_ms_teams_webhook.php index d49f08824..785d399f6 100644 --- a/app/Model/WorkflowModules/action/Module_ms_teams_webhook.php +++ b/app/Model/WorkflowModules/action/Module_ms_teams_webhook.php @@ -5,6 +5,7 @@ class Module_ms_teams_webhook extends Module_webhook { public $id = 'ms-teams-webhook'; public $name = 'MS Teams Webhook'; + public $version = '0.2'; public $description = 'Perform callbacks to the MS Teams webhook provided by the "Incoming Webhook" connector'; public $icon_path = 'MS_Teams.png'; @@ -37,7 +38,7 @@ class Module_ms_teams_webhook extends Module_webhook ]; } - protected function doRequest($url, $contentType, $data) + protected function doRequest($url, $contentType, $data, $headers = [], $serverConfig = null) { $data = '{"text":"' . implode($data) . '"}'; return parent::doRequest($url, $contentType, $data); diff --git a/app/Model/WorkflowModules/action/Module_webhook.php b/app/Model/WorkflowModules/action/Module_webhook.php index 51626983b..ef4e59d45 100644 --- a/app/Model/WorkflowModules/action/Module_webhook.php +++ b/app/Model/WorkflowModules/action/Module_webhook.php @@ -8,7 +8,7 @@ class Module_webhook extends WorkflowBaseActionModule { public $id = 'webhook'; public $name = 'Webhook'; - public $version = '0.2'; + public $version = '0.3'; public $description = 'Allow to perform custom callbacks to the provided URL'; public $icon_path = 'webhook.png'; public $inputs = 1; @@ -106,21 +106,22 @@ class Module_webhook extends WorkflowBaseActionModule return false; } - protected function doRequest($url, $contentType, $data) + protected function doRequest($url, $contentType, $data, $headers = [], $serverConfig = null) { $this->Event = ClassRegistry::init('Event'); // We just need a model to use AppModel functions $version = implode('.', $this->Event->checkMISPVersion()); $commit = $this->Event->checkMIPSCommit(); $request = [ - 'header' => [ + 'header' => array_merge([ 'Accept' => 'application/json', 'Content-Type' => 'application/json', 'User-Agent' => 'MISP ' . $version . (empty($commit) ? '' : ' - #' . $commit), - ] + ], $headers) ]; $syncTool = new SyncTool(); - $HttpSocket = $syncTool->setupHttpSocket(null, $this->timeout); + $serverConfig = !empty($serverConfig['Server']) ? $serverConfig : ['Server' => $serverConfig]; + $HttpSocket = $syncTool->setupHttpSocket($serverConfig, $this->timeout); if ($contentType == 'form') { $request['header']['Content-Type'] = 'application/x-www-form-urlencoded'; $response = $HttpSocket->post($url, $data, $request); From f179b2db7a3554ae23c80efe98fd361e08df0b8c Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 23 Feb 2023 10:47:56 +0100 Subject: [PATCH 50/54] chg: [workflow:module_splunk_hec_export] Small refactoring --- .../action/Module_splunk_hec_export.php | 139 +++++++----------- 1 file changed, 52 insertions(+), 87 deletions(-) diff --git a/app/Model/WorkflowModules/action/Module_splunk_hec_export.php b/app/Model/WorkflowModules/action/Module_splunk_hec_export.php index c351f529d..6046f7533 100644 --- a/app/Model/WorkflowModules/action/Module_splunk_hec_export.php +++ b/app/Model/WorkflowModules/action/Module_splunk_hec_export.php @@ -4,20 +4,17 @@ include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php'; App::uses('SyncTool', 'Tools'); App::uses('JsonTool', 'Tools'); -class Module_splunk_hec_export extends WorkflowBaseActionModule +class Module_splunk_hec_export extends Module_webhook { public $id = 'splunk-hec-export'; public $name = 'Splunk HEC export'; - public $version = '0.1'; - public $description = 'Export Event Data to Splunk HTTP Event Collector'; + public $version = '0.2'; + public $description = 'Export Event Data to Splunk HTTP Event Collector. Due to the potential high amount of requests, it\'s recommanded to put this module after a `concurrent_task` logic module.'; public $icon_path = 'Splunk.png'; - public $inputs = 1; - public $outputs = 0; public $support_filters = false; + public $expect_misp_core_format = true; public $params = []; - - private $timeout = false; - private $Event; + public $outputs = 0; public function __construct() { @@ -34,10 +31,10 @@ class Module_splunk_hec_export extends WorkflowBaseActionModule 'label' => __('Verify HTTPS Certificate'), 'type' => 'select', 'options' => [ - 'true' => __('True'), - 'false' => __('False'), + '1' => __('True'), + '0' => __('False'), ], - 'default' => 'true', + 'default' => 1, ], [ 'id' => 'hec_token', @@ -51,10 +48,10 @@ class Module_splunk_hec_export extends WorkflowBaseActionModule 'label' => __('Create one Splunk Event per Attribute'), 'type' => 'select', 'options' => [ - 'true' => __('True'), - 'false' => __('False'), + '1' => __('True'), + '0' => __('False'), ], - 'default' => 'false', + 'default' => 0, ], [ 'id' => 'data_extraction_model', @@ -66,28 +63,8 @@ class Module_splunk_hec_export extends WorkflowBaseActionModule ]; } - public function diagnostic(): array - { - $errors = array_merge(parent::diagnostic(), []); - if (empty(Configure::read('Security.rest_client_enable_arbitrary_urls'))) { - $errors = $this->addNotification( - $errors, - 'error', - __('`rest_client_enable_arbitrary_urls` is turned off.'), - __('The module will not send any request as long as `Security.rest_client_enable_arbitrary_urls` is turned off.'), - [ - __('This is a security measure to ensure a site-admin do not send arbitrary request to internal services') - ], - true, - true - ); - } - return $errors; - } - public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool { - parent::exec($node, $roamingData, $errors); if (empty(Configure::read('Security.rest_client_enable_arbitrary_urls'))) { $errors[] = __('`Security.rest_client_enable_arbitrary_urls` is turned off'); return false; @@ -97,45 +74,62 @@ class Module_splunk_hec_export extends WorkflowBaseActionModule $errors[] = __('URL not provided.'); return false; } + if (empty($params['hec_token']['value'])) { + $errors[] = __('Authorization token not provided.'); + return false; + } $rData = $roamingData->getData(); - - //$path = $params['data_extraction_path']['value']; - //$extracted = !empty($params['data_extraction_path']['value']) ? $this->extractData($rData, $path) : $rData; - $event_without_attributes = $rData['Event']; unset($event_without_attributes['Attribute']); unset($event_without_attributes['_AttributeFlattened']); $splunk_events = []; - if ($params['event_per_attribute']['value'] == 'true') { + if (!empty($params['event_per_attribute']['value'])) { foreach ($rData['Event']['Attribute'] as $attribute) { - array_push($splunk_events, [ - 'Attribute' => $attribute, - 'Event' => $event_without_attributes - ]); + $splunk_events[] = [ + 'Attribute' => $attribute, + 'Event' => $event_without_attributes + ]; } } else { - array_push($splunk_events, $rData); + $splunk_events[] = $rData; } - if (!empty($params['data_extraction_model']['value'])) { - $data_extraction_model = JsonTool::decode($params['data_extraction_model']['value']); - $extracted_events = []; - foreach ($splunk_events as $splunk_event) { - $event = array(); - foreach ($data_extraction_model as $field => $path) { - $field_data = $this->extractData($splunk_event, $path); - $event[$field] = count($field_data) == 1 ? $field_data[0] : $field_data; // unpack if only one element - } - array_push($extracted_events, $event); - } - $splunk_events = $extracted_events; - } + if (!empty($params['data_extraction_model']['value'])) { + $data_extraction_model = JsonTool::decode($params['data_extraction_model']['value']); + $extracted_events = []; + foreach ($splunk_events as $splunk_event) { + $event = []; + foreach ($data_extraction_model as $field => $path) { + $field_data = $this->extractData($splunk_event, $path); + $event[$field] = count($field_data) == 1 ? $field_data[0] : $field_data; // unpack if only one element + } + $extracted_events[] = $event; + } + $splunk_events = $extracted_events; + } + return $this->sendToSplunk($splunk_events, $params['hec_token']['value'], $params['url']['value']); + } + + protected function sendToSplunk(array $splunk_events, $token, $url): bool + { foreach ($splunk_events as $splunk_event) { try { - $response = $this->doRequest($params['url']['value'], $params['hec_token']['value'], $params['verify_tls']['value'], $splunk_event); + $headers = [ + 'Authorization' => "Splunk {$token}", + ]; + $serverConfig = [ + 'Server' => ['self_signed' => empty($params['verify_tls']['value'])] + ]; + $response = $this->doRequest( + $url, + 'json', + $splunk_event, + $headers, + $serverConfig, + ); if (!$response->isOk()) { if ($response->code === 403 || $response->code === 401) { $errors[] = __('Authentication failed.'); @@ -154,33 +148,4 @@ class Module_splunk_hec_export extends WorkflowBaseActionModule } return true; } - - protected function doRequest($url, $hec_token, $verify_tls, $data) - { - $this->Event = ClassRegistry::init('Event'); // We just need a model to use AppModel functions - $version = implode('.', $this->Event->checkMISPVersion()); - $commit = $this->Event->checkMIPSCommit(); - - $request = [ - 'header' => [ - 'Authorization' => 'Splunk ' . $hec_token, - ] - ]; - $syncTool = new SyncTool(); - - $server = []; - if ($verify_tls == 'false') { - $server['Server'] = ['self_signed' => true]; - } - - $HttpSocket = $syncTool->setupHttpSocket($server, $this->timeout, 'Server'); - - - $HEC_Event = [ - 'event' => $data, - ]; - - $response = $HttpSocket->post($url, JsonTool::encode($HEC_Event), $request); - return $response; - } } From 738fd95db218c94bbcb39ad99d0cf6ea61df0d01 Mon Sep 17 00:00:00 2001 From: iglocska Date: Fri, 24 Feb 2023 02:00:16 +0100 Subject: [PATCH 51/54] fix: [taxii servers] invalid baseurl field type - copy pasta strikes again --- app/Model/AppModel.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 52b8ba552..dda95b94c 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -84,7 +84,7 @@ class AppModel extends Model 87 => false, 88 => false, 89 => false, 90 => false, 91 => false, 92 => false, 93 => false, 94 => false, 95 => true, 96 => false, 97 => true, 98 => false, 99 => false, 100 => false, 101 => false, 102 => false, 103 => false, 104 => false, - 105 => false + 105 => false, 106 => false ); const ADVANCED_UPDATES_DESCRIPTION = array( @@ -1910,7 +1910,7 @@ class AppModel extends Model `uuid` varchar(40) COLLATE utf8_bin NOT NULL , `name` varchar(191) NOT NULL, `owner` varchar(191) NOT NULL, - `baseurl` int(11) NOT NULL DEFAULT 0, + `baseurl` varchar(191) NOT NULL, `api_root` varchar(191) NOT NULL DEFAULT 0, `description` text, `filters` text, @@ -1941,6 +1941,9 @@ class AppModel extends Model } } break; + case 106: + $sqlArray[] = "ALTER TABLE `taxii_servers` MODIFY `baseurl` varchar(191) NOT NULL;"; + break; case 'fixNonEmptySharingGroupID': $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; From 5b71767f18d13158bbf8a8fffd97dac4d21e023d Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Sat, 25 Feb 2023 20:28:58 +0800 Subject: [PATCH 52/54] fix: [tools] corrected path to misp-website --- tools/gen_misp_types_categories.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/gen_misp_types_categories.py b/tools/gen_misp_types_categories.py index f609bdad7..0bde31517 100755 --- a/tools/gen_misp_types_categories.py +++ b/tools/gen_misp_types_categories.py @@ -167,10 +167,10 @@ with open('../../misp-book/categories-and-types/README.md', 'w') as f: # Find the offset of the start header: "### MISP default attributes and categories" # Find the offset of the end/next header: "## MISP objects" # Replace our new content in between -print("Updating MISP website - ../../misp-website/_pages/datamodels.md") +print("Updating MISP website - ../../misp-website/content/datamodels.md") misp_website = [] store_lines = True -with open('../../misp-website/_pages/datamodels.md', 'r') as f: +with open('../../misp-website/content/datamodels.md', 'r') as f: for line in f: # start marker if store_lines: @@ -183,7 +183,7 @@ with open('../../misp-website/_pages/datamodels.md', 'r') as f: elif line.startswith('## MISP objects'): store_lines = True misp_website.append(line) -with open('../../misp-website/_pages/datamodels.md', 'w') as f: +with open('../../misp-website/content/datamodels.md', 'w') as f: f.write(''.join(misp_website)) From 6ae140e5a9a07db9aa8b5e215fdf5b52761f99bc Mon Sep 17 00:00:00 2001 From: benni0 Date: Sun, 26 Feb 2023 12:23:54 +0000 Subject: [PATCH 53/54] fix: [workflow:module_splunk_hec_export] top level event required by splunk hec --- .../WorkflowModules/action/Module_splunk_hec_export.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/Model/WorkflowModules/action/Module_splunk_hec_export.php b/app/Model/WorkflowModules/action/Module_splunk_hec_export.php index 6046f7533..23a971722 100644 --- a/app/Model/WorkflowModules/action/Module_splunk_hec_export.php +++ b/app/Model/WorkflowModules/action/Module_splunk_hec_export.php @@ -123,10 +123,15 @@ class Module_splunk_hec_export extends Module_webhook $serverConfig = [ 'Server' => ['self_signed' => empty($params['verify_tls']['value'])] ]; + + $hec_event = [ + 'event' => $splunk_event + ]; + $response = $this->doRequest( $url, 'json', - $splunk_event, + $hec_event, $headers, $serverConfig, ); From 8eb26b7a9d67c35b1ba90046d1a276bb92654138 Mon Sep 17 00:00:00 2001 From: benni0 Date: Sun, 26 Feb 2023 12:25:24 +0000 Subject: [PATCH 54/54] chg: [workflow:module_splunk_hec_export] added parameter for source type specification --- .../action/Module_splunk_hec_export.php | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/app/Model/WorkflowModules/action/Module_splunk_hec_export.php b/app/Model/WorkflowModules/action/Module_splunk_hec_export.php index 23a971722..815edd942 100644 --- a/app/Model/WorkflowModules/action/Module_splunk_hec_export.php +++ b/app/Model/WorkflowModules/action/Module_splunk_hec_export.php @@ -43,6 +43,14 @@ class Module_splunk_hec_export extends Module_webhook 'type' => 'input', 'placeholder' => '00000000-0000-0000-000000000000' ], + [ + 'id' => 'source_type', + 'label' => __('Source Type'), + 'type' => 'select', + 'type' => 'input', + 'default' => '', + 'placeholder' => 'misp:event' + ], [ 'id' => 'event_per_attribute', 'label' => __('Create one Splunk Event per Attribute'), @@ -110,10 +118,10 @@ class Module_splunk_hec_export extends Module_webhook $splunk_events = $extracted_events; } - return $this->sendToSplunk($splunk_events, $params['hec_token']['value'], $params['url']['value']); + return $this->sendToSplunk($splunk_events, $params['hec_token']['value'], $params['url']['value'], $params['source_type']['value']); } - protected function sendToSplunk(array $splunk_events, $token, $url): bool + protected function sendToSplunk(array $splunk_events, $token, $url, $source_type): bool { foreach ($splunk_events as $splunk_event) { try { @@ -127,6 +135,9 @@ class Module_splunk_hec_export extends Module_webhook $hec_event = [ 'event' => $splunk_event ]; + if (!empty($source_type)) { + $hec_event['sourcetype'] = $source_type; + } $response = $this->doRequest( $url,