From bc4d5c35def2248711ec9a726d86aaf4daac6bc1 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Wed, 24 Feb 2021 20:28:37 +0100 Subject: [PATCH 01/17] chg: [internal] When caching feed, save progress to db less often --- app/Model/Feed.php | 4 ++-- app/Model/Job.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/Model/Feed.php b/app/Model/Feed.php index 4d9cdb8af..037bc3dd8 100644 --- a/app/Model/Feed.php +++ b/app/Model/Feed.php @@ -1226,7 +1226,7 @@ class Feed extends AppModel $md5Values = array_map('md5', array_column($values, 'value')); $redis->del('misp:feed_cache:' . $feedId); - foreach (array_chunk($md5Values, 1000) as $k => $chunk) { + foreach (array_chunk($md5Values, 5000) as $k => $chunk) { $pipe = $redis->multi(Redis::PIPELINE); if (method_exists($redis, 'sAddArray')) { $redis->sAddArray('misp:feed_cache:' . $feedId, $chunk); @@ -1238,7 +1238,7 @@ class Feed extends AppModel } } $pipe->exec(); - $this->jobProgress($jobId, __('Feed %s: %s/%s values cached.', $feedId, $k * 1000, count($md5Values))); + $this->jobProgress($jobId, __('Feed %s: %s/%s values cached.', $feedId, $k * 5000, count($md5Values))); } $redis->set('misp:feed_cache_timestamp:' . $feedId, time()); return true; diff --git a/app/Model/Job.php b/app/Model/Job.php index ecb8a846c..9193a2366 100644 --- a/app/Model/Job.php +++ b/app/Model/Job.php @@ -92,7 +92,7 @@ class Job extends AppModel } } try { - if ($this->save($jobData)) { + if ($this->save($jobData, ['atomic' => false])) { return true; } $this->log("Could not save progress for job $jobId because of validation errors: " . json_encode($this->validationErrors), LOG_NOTICE); From 98ec79db60c3b710869ec115cb9de77eb77584e3 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 1 Mar 2021 16:55:40 +0100 Subject: [PATCH 02/17] chg: [internal] Set cookie name just when no name is set --- app/Controller/AppController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index c2fa205d3..e8b175373 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -1260,7 +1260,7 @@ class AppController extends Controller private function __sessionMassage() { - if (!empty(Configure::read('MISP.uuid'))) { + if (empty(Configure::read('Session.cookie')) && !empty(Configure::read('MISP.uuid'))) { Configure::write('Session.cookie', 'MISP-' . Configure::read('MISP.uuid')); } if (!empty(Configure::read('Session.cookieTimeout')) || !empty(Configure::read('Session.timeout'))) { From 99ca948555e41ad1fc7d297bae4cd4bc44040bc0 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 1 Mar 2021 19:37:32 +0100 Subject: [PATCH 03/17] new: [UI] Render galaxy cluster description as markdown --- app/View/GalaxyClusters/view.ctp | 172 +++++++++++++++-------------- app/View/Helper/MarkdownHelper.php | 27 +++++ 2 files changed, 119 insertions(+), 80 deletions(-) create mode 100644 app/View/Helper/MarkdownHelper.php diff --git a/app/View/GalaxyClusters/view.ctp b/app/View/GalaxyClusters/view.ctp index cf91eff31..b57d4d184 100755 --- a/app/View/GalaxyClusters/view.ctp +++ b/app/View/GalaxyClusters/view.ctp @@ -1,85 +1,85 @@ element('/genericElements/SideMenu/side_menu', array('menuList' => 'galaxies', 'menuItem' => 'view_cluster')); - - $extendedFromHtml = ''; - if (!empty($cluster['GalaxyCluster']['extended_from'])) { - $element = $this->element('genericElements/IndexTable/Fields/links', array( - 'url' => $baseurl . '/galaxy_clusters/view/', - 'row' => $cluster, - 'field' => array( - 'data_path' => 'GalaxyCluster.extended_from.GalaxyCluster.id', - 'title' => sprintf(__('%s (version: %s)'), $cluster['GalaxyCluster']['extended_from']['GalaxyCluster']['value'], $cluster['GalaxyCluster']['extends_version']) - ), - )); - $extendedFromHtml = sprintf('
  • %s
', $element); - } - if ($newVersionAvailable) { - $extendedFromHtml .= sprintf('
%s
', sprintf(__('New version available! Update cluster to version %s'), - '/galaxy_clusters/updateCluster/' . $cluster['GalaxyCluster']['id'], - h($cluster['GalaxyCluster']['extended_from']['GalaxyCluster']['version']) - )); - } - - $extendedByHtml = ''; - $extendByLinks = array(); - foreach ($cluster['GalaxyCluster']['extended_by'] as $extendCluster) { - $element = $this->element('genericElements/IndexTable/Fields/links', array( - 'url' => '/galaxy_clusters/view/', - 'row' => $extendCluster, - 'field' => array( - 'data_path' => 'GalaxyCluster.id', - 'title' => sprintf(__('%s (parent version: %s)'), $extendCluster['GalaxyCluster']['value'], $extendCluster['GalaxyCluster']['extends_version']) - ), - )); - $extendByLinks[] = sprintf('
  • %s
    • ', $element); - } - if (!empty($extendByLinks)) { - $extendedByHtml = sprintf('
      %s
    ', implode('', $extendByLinks)); - } - - $table_data = array(); - $table_data[] = array('key' => __('Cluster ID'), 'value' => $cluster['GalaxyCluster']['id']); - $table_data[] = array('key' => __('Name'), 'value' => $cluster['GalaxyCluster']['value']); - $table_data[] = array('key' => __('Parent Galaxy'), 'value' => $cluster['GalaxyCluster']['Galaxy']['name'] ? $cluster['GalaxyCluster']['Galaxy']['name'] : $cluster['GalaxyCluster']['Galaxy']['type']); - $table_data[] = array('key' => __('Description'), 'value' => $cluster['GalaxyCluster']['description']); - $table_data[] = array('key' => __('Published'), 'boolean' => $cluster['GalaxyCluster']['published'], 'class' => (!$cluster['GalaxyCluster']['published'] ? 'background-red bold': '')); - $table_data[] = array('key' => __('Default'), 'boolean' => $cluster['GalaxyCluster']['default'], 'class' => (!$cluster['GalaxyCluster']['published'] ? 'black': 'black')); - $table_data[] = array('key' => __('Version'), 'value' => $cluster['GalaxyCluster']['version']); - $table_data[] = array('key' => __('UUID'), 'value' => $cluster['GalaxyCluster']['uuid'], 'value_class' => 'quickSelect'); - $table_data[] = array('key' => __('Collection UUID'), 'value' => $cluster['GalaxyCluster']['collection_uuid'], 'value_class' => 'quickSelect'); - $table_data[] = array( - 'key' => __('Source'), - 'html' => filter_var($cluster['GalaxyCluster']['source'], FILTER_VALIDATE_URL) ? - '' . h($cluster['GalaxyCluster']['source']) : - h($cluster['GalaxyCluster']['source']), - ); - $table_data[] = array('key' => __('Authors'), 'value' => !empty($cluster['GalaxyCluster']['authors']) ? implode(', ', $cluster['GalaxyCluster']['authors']) : __('N/A')); - $table_data[] = array('key' => __('Distribution'), 'element' => 'genericElements/IndexTable/Fields/distribution_levels', 'element_params' => array( - 'row' => $cluster['GalaxyCluster'], - 'field' => array('data_path' => 'distribution') +$extendedFromHtml = ''; +if (!empty($cluster['GalaxyCluster']['extended_from'])) { + $element = $this->element('genericElements/IndexTable/Fields/links', array( + 'url' => $baseurl . '/galaxy_clusters/view/', + 'row' => $cluster, + 'field' => array( + 'data_path' => 'GalaxyCluster.extended_from.GalaxyCluster.id', + 'title' => sprintf(__('%s (version: %s)'), $cluster['GalaxyCluster']['extended_from']['GalaxyCluster']['value'], $cluster['GalaxyCluster']['extends_version']) + ), )); - $table_data[] = array( - 'key' => __('Owner Organisation'), - 'html' => $this->OrgImg->getOrgImg(array('name' => $cluster['GalaxyCluster']['Org']['name'], 'id' => $cluster['GalaxyCluster']['Org']['id'], 'size' => 18), true), - ); - $table_data[] = array( - 'key' => __('Creator Organisation'), - 'html' => $this->OrgImg->getOrgImg(array('name' => $cluster['GalaxyCluster']['Orgc']['name'], 'id' => $cluster['GalaxyCluster']['Orgc']['id'], 'size' => 18), true), - ); - $table_data[] = array('key' => __('Connector tag'), 'value' => $cluster['GalaxyCluster']['tag_name']); - $table_data[] = array('key' => __('Events'), 'html' => isset($cluster['GalaxyCluster']['tag_count']) ? - sprintf('%s', - sprintf('%s/events/index/searchtag:%s', $baseurl, h($cluster['GalaxyCluster']['tag_id'])), - __n('%s event', '%s events', $cluster['GalaxyCluster']['tag_count'], h($cluster['GalaxyCluster']['tag_count'])) - ): - '0' - ); - if (!empty($extendedFromHtml)) { - $table_data[] = array('key' => __('Forked From'), 'html' => $extendedFromHtml); - } - if (!empty($extendedByHtml)) { - $table_data[] = array('key' => __('Forked By'), 'html' => $extendedByHtml); - } + $extendedFromHtml = sprintf('
    • %s
    ', $element); +} +if ($newVersionAvailable) { + $extendedFromHtml .= sprintf('
    %s
    ', sprintf(__('New version available! Update cluster to version %s'), + '/galaxy_clusters/updateCluster/' . $cluster['GalaxyCluster']['id'], + h($cluster['GalaxyCluster']['extended_from']['GalaxyCluster']['version']) + )); +} + +$extendedByHtml = ''; +$extendByLinks = array(); +foreach ($cluster['GalaxyCluster']['extended_by'] as $extendCluster) { + $element = $this->element('genericElements/IndexTable/Fields/links', array( + 'url' => '/galaxy_clusters/view/', + 'row' => $extendCluster, + 'field' => array( + 'data_path' => 'GalaxyCluster.id', + 'title' => sprintf(__('%s (parent version: %s)'), $extendCluster['GalaxyCluster']['value'], $extendCluster['GalaxyCluster']['extends_version']) + ), + )); + $extendByLinks[] = sprintf('
  • %s
    • ', $element); +} +if (!empty($extendByLinks)) { + $extendedByHtml = sprintf('
      %s
    ', implode('', $extendByLinks)); +} + +$description = $this->Markdown->cleanup($cluster['GalaxyCluster']['description']); + +$table_data = array(); +$table_data[] = array('key' => __('Cluster ID'), 'value' => $cluster['GalaxyCluster']['id']); +$table_data[] = array('key' => __('Name'), 'value' => $cluster['GalaxyCluster']['value']); +$table_data[] = array('key' => __('Parent Galaxy'), 'value' => $cluster['GalaxyCluster']['Galaxy']['name'] ? $cluster['GalaxyCluster']['Galaxy']['name'] : $cluster['GalaxyCluster']['Galaxy']['type']); +$table_data[] = array('key' => __('Description'), 'value' => $description, 'value_class' => 'md'); +$table_data[] = array('key' => __('Published'), 'boolean' => $cluster['GalaxyCluster']['published'], 'class' => (!$cluster['GalaxyCluster']['published'] ? 'background-red bold': '')); +$table_data[] = array('key' => __('Default'), 'boolean' => $cluster['GalaxyCluster']['default'], 'class' => (!$cluster['GalaxyCluster']['published'] ? 'black': 'black')); +$table_data[] = array('key' => __('Version'), 'value' => $cluster['GalaxyCluster']['version']); +$table_data[] = array('key' => __('UUID'), 'value' => $cluster['GalaxyCluster']['uuid'], 'value_class' => 'quickSelect'); +$table_data[] = array('key' => __('Collection UUID'), 'value' => $cluster['GalaxyCluster']['collection_uuid'], 'value_class' => 'quickSelect'); +$table_data[] = array( + 'key' => __('Source'), + 'html' => filter_var($cluster['GalaxyCluster']['source'], FILTER_VALIDATE_URL) ? + '' . h($cluster['GalaxyCluster']['source']) : + h($cluster['GalaxyCluster']['source']), +); +$table_data[] = array('key' => __('Authors'), 'value' => !empty($cluster['GalaxyCluster']['authors']) ? implode(', ', $cluster['GalaxyCluster']['authors']) : __('N/A')); +$table_data[] = array('key' => __('Distribution'), 'element' => 'genericElements/IndexTable/Fields/distribution_levels', 'element_params' => array( + 'row' => $cluster['GalaxyCluster'], + 'field' => array('data_path' => 'distribution') +)); +$table_data[] = array( + 'key' => __('Owner Organisation'), + 'html' => $this->OrgImg->getOrgImg(array('name' => $cluster['GalaxyCluster']['Org']['name'], 'id' => $cluster['GalaxyCluster']['Org']['id'], 'size' => 18), true), +); +$table_data[] = array( + 'key' => __('Creator Organisation'), + 'html' => $this->OrgImg->getOrgImg(array('name' => $cluster['GalaxyCluster']['Orgc']['name'], 'id' => $cluster['GalaxyCluster']['Orgc']['id'], 'size' => 18), true), +); +$table_data[] = array('key' => __('Connector tag'), 'value' => $cluster['GalaxyCluster']['tag_name']); +$table_data[] = array('key' => __('Events'), 'html' => isset($cluster['GalaxyCluster']['tag_count']) ? + sprintf('%s', + sprintf('%s/events/index/searchtag:%s', $baseurl, h($cluster['GalaxyCluster']['tag_id'])), + __n('%s event', '%s events', $cluster['GalaxyCluster']['tag_count'], h($cluster['GalaxyCluster']['tag_count'])) + ): + '0' + ); +if (!empty($extendedFromHtml)) { + $table_data[] = array('key' => __('Forked From'), 'html' => $extendedFromHtml); +} +if (!empty($extendedByHtml)) { + $table_data[] = array('key' => __('Forked By'), 'html' => $extendedByHtml); +} ?>
    @@ -98,6 +98,12 @@
    +element('genericElements/assetLoader', array( + 'js' => array( + 'markdown-it', + ), +)); +?> +element('/genericElements/SideMenu/side_menu', array('menuList' => 'galaxies', 'menuItem' => 'view_cluster')); diff --git a/app/View/Helper/MarkdownHelper.php b/app/View/Helper/MarkdownHelper.php new file mode 100644 index 000000000..ae9318428 --- /dev/null +++ b/app/View/Helper/MarkdownHelper.php @@ -0,0 +1,27 @@ +cleanup($string); + // Remove markdown style links + $string = preg_replace('/\[([^]]+)]\([^)]+\)/', '$1', $string); + // Remove citations + $string = preg_replace('/\(Citation: [^)]+\)/', '', $string); + return $string; + } + + public function cleanup($string) + { + // Remove blocks and replace by ticks + $string = preg_replace('/([^<]+)<\/code>/', '`$1`', $string); + return $string; + } +} From ce98ce48acdb80938ac8ccaac782795fcdcd7f01 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 1 Mar 2021 19:48:54 +0100 Subject: [PATCH 04/17] chg: [UI] Do not show published for default galaxy clusters --- app/View/GalaxyClusters/view.ctp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/app/View/GalaxyClusters/view.ctp b/app/View/GalaxyClusters/view.ctp index b57d4d184..f7d869b04 100755 --- a/app/View/GalaxyClusters/view.ctp +++ b/app/View/GalaxyClusters/view.ctp @@ -42,8 +42,14 @@ $table_data[] = array('key' => __('Cluster ID'), 'value' => $cluster['GalaxyClus $table_data[] = array('key' => __('Name'), 'value' => $cluster['GalaxyCluster']['value']); $table_data[] = array('key' => __('Parent Galaxy'), 'value' => $cluster['GalaxyCluster']['Galaxy']['name'] ? $cluster['GalaxyCluster']['Galaxy']['name'] : $cluster['GalaxyCluster']['Galaxy']['type']); $table_data[] = array('key' => __('Description'), 'value' => $description, 'value_class' => 'md'); -$table_data[] = array('key' => __('Published'), 'boolean' => $cluster['GalaxyCluster']['published'], 'class' => (!$cluster['GalaxyCluster']['published'] ? 'background-red bold': '')); -$table_data[] = array('key' => __('Default'), 'boolean' => $cluster['GalaxyCluster']['default'], 'class' => (!$cluster['GalaxyCluster']['published'] ? 'black': 'black')); +if (!$cluster['GalaxyCluster']['default']) { + $table_data[] = [ + 'key' => __('Published'), + 'boolean' => $cluster['GalaxyCluster']['published'], + 'class' => !$cluster['GalaxyCluster']['published'] ? 'background-red bold' : '' + ]; +} +$table_data[] = array('key' => __('Default'), 'boolean' => $cluster['GalaxyCluster']['default'], 'class' => 'black'); $table_data[] = array('key' => __('Version'), 'value' => $cluster['GalaxyCluster']['version']); $table_data[] = array('key' => __('UUID'), 'value' => $cluster['GalaxyCluster']['uuid'], 'value_class' => 'quickSelect'); $table_data[] = array('key' => __('Collection UUID'), 'value' => $cluster['GalaxyCluster']['collection_uuid'], 'value_class' => 'quickSelect'); From 6f74097c37abdca908748cef4bbe0cb5257325f7 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 1 Mar 2021 22:41:38 +0100 Subject: [PATCH 05/17] chg: [optimise] Faster loading galaxy cluster index --- app/Controller/GalaxiesController.php | 2 +- app/Controller/GalaxyClustersController.php | 116 ++++++++++---------- app/Model/GalaxyCluster.php | 27 ++++- 3 files changed, 84 insertions(+), 61 deletions(-) diff --git a/app/Controller/GalaxiesController.php b/app/Controller/GalaxiesController.php index 15e609fc6..917a6fe01 100644 --- a/app/Controller/GalaxiesController.php +++ b/app/Controller/GalaxiesController.php @@ -558,8 +558,8 @@ class GalaxiesController extends AppController if (empty($clusters)) { throw new MethodNotAllowedException('Invalid Galaxy.'); } + $this->Galaxy->GalaxyCluster->attachExtendByInfo($this->Auth->user(), $clusters); foreach ($clusters as $k => $cluster) { - $clusters[$k] = $this->Galaxy->GalaxyCluster->attachExtendByInfo($this->Auth->user(), $clusters[$k]); $clusters[$k] = $this->Galaxy->GalaxyCluster->attachExtendFromInfo($this->Auth->user(), $clusters[$k]); } $galaxy = $this->Galaxy->find('first', array( diff --git a/app/Controller/GalaxyClustersController.php b/app/Controller/GalaxyClustersController.php index db5ef3108..dcd313c7d 100644 --- a/app/Controller/GalaxyClustersController.php +++ b/app/Controller/GalaxyClustersController.php @@ -96,62 +96,64 @@ class GalaxyClustersController extends AppController ) ); return $this->RestResponse->viewData($clusters, $this->response->type()); - } else { - $this->paginate['conditions']['AND'][] = $contextConditions; - $this->paginate['conditions']['AND'][] = $searchConditions; - $this->paginate['conditions']['AND'][] = $aclConditions; - $this->paginate['contain'] = array_merge($this->paginate['contain'], array('Org', 'Orgc', 'SharingGroup', 'GalaxyClusterRelation', 'TargetingClusterRelation')); - $clusters = $this->paginate(); - - $tagIds = array(); - foreach ($clusters as $k => $cluster) { - $clusters[$k] = $this->GalaxyCluster->attachExtendByInfo($this->Auth->user(), $clusters[$k]); - $clusters[$k] = $this->GalaxyCluster->attachExtendFromInfo($this->Auth->user(), $clusters[$k]); - $clusters[$k]['GalaxyCluster']['relation_counts'] = array( - 'out' => count($clusters[$k]['GalaxyClusterRelation']), - 'in' => count($clusters[$k]['TargetingClusterRelation']), - ); - - if (isset($cluster['Tag']['id'])) { - $tagIds[] = $cluster['Tag']['id']; - $clusters[$k]['GalaxyCluster']['tag_id'] = $cluster['Tag']['id']; - } - $clusters[$k]['GalaxyCluster']['synonyms'] = array(); - foreach ($cluster['GalaxyElement'] as $element) { - $clusters[$k]['GalaxyCluster']['synonyms'][] = $element['value']; - } - $clusters[$k]['GalaxyCluster']['event_count'] = 0; // real number is assigned later - } - - $eventCountsForTags = $this->GalaxyCluster->Tag->EventTag->countForTags($tagIds, $this->Auth->user()); - - $this->loadModel('Sighting'); - $csvForTags = $this->Sighting->tagsSparkline($tagIds, $this->Auth->user(), '0'); - foreach ($clusters as $k => $cluster) { - if (isset($cluster['GalaxyCluster']['tag_id'])) { - if (isset($csvForTags[$cluster['GalaxyCluster']['tag_id']])) { - $clusters[$k]['csv'] = $csvForTags[$cluster['GalaxyCluster']['tag_id']]; - } - if (isset($eventCountsForTags[$cluster['GalaxyCluster']['tag_id']])) { - $clusters[$k]['GalaxyCluster']['event_count'] = $eventCountsForTags[$cluster['GalaxyCluster']['tag_id']]; - } - } - } - $customClusterCount = $this->GalaxyCluster->fetchGalaxyClusters($this->Auth->user(), [ - 'count' => true, - 'conditions' => [ - 'AND' => [$searchConditions, $aclConditions], - 'GalaxyCluster.default' => 0, - ] - ]); - $this->loadModel('Attribute'); - $distributionLevels = $this->Attribute->distributionLevels; - unset($distributionLevels[5]); - $this->set('distributionLevels', $distributionLevels); - $this->set('list', $clusters); - $this->set('galaxy_id', $galaxyId); - $this->set('custom_cluster_count', $customClusterCount); } + + $this->paginate['conditions']['AND'][] = $contextConditions; + $this->paginate['conditions']['AND'][] = $searchConditions; + $this->paginate['conditions']['AND'][] = $aclConditions; + $this->paginate['contain'] = array_merge($this->paginate['contain'], array('Org', 'Orgc', 'SharingGroup', 'GalaxyClusterRelation', 'TargetingClusterRelation')); + $clusters = $this->paginate(); + + $this->GalaxyCluster->attachExtendByInfo($this->Auth->user(), $clusters); + + $tagIds = array(); + foreach ($clusters as $k => $cluster) { + $clusters[$k] = $this->GalaxyCluster->attachExtendFromInfo($this->Auth->user(), $clusters[$k]); + $clusters[$k]['GalaxyCluster']['relation_counts'] = array( + 'out' => count($clusters[$k]['GalaxyClusterRelation']), + 'in' => count($clusters[$k]['TargetingClusterRelation']), + ); + + if (isset($cluster['Tag']['id'])) { + $tagIds[] = $cluster['Tag']['id']; + $clusters[$k]['GalaxyCluster']['tag_id'] = $cluster['Tag']['id']; + } + $clusters[$k]['GalaxyCluster']['synonyms'] = array(); + foreach ($cluster['GalaxyElement'] as $element) { + $clusters[$k]['GalaxyCluster']['synonyms'][] = $element['value']; + } + $clusters[$k]['GalaxyCluster']['event_count'] = 0; // real number is assigned later + } + + $eventCountsForTags = $this->GalaxyCluster->Tag->EventTag->countForTags($tagIds, $this->Auth->user()); + + $this->loadModel('Sighting'); + $csvForTags = $this->Sighting->tagsSparkline($tagIds, $this->Auth->user(), '0'); + foreach ($clusters as $k => $cluster) { + if (isset($cluster['GalaxyCluster']['tag_id'])) { + if (isset($csvForTags[$cluster['GalaxyCluster']['tag_id']])) { + $clusters[$k]['csv'] = $csvForTags[$cluster['GalaxyCluster']['tag_id']]; + } + if (isset($eventCountsForTags[$cluster['GalaxyCluster']['tag_id']])) { + $clusters[$k]['GalaxyCluster']['event_count'] = $eventCountsForTags[$cluster['GalaxyCluster']['tag_id']]; + } + } + } + $customClusterCount = $this->GalaxyCluster->fetchGalaxyClusters($this->Auth->user(), [ + 'count' => true, + 'conditions' => [ + 'AND' => [$searchConditions, $aclConditions], + 'GalaxyCluster.default' => 0, + ] + ]); + $this->loadModel('Attribute'); + $distributionLevels = $this->Attribute->distributionLevels; + unset($distributionLevels[5]); + $this->set('distributionLevels', $distributionLevels); + $this->set('list', $clusters); + $this->set('galaxy_id', $galaxyId); + $this->set('custom_cluster_count', $customClusterCount); + if ($this->request->is('ajax')) { $this->layout = 'ajax'; $this->render('ajax/index'); @@ -179,7 +181,9 @@ class GalaxyClustersController extends AppController if ($this->_isRest()) { return $this->RestResponse->viewData($cluster, $this->response->type()); } else { - $cluster = $this->GalaxyCluster->attachExtendByInfo($this->Auth->user(), $cluster); + $clusters = [$cluster]; + $this->GalaxyCluster->attachExtendByInfo($this->Auth->user(), $clusters); + $cluster = $clusters[0]; $cluster = $this->GalaxyCluster->attachExtendFromInfo($this->Auth->user(), $cluster); $this->set('id', $id); $this->set('galaxy', ['Galaxy' => $cluster['GalaxyCluster']['Galaxy']]); diff --git a/app/Model/GalaxyCluster.php b/app/Model/GalaxyCluster.php index 5ea05df2d..482c2d737 100644 --- a/app/Model/GalaxyCluster.php +++ b/app/Model/GalaxyCluster.php @@ -822,11 +822,30 @@ class GalaxyCluster extends AppModel return $element; } - public function attachExtendByInfo($user, $cluster) + /** + * @param array $user + * @param array $clusters + * @return void + */ + public function attachExtendByInfo(array $user, array &$clusters) { - $extensions = $this->fetchGalaxyClusters($user, array('conditions' => array('extends_uuid' => $cluster['GalaxyCluster']['uuid']))); - $cluster['GalaxyCluster']['extended_by'] = $extensions; - return $cluster; + if (empty($clusters)) { + return; + } + + $clusterUuids = array_column(array_column($clusters, 'GalaxyCluster'), 'uuid'); + $extensions = $this->fetchGalaxyClusters($user, [ + 'conditions' => ['extends_uuid' => $clusterUuids], + ]); + foreach ($clusters as &$cluster) { + $extendedBy = []; + foreach ($extensions as $extension) { + if ($cluster['GalaxyCluster']['uuid'] === $extension['GalaxyCluster']['extends_uuid']) { + $extendedBy[] = $extension; + } + } + $cluster['GalaxyCluster']['extended_by'] = $extendedBy; + } } public function attachExtendFromInfo($user, $cluster) From 55d695bd10dfb62bb47a9dba1721ac5cb12705a8 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Tue, 2 Mar 2021 10:04:32 +0100 Subject: [PATCH 06/17] chg: [UI] Use Page Visibility API --- app/View/Jobs/index.ctp | 2 +- app/View/Layouts/dashboard.ctp | 9 --------- app/View/Layouts/default.ctp | 8 -------- app/webroot/js/misp.js | 4 ++-- 4 files changed, 3 insertions(+), 20 deletions(-) diff --git a/app/View/Jobs/index.ctp b/app/View/Jobs/index.ctp index fd1fd4b30..162e9a22d 100644 --- a/app/View/Jobs/index.ctp +++ b/app/View/Jobs/index.ctp @@ -32,7 +32,7 @@ function queueInterval(k, id) { intervalArray[k] = setInterval(function() { - if (tabIsActive) { + if (!document.hidden) { $.getJSON('/jobs/getGenerateCorrelationProgress/' + id, function(data) { var x = document.getElementById("bar" + id); x.style.width = data+"%"; diff --git a/app/View/Layouts/dashboard.ctp b/app/View/Layouts/dashboard.ctp index 839d0dad8..479080ccb 100644 --- a/app/View/Layouts/dashboard.ctp +++ b/app/View/Layouts/dashboard.ctp @@ -102,7 +102,6 @@ - var tabIsActive = true; var baseurl = ''; var here = 'params['action'], 0, 6) === 'admin_') { @@ -111,13 +110,6 @@ echo $baseurl . '/' . h($this->params['controller']) . '/' . h($this->params['action']); } ?>'; - $(document).ready(function(){ - $(window).blur(function() { - tabIsActive = false; - }); - $(window).focus(function() { - tabIsActive = true; - }); @@ -125,7 +117,6 @@ - }); diff --git a/app/View/Layouts/default.ctp b/app/View/Layouts/default.ctp index 3b79002e4..cea81bbc7 100644 --- a/app/View/Layouts/default.ctp +++ b/app/View/Layouts/default.ctp @@ -98,7 +98,6 @@ - var tabIsActive = true; var baseurl = ''; var here = 'params['action'], 0, 6) === 'admin_') { @@ -107,12 +106,6 @@ echo $baseurl . '/' . h($this->params['controller']) . '/' . h($this->params['action']); } ?>'; - $(function(){ - $(window).blur(function() { - tabIsActive = false; - }).focus(function() { - tabIsActive = true; - }); @@ -120,7 +113,6 @@ - }); diff --git a/app/webroot/js/misp.js b/app/webroot/js/misp.js index 2664d58ad..24e4f0880 100644 --- a/app/webroot/js/misp.js +++ b/app/webroot/js/misp.js @@ -4808,7 +4808,7 @@ $(document.body).on('click', 'a[data-paginator]', function (e) { }); function queryEventLock(event_id) { - if (tabIsActive) { + if (!document.hidden) { $.ajax({ url: baseurl + "/events/checkLocks/" + event_id, type: "get", @@ -4826,7 +4826,7 @@ function queryEventLock(event_id) { } function checkIfLoggedIn() { - if (tabIsActive) { + if (!document.hidden) { $.get(baseurl + "/users/checkIfLoggedIn.json") .fail(function (xhr) { if (xhr.status === 403) { From 4bcf270233fb7eec07a70ea7dfbe120d61115c00 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Tue, 2 Mar 2021 10:54:39 +0100 Subject: [PATCH 07/17] chg: [UI] Simplify keyboard-shortcuts.js --- app/View/Elements/global_menu.ctp | 1 - app/View/Events/index.ctp | 1 - app/View/Events/view.ctp | 1 - app/View/Layouts/default.ctp | 5 +- .../js/keyboard-shortcuts-definition.js | 61 +++++++++++++++++++ app/webroot/js/keyboard-shortcuts.js | 39 ++---------- app/webroot/shortcuts/event_index.json | 9 --- app/webroot/shortcuts/event_view.json | 24 -------- app/webroot/shortcuts/global_menu.json | 15 ----- 9 files changed, 68 insertions(+), 88 deletions(-) create mode 100644 app/webroot/js/keyboard-shortcuts-definition.js delete mode 100644 app/webroot/shortcuts/event_index.json delete mode 100644 app/webroot/shortcuts/event_view.json delete mode 100644 app/webroot/shortcuts/global_menu.json diff --git a/app/View/Elements/global_menu.ctp b/app/View/Elements/global_menu.ctp index 60feef643..1935e0c87 100755 --- a/app/View/Elements/global_menu.ctp +++ b/app/View/Elements/global_menu.ctp @@ -526,4 +526,3 @@ - diff --git a/app/View/Events/index.ctp b/app/View/Events/index.ctp index cacdfb804..f54efefbe 100644 --- a/app/View/Events/index.ctp +++ b/app/View/Events/index.ctp @@ -127,6 +127,5 @@ echo $this->Html->css('distribution-graph'); echo $this->Html->script('network-distribution-graph'); ?> - element('/genericElements/SideMenu/side_menu', array('menuList' => 'event-collection', 'menuItem' => 'index')); diff --git a/app/View/Events/view.ctp b/app/View/Events/view.ctp index f604f344d..06be58a8c 100644 --- a/app/View/Events/view.ctp +++ b/app/View/Events/view.ctp @@ -561,4 +561,3 @@ $(function () { }); }); - diff --git a/app/View/Layouts/default.ctp b/app/View/Layouts/default.ctp index 3b79002e4..572ff527f 100644 --- a/app/View/Layouts/default.ctp +++ b/app/View/Layouts/default.ctp @@ -33,7 +33,7 @@ )); ?> - +
    @@ -71,7 +71,8 @@ 'bootstrap-datepicker', 'bootstrap-colorpicker', 'misp', - 'keyboard-shortcuts' + 'keyboard-shortcuts-definition', + 'keyboard-shortcuts', ) )); echo $this->element('footer'); diff --git a/app/webroot/js/keyboard-shortcuts-definition.js b/app/webroot/js/keyboard-shortcuts-definition.js new file mode 100644 index 000000000..bd6acbe3f --- /dev/null +++ b/app/webroot/js/keyboard-shortcuts-definition.js @@ -0,0 +1,61 @@ +function getShortcutsDefinition() { + var shortcuts = [ + { + "key": "l", + "description": "Go to event list", + "action": function () { + document.location.href = baseurl + '/events/index'; + } + }, + { + "key": "e", + "description": "Go to add event page", + "action": function () { + document.location.href = baseurl + '/events/add'; + } + } + ]; + + var $body = $(document.body); + if ($body.data('controller') === 'events' && $body.data('action') === 'view') { + shortcuts.push({ + "key": "t", + "description": "Open the tag selection modal", + "action": function () { + $('.addTagButton').first().click(); + } + }); + shortcuts.push({ + "key": "f", + "description": "Open the freetext import modal", + "action": function () { + $('#freetext-button').click(); + } + }); + shortcuts.push({ + "key": "a", + "description": "Add an attribute", + "action": function () { + $('#create-button').click(); + } + }); + shortcuts.push({ + "key": "s", + "description": "Focus the filter attribute bar", + "action": function () { + $('#quickFilterField').focus(); + } + }); + } + + if ($body.data('controller') === 'events' && $body.data('action') === 'index') { + shortcuts.push({ + "key": "s", + "description": "Focus the filter events bar", + "action": function () { + $('#quickFilterField').focus(); + } + }); + } + return shortcuts; +} diff --git a/app/webroot/js/keyboard-shortcuts.js b/app/webroot/js/keyboard-shortcuts.js index 1895bf0b0..0375a15ec 100644 --- a/app/webroot/js/keyboard-shortcuts.js +++ b/app/webroot/js/keyboard-shortcuts.js @@ -21,13 +21,7 @@ let keyboardShortcutsManager = { init() { /* Codacy comment to notify that baseurl is a read-only global variable. */ /* global baseurl */ - let shortcutURIs = []; - for(let keyboardShortcutElement of $('.keyboardShortcutsConfig')) { - shortcutURIs.push(keyboardShortcutElement.value); - this.ajaxGet(baseurl + keyboardShortcutElement.value).then((response) => { - this.mapKeyboardShortcuts(JSON.parse(response)); - }); - } + this.mapKeyboardShortcuts(getShortcutsDefinition()); this.setKeyboardListener(); }, @@ -61,7 +55,7 @@ let keyboardShortcutsManager = { * @param {} config The shortcut JSON list: [{key: string, description: string, action: string(eval-able JS code)}] */ mapKeyboardShortcuts(config) { - for(let shortcut of config.shortcuts) { + for(let shortcut of config) { this.shortcutKeys.set(shortcut.key, shortcut); } this.addShortcutListToHTML(); @@ -76,39 +70,14 @@ let keyboardShortcutsManager = { window.onkeyup = (keyboardEvent) => { if(this.shortcutKeys.has(keyboardEvent.key)) { let activeElement = document.activeElement.tagName; - if( !this.ESCAPED_TAG_NAMES.includes(activeElement)) { - eval(this.shortcutKeys.get(keyboardEvent.key).action); + if(!this.ESCAPED_TAG_NAMES.includes(activeElement)) { + this.shortcutKeys.get(keyboardEvent.key).action(); } } else if(this.NAVIGATION_KEYS.includes(keyboardEvent.key)) { window.dispatchEvent(new CustomEvent(this.EVENTS[keyboardEvent.key], {detail: keyboardEvent})); } } }, - - /** - * Queries the given URL with a GET request and returns a Promise - * that resolves when the response arrives. - * @param string url The URL to fetch. - */ - ajaxGet(url) { - return new Promise(function(resolve, reject) { - let req = new XMLHttpRequest(); - req.open("GET", url); - req.onload = function() { - if (req.status === 200) { - resolve(req.response); - } else { - reject(new Error(req.statusText)); - } - }; - - req.onerror = function() { - reject(new Error("Network error")); - }; - - req.send(); - }); - } } // Inits the keyboard shortcut manager's main routine. diff --git a/app/webroot/shortcuts/event_index.json b/app/webroot/shortcuts/event_index.json deleted file mode 100644 index 15e6fa1cb..000000000 --- a/app/webroot/shortcuts/event_index.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "shortcuts": [ - { - "key": "s", - "description": "Focus the filter events bar", - "action": "$('#quickFilterField').focus()" - } - ] -} \ No newline at end of file diff --git a/app/webroot/shortcuts/event_view.json b/app/webroot/shortcuts/event_view.json deleted file mode 100644 index c411c8802..000000000 --- a/app/webroot/shortcuts/event_view.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "shortcuts": [ - { - "key": "t", - "description": "Open the tag selection modal", - "action": "$('.addTagButton').first().click()" - }, - { - "key": "f", - "description": "Open the freetext import modal", - "action": "$('#freetext-button').click()" - }, - { - "key": "a", - "description": "Add an attribute", - "action": "$('#create-button').click()" - }, - { - "key": "s", - "description": "Focus the filter attribute bar", - "action": "$('#quickFilterField').focus()" - } - ] -} diff --git a/app/webroot/shortcuts/global_menu.json b/app/webroot/shortcuts/global_menu.json deleted file mode 100644 index 6ee3ca893..000000000 --- a/app/webroot/shortcuts/global_menu.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "shortcuts": [ - { - "key": "l", - "description": "Go to event list", - "action": "document.location.href = baseurl + '/events/index'" - }, - { - "key": "e", - "description": "Go to add event page", - "action": "document.location.href = baseurl + '/events/add'" - } - ] -} - From 62537961f0f1caafe171d977b5af2643137726e4 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Tue, 2 Mar 2021 14:44:41 +0100 Subject: [PATCH 08/17] fix: [internal] Undefined index when importing from module --- app/Controller/AppController.php | 4 ++-- app/Controller/EventsController.php | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index c2fa205d3..63c388cb1 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -1453,10 +1453,10 @@ class AppController extends Controller if ($this->userRole['perm_site_admin']) { return true; } - if ($this->userRole['perm_modify_org'] && $event['Event']['orgc_id'] == $this->Auth->user('org_id')) { + if ($this->userRole['perm_modify_org'] && $event['Event']['orgc_id'] == $this->Auth->user()['org_id']) { return true; } - if ($this->userRole['perm_modify'] && $event['Event']['user_id'] == $this->Auth->user('id')) { + if ($this->userRole['perm_modify'] && $event['Event']['user_id'] == $this->Auth->user()['id']) { return true; } return false; diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 3a64d2f49..829853d94 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -4938,6 +4938,7 @@ class EventsController extends AppController if (!$event) { throw new NotFoundException(__('Invalid event.')); } + $mayModify = $this->__canModifyEvent($event); $eventId = $event['Event']['id']; $this->loadModel('Module'); @@ -5108,7 +5109,7 @@ class EventsController extends AppController $this->set('module', $module); $this->set('eventId', $eventId); $this->set('event', $event); - $this->set('mayModify', $this->__canModifyEvent($event)); + $this->set('mayModify', $mayModify); } public function exportModule($module, $id, $standard = false) From c8533ead788ad08aa5995c1bb84eaeb31e3d5198 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Tue, 2 Mar 2021 18:04:49 +0100 Subject: [PATCH 09/17] chg: [internal] Cleanup code that is resposible for fetching server setting --- app/Model/Server.php | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/app/Model/Server.php b/app/Model/Server.php index f9d07cc92..554addd56 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -4,6 +4,7 @@ App::uses('GpgTool', 'Tools'); /** * @property-read array $serverSettings + * @property Organisation $Organisation */ class Server extends AppModel { @@ -1235,16 +1236,23 @@ class Server extends AppModel return true; } + /** + * @return array + */ public function getCurrentServerSettings() { - $this->Module = ClassRegistry::init('Module'); $serverSettings = $this->serverSettings; $moduleTypes = array('Enrichment', 'Import', 'Export', 'Cortex'); $serverSettings = $this->readModuleSettings($serverSettings, $moduleTypes); return $serverSettings; } - private function readModuleSettings($serverSettings, $moduleTypes) + /** + * @param array $serverSettings + * @param array $moduleTypes + * @return array + */ + private function readModuleSettings(array $serverSettings, array $moduleTypes) { $this->Module = ClassRegistry::init('Module'); foreach ($moduleTypes as $moduleType) { @@ -1253,12 +1261,12 @@ class Server extends AppModel foreach ($results as $module => $data) { foreach ($data as $result) { $setting = array('level' => 1, 'errorMessage' => ''); - if ($result['type'] == 'boolean') { + if ($result['type'] === 'boolean') { $setting['test'] = 'testBool'; $setting['type'] = 'boolean'; $setting['description'] = __('Enable or disable the %s module.', $module); $setting['value'] = false; - } elseif ($result['type'] == 'orgs') { + } elseif ($result['type'] === 'orgs') { $setting['description'] = __('Restrict the %s module to the given organisation.', $module); $setting['value'] = 0; $setting['test'] = 'testLocalOrg'; @@ -1335,15 +1343,11 @@ class Server extends AppModel public function serverSettingsRead($unsorted = false) { - $this->Module = ClassRegistry::init('Module'); $serverSettings = $this->getCurrentServerSettings(); $currentSettings = Configure::read(); - if (Configure::read('Plugin.Enrichment_services_enable')) { - $this->readModuleSettings($serverSettings, array('Enrichment')); - } $finalSettingsUnsorted = $this->__serverSettingsRead($serverSettings, $currentSettings); foreach ($finalSettingsUnsorted as $key => $temp) { - if (in_array($temp['tab'], array_keys($this->__settingTabMergeRules))) { + if (isset($this->__settingTabMergeRules[$temp['tab']])) { $finalSettingsUnsorted[$key]['tab'] = $this->__settingTabMergeRules[$temp['tab']]; } } From 874ec66c9bcaa702013f390eb638b4fd679212c2 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 22 Feb 2021 10:33:16 +0100 Subject: [PATCH 10/17] chg: [schema] Convert GalaxyCluster tag name to case insensitive --- INSTALL/MYSQL.sql | 2 +- app/Model/AppModel.php | 3 +++ app/Model/GalaxyCluster.php | 8 ++++---- db_schema.json | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/INSTALL/MYSQL.sql b/INSTALL/MYSQL.sql index 4c0dee91e..8ffd9e0dc 100644 --- a/INSTALL/MYSQL.sql +++ b/INSTALL/MYSQL.sql @@ -455,7 +455,7 @@ CREATE TABLE IF NOT EXISTS `galaxy_clusters` ( `collection_uuid` varchar(255) COLLATE utf8_bin NOT NULL, `type` varchar(255) COLLATE utf8_bin NOT NULL, `value` text COLLATE utf8_bin NOT NULL, - `tag_name` varchar(255) COLLATE utf8_bin NOT NULL DEFAULT '', + `tag_name` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '', `description` text COLLATE utf8_bin NOT NULL, `galaxy_id` int(11) NOT NULL, `source` varchar(255) COLLATE utf8_bin NOT NULL DEFAULT '', diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 3348e9ecc..6c7f225b1 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -1568,6 +1568,9 @@ class AppModel extends Model INDEX `value` (`value`(255)) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"; break; + case 66: + $sqlArray[] = "ALTER TABLE `galaxy_clusters` MODIFY COLUMN `tag_name` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '';"; + break; case 'fixNonEmptySharingGroupID': $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; diff --git a/app/Model/GalaxyCluster.php b/app/Model/GalaxyCluster.php index 18960ca15..0fb8d4d30 100644 --- a/app/Model/GalaxyCluster.php +++ b/app/Model/GalaxyCluster.php @@ -915,7 +915,7 @@ class GalaxyCluster extends AppModel if (!$isGalaxyTag) { return null; } - $conditions = array('LOWER(GalaxyCluster.tag_name)' => strtolower($name)); + $conditions = array('GalaxyCluster.tag_name' => $name); } $cluster = $this->fetchGalaxyClusters($user, array( 'conditions' => $conditions, @@ -943,7 +943,7 @@ class GalaxyCluster extends AppModel if (count(array_filter($namesOrIds, 'is_numeric')) === count($namesOrIds)) { // all elements are numeric $conditions = array('GalaxyCluster.id' => $namesOrIds); } else { - $conditions = array('LOWER(GalaxyCluster.tag_name)' => array_map('strtolower', $namesOrIds)); + $conditions = array('GalaxyCluster.tag_name' => $namesOrIds); } $options = ['conditions' => $conditions]; @@ -1470,7 +1470,7 @@ class GalaxyCluster extends AppModel foreach ($events as $event) { foreach ($event['EventTag'] as $eventTag) { if ($eventTag['Tag']['is_galaxy']) { - $clusterTagNames[$eventTag['Tag']['id']] = strtolower($eventTag['Tag']['name']); + $clusterTagNames[$eventTag['Tag']['id']] = $eventTag['Tag']['name']; } } } @@ -1480,7 +1480,7 @@ class GalaxyCluster extends AppModel } $options = [ - 'conditions' => ['LOWER(GalaxyCluster.tag_name)' => $clusterTagNames], + 'conditions' => ['GalaxyCluster.tag_name' => $clusterTagNames], 'contain' => ['Galaxy', 'GalaxyElement'], ]; $clusters = $this->fetchGalaxyClusters($user, $options); diff --git a/db_schema.json b/db_schema.json index f2427a0c0..86641ec5f 100644 --- a/db_schema.json +++ b/db_schema.json @@ -2356,7 +2356,7 @@ "data_type": "varchar", "character_maximum_length": "255", "numeric_precision": null, - "collation_name": "utf8_bin", + "collation_name": "utf8_unicode_ci", "column_type": "varchar(255)", "column_default": "", "extra": "" From e3b2a0a40cf62485d80015e567764c4ebab331c4 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 22 Feb 2021 13:56:30 +0100 Subject: [PATCH 11/17] chg: [schema] Add index for EventReport.event_id --- INSTALL/MYSQL.sql | 3 ++- app/Model/AppModel.php | 12 +++++++----- db_schema.json | 3 ++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/INSTALL/MYSQL.sql b/INSTALL/MYSQL.sql index 8ffd9e0dc..284128120 100644 --- a/INSTALL/MYSQL.sql +++ b/INSTALL/MYSQL.sql @@ -243,7 +243,8 @@ CREATE TABLE IF NOT EXISTS event_reports ( `deleted` tinyint(1) NOT NULL DEFAULT 0, PRIMARY KEY (id), CONSTRAINT u_uuid UNIQUE (uuid), - INDEX `name` (`name`) + INDEX `name` (`name`), + INDEX `event_id` (`event_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -- -------------------------------------------------------- diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 6c7f225b1..60c220c87 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -1570,6 +1570,7 @@ class AppModel extends Model break; case 66: $sqlArray[] = "ALTER TABLE `galaxy_clusters` MODIFY COLUMN `tag_name` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '';"; + $indexArray[] = ['event_reports', 'event_id']; break; case 'fixNonEmptySharingGroupID': $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; @@ -1824,18 +1825,19 @@ class AppModel extends Model } } - private function __addIndex($table, $field, $length = false) + private function __addIndex($table, $field, $length = null, $unique = false) { $dataSourceConfig = ConnectionManager::getDataSource('default')->config; $dataSource = $dataSourceConfig['datasource']; $this->Log = ClassRegistry::init('Log'); + $index = $unique ? 'UNIQUE INDEX' : 'INDEX'; if ($dataSource == 'Database/Postgres') { - $addIndex = "CREATE INDEX idx_" . $table . "_" . $field . " ON " . $table . " (" . $field . ");"; + $addIndex = "CREATE $index idx_" . $table . "_" . $field . " ON " . $table . " (" . $field . ");"; } else { if (!$length) { - $addIndex = "ALTER TABLE `" . $table . "` ADD INDEX `" . $field . "` (`" . $field . "`);"; + $addIndex = "ALTER TABLE `" . $table . "` ADD $index `" . $field . "` (`" . $field . "`);"; } else { - $addIndex = "ALTER TABLE `" . $table . "` ADD INDEX `" . $field . "` (`" . $field . "`(" . $length . "));"; + $addIndex = "ALTER TABLE `" . $table . "` ADD $index `" . $field . "` (`" . $field . "`(" . $length . "));"; } } $result = true; @@ -1844,7 +1846,7 @@ class AppModel extends Model try { $this->query($addIndex); } catch (Exception $e) { - $duplicate = (strpos($e->getMessage(), '1061') !== false); + $duplicate = strpos($e->getMessage(), '1061') !== false; $errorMessage = $e->getMessage(); $result = false; } diff --git a/db_schema.json b/db_schema.json index 86641ec5f..80cdb3c39 100644 --- a/db_schema.json +++ b/db_schema.json @@ -7654,7 +7654,8 @@ "event_reports": { "id": true, "uuid": true, - "name": false + "name": false, + "event_id": false }, "event_tags": { "id": true, From 599819f7f9cdae2e5d9d3a32caa9f25e1a5a2d7c Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 27 Feb 2021 11:13:47 +0100 Subject: [PATCH 12/17] new: [authkeys] Allowed IPs --- app/Controller/AppController.php | 95 +++++++++---- app/Controller/AuthKeysController.php | 30 +++- app/Controller/Component/CRUDComponent.php | 31 +++-- app/Controller/ServersController.php | 6 +- app/Lib/Tools/CidrTool.php | 20 +++ app/Model/AppModel.php | 5 +- app/Model/AuthKey.php | 101 +++++++++++--- app/Model/Log.php | 6 +- app/View/AuthKeys/add.ctp | 11 +- app/View/AuthKeys/index.ctp | 16 ++- app/View/AuthKeys/view.ctp | 129 +++++++++--------- .../IndexTable/Fields/datetime.ctp | 41 +++--- .../SingleViews/Fields/customField.ctp | 1 + app/webroot/css/main.css | 6 +- app/webroot/js/misp.js | 3 +- db_schema.json | 13 +- tests/testlive_security.py | 23 ++++ 17 files changed, 380 insertions(+), 157 deletions(-) create mode 100644 app/View/Elements/genericElements/SingleViews/Fields/customField.ctp diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index 63c388cb1..7601603a4 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -125,8 +125,8 @@ class AppController extends Controller } if (!$this->_isRest()) { $this->__contentSecurityPolicy(); + $this->response->header('X-XSS-Protection', '1; mode=block'); } - $this->response->header('X-XSS-Protection', '1; mode=block'); if (!empty($this->params['named']['sql'])) { $this->sql_dump = intval($this->params['named']['sql']); @@ -446,22 +446,9 @@ class AppController extends Controller } else { // User not authenticated correctly // reset the session information - $redis = $this->User->setupRedis(); - // Do not log every fail, but just once per hour - if ($redis && !$redis->exists('misp:auth_fail_throttling:' . $authKeyToStore)) { - $redis->setex('misp:auth_fail_throttling:' . $authKeyToStore, 3600, 1); + if ($this->_shouldLog($authKeyToStore)) { $this->loadModel('Log'); - $this->Log->create(); - $log = array( - 'org' => 'SYSTEM', - 'model' => 'User', - 'model_id' => 0, - 'email' => 'SYSTEM', - 'action' => 'auth_fail', - 'title' => "Failed authentication using API key ($authKeyToStore)", - 'change' => null, - ); - $this->Log->save($log); + $this->Log->createLogEntry('SYSTEM', 'auth_fail', 'User', 0, "Failed authentication using API key ($authKeyToStore)"); } $this->Session->destroy(); } @@ -548,8 +535,10 @@ class AppController extends Controller } if ($user['disabled']) { - $this->Log = ClassRegistry::init('Log'); - $this->Log->createLogEntry($user, 'auth_fail', 'User', $user['id'], 'Login attempt by disabled user.'); + if ($this->_shouldLog('disabled:' . $user['id'])) { + $this->Log = ClassRegistry::init('Log'); + $this->Log->createLogEntry($user, 'auth_fail', 'User', $user['id'], 'Login attempt by disabled user.'); + } $this->Auth->logout(); if ($this->_isRest()) { @@ -565,11 +554,33 @@ class AppController extends Controller if (isset($user['authkey_expiration']) && $user['authkey_expiration']) { $time = isset($_SERVER['REQUEST_TIME']) ? $_SERVER['REQUEST_TIME'] : time(); if ($user['authkey_expiration'] < $time) { + if ($this->_shouldLog('expired:' . $user['authkey_id'])) { + $this->Log = ClassRegistry::init('Log'); + $this->Log->createLogEntry($user, 'auth_fail', 'User', $user['id'], "Login attempt by expired auth key {$user['authkey_id']}."); + } $this->Auth->logout(); throw new ForbiddenException('Auth key is expired'); } } + if (!empty($user['allowed_ips'])) { + App::uses('CidrTool', 'Tools'); + $cidrTool = new CidrTool($user['allowed_ips']); + $remoteIp = $this->_remoteIp(); + if ($remoteIp === null) { + $this->Auth->logout(); + throw new ForbiddenException('Auth key is limited to IP address, but IP address not found'); + } + if (!$cidrTool->contains($remoteIp)) { + if ($this->_shouldLog('not_allowed_ip:' . $user['authkey_id'] . ':' . $remoteIp)) { + $this->Log = ClassRegistry::init('Log'); + $this->Log->createLogEntry($user, 'auth_fail', 'User', $user['id'], "Login attempt from not allowed IP address for auth key {$user['authkey_id']}."); + } + $this->Auth->logout(); + throw new ForbiddenException('It is not possible to use this Auth key from your IP address'); + } + } + $isUserRequest = !$this->_isRest() && !$this->request->is('ajax') && !$this->_isAutomation(); // Next checks makes sense just for user direct HTTP request, so skip REST and AJAX calls if (!$isUserRequest) { @@ -632,7 +643,7 @@ class AppController extends Controller return; } - $remoteAddress = trim($_SERVER['REMOTE_ADDR']); + $remoteAddress = $this->_remoteIp(); $pipe = $redis->multi(Redis::PIPELINE); // keep for 30 days @@ -680,11 +691,7 @@ class AppController extends Controller $change .= PHP_EOL . 'Request body: ' . $payload; } $this->Log = ClassRegistry::init('Log'); - try { - $this->Log->createLogEntry($user, 'request', 'User', $user['id'], 'Paranoid log entry', $change); - } catch (Exception $e) { - // When `MISP.log_skip_db_logs_completely` is enabled, Log::createLogEntry method throws exception - } + $this->Log->createLogEntry($user, 'request', 'User', $user['id'], 'Paranoid log entry', $change); } } @@ -1500,17 +1507,47 @@ class AppController extends Controller throw new RuntimeException("User with ID {$sessionUser['id']} not exists."); } if (isset($sessionUser['authkey_id'])) { + // Reload authkey $this->loadModel('AuthKey'); - if (!$this->AuthKey->exists($sessionUser['authkey_id'])) { + $authKey = $this->AuthKey->find('first', [ + 'conditions' => ['id' => $sessionUser['authkey_id'], 'user_id' => $user['id']], + 'fields' => ['id', 'expiration', 'allowed_ips'], + 'recursive' => -1, + ]); + if (empty($authKey)) { throw new RuntimeException("Auth key with ID {$sessionUser['authkey_id']} not exists."); } + $user['authkey_id'] = $authKey['AuthKey']['id']; + $user['authkey_expiration'] = $authKey['AuthKey']['expiration']; + $user['allowed_ips'] = $authKey['AuthKey']['allowed_ips']; } - foreach (['authkey_id', 'authkey_expiration', 'logged_by_authkey'] as $copy) { - if (isset($sessionUser[$copy])) { - $user[$copy] = $sessionUser[$copy]; - } + if (isset($sessionUser['logged_by_authkey'])) { + $user['logged_by_authkey'] = $sessionUser['logged_by_authkey']; } $this->Auth->login($user); return $user; } + + /** + * @return string|null + */ + protected function _remoteIp() + { + $ipHeader = Configure::read('MISP.log_client_ip_header') ?: 'REMOTE_ADDR'; + return isset($_SERVER[$ipHeader]) ? trim($_SERVER[$ipHeader]) : null; + } + + /** + * @param string $key + * @return bool Returns true if the same log defined by $key was not stored in last hour + */ + protected function _shouldLog($key) + { + $redis = $this->User->setupRedis(); + if ($redis && !$redis->exists('misp:auth_fail_throttling:' . $key)) { + $redis->setex('misp:auth_fail_throttling:' . $key, 3600, 1); + return true; + } + return false; + } } diff --git a/app/Controller/AuthKeysController.php b/app/Controller/AuthKeysController.php index b90bb2fa4..47e1fef57 100644 --- a/app/Controller/AuthKeysController.php +++ b/app/Controller/AuthKeysController.php @@ -71,8 +71,34 @@ class AuthKeysController extends AppController public function edit($id) { - $this->set('metaGroup', 'admin'); - $this->set('metaAction', 'authkeys_edit'); + $this->CRUD->edit($id, [ + 'conditions' => $this->__prepareConditions(), + 'afterFind' => function (array $authKey) { + unset($authKey['AuthKey']['authkey']); + if (is_array($authKey['AuthKey']['allowed_ips'])) { + $authKey['AuthKey']['allowed_ips'] = implode("\n", $authKey['AuthKey']['allowed_ips']); + } + $authKey['AuthKey']['expiration'] = date('Y-m-d H:i:s', $authKey['AuthKey']['expiration']); + return $authKey; + }, + 'fields' => ['comment', 'allowed_ips', 'expiration'], + ]); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + $this->set('dropdownData', [ + 'user' => $this->User->find('list', [ + 'sort' => ['username' => 'asc'], + 'conditions' => ['id' => $this->request->data['AuthKey']['user_id']], + ]) + ]); + $this->set('menuData', [ + 'menuList' => $this->_isSiteAdmin() ? 'admin' : 'globalActions', + 'menuItem' => 'authKeyAdd', + ]); + $this->set('edit', true); + $this->set('validity', Configure::read('Security.advanced_authkeys_validity')); + $this->render('add'); } public function add($user_id = false) diff --git a/app/Controller/Component/CRUDComponent.php b/app/Controller/Component/CRUDComponent.php index 1e682c197..5a63087c4 100644 --- a/app/Controller/Component/CRUDComponent.php +++ b/app/Controller/Component/CRUDComponent.php @@ -145,13 +145,21 @@ class CRUDComponent extends Component if (empty($id)) { throw new NotFoundException(__('Invalid %s.', $modelName)); } - $data = $this->Controller->{$modelName}->find('first', - isset($params['get']) ? $params['get'] : [ - 'recursive' => -1, - 'conditions' => [ - 'id' => $id - ] - ]); + $query = isset($params['get']) ? $params['get'] : [ + 'recursive' => -1, + 'conditions' => [ + 'id' => $id + ], + ]; + if (!empty($params['conditions'])) { + $query['conditions']['AND'][] = $params['conditions']; + } + /** @var Model $model */ + $model = $this->Controller->{$modelName}; + $data = $model->find('first', $query); + if (isset($params['afterFind'])) { + $data = $params['afterFind']($data); + } if ($this->Controller->request->is('post') || $this->Controller->request->is('put')) { $input = $this->Controller->request->data; if (empty($input[$modelName])) { @@ -171,7 +179,10 @@ class CRUDComponent extends Component $data[$modelName][$field] = $fieldData; } } - if ($this->Controller->{$modelName}->save($data)) { + if (isset($params['beforeSave'])) { + $data = $params['beforeSave']($data); + } + if ($model->save($data)) { $message = __('%s updated.', $modelName); if ($this->Controller->IndexFilter->isRest()) { $this->Controller->restResponsePayload = $this->Controller->RestResponse->viewData($data, 'json'); @@ -182,7 +193,9 @@ class CRUDComponent extends Component } } else { if ($this->Controller->IndexFilter->isRest()) { - + $controllerName = $this->Controller->params['controller']; + $actionName = $this->Controller->params['action']; + $this->Controller->restResponsePayload = $this->Controller->RestResponse->saveFailResponse($controllerName, $actionName, false, $model->validationErrors, 'json'); } } } else { diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index 7e07d5b3d..52a95d388 100644 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -2434,9 +2434,9 @@ misp.direct_call(relative_path, body) } $message = 'CSP reported violation'; - $ipHeader = Configure::read('MISP.log_client_ip_header') ?: 'REMOTE_ADDR'; - if (isset($_SERVER[$ipHeader])) { - $message .= ' from IP ' . $_SERVER[$ipHeader]; + $remoteIp = $this->_remoteIp(); + if ($remoteIp) { + $message .= ' from IP ' . $remoteIp; } $this->log("$message: " . json_encode($report['csp-report'], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)); diff --git a/app/Lib/Tools/CidrTool.php b/app/Lib/Tools/CidrTool.php index faf978f3d..b21351c0c 100644 --- a/app/Lib/Tools/CidrTool.php +++ b/app/Lib/Tools/CidrTool.php @@ -66,6 +66,26 @@ class CidrTool return $match; } + /** + * @param string $cidr + * @return bool + */ + public static function validate($cidr) + { + $parts = explode('/', $cidr, 2); + $ipBytes = inet_pton($parts[0]); + if ($ipBytes === false) { + return false; + } + + $maximumNetmask = strlen($ipBytes) === 4 ? 32 : 128; + if (isset($parts[1]) && ($parts[1] > $maximumNetmask || $parts[1] < 0)) { + return false; // Netmask part of CIDR is invalid + } + + return true; + } + /** * Using solution from https://github.com/symfony/symfony/blob/master/src/Symfony/Component/HttpFoundation/IpUtils.php * diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 60c220c87..3f4e0d216 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -89,7 +89,7 @@ class AppModel extends Model 45 => false, 46 => false, 47 => false, 48 => false, 49 => false, 50 => false, 51 => false, 52 => false, 53 => false, 54 => false, 55 => false, 56 => false, 57 => false, 58 => false, 59 => false, 60 => false, 61 => false, 62 => false, - 63 => true, 64 => false, 65 => false + 63 => true, 64 => false, 65 => false, 66 => false, 67 => false, ); public $advanced_updates_description = array( @@ -1572,6 +1572,9 @@ class AppModel extends Model $sqlArray[] = "ALTER TABLE `galaxy_clusters` MODIFY COLUMN `tag_name` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '';"; $indexArray[] = ['event_reports', 'event_id']; break; + case 67: + $sqlArray[] = "ALTER TABLE `auth_keys` ADD `allowed_ips` text DEFAULT NULL;"; + break; case 'fixNonEmptySharingGroupID': $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; diff --git a/app/Model/AuthKey.php b/app/Model/AuthKey.php index f4711b6b7..50de35909 100644 --- a/app/Model/AuthKey.php +++ b/app/Model/AuthKey.php @@ -1,6 +1,7 @@ data['AuthKey']['id'])) { if (empty($this->data['AuthKey']['uuid'])) { $this->data['AuthKey']['uuid'] = CakeText::uuid(); @@ -42,22 +42,66 @@ class AuthKey extends AppModel $this->data['AuthKey']['authkey_end'] = substr($authkey, -4); $this->data['AuthKey']['authkey_raw'] = $authkey; $this->authkey_raw = $authkey; + } - $validity = Configure::read('Security.advanced_authkeys_validity'); - if (empty($this->data['AuthKey']['expiration'])) { - $this->data['AuthKey']['expiration'] = $validity ? strtotime("+$validity days") : 0; + if (!empty($this->data['AuthKey']['allowed_ips'])) { + if (is_string($this->data['AuthKey']['allowed_ips'])) { + $this->data['AuthKey']['allowed_ips'] = trim($this->data['AuthKey']['allowed_ips']); + if (empty($this->data['AuthKey']['allowed_ips'])) { + $this->data['AuthKey']['allowed_ips'] = []; + } else { + $this->data['AuthKey']['allowed_ips'] = explode("\n", $this->data['AuthKey']['allowed_ips']); + $this->data['AuthKey']['allowed_ips'] = array_map('trim', $this->data['AuthKey']['allowed_ips']); + } + } + if (!is_array($this->data['AuthKey']['allowed_ips'])) { + $this->invalidate('allowed_ips', 'Allowed IPs must be array'); + } + foreach ($this->data['AuthKey']['allowed_ips'] as $cidr) { + if (!CidrTool::validate($cidr)) { + $this->invalidate('allowed_ips', "$cidr is not valid IP range"); + } + } + } + + $creationTime = isset($this->data['AuthKey']['created']) ? $this->data['AuthKey']['created'] : time(); + $validity = Configure::read('Security.advanced_authkeys_validity'); + if (empty($this->data['AuthKey']['expiration'])) { + $this->data['AuthKey']['expiration'] = $validity ? strtotime("+$validity days", $creationTime) : 0; + } else { + $expiration = is_numeric($this->data['AuthKey']['expiration']) ? + (int)$this->data['AuthKey']['expiration'] : + strtotime($this->data['AuthKey']['expiration']); + + if ($expiration === false) { + $this->invalidate('expiration', __('Expiration must be in YYYY-MM-DD format.')); + } + if ($validity && $expiration > strtotime("+$validity days", $creationTime)) { + $this->invalidate('expiration', __('Maximal key validity is %s days.', $validity)); + } + $this->data['AuthKey']['expiration'] = $expiration; + } + + return true; + } + + public function afterFind($results, $primary = false) + { + foreach ($results as $key => $val) { + if (isset($val['AuthKey']['allowed_ips'])) { + $results[$key]['AuthKey']['allowed_ips'] = $this->jsonDecode($val['AuthKey']['allowed_ips']); + } + } + return $results; + } + + public function beforeSave($options = array()) + { + if (isset($this->data['AuthKey']['allowed_ips'])) { + if (empty($this->data['AuthKey']['allowed_ips'])) { + $this->data['AuthKey']['allowed_ips'] = null; } else { - $expiration = is_numeric($this->data['AuthKey']['expiration']) ? - (int)$this->data['AuthKey']['expiration'] : - strtotime($this->data['AuthKey']['expiration']); - - if ($expiration === false) { - $this->invalidate('expiration', __('Expiration must be in YYYY-MM-DD format.')); - } - if ($validity && $expiration > strtotime("+$validity days")) { - $this->invalidate('expiration', __('Maximal key validity is %s days.', $validity)); - } - $this->data['AuthKey']['expiration'] = $expiration; + $this->data['AuthKey']['allowed_ips'] = json_encode($this->data['AuthKey']['allowed_ips']); } } return true; @@ -71,9 +115,9 @@ class AuthKey extends AppModel { $start = substr($authkey, 0, 4); $end = substr($authkey, -4); - $existing_authkeys = $this->find('all', [ + $possibleAuthkeys = $this->find('all', [ 'recursive' => -1, - 'fields' => ['id', 'authkey', 'user_id', 'expiration'], + 'fields' => ['id', 'authkey', 'user_id', 'expiration', 'allowed_ips'], 'conditions' => [ 'OR' => [ 'expiration >' => time(), @@ -84,12 +128,13 @@ class AuthKey extends AppModel ] ]); $passwordHasher = $this->getHasher(); - foreach ($existing_authkeys as $existing_authkey) { - if ($passwordHasher->check($authkey, $existing_authkey['AuthKey']['authkey'])) { - $user = $this->User->getAuthUser($existing_authkey['AuthKey']['user_id']); + foreach ($possibleAuthkeys as $possibleAuthkey) { + if ($passwordHasher->check($authkey, $possibleAuthkey['AuthKey']['authkey'])) { + $user = $this->User->getAuthUser($possibleAuthkey['AuthKey']['user_id']); if ($user) { - $user['authkey_id'] = $existing_authkey['AuthKey']['id']; - $user['authkey_expiration'] = $existing_authkey['AuthKey']['expiration']; + $user['authkey_id'] = $possibleAuthkey['AuthKey']['id']; + $user['authkey_expiration'] = $possibleAuthkey['AuthKey']['expiration']; + $user['allowed_ips'] = $possibleAuthkey['AuthKey']['allowed_ips']; } return $user; } @@ -175,6 +220,18 @@ class AuthKey extends AppModel return $output; } + /** + * When key is modified, update `date_modified` for user that was assigned to that key, so session data + * will be realoaded. + * @see AppController::_refreshAuth + */ + public function afterSave($created, $options = array()) + { + parent::afterSave($created, $options); + $userId = $this->data['AuthKey']['user_id']; + $this->User->updateAll(['date_modified' => time()], ['User.id' => $userId]); + } + /** * When key is deleted, update after `date_modified` for user that was assigned to that key, so session data * will be realoaded and canceled. diff --git a/app/Model/Log.php b/app/Model/Log.php index 1cf7aee90..19b7aeb64 100644 --- a/app/Model/Log.php +++ b/app/Model/Log.php @@ -199,7 +199,7 @@ class Log extends AppModel * @param int $modelId * @param string $title * @param string|array $change - * @return array + * @return array|null * @throws Exception * @throws InvalidArgumentException */ @@ -238,6 +238,10 @@ class Log extends AppModel )); if (!$result) { + if ($action === 'request' && !empty(Configure::read('MISP.log_paranoid_skip_db'))) { + return null; + } + throw new Exception("Cannot save log because of validation errors: " . json_encode($this->validationErrors)); } diff --git a/app/View/AuthKeys/add.ctp b/app/View/AuthKeys/add.ctp index 62e5a40db..a3cd94042 100644 --- a/app/View/AuthKeys/add.ctp +++ b/app/View/AuthKeys/add.ctp @@ -1,7 +1,7 @@ element('genericElements/Form/genericForm', [ 'data' => [ - 'title' => __('Add auth key'), + 'title' => isset($edit) ? __('Edit auth key') : __('Add auth key'), 'description' => __('Auth keys are used for API access. A user can have more than one authkey, so if you would like to use separate keys per tool that queries MISP, add additional keys. Use the comment field to make identifying your keys easier.'), 'fields' => [ [ @@ -13,7 +13,14 @@ echo $this->element('genericElements/Form/genericForm', [ [ 'field' => 'comment', 'label' => __('Comment'), - 'class' => 'span6' + 'class' => 'span6', + 'rows' => 4, + ], + [ + 'field' => 'allowed_ips', + 'label' => __('Allowed IPs'), + 'class' => 'span6', + 'rows' => 4, ], [ 'field' => 'expiration', diff --git a/app/View/AuthKeys/index.ctp b/app/View/AuthKeys/index.ctp index 127132961..77c3ab45f 100644 --- a/app/View/AuthKeys/index.ctp +++ b/app/View/AuthKeys/index.ctp @@ -63,12 +63,17 @@ 'data_path' => 'AuthKey.last_used', 'element' => 'datetime', 'requirements' => $keyUsageEnabled, + 'empty' => __('Never'), ], [ 'name' => __('Comment'), 'sort' => 'AuthKey.comment', 'data_path' => 'AuthKey.comment', ], + [ + 'name' => __('Allowed IPs'), + 'data_path' => 'AuthKey.allowed_ips', + ], ], 'title' => empty($ajax) ? __('Authentication key Index') : false, 'description' => empty($ajax) ? __('A list of API keys bound to a user.') : false, @@ -80,7 +85,16 @@ 'AuthKey.id' ), 'icon' => 'eye', - 'dbclickAction' => true + 'dbclickAction' => true, + 'title' => 'View auth key', + ], + [ + 'url' => $baseurl . '/auth_keys/edit', + 'url_params_data_paths' => array( + 'AuthKey.id' + ), + 'icon' => 'edit', + 'title' => 'Edit auth key', ], [ 'onclick' => sprintf( diff --git a/app/View/AuthKeys/view.ctp b/app/View/AuthKeys/view.ctp index 37f786371..43b79780b 100644 --- a/app/View/AuthKeys/view.ctp +++ b/app/View/AuthKeys/view.ctp @@ -15,65 +15,72 @@ if (isset($keyUsage)) { $uniqueIps = null; } -echo $this->element( - 'genericElements/SingleViews/single_view', - [ - 'title' => 'Auth key view', - 'data' => $data, - 'fields' => [ - [ - 'key' => __('ID'), - 'path' => 'AuthKey.id' - ], - [ - 'key' => __('UUID'), - 'path' => 'AuthKey.uuid', - ], - [ - 'key' => __('Auth Key'), - 'path' => 'AuthKey', - 'type' => 'authkey' - ], - [ - 'key' => __('User'), - 'path' => 'User.id', - 'pathName' => 'User.email', - 'model' => 'users', - 'type' => 'model' - ], - [ - 'key' => __('Comment'), - 'path' => 'AuthKey.comment' - ], - [ - 'key' => __('Created'), - 'path' => 'AuthKey.created', - 'type' => 'datetime' - ], - [ - 'key' => __('Expiration'), - 'path' => 'AuthKey.expiration', - 'type' => 'expiration' - ], - [ - 'key' => __('Key usage'), - 'type' => 'sparkline', - 'path' => 'AuthKey.id', - 'csv' => [ - 'data' => $keyUsageCsv, - ], - 'requirement' => isset($keyUsage), - ], - [ - 'key' => __('Last used'), - 'raw' => $lastUsed ? $this->Time->time($lastUsed) : __('Not used yet'), - 'requirement' => isset($keyUsage), - ], - [ - 'key' => __('Unique IPs'), - 'raw' => $uniqueIps, - 'requirement' => isset($keyUsage), - ] +echo $this->element('genericElements/SingleViews/single_view', [ + 'title' => 'Auth key view', + 'data' => $data, + 'fields' => [ + [ + 'key' => __('ID'), + 'path' => 'AuthKey.id' ], - ] -); + [ + 'key' => __('UUID'), + 'path' => 'AuthKey.uuid', + ], + [ + 'key' => __('Auth Key'), + 'path' => 'AuthKey', + 'type' => 'authkey' + ], + [ + 'key' => __('User'), + 'path' => 'User.id', + 'pathName' => 'User.email', + 'model' => 'users', + 'type' => 'model' + ], + [ + 'key' => __('Comment'), + 'path' => 'AuthKey.comment' + ], + [ + 'key' => __('Allowed IPs'), + 'type' => 'custom', + 'function' => function (array $data) { + if (is_array($data['AuthKey']['allowed_ips'])) { + return implode("
    ", array_map('h', $data['AuthKey']['allowed_ips'])); + } + return __('All'); + } + ], + [ + 'key' => __('Created'), + 'path' => 'AuthKey.created', + 'type' => 'datetime' + ], + [ + 'key' => __('Expiration'), + 'path' => 'AuthKey.expiration', + 'type' => 'expiration' + ], + [ + 'key' => __('Key usage'), + 'type' => 'sparkline', + 'path' => 'AuthKey.id', + 'csv' => [ + 'data' => $keyUsageCsv, + ], + 'requirement' => isset($keyUsage), + ], + [ + 'key' => __('Last used'), + 'raw' => $lastUsed ? $this->Time->time($lastUsed) : __('Not used yet'), + 'requirement' => isset($keyUsage), + ], + [ + 'key' => __('Unique IPs'), + 'raw' => $uniqueIps, + 'requirement' => isset($keyUsage), + ] + ], +]); diff --git a/app/View/Elements/genericElements/IndexTable/Fields/datetime.ctp b/app/View/Elements/genericElements/IndexTable/Fields/datetime.ctp index 34d3f3bbf..b78c98c5a 100644 --- a/app/View/Elements/genericElements/IndexTable/Fields/datetime.ctp +++ b/app/View/Elements/genericElements/IndexTable/Fields/datetime.ctp @@ -1,26 +1,27 @@ 1) { - $data = implode(', ', $data); +$data = Hash::extract($row, $field['data_path']); +if (is_array($data)) { + if (count($data) > 1) { + $data = implode(', ', $data); + } else { + if (count($data) > 0) { + $data = $data[0]; } else { - if (count($data) > 0) { - $data = $data[0]; - } else { - $data = ''; - } + $data = ''; } } - if (empty($data) && !empty($field['empty'])) { - $data = $field['empty']; - } +} +if (empty($data) && !empty($field['empty'])) { + $data = $field['empty']; +} else { $data = $this->Time->time($data); - if (!empty($field['onClick'])) { - $data = sprintf( - '%s', - $field['onClick'], - $data - ); - } - echo $data; +} +if (!empty($field['onClick'])) { + $data = sprintf( + '%s', + $field['onClick'], + $data + ); +} +echo $data; diff --git a/app/View/Elements/genericElements/SingleViews/Fields/customField.ctp b/app/View/Elements/genericElements/SingleViews/Fields/customField.ctp new file mode 100644 index 000000000..a056047b4 --- /dev/null +++ b/app/View/Elements/genericElements/SingleViews/Fields/customField.ctp @@ -0,0 +1 @@ + Date: Mon, 1 Mar 2021 15:25:18 +0100 Subject: [PATCH 13/17] new: [authkeys] Copy key info when resetting key --- app/Controller/Component/ACLComponent.php | 2 +- app/Controller/UsersController.php | 16 +++--- app/Model/AuthKey.php | 67 ++++++++++++++++++----- app/Model/User.php | 7 +-- 4 files changed, 65 insertions(+), 27 deletions(-) diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php index fb398c910..783329be2 100644 --- a/app/Controller/Component/ACLComponent.php +++ b/app/Controller/Component/ACLComponent.php @@ -687,7 +687,7 @@ class ACLComponent extends Component 'register' => array('*'), 'registrations' => array('perm_site_admin'), 'resetAllSyncAuthKeys' => array(), - 'resetauthkey' => array('*'), + 'resetauthkey' => ['AND' => ['self_management_enabled', 'perm_auth']], 'request_API' => array('*'), 'routeafterlogin' => array('*'), 'statistics' => array('*'), diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php index 966eb329f..3d3c47d04 100644 --- a/app/Controller/UsersController.php +++ b/app/Controller/UsersController.php @@ -1294,24 +1294,22 @@ class UsersController extends AppController public function resetauthkey($id = null, $alert = false) { - if (!$this->_isAdmin() && Configure::read('MISP.disableUserSelfManagement')) { - throw new MethodNotAllowedException('User self-management has been disabled on this instance.'); - } if (!$this->request->is('post') && !$this->request->is('put')) { throw new MethodNotAllowedException(__('This functionality is only accessible via POST requests.')); } - if ($id == 'me') { + if ($id === 'me') { $id = $this->Auth->user('id'); + // Reset just current auth key + $keyId = isset($this->Auth->user()['authkey_id']) ? $this->Auth->user()['authkey_id'] : null; + } else { + $keyId = null; } - if (!$this->userRole['perm_auth']) { - throw new MethodNotAllowedException(__('Invalid action.')); - } - $newkey = $this->User->resetauthkey($this->Auth->user(), $id, $alert); + $newkey = $this->User->resetauthkey($this->Auth->user(), $id, $alert, $keyId); if ($newkey === false) { throw new MethodNotAllowedException(__('Invalid user.')); } if (!$this->_isRest()) { - $this->Flash->success(__('New authkey generated.', true)); + $this->Flash->success(__('New authkey generated.')); $this->redirect($this->referer()); } else { return $this->RestResponse->saveSuccessResponse('User', 'resetauthkey', $id, $this->response->type(), 'Authkey updated: ' . $newkey); diff --git a/app/Model/AuthKey.php b/app/Model/AuthKey.php index 50de35909..5dbfb8ac2 100644 --- a/app/Model/AuthKey.php +++ b/app/Model/AuthKey.php @@ -142,26 +142,67 @@ class AuthKey extends AppModel return false; } - public function resetauthkey($id) + /** + * @param int $userId + * @param int|null $keyId + * @return false|string + * @throws Exception + */ + public function resetAuthKey($userId, $keyId = null) { - $existing_authkeys = $this->find('all', [ - 'recursive' => -1, - 'conditions' => [ - 'user_id' => $id - ] - ]); - foreach ($existing_authkeys as $key) { - $key['AuthKey']['expiration'] = time(); - $this->save($key); + $time = time(); + + if ($keyId) { + $currentAuthkey = $this->find('first', [ + 'recursive' => -1, + 'conditions' => [ + 'id' => $keyId, + 'user_id' => $userId, + ], + ]); + if (empty($currentAuthkey)) { + throw new RuntimeException("Key with ID $keyId for user with ID $userId not found."); + } + $currentAuthkey['AuthKey']['expiration'] = $time; + if (!$this->save($currentAuthkey)) { + throw new RuntimeException("Key with ID $keyId could not be saved."); + } + $comment = __("Created by resetting auth key %s\n%s", $keyId, $currentAuthkey['AuthKey']['comment']); + $allowedIps = isset($currentAuthkey['AuthKey']['allowed_ips']) ? $currentAuthkey['AuthKey']['allowed_ips'] : []; + return $this->createnewkey($userId, $comment, $allowedIps); + } else { + $existingAuthkeys = $this->find('all', [ + 'recursive' => -1, + 'conditions' => [ + 'OR' => [ + 'expiration >' => $time, + 'expiration' => 0 + ], + 'user_id' => $userId + ] + ]); + foreach ($existingAuthkeys as $key) { + $key['AuthKey']['expiration'] = $time; + $this->save($key); + } + return $this->createnewkey($userId); } - return $this->createnewkey($id); } - public function createnewkey($id) + /** + * @param int $userId + * @param string $comment + * @param array $allowedIps + * @return false|string + * @throws Exception + */ + public function createnewkey($userId, $comment = '', array $allowedIps = []) { $newKey = [ 'authkey' => (new RandomTool())->random_str(true, 40), - 'user_id' => $id + 'user_id' => $userId, + 'comment' => $comment, + 'allowed_ips' => empty($allowedIps) ? null : $allowedIps, ]; $this->create(); if ($this->save($newKey)) { diff --git a/app/Model/User.php b/app/Model/User.php index e8f70527d..ca505519c 100644 --- a/app/Model/User.php +++ b/app/Model/User.php @@ -1106,7 +1106,7 @@ class User extends AppModel return $results; } - public function resetauthkey($user, $id, $alert = false) + public function resetauthkey($user, $id, $alert = false, $keyId = null) { $this->id = $id; if (!$id || !$this->exists($id)) { @@ -1123,8 +1123,7 @@ class User extends AppModel $this->extralog( $user, 'reset_auth_key', - sprintf( - __('Authentication key for user %s (%s) updated.'), + __('Authentication key for user %s (%s) updated.', $updatedUser['User']['id'], $updatedUser['User']['email'] ), @@ -1133,7 +1132,7 @@ class User extends AppModel ); } else { $this->AuthKey = ClassRegistry::init('AuthKey'); - $newkey = $this->AuthKey->resetauthkey($id); + $newkey = $this->AuthKey->resetAuthKey($id, $keyId); } if ($alert) { $baseurl = Configure::read('MISP.external_baseurl'); From 27eb90f3d56b04cc6e23a55e95fe069b82bb3983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 3 Mar 2021 09:46:40 +0100 Subject: [PATCH 14/17] chg: [PyMISP] Bump before release --- PyMISP | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyMISP b/PyMISP index 4a2367ec9..fe87d4293 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit 4a2367ec965d70d84a0091ea3a6978916a7df25a +Subproject commit fe87d4293bac938f9601ea44b6738fba0f3586da From 12dec25144aea5dc3ba243ccc3d37e69454bdb2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 3 Mar 2021 10:23:45 +0100 Subject: [PATCH 15/17] chg: [PyMISP] Fix tests --- PyMISP | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyMISP b/PyMISP index fe87d4293..ef9efce5c 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit fe87d4293bac938f9601ea44b6738fba0f3586da +Subproject commit ef9efce5c348379ceeaacf91b604d6cde5190e6c From 9923e30c848d40e03d0b143b101345b5240103c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 3 Mar 2021 10:41:50 +0100 Subject: [PATCH 16/17] chg: [PyMISP] Bump version --- PyMISP | 2 +- app/Controller/AppController.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PyMISP b/PyMISP index ef9efce5c..39a7b8242 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit ef9efce5c348379ceeaacf91b604d6cde5190e6c +Subproject commit 39a7b8242f0d3022276d417ec334bb46b890ff23 diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index 5d46a29ac..4c919f55d 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -26,7 +26,7 @@ class AppController extends Controller public $helpers = array('OrgImg', 'FontAwesome', 'UserName', 'DataPathCollector'); private $__queryVersion = '126'; - public $pyMispVersion = '2.4.138'; + public $pyMispVersion = '2.4.140'; public $phpmin = '7.2'; public $phprec = '7.4'; public $phptoonew = '8.0'; From 6a553d39da44a43eae02fc7a6fd374152eed311f Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Wed, 3 Mar 2021 13:37:08 +0100 Subject: [PATCH 17/17] fix: [OIDC] Change algo how roles are assigned to users --- .../Component/Auth/OidcAuthenticate.php | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/app/Plugin/OidcAuth/Controller/Component/Auth/OidcAuthenticate.php b/app/Plugin/OidcAuth/Controller/Component/Auth/OidcAuthenticate.php index 5031c65c9..4ec5befbe 100644 --- a/app/Plugin/OidcAuth/Controller/Component/Auth/OidcAuthenticate.php +++ b/app/Plugin/OidcAuth/Controller/Component/Auth/OidcAuthenticate.php @@ -72,9 +72,9 @@ class OidcAuthenticate extends BaseAuthenticate } if ($user['role_id'] != $roleId) { - $user['role_id'] = $roleId; $this->userModel()->updateField($user, 'role_id', $roleId); $this->log($mispUsername, "User role changed from {$user['role_id']} to $roleId."); + $user['role_id'] = $roleId; } $this->log($mispUsername, 'Logged in.'); @@ -182,20 +182,18 @@ class OidcAuthenticate extends BaseAuthenticate ]); $roleNameToId = array_change_key_case($roleNameToId); // normalize role names to lowercase - $userRole = null; - foreach ($roles as $role) { - if (isset($roleMapper[$role])) { - $roleId = $roleMapper[$role]; - if (!is_numeric($roleId)) { - $roleId = mb_strtolower($roleId); - if (isset($roleNameToId[$roleId])) { - $roleId = $roleNameToId[$roleId]; + foreach ($roleMapper as $oidcRole => $mispRole) { + if (in_array($oidcRole, $roles, true)) { + if (!is_numeric($mispRole)) { + $mispRole = mb_strtolower($mispRole); + if (isset($roleNameToId[$mispRole])) { + $mispRole = $roleNameToId[$mispRole]; } else { - $this->log($mispUsername, "MISP Role with name `$roleId` not found, skipping."); + $this->log($mispUsername, "MISP Role with name `$mispRole` not found, skipping."); continue; } } - return $roleId; // first match wins + return $mispRole; // first match wins } }