From 53a8029ea5122d9668fd114423e1d7f8f456978f Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 8 Jul 2021 10:34:32 +0900 Subject: [PATCH 01/78] chg: [doc] Added 3 more optionals --- docs/generic/MISP_CAKE_init.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/generic/MISP_CAKE_init.md b/docs/generic/MISP_CAKE_init.md index 4385ca117..b6fd02a23 100644 --- a/docs/generic/MISP_CAKE_init.md +++ b/docs/generic/MISP_CAKE_init.md @@ -212,6 +212,9 @@ coreCAKE () { ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.block_old_event_alert" false ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.block_old_event_alert_age" "" ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.block_old_event_alert_by_date" "" + ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.event_alert_republish_ban" false + ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.event_alert_republish_ban_threshold" 5 + ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.event_alert_republish_ban_refresh_on_retry" false ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.incoming_tags_disabled_by_default" false ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.maintenance_message" "Great things are happening! MISP is undergoing maintenance, but will return shortly. You can contact the administration at \$email." ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.footermidleft" "This is an initial install" From 82ec738770377f53cc0425397455a9165741f741 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Mon, 26 Jul 2021 15:58:30 +0200 Subject: [PATCH 02/78] chg: initial migration of attributes/index view to factory --- app/View/Attributes/add.ctp.bk | 189 ------------ app/View/Attributes/index.ctp | 278 ++++++------------ .../IndexTable/Fields/correlate.ctp | 22 ++ 3 files changed, 111 insertions(+), 378 deletions(-) delete mode 100644 app/View/Attributes/add.ctp.bk create mode 100644 app/View/Elements/genericElements/IndexTable/Fields/correlate.ctp diff --git a/app/View/Attributes/add.ctp.bk b/app/View/Attributes/add.ctp.bk deleted file mode 100644 index 89bf37355..000000000 --- a/app/View/Attributes/add.ctp.bk +++ /dev/null @@ -1,189 +0,0 @@ -
-Form->create('Attribute', array('id', 'url' => '/attributes/' . $url_params)); -?> -
- -
- -
- Form->hidden('event_id'); - echo $this->Form->input('category', array( - 'empty' => __('(choose one)'), - 'label' => __('Category ') . $this->element('formInfo', array('type' => 'category')), - )); - echo $this->Form->input('type', array( - 'empty' => __('(first choose category)'), - 'label' => __('Type ') . $this->element('formInfo', array('type' => 'type')), - )); - - $initialDistribution = 5; - if (Configure::read('MISP.default_attribute_distribution') != null) { - if (Configure::read('MISP.default_attribute_distribution') === 'event') { - $initialDistribution = 5; - } else { - $initialDistribution = Configure::read('MISP.default_attribute_distribution'); - } - } - - ?> -
- array($distributionLevels), - 'label' => __('Distribution ') . $this->element('formInfo', array('type' => 'distribution')), - ); - - if ($action == 'add') { - $distArray['selected'] = $initialDistribution; - } - - echo $this->Form->input('distribution', $distArray); - ?> - - Form->input('value', array( - 'type' => 'textarea', - 'error' => array('escape' => false), - 'div' => 'input clear', - 'class' => 'input-xxlarge' - )); - ?> -
- Form->input('comment', array( - 'type' => 'text', - 'label' => __('Contextual Comment'), - 'error' => array('escape' => false), - 'div' => 'input clear', - 'class' => 'input-xxlarge' - )); - ?> -
- Form->input('to_ids', array( - 'label' => __('for Intrusion Detection System'), - 'type' => 'checkbox' - )); - echo $this->Form->input('batch_import', array( - 'type' => 'checkbox' - )); - echo '
'; - echo $this->Form->input('disable_correlation', array( - 'type' => 'checkbox' - )); - ?> -
-
- - -
- - -
- - Form->button('Submit', array('class' => 'btn btn-primary')); - endif; - echo $this->Form->end(); - ?> -
-element('/genericElements/SideMenu/side_menu', array('menuList' => 'event', 'menuItem' => 'addAttribute', 'event' => $event)); - } -?> - -Js->writeBuffer(); // Write cached scripts diff --git a/app/View/Attributes/index.ctp b/app/View/Attributes/index.ctp index b8d55324d..016c6e62d 100755 --- a/app/View/Attributes/index.ctp +++ b/app/View/Attributes/index.ctp @@ -1,193 +1,93 @@ -
-

- __(" with the value containing "), - 'tags' => __(" being tagged with "), - 'id' => __(" from the events "), - 'tag' => __(" carrying the tag(s) "), - 'type' => __(" of type "), - 'category' => __(" of category "), - 'org' => __(" created by organisation ") - ); - $temp = ''; - foreach ($filterOptions as $fo => $text) { - if (!empty($filters[$fo])) { - $filter_options_string = $filters[$fo]; - if (is_array($filter_options_string)) { - $filter_options_string = implode(' OR ', $filter_options_string); - } - $temp .= sprintf('%s %s', $text, h($filter_options_string)); - } - } - echo sprintf("

%s%s

", __("Results for all attributes"), $temp); - } - ?> - - Paginator->sort('timestamp', __('Date')), - $this->Paginator->sort('event_id'), - $this->Paginator->sort('Event.orgc_id', __('Org')), - $this->Paginator->sort('category'), - $this->Paginator->sort('type'), - $this->Paginator->sort('value'), - __('Tags'), - __('Galaxies'), - $this->Paginator->sort('comment'), - __('Correlate'), - __('Related Events'), - __('Feed hits'), - sprintf('%s', $attrDescriptions['signature']['desc'], $this->Paginator->sort('IDS')), - sprintf('%s', $attrDescriptions['distribution']['desc'], $this->Paginator->sort('distribution')), - __('Sightings'), - __('Activity'), - __('Actions') - ); - foreach ($headers as $k => &$header) { - $header = "$header"; - } - ?> - - - Highlight->build_replace_pairs($keywordArray); - } - foreach ($attributes as $k => $attribute) { - $event = array( - 'Event' => $attribute['Event'], - 'Orgc' => $attribute['Event']['Orgc'], - ); - $mayModify = ($isSiteAdmin || ($isAclModify && $event['Event']['user_id'] == $me['id'] && $attribute['Event']['orgc_id'] == $me['org_id']) || ($isAclModifyOrg && $attribute['Event']['orgc_id'] == $me['org_id'])); - $mayPublish = ($isAclPublish && $attribute['Event']['orgc_id'] == $me['org_id']); - $mayChangeCorrelation = !Configure::read('MISP.completely_disable_correlation') && ($isSiteAdmin || ($mayModify && Configure::read('MISP.allow_disabling_correlation'))); - if (!empty($attribute['Attribute']['RelatedAttribute'])) { - $event['RelatedAttribute'] = array($attribute['Attribute']['id'] => $attribute['Attribute']['RelatedAttribute']); - } - $attribute['Attribute']['objectType'] = 'attribute'; - echo $this->element('/Events/View/row_attribute', array( - 'object' => $attribute['Attribute'], - 'k' => $k, - 'mayModify' => $mayModify, - 'mayChangeCorrelation' => $mayChangeCorrelation, - 'page' => 1, - 'fieldCount' => 11, - 'includeRelatedTags' => 0, - 'event' => $event, - 'me' => $me, - 'extended' => 1, - 'disable_multi_select' => 1, - 'context' => 'list' - )); - } - ?> -
- Form->create('Sighting', ['id' => 'SightingForm', 'url' => $baseurl . '/sightings/add/', 'style' => 'display:none;']); - echo $this->Form->input('id', ['label' => false, 'type' => 'number']); - echo $this->Form->input('type', ['label' => false]); - echo $this->Form->end(); - ?> -

- Paginator->counter(array( - 'format' => __('Page {:page} of {:pages}, showing {:current} records out of {:count} total, starting on record {:start}, ending on {:end}') - )); - ?> -

- -
'; +echo $this->element('/genericElements/IndexTable/index_table', [ + 'data' => [ + 'title' => __('Attributes'), + 'data' => $attributes, + 'fields' => [ + [ + 'name' => __('Date'), + 'sort' => 'Attribute.timestamp', + 'class' => 'short', + 'element' => 'timestamp', + 'time_format' => 'Y-m-d', + 'data_path' => 'Attribute.timestamp', + ], + [ + 'name' => __('Event'), + 'sort' => 'Attribute.event_id', + 'class' => 'short', + 'data_path' => 'Attribute.event_id', + ], + [ + 'name' => __('Org'), + 'sort' => 'Event.Orgc.name', + 'class' => 'short', + 'data_path' => 'Event.Orgc', + 'element' => 'org' + ], + [ + 'name' => __('Category'), + 'sort' => 'Attribute.category', + 'class' => 'short', + 'data_path' => 'Attribute.category', + ], + [ + 'name' => __('Type'), + 'sort' => 'Attribute.type', + 'class' => 'short', + 'data_path' => 'Attribute.type', + ], + [ + 'name' => __('Value'), + 'sort' => 'Attribute.value', + 'class' => 'short', + 'data_path' => 'Attribute.value', + ], + [ + 'name' => __('Tags'), + 'class' => 'short', + 'data_path' => 'Attribute.AttributeTag', + ], + [ + 'name' => __('Galaxies'), + 'class' => 'short', + 'data_path' => 'Attribute.Galaxy', + ], + [ + 'name' => __('Comment'), + 'class' => 'shortish', + 'data_path' => 'Attribute.comment', + ], + [ + 'name' => __('Correlate'), + 'class' => 'shortish', + 'data_path' => 'Attribute.disable_correlation', + 'data' => [ + 'object' => [ + 'value_path' => 'Attribute' + ], + ], + 'element' => 'correlate', + 'scope' => 'Attribute', + ], + ], + ] +]); +echo ''; + +$class = $isSearch == 1 ? 'searchAttributes2' : 'listAttributes'; +echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event-collection', 'menuItem' => $class)); + ?> -element('/genericElements/SideMenu/side_menu', array('menuList' => 'event-collection', 'menuItem' => $class)); ?> + + \ No newline at end of file diff --git a/app/View/Elements/genericElements/IndexTable/Fields/correlate.ctp b/app/View/Elements/genericElements/IndexTable/Fields/correlate.ctp new file mode 100644 index 000000000..af9dfd303 --- /dev/null +++ b/app/View/Elements/genericElements/IndexTable/Fields/correlate.ctp @@ -0,0 +1,22 @@ +DataPathCollector->extract($row, $field['data_path']); +$object = Hash::extract($row, $field['data']['object']['value_path']); +$event = Hash::extract($row, 'Event'); +$mayModify = ($isSiteAdmin || ($isAclModify && $event['user_id'] == $me['id'] && $event['orgc_id'] == $me['org_id']) || ($isAclModifyOrg && $event['orgc_id'] == $me['org_id'])); +$mayChangeCorrelation = !Configure::read('MISP.completely_disable_correlation') && ($isSiteAdmin || ($mayModify && Configure::read('MISP.allow_disabling_correlation'))); + +?> + + +/> \ No newline at end of file From 01ec762045536906a66c824622896c740f79f4fe Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Wed, 28 Jul 2021 11:37:10 +0200 Subject: [PATCH 03/78] chg: add attributes index custom fields --- app/Controller/AttributesController.php | 3 + app/View/Attributes/index.ctp | 116 ++++++++++++++++-- .../Events/View/attribute_correlations.ctp | 12 +- .../IndexTable/Fields/correlate.ctp | 6 +- .../IndexTable/Fields/distribution_levels.ctp | 48 ++++++-- .../IndexTable/Fields/feedHits.ctp | 87 +++++++++++++ .../IndexTable/Fields/relatedEvents.ctp | 18 +++ .../IndexTable/Fields/toIds.ctp | 13 ++ 8 files changed, 269 insertions(+), 34 deletions(-) create mode 100644 app/View/Elements/genericElements/IndexTable/Fields/feedHits.ctp create mode 100644 app/View/Elements/genericElements/IndexTable/Fields/relatedEvents.ctp create mode 100644 app/View/Elements/genericElements/IndexTable/Fields/toIds.ctp diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 2297ef7af..d8e6f9ef2 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -93,6 +93,8 @@ class AttributesController extends AppController } } + $distributionLevels = $this->Attribute->distributionLevels; + list($attributes, $sightingsData) = $this->__searchUI($attributes); $this->set('sightingsData', $sightingsData); $this->set('orgTable', array_column($orgTable, 'name', 'id')); @@ -101,6 +103,7 @@ class AttributesController extends AppController $this->set('attrDescriptions', $this->Attribute->fieldDescriptions); $this->set('typeDefinitions', $this->Attribute->typeDefinitions); $this->set('categoryDefinitions', $this->Attribute->categoryDefinitions); + $this->set('distributionLevels', $distributionLevels); } public function add($eventId = false) diff --git a/app/View/Attributes/index.ctp b/app/View/Attributes/index.ctp index 016c6e62d..e83a13171 100755 --- a/app/View/Attributes/index.ctp +++ b/app/View/Attributes/index.ctp @@ -1,5 +1,7 @@ '; + echo $this->element('/genericElements/IndexTable/index_table', [ 'data' => [ 'title' => __('Attributes'), @@ -11,13 +13,13 @@ echo $this->element('/genericElements/IndexTable/index_table', [ 'class' => 'short', 'element' => 'timestamp', 'time_format' => 'Y-m-d', - 'data_path' => 'Attribute.timestamp', + 'data_path' => 'Attribute.timestamp' ], [ 'name' => __('Event'), 'sort' => 'Attribute.event_id', 'class' => 'short', - 'data_path' => 'Attribute.event_id', + 'data_path' => 'Attribute.event_id' ], [ 'name' => __('Org'), @@ -30,50 +32,94 @@ echo $this->element('/genericElements/IndexTable/index_table', [ 'name' => __('Category'), 'sort' => 'Attribute.category', 'class' => 'short', - 'data_path' => 'Attribute.category', + 'data_path' => 'Attribute.category' ], [ 'name' => __('Type'), 'sort' => 'Attribute.type', 'class' => 'short', - 'data_path' => 'Attribute.type', + 'data_path' => 'Attribute.type' ], [ 'name' => __('Value'), 'sort' => 'Attribute.value', 'class' => 'short', - 'data_path' => 'Attribute.value', + 'data_path' => 'Attribute.value' ], [ 'name' => __('Tags'), 'class' => 'short', - 'data_path' => 'Attribute.AttributeTag', + 'data_path' => 'Attribute.AttributeTag' ], [ 'name' => __('Galaxies'), 'class' => 'short', - 'data_path' => 'Attribute.Galaxy', + 'data_path' => 'Attribute.Galaxy' ], [ 'name' => __('Comment'), 'class' => 'shortish', - 'data_path' => 'Attribute.comment', + 'data_path' => 'Attribute.comment' ], [ 'name' => __('Correlate'), - 'class' => 'shortish', - 'data_path' => 'Attribute.disable_correlation', + 'class' => 'short', + 'element' => 'correlate', + 'data' => [ + 'object' => [ + 'value_path' => 'Attribute' + ] + ] + ], + [ + 'name' => __('Related Events'), + 'class' => 'short', + 'element' => 'relatedEvents', 'data' => [ 'object' => [ 'value_path' => 'Attribute' ], + 'scope' => 'Attribute' + ] + ], + [ + 'name' => __('Feed hits'), + 'class' => 'short', + 'element' => 'feedHits', + 'data' => [ + 'object' => [ + 'value_path' => 'Attribute' + ] + ] + ], + [ + 'name' => __('IDS'), + 'class' => 'short', + 'element' => 'toIds', + 'data' => [ + 'object' => [ + 'value_path' => 'Attribute' + ] ], - 'element' => 'correlate', - 'scope' => 'Attribute', + ], + [ + 'name' => __('Distribution'), + 'element' => 'distribution_levels', + 'class' => 'short', + 'data_path' => 'Attribute.distribution', + 'distributionLevels' => $distributionLevels, + 'data' => [ + 'object' => [ + 'value_path' => 'Attribute' + ], + 'scope' => 'Attribute' + ], + 'quickedit' => true ], ], ] ]); + echo ''; $class = $isSearch == 1 ? 'searchAttributes2' : 'listAttributes'; @@ -84,10 +130,56 @@ echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => ' \ No newline at end of file diff --git a/app/View/Elements/Events/View/attribute_correlations.ctp b/app/View/Elements/Events/View/attribute_correlations.ctp index 7880d8f01..7fa6b41cc 100644 --- a/app/View/Elements/Events/View/attribute_correlations.ctp +++ b/app/View/Elements/Events/View/attribute_correlations.ctp @@ -4,11 +4,11 @@ $linkColour = ($scope == 'Attribute') ? 'red' : 'white'; // remove duplicates $relatedEvents = array(); foreach ($event['Related' . $scope][$object['id']] as $k => $relatedAttribute) { - if (isset($relatedEvents[$relatedAttribute['id']])) { - unset($event['Related' . $scope][$object['id']][$k]); - } else { - $relatedEvents[$relatedAttribute['id']] = true; - } + if (isset($relatedEvents[$relatedAttribute['id']])) { + unset($event['Related' . $scope][$object['id']][$k]); + } else { + $relatedEvents[$relatedAttribute['id']] = true; + } } $event['Related' . $scope][$object['id']] = array_values($event['Related' . $scope][$object['id']]); $count = count($event['Related' . $scope][$object['id']]); @@ -33,7 +33,7 @@ foreach ($event['Related' . $scope][$object['id']] as $relatedAttribute) { } $link = $this->Html->link( $relatedAttribute['id'], - array('controller' => 'events', 'action' => 'view', $relatedAttribute['id'], true, $event['Event']['id']), + array('controller' => 'events', 'action' => 'view', $relatedAttribute['id'], true, $event['id']), array('class' => ($relatedAttribute['org_id'] == $me['org_id']) ? $linkColour : 'blue') ); echo sprintf( diff --git a/app/View/Elements/genericElements/IndexTable/Fields/correlate.ctp b/app/View/Elements/genericElements/IndexTable/Fields/correlate.ctp index af9dfd303..640c34b85 100644 --- a/app/View/Elements/genericElements/IndexTable/Fields/correlate.ctp +++ b/app/View/Elements/genericElements/IndexTable/Fields/correlate.ctp @@ -1,20 +1,20 @@ DataPathCollector->extract($row, $field['data_path']); $object = Hash::extract($row, $field['data']['object']['value_path']); $event = Hash::extract($row, 'Event'); $mayModify = ($isSiteAdmin || ($isAclModify && $event['user_id'] == $me['id'] && $event['orgc_id'] == $me['org_id']) || ($isAclModifyOrg && $event['orgc_id'] == $me['org_id'])); $mayChangeCorrelation = !Configure::read('MISP.completely_disable_correlation') && ($isSiteAdmin || ($mayModify && Configure::read('MISP.allow_disabling_correlation'))); +$objectId = h($object['id']); ?> %s', - $distributionLevel == 0 ? 'red' : '', - $distributionLevel != 4 ? $distributionLevels[$distributionLevel] : - sprintf( - '%s', - $baseurl, - h($row['SharingGroup']['id']), - h($row['SharingGroup']['name']) - ) - ); +$quickedit = isset($field['quickedit']) && $field['quickedit']; +if ($quickedit) { + $object = Hash::extract($row, $field['data']['object']['value_path']); + $event = Hash::extract($row, 'Event'); + $objectId = h($object['id']); + $scope = $field['data']['scope']; +} -?> +$distributionLevel = (Hash::extract($row, $field['data_path'])[0]); + +echo sprintf('
', $quickedit ? sprintf( + " onmouseenter=\"quickEditHover(this, '%s', %s, 'distribution', %s);\"", + $scope, + $objectId, + $event['id'] +) : ''); + +if ($quickedit) { + echo sprintf("
", $scope, $objectId); + echo sprintf("
", $scope, $objectId); +} + +echo sprintf( + '%s', + $distributionLevel == 0 ? 'red' : '', + $distributionLevel != 4 ? $distributionLevels[$distributionLevel] : + sprintf( + '%s', + $baseurl, + h($row['SharingGroup']['id']), + h($row['SharingGroup']['name']) + ) +); +if ($quickedit) { + echo '
'; +} diff --git a/app/View/Elements/genericElements/IndexTable/Fields/feedHits.ctp b/app/View/Elements/genericElements/IndexTable/Fields/feedHits.ctp new file mode 100644 index 000000000..a6c285810 --- /dev/null +++ b/app/View/Elements/genericElements/IndexTable/Fields/feedHits.ctp @@ -0,0 +1,87 @@ + h($feed['name']), + __('Provider') => h($feed['provider']), + ); + if (isset($feed['event_uuids'])) { + $relatedData[__('Event UUIDs')] = implode('
', array_map('h', $feed['event_uuids'])); + } + $popover = ''; + foreach ($relatedData as $k => $v) { + $popover .= '' . $k . ': ' . $v . '
'; + } + if ($isSiteAdmin || $hostOrgUser) { + if ($feed['source_format'] === 'misp') { + $liContents = sprintf( + '
%s%s
', + $baseurl, + h($feed['id']), + sprintf( + '', + h(json_encode($feed['event_uuids'])) + ), + sprintf( + '', + h($feed['id']), + h($popover) + ) + ); + } else { + $liContents = sprintf( + '%s', + $baseurl, + h($feed['id']), + h($popover), + h($feed['id']) + ); + } + } else { + $liContents = sprintf( + '%s', + h($feed['id']) + ); + } + echo "
  • $liContents
    • "; + } +} +if (isset($object['Server'])) { + foreach ($object['Server'] as $server) { + $popover = ''; + foreach ($server as $k => $v) { + if ($k == 'id') continue; + if (is_array($v)) { + foreach ($v as $k2 => $v2) { + $v[$k2] = h($v2); + } + $v = implode('
    ', $v); + } else { + $v = h($v); + } + $popover .= '' . Inflector::humanize(h($k)) . ': ' . $v . '
    '; + } + foreach ($server['event_uuids'] as $k => $event_uuid) { + $liContents = ''; + if ($isSiteAdmin) { + $liContents .= sprintf( + '%s ', + $baseurl, + h($server['id']), + h($event_uuid), + h($popover), + 'S' . h($server['id']) . ':' . ($k + 1) + ); + } else { + $liContents .= sprintf( + '%s', + 'S' . h($server['id']) . ':' . ($k + 1) + ); + } + echo "
  • $liContents
    • "; + } + } +} diff --git a/app/View/Elements/genericElements/IndexTable/Fields/relatedEvents.ctp b/app/View/Elements/genericElements/IndexTable/Fields/relatedEvents.ctp new file mode 100644 index 000000000..f0aa02506 --- /dev/null +++ b/app/View/Elements/genericElements/IndexTable/Fields/relatedEvents.ctp @@ -0,0 +1,18 @@ + $object['RelatedAttribute']); +} + +if (!empty($event['RelatedAttribute'][$object['id']])) { + echo '
      '; + echo $this->element('Events/View/attribute_correlations', array( + 'scope' => $field['data']['scope'], + 'object' => $object, + 'event' => $event, + )); + echo '
    '; +} diff --git a/app/View/Elements/genericElements/IndexTable/Fields/toIds.ctp b/app/View/Elements/genericElements/IndexTable/Fields/toIds.ctp new file mode 100644 index 000000000..033fab3a9 --- /dev/null +++ b/app/View/Elements/genericElements/IndexTable/Fields/toIds.ctp @@ -0,0 +1,13 @@ + + +
    +
    + > +
    \ No newline at end of file From 89a074cd5a3697f4b6ec5ee080fb84796213c13a Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Wed, 28 Jul 2021 16:50:22 +0200 Subject: [PATCH 04/78] chg: add sightings cols and actions. --- app/View/Attributes/index.ctp | 62 +++++++++++++++++++ .../IndexTable/Fields/sightings.ctp | 40 ++++++++++++ .../IndexTable/Fields/sightingsActivity.ctp | 9 +++ 3 files changed, 111 insertions(+) create mode 100644 app/View/Elements/genericElements/IndexTable/Fields/sightings.ctp create mode 100644 app/View/Elements/genericElements/IndexTable/Fields/sightingsActivity.ctp diff --git a/app/View/Attributes/index.ctp b/app/View/Attributes/index.ctp index e83a13171..bb0e87af5 100755 --- a/app/View/Attributes/index.ctp +++ b/app/View/Attributes/index.ctp @@ -116,12 +116,74 @@ echo $this->element('/genericElements/IndexTable/index_table', [ ], 'quickedit' => true ], + [ + 'name' => __('Sightings'), + 'element' => 'sightings', + 'class' => 'short', + 'data' => [ + 'object' => [ + 'value_path' => 'Attribute' + ], + ], + 'sightings' => $sightingsData + ], + [ + 'name' => __('Activity'), + 'element' => 'sightingsActivity', + 'class' => 'short', + 'data' => [ + 'object' => [ + 'value_path' => 'Attribute' + ], + ], + 'sightings' => $sightingsData + ], ], + 'actions' => array( + [ + 'url' => $baseurl . '/shadow_attributes/edit', + 'url_params_data_paths' => array( + 'Attribute.id' + ), + 'icon' => 'comment' + ], + [ + 'onclick' => "deleteObject('shadow_attributes', 'delete', '[onclick_params_data_path]');", + 'onclick_params_data_path' => 'Attribute.id', + 'icon' => 'trash', + 'title' => __('Propose deletion'), + 'requirement' => $isSiteAdmin, + ], + [ + 'url' => $baseurl . '/attributes', + 'icon' => 'grip-lines-vertical' + ], + [ + 'url' => $baseurl . '/attributes/edit', + 'url_params_data_paths' => array( + 'Attribute.id' + ), + 'icon' => 'edit' + ], + [ + 'onclick' => "deleteObject('attributes', 'delete', '[onclick_params_data_path]/true');", + 'onclick_params_data_path' => 'Attribute.id', + 'icon' => 'trash', + 'title' => __('Permanently delete attribute'), + 'requirement' => $isSiteAdmin, + ] + ) ] ]); echo ''; +// Generate form for adding sighting just once, generation for every attribute is surprisingly too slow +echo $this->Form->create('Sighting', ['id' => 'SightingForm', 'url' => $baseurl . '/sightings/add/', 'style' => 'display:none;']); +echo $this->Form->input('id', ['label' => false, 'type' => 'number']); +echo $this->Form->input('type', ['label' => false]); +echo $this->Form->end(); + $class = $isSearch == 1 ? 'searchAttributes2' : 'listAttributes'; echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event-collection', 'menuItem' => $class)); diff --git a/app/View/Elements/genericElements/IndexTable/Fields/sightings.ctp b/app/View/Elements/genericElements/IndexTable/Fields/sightings.ctp new file mode 100644 index 000000000..82eb462c0 --- /dev/null +++ b/app/View/Elements/genericElements/IndexTable/Fields/sightings.ctp @@ -0,0 +1,40 @@ + $typeData) { + $name = $type !== 'expiration' ? Inflector::pluralize($type) : $type; + $html .= '' . ucfirst(h($name)) . '
    '; + foreach ($typeData['orgs'] as $org => $orgData) { + $extra = $org === $me['Organisation']['name'] ? ' class="bold"' : ""; + if ($type == 'expiration') { + $html .= '' . h($org) . ': ' . $this->Time->time($orgData['date']) . '
    '; + } else { + $html .= '' . h($org) . ': ' . h($orgData['count']) . ' (' . $this->Time->time($orgData['date']) . ')
    '; + } + } + } + + $s = isset($objectSighting['sighting']['count']) ? intval($objectSighting['sighting']['count']) : 0; + $f = isset($objectSighting['false-positive']['count']) ? intval($objectSighting['false-positive']['count']) : 0; + $e = isset($objectSighting['expiration']['count']) ? intval($objectSighting['expiration']['count']) : 0; +} else { + $s = $f = $e = 0; +} + +if ($isAclSighting) : +?> +   +   +   + + + (//) + \ No newline at end of file diff --git a/app/View/Elements/genericElements/IndexTable/Fields/sightingsActivity.ctp b/app/View/Elements/genericElements/IndexTable/Fields/sightingsActivity.ctp new file mode 100644 index 000000000..9310349b0 --- /dev/null +++ b/app/View/Elements/genericElements/IndexTable/Fields/sightingsActivity.ctp @@ -0,0 +1,9 @@ +element('sparkline', array('scope' => 'object', 'id' => $objectId, 'csv' => $sightings['csv'][$objectId])); +} From 1b0a02e1a04a326f3732b7f5b7c24496d2fd75b0 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Thu, 29 Jul 2021 11:58:08 +0200 Subject: [PATCH 05/78] chg: add tags and galaxies col --- app/View/Attributes/index.ctp | 24 ++++--------- .../IndexTable/Fields/attributeGalaxies.ctp | 14 ++++++++ .../IndexTable/Fields/attributeTags.ctp | 36 +++++++++++++++++++ 3 files changed, 56 insertions(+), 18 deletions(-) create mode 100644 app/View/Elements/genericElements/IndexTable/Fields/attributeGalaxies.ctp create mode 100644 app/View/Elements/genericElements/IndexTable/Fields/attributeTags.ctp diff --git a/app/View/Attributes/index.ctp b/app/View/Attributes/index.ctp index bb0e87af5..b147f7b4e 100755 --- a/app/View/Attributes/index.ctp +++ b/app/View/Attributes/index.ctp @@ -10,7 +10,6 @@ echo $this->element('/genericElements/IndexTable/index_table', [ [ 'name' => __('Date'), 'sort' => 'Attribute.timestamp', - 'class' => 'short', 'element' => 'timestamp', 'time_format' => 'Y-m-d', 'data_path' => 'Attribute.timestamp' @@ -18,52 +17,47 @@ echo $this->element('/genericElements/IndexTable/index_table', [ [ 'name' => __('Event'), 'sort' => 'Attribute.event_id', - 'class' => 'short', 'data_path' => 'Attribute.event_id' ], [ 'name' => __('Org'), 'sort' => 'Event.Orgc.name', - 'class' => 'short', 'data_path' => 'Event.Orgc', 'element' => 'org' ], [ 'name' => __('Category'), 'sort' => 'Attribute.category', - 'class' => 'short', 'data_path' => 'Attribute.category' ], [ 'name' => __('Type'), 'sort' => 'Attribute.type', - 'class' => 'short', 'data_path' => 'Attribute.type' ], [ 'name' => __('Value'), 'sort' => 'Attribute.value', - 'class' => 'short', 'data_path' => 'Attribute.value' ], [ 'name' => __('Tags'), - 'class' => 'short', - 'data_path' => 'Attribute.AttributeTag' + 'element' => 'attributeTags', + 'data_path' => 'Attribute.AttributeTag', + 'class' => 'short' ], [ 'name' => __('Galaxies'), - 'class' => 'short', - 'data_path' => 'Attribute.Galaxy' + 'element' => 'attributeGalaxies', + 'data_path' => 'Attribute.Galaxy', + 'class' => 'short' ], [ 'name' => __('Comment'), - 'class' => 'shortish', 'data_path' => 'Attribute.comment' ], [ 'name' => __('Correlate'), - 'class' => 'short', 'element' => 'correlate', 'data' => [ 'object' => [ @@ -73,7 +67,6 @@ echo $this->element('/genericElements/IndexTable/index_table', [ ], [ 'name' => __('Related Events'), - 'class' => 'short', 'element' => 'relatedEvents', 'data' => [ 'object' => [ @@ -84,7 +77,6 @@ echo $this->element('/genericElements/IndexTable/index_table', [ ], [ 'name' => __('Feed hits'), - 'class' => 'short', 'element' => 'feedHits', 'data' => [ 'object' => [ @@ -94,7 +86,6 @@ echo $this->element('/genericElements/IndexTable/index_table', [ ], [ 'name' => __('IDS'), - 'class' => 'short', 'element' => 'toIds', 'data' => [ 'object' => [ @@ -105,7 +96,6 @@ echo $this->element('/genericElements/IndexTable/index_table', [ [ 'name' => __('Distribution'), 'element' => 'distribution_levels', - 'class' => 'short', 'data_path' => 'Attribute.distribution', 'distributionLevels' => $distributionLevels, 'data' => [ @@ -119,7 +109,6 @@ echo $this->element('/genericElements/IndexTable/index_table', [ [ 'name' => __('Sightings'), 'element' => 'sightings', - 'class' => 'short', 'data' => [ 'object' => [ 'value_path' => 'Attribute' @@ -130,7 +119,6 @@ echo $this->element('/genericElements/IndexTable/index_table', [ [ 'name' => __('Activity'), 'element' => 'sightingsActivity', - 'class' => 'short', 'data' => [ 'object' => [ 'value_path' => 'Attribute' diff --git a/app/View/Elements/genericElements/IndexTable/Fields/attributeGalaxies.ctp b/app/View/Elements/genericElements/IndexTable/Fields/attributeGalaxies.ctp new file mode 100644 index 000000000..a68099fad --- /dev/null +++ b/app/View/Elements/genericElements/IndexTable/Fields/attributeGalaxies.ctp @@ -0,0 +1,14 @@ +element('galaxyQuickViewNew', array( + 'mayModify' => $mayModify, + 'isAclTagger' => $isAclTagger, + 'data' => (!empty($attribute['Galaxy']) ? $attribute['Galaxy'] : array()), + 'event' => $event, + 'target_id' => $attribute['id'], + 'target_type' => 'attribute', +)); diff --git a/app/View/Elements/genericElements/IndexTable/Fields/attributeTags.ctp b/app/View/Elements/genericElements/IndexTable/Fields/attributeTags.ctp new file mode 100644 index 000000000..36f5265b5 --- /dev/null +++ b/app/View/Elements/genericElements/IndexTable/Fields/attributeTags.ctp @@ -0,0 +1,36 @@ + + +
    + element( + 'ajaxTags', + array( + 'attributeId' => $attribute['id'], + 'tags' => $attribute['AttributeTag'], + 'tagAccess' => ($isSiteAdmin || $mayModify), + 'localTagAccess' => ($isSiteAdmin || $mayModify || $me['org_id'] == $event['Event']['org_id'] || (int)$me['org_id'] === Configure::read('MISP.host_org_id')), + 'context' => 'event', + 'scope' => 'attribute', + 'tagConflicts' => isset($attribute['tagConflicts']) ? $attribute['tagConflicts'] : array() + ) + ); ?> +
    + +element('ajaxAttributeTags', array('attributeId' => $attribute['id'], 'attributeTags' => $attribute['RelatedTags'], 'tagAccess' => false)); + } + echo sprintf( + '
    %s
    ', + 'class="attributeRelatedTagContainer" id="#Attribute_' . $objectId . 'Related_tr .attributeTagContainer"', + $element + ); +} From 7c85d3a1739bee7d77d33f8441d3fa28528b458f Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Thu, 29 Jul 2021 15:59:26 +0200 Subject: [PATCH 06/78] chg: add missing action buttons. --- app/Controller/AttributesController.php | 5 +- app/View/Attributes/index.ctp | 102 ++++++++++++++++-- .../IndexTable/Fields/attributeTags.ctp | 2 +- app/webroot/js/misp.js | 2 +- 4 files changed, 95 insertions(+), 16 deletions(-) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index d8e6f9ef2..4b71924a1 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -93,8 +93,6 @@ class AttributesController extends AppController } } - $distributionLevels = $this->Attribute->distributionLevels; - list($attributes, $sightingsData) = $this->__searchUI($attributes); $this->set('sightingsData', $sightingsData); $this->set('orgTable', array_column($orgTable, 'name', 'id')); @@ -103,7 +101,7 @@ class AttributesController extends AppController $this->set('attrDescriptions', $this->Attribute->fieldDescriptions); $this->set('typeDefinitions', $this->Attribute->typeDefinitions); $this->set('categoryDefinitions', $this->Attribute->categoryDefinitions); - $this->set('distributionLevels', $distributionLevels); + $this->set('distributionLevels', $this->Attribute->distributionLevels); } public function add($eventId = false) @@ -1652,6 +1650,7 @@ class AttributesController extends AppController $this->set('isSearch', 1); $this->set('attrDescriptions', $this->Attribute->fieldDescriptions); $this->set('shortDist', $this->Attribute->shortDist); + $this->set('distributionLevels', $this->Attribute->distributionLevels); $this->render('index'); } if (isset($attributeTags)) { diff --git a/app/View/Attributes/index.ctp b/app/View/Attributes/index.ctp index b147f7b4e..837118849 100755 --- a/app/View/Attributes/index.ctp +++ b/app/View/Attributes/index.ctp @@ -1,7 +1,9 @@ '; +$modules = isset($modules) ? $modules : null; +$cortex_modules = isset($cortex_modules) ? $cortex_modules : null; +echo '
    '; echo $this->element('/genericElements/IndexTable/index_table', [ 'data' => [ 'title' => __('Attributes'), @@ -43,13 +45,11 @@ echo $this->element('/genericElements/IndexTable/index_table', [ [ 'name' => __('Tags'), 'element' => 'attributeTags', - 'data_path' => 'Attribute.AttributeTag', 'class' => 'short' ], [ 'name' => __('Galaxies'), 'element' => 'attributeGalaxies', - 'data_path' => 'Attribute.Galaxy', 'class' => 'short' ], [ @@ -127,12 +127,12 @@ echo $this->element('/genericElements/IndexTable/index_table', [ 'sightings' => $sightingsData ], ], - 'actions' => array( + 'actions' => [ [ 'url' => $baseurl . '/shadow_attributes/edit', - 'url_params_data_paths' => array( + 'url_params_data_paths' => [ 'Attribute.id' - ), + ], 'icon' => 'comment' ], [ @@ -143,24 +143,104 @@ echo $this->element('/genericElements/IndexTable/index_table', [ 'requirement' => $isSiteAdmin, ], [ - 'url' => $baseurl . '/attributes', + 'title' => __('Propose enrichment'), + 'icon' => 'asterisk', + 'onclick' => 'simplePopup(\'' . $baseurl . '/events/queryEnrichment/[onclick_params_data_path]/ShadowAttribute\');', + 'onclick_params_data_path' => 'Attribute.id', + 'complex_requirement' => [ + 'function' => function ($object) use ($modules) { + return isset($modules) && isset($object['Attribute']['type']); + }, + 'options' => [ + 'datapath' => [ + 'type' => 'Attribute.type' + ] + ], + ], + ], + [ + 'title' => __('Propose enrichment through Cortex'), + 'icon' => 'eye', + 'onclick' => 'simplePopup(\'' . $baseurl . '/events/queryEnrichment/[onclick_params_data_path]/ShadowAttribute/Cortex\');', + 'onclick_params_data_path' => 'Attribute.id', + 'complex_requirement' => [ + 'function' => function ($object) use ($cortex_modules) { + return isset($cortex_modules) && isset($cortex_modules['types'][$object['type']]); + }, + 'options' => [ + 'datapath' => [ + 'type' => 'Attribute.type' + ] + ], + ], + ], + [ 'icon' => 'grip-lines-vertical' ], + [ + 'title' => __('Add enrichment'), + 'icon' => 'asterisk', + 'onclick' => 'simplePopup(\'' . $baseurl . '/events/queryEnrichment/[onclick_params_data_path]/Attribute\');', + 'onclick_params_data_path' => 'Attribute.id', + 'complex_requirement' => [ + 'function' => function ($object) use ($modules) { + return isset($modules) && isset($object['Attribute']['type']); + }, + 'options' => [ + 'datapath' => [ + 'type' => 'Attribute.type' + ] + ], + ], + ], + [ + 'title' => __('Add enrichment via Cortex'), + 'icon' => 'eye', + 'onclick' => 'simplePopup(\'' . $baseurl . '/events/queryEnrichment/[onclick_params_data_path]/Attribute/Cortex\');', + 'onclick_params_data_path' => 'Attribute.id', + 'complex_requirement' => [ + 'function' => function ($object) use ($cortex_modules) { + return isset($cortex_modules) && isset($cortex_modules['types'][$object['type']]); + }, + 'options' => [ + 'datapath' => [ + 'type' => 'Attribute.type' + ] + ], + ], + ], [ 'url' => $baseurl . '/attributes/edit', - 'url_params_data_paths' => array( + 'url_params_data_paths' => [ 'Attribute.id' - ), + ], 'icon' => 'edit' ], + [ + 'onclick' => "deleteObject('attributes', 'delete', '[onclick_params_data_path]');", + 'onclick_params_data_path' => 'Attribute.id', + 'icon' => 'trash', + 'title' => __('Soft delete attribute'), + 'requirement' => $isSiteAdmin, + 'complex_requirement' => [ + 'function' => function ($object) { + return !empty($object['Event']['publish_timestamp']); + }, + ] + ], [ 'onclick' => "deleteObject('attributes', 'delete', '[onclick_params_data_path]/true');", 'onclick_params_data_path' => 'Attribute.id', 'icon' => 'trash', 'title' => __('Permanently delete attribute'), 'requirement' => $isSiteAdmin, + 'complex_requirement' => [ + 'function' => function ($object) { + return empty($object['Event']['publish_timestamp']); + }, + ] ] - ) + ] ] ]); @@ -173,7 +253,7 @@ echo $this->Form->input('type', ['label' => false]); echo $this->Form->end(); $class = $isSearch == 1 ? 'searchAttributes2' : 'listAttributes'; -echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event-collection', 'menuItem' => $class)); +echo $this->element('/genericElements/SideMenu/side_menu', ['menuList' => 'event-collection', 'menuItem' => $class]); ?> diff --git a/app/View/Elements/genericElements/IndexTable/Fields/attributeTags.ctp b/app/View/Elements/genericElements/IndexTable/Fields/attributeTags.ctp index 36f5265b5..c12f24afc 100644 --- a/app/View/Elements/genericElements/IndexTable/Fields/attributeTags.ctp +++ b/app/View/Elements/genericElements/IndexTable/Fields/attributeTags.ctp @@ -7,7 +7,7 @@ $objectId = intval($attribute['id']); ?> -
    +
    element( 'ajaxTags', array( diff --git a/app/webroot/js/misp.js b/app/webroot/js/misp.js index 8900308a3..106243015 100644 --- a/app/webroot/js/misp.js +++ b/app/webroot/js/misp.js @@ -1127,7 +1127,7 @@ function loadAttributeTags(id) { dataType:"html", cache: false, success:function (data) { - $("#Attribute_"+id+"_tr .attributeTagContainer").html(data); + $("#Attribute_"+id+".attributeTagContainer").html(data); }, error: xhrFailCallback, url: baseurl + "/tags/showAttributeTag/" + id From f73a1e710e1465894e77d7109d77463d4e094b00 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Mon, 9 Aug 2021 16:36:07 +0200 Subject: [PATCH 07/78] chg: migrate news views to factory. --- app/Controller/NewsController.php | 29 +++----- app/View/News/add.ctp | 79 +++++++++++---------- app/View/News/edit.ctp | 40 ----------- app/View/News/index.ctp | 113 ++++++++++++++++-------------- 4 files changed, 112 insertions(+), 149 deletions(-) delete mode 100644 app/View/News/edit.ctp diff --git a/app/Controller/NewsController.php b/app/Controller/NewsController.php index 2601a3edd..e7ce8c257 100755 --- a/app/Controller/NewsController.php +++ b/app/Controller/NewsController.php @@ -1,16 +1,16 @@ 5, - 'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page. - 'order' => array( - 'News.id' => 'DESC' - ), + 'limit' => 5, + 'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page. + 'order' => array( + 'News.id' => 'DESC' + ), ); public function index() @@ -69,22 +69,15 @@ class NewsController extends AppController $this->request->data = $this->News->read(null, $id); $this->set('newsItem', $this->request->data); } + $this->render('add'); } public function delete($id) { - if (!$this->request->is('post')) { - throw new MethodNotAllowedException(); + $this->defaultModel = 'News'; + $this->CRUD->delete($id); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; } - $this->News->id = $id; - if (!$this->News->exists()) { - throw new NotFoundException('Invalid news item'); - } - if ($this->News->delete()) { - $this->Flash->success(__('News item deleted.')); - $this->redirect(array('action' => 'index')); - } - $this->Flash->error(__('News item could not be deleted.')); - $this->redirect(array('action' => 'index')); } } diff --git a/app/View/News/add.ctp b/app/View/News/add.ctp index 4d1ea7661..376f5505d 100755 --- a/app/View/News/add.ctp +++ b/app/View/News/add.ctp @@ -1,39 +1,42 @@ -
    Form->create('News'); -?> -
    - - Form->input('title', array( - 'type' => 'text', - 'error' => array('escape' => false), - 'div' => 'input clear', - 'class' => 'input-xxlarge' - )); - ?> -
    - Form->input('message', array( - 'type' => 'textarea', - 'error' => array('escape' => false), - 'div' => 'input clear', - 'class' => 'input-xxlarge' - )); - ?> -
    - Form->input('anonymise', array( - 'checked' => false, - 'label' => __('Create anonymously'), - )); - ?> -
    - Form->button(__('Submit'), array('class' => 'btn btn-primary')); - echo $this->Form->end(); - ?> -
    -element('/genericElements/SideMenu/side_menu', array('menuList' => 'news', 'menuItem' => 'add')); -?> +$edit = $this->request->params['action'] === 'edit' ? true : false; +echo $this->element( + '/genericElements/SideMenu/side_menu', + [ + 'menuList' => 'news', + 'menuItem' => $edit ? 'edit' : 'add' + ] +); + +echo $this->element('genericElements/Form/genericForm', [ + 'data' => [ + 'title' => $edit ? __('Edit News Item') : __('Add News Item'), + 'fields' => [ + [ + 'field' => 'title', + 'label' => __('Title'), + 'type' => 'text', + 'error' => ['escape' => false], + 'div' => 'input clear', + 'class' => 'input-xxlarge', + ], + [ + 'field' => 'message', + 'label' => __('Message'), + 'type' => 'textarea', + 'error' => ['escape' => false], + 'div' => 'input clear', + 'class' => 'input-xxlarge' + ], + [ + 'field' => 'anonymise', + 'label' => __('Create anonymously'), + 'type' => 'checkbox', + ], + ], + 'submit' => [ + 'action' => $this->request->params['action'], + 'ajaxSubmit' => 'submitGenericFormInPlace();' + ] + ] +]); diff --git a/app/View/News/edit.ctp b/app/View/News/edit.ctp deleted file mode 100644 index 2f4d0e8b5..000000000 --- a/app/View/News/edit.ctp +++ /dev/null @@ -1,40 +0,0 @@ -
    -Form->create('News'); -?> -
    - - Form->input('title', array( - 'type' => 'text', - 'error' => array('escape' => false), - 'div' => 'input clear', - 'class' => 'input-xxlarge' - )); - ?> -
    - Form->input('message', array( - 'type' => 'textarea', - 'error' => array('escape' => false), - 'div' => 'input clear', - 'class' => 'input-xxlarge' - )); - ?> -
    - Form->input('anonymise', array( - 'type' => 'checkbox', - 'checked' => $newsItem['News']['user_id'] == 0, - 'label' => __('Create anonymously'), - )); - ?> -
    - Form->button(__('Submit'), array('class' => 'btn btn-primary')); - echo $this->Form->end(); - ?> -
    -element('/genericElements/SideMenu/side_menu', array('menuList' => 'news', 'menuItem' => 'edit')); -?> diff --git a/app/View/News/index.ctp b/app/View/News/index.ctp index 6b9c8b698..cfd9ea19f 100644 --- a/app/View/News/index.ctp +++ b/app/View/News/index.ctp @@ -1,54 +1,61 @@ -
    -

    -
    - -
    -
    -
    -
    -
    -
    Time->time($newsItem['News']['date_created']); ?>
    -
    -
    -
    -

    - $0', $message)); - if ($isSiteAdmin): - ?> -
    - Form->postLink('', array('action' => 'delete', $newsItem['News']['id']), array('class' => 'fa fa-trash', 'title' => __('Delete'), 'aria-label' => __('Delete')), __('Are you sure you want to delete news item #%s?', $newsItem['News']['id'])); - endif; - ?> -
    -
    -
    - Paginator->counter(array( - 'format' => __('Page {:page} of {:pages}, showing {:current} records out of {:count} total, starting on record {:start}, ending on {:end}') - )); - ?> - - -
    -
    element('/genericElements/SideMenu/side_menu', array('menuList' => 'news', 'menuItem' => 'index')); -?> + +$this->set('menuData', ['menuList' => 'news', 'menuItem' => 'index']); + +echo $this->element('genericElements/IndexTable/scaffold', [ + 'scaffold_data' => [ + 'data' => [ + 'data' => $newsItems, + 'fields' => [ + [ + 'name' => __('Id'), + 'sort' => 'id', + 'data_path' => 'News.id' + ], + [ + 'name' => __('User'), + 'sort' => 'User', + 'data_path' => 'User.email' + ], + [ + 'name' => __('Title'), + 'sort' => 'News.title', + 'data_path' => 'News.title' + ], + [ + 'name' => __('Message'), + 'sort' => 'Message', + 'data_path' => 'News.message' + ], + [ + 'name' => __('Created at'), + 'sort' => 'date_created', + 'data_path' => 'News.date_created', + 'element' => 'datetime' + ], + + ], + 'title' => empty($ajax) ? __('News') : false, + 'pull' => 'right', + 'actions' => [ + [ + 'url' => $baseurl . '/news/edit', + 'url_params_data_paths' => [ + 'News.id' + ], + 'icon' => 'edit', + 'title' => 'Edit News', + ], + [ + 'onclick' => sprintf( + 'openGenericModal(\'%s/news/delete/[onclick_params_data_path]\');', + $baseurl + ), + 'onclick_params_data_path' => 'News.id', + 'icon' => 'trash', + 'title' => __('Delete news'), + ] + ] + ] + ] +]); From ca0e54d341732e946d7e4faf4744e5858aa05a0b Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Tue, 10 Aug 2021 12:12:28 +0200 Subject: [PATCH 08/78] fix: incorrect sort keys --- app/View/News/index.ctp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/View/News/index.ctp b/app/View/News/index.ctp index cfd9ea19f..74ebd33bd 100644 --- a/app/View/News/index.ctp +++ b/app/View/News/index.ctp @@ -14,17 +14,17 @@ echo $this->element('genericElements/IndexTable/scaffold', [ ], [ 'name' => __('User'), - 'sort' => 'User', + 'sort' => 'email', 'data_path' => 'User.email' ], [ 'name' => __('Title'), - 'sort' => 'News.title', + 'sort' => 'title', 'data_path' => 'News.title' ], [ 'name' => __('Message'), - 'sort' => 'Message', + 'sort' => 'message', 'data_path' => 'News.message' ], [ From 66a918245bcf499c0379566cd797481c417b5b02 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Wed, 11 Aug 2021 09:38:57 +0200 Subject: [PATCH 09/78] fix: [API] Return correct error message if event is blocklisted --- app/Controller/EventsController.php | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 93807891f..642bc47f9 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -1993,9 +1993,6 @@ class EventsController extends AppController public function add() { - if (!$this->userRole['perm_add']) { - throw new MethodNotAllowedException(__('You do not have permissions to create events.')); - } $sgs = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1); if ($this->request->is('post')) { if ($this->_isRest()) { @@ -2065,11 +2062,8 @@ class EventsController extends AppController $validationErrors = array(); $created_id = 0; $add = $this->Event->_add($this->request->data, $this->_isRest(), $this->Auth->user(), '', null, false, null, $created_id, $validationErrors); - if ($add === true && !is_numeric($add)) { + if ($add === true) { if ($this->_isRest()) { - if ($add === 'blocked') { - throw new ForbiddenException(__('Event blocked by local blocklist.')); - } // REST users want to see the newly created event $metadata = $this->request->param('named.metadata'); $results = $this->Event->fetchEvent($this->Auth->user(), ['eventid' => $created_id, 'metadata' => $metadata]); @@ -2090,6 +2084,15 @@ class EventsController extends AppController $this->response->send(); throw new NotFoundException(__('Event already exists, if you would like to edit it, use the url in the location header.')); } + + if ($add === 'blocked') { + throw new ForbiddenException(__('Event blocked by organisation blocklist.')); + } else if ($add === 'Blocked by blocklist') { + throw new ForbiddenException(__('Event blocked by event blocklist.')); + } else if ($add === 'Blocked by event block rules') { + throw new ForbiddenException(__('Blocked by event block rules.')); + } + // # TODO i18n? return $this->RestResponse->saveFailResponse('Events', 'add', false, $validationErrors, $this->response->type()); } else { @@ -2132,9 +2135,9 @@ class EventsController extends AppController } // combobox for risks - $threat_levels = $this->Event->ThreatLevel->find('all'); - $this->set('threatLevels', Set::combine($threat_levels, '{n}.ThreatLevel.id', '{n}.ThreatLevel.name')); - $fieldDesc['threat_level_id'] = Set::combine($threat_levels, '{n}.ThreatLevel.id', '{n}.ThreatLevel.description'); + $threat_levels = array_column($this->Event->ThreatLevel->find('all'), 'ThreatLevel'); + $this->set('threatLevels', array_column($threat_levels, 'name', 'id')); + $fieldDesc['threat_level_id'] = array_column($threat_levels, 'description', 'id'); // combobox for analysis $this->set('sharingGroups', $sgs); @@ -2144,9 +2147,7 @@ class EventsController extends AppController foreach ($analysisLevels as $key => $value) { $fieldDesc['analysis'][$key] = $this->Event->analysisDescriptions[$key]['formdesc']; } - if (!$this->_isRest()) { - $this->Flash->info(__('The event created will be visible to the organisations having an account on this platform, but not synchronised to other MISP instances until it is published.')); - } + $this->Flash->info(__('The event created will be visible to the organisations having an account on this platform, but not synchronised to other MISP instances until it is published.')); $this->set('fieldDesc', $fieldDesc); if (isset($this->params['named']['extends'])) { $this->set('extends_uuid', $this->params['named']['extends']); From 4ef3d8889514c59f1d4bd42df422caa0a4a72695 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Tue, 31 Aug 2021 15:06:02 +0200 Subject: [PATCH 10/78] fix: [attribute] Use `filename-pattern` --- app/Model/Attribute.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index cc7c1e7f3..3f5ba17e4 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -4420,7 +4420,7 @@ class Attribute extends AppModel 'pattern-in-file' => array('desc' => __('Pattern in file that identifies the malware'), 'default_category' => 'Payload installation', 'to_ids' => 1), 'pattern-in-traffic' => array('desc' => __('Pattern in network traffic that identifies the malware'), 'default_category' => 'Network activity', 'to_ids' => 1), 'pattern-in-memory' => array('desc' => __('Pattern in memory dump that identifies the malware'), 'default_category' => 'Payload installation', 'to_ids' => 1), - 'pattern-filename' => array('desc' => __('A pattern in the name of a file'), 'default_category' => 'Payload installation', 'to_ids' => 1), + 'filename-pattern' => array('desc' => __('A pattern in the name of a file'), 'default_category' => 'Payload installation', 'to_ids' => 1), 'pgp-public-key' => array('desc' => __('A PGP public key'), 'default_category' => 'Person', 'to_ids' => 0), 'pgp-private-key' => array('desc' => __('A PGP private key'), 'default_category' => 'Person', 'to_ids' => 0), 'yara' => array('desc' => __('Yara signature'), 'default_category' => 'Payload installation', 'to_ids' => 1), From 96d2199726d0294701e1a48c8f4992914c5d006a Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Fri, 8 Oct 2021 12:33:09 +0200 Subject: [PATCH 11/78] fix: [internal] Server save setting file --- app/Console/Command/AdminShell.php | 24 ++++---- app/Lib/Tools/FileAccessTool.php | 40 +++++++++--- app/Model/Server.php | 98 +++++++++++++----------------- 3 files changed, 86 insertions(+), 76 deletions(-) diff --git a/app/Console/Command/AdminShell.php b/app/Console/Command/AdminShell.php index fbda66383..96d1a78c0 100644 --- a/app/Console/Command/AdminShell.php +++ b/app/Console/Command/AdminShell.php @@ -397,20 +397,18 @@ class AdminShell extends AppShell $cli_user = array('id' => 0, 'email' => 'SYSTEM', 'Organisation' => array('name' => 'SYSTEM')); if (empty($setting_name) || $value === null) { die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Set setting'] . PHP_EOL); + } + $setting = $this->Server->getSettingData($setting_name); + if (empty($setting)) { + $message = 'Invalid setting "' . $setting_name . '". Please make sure that the setting that you are attempting to change exists and if a module parameter, the modules are running.' . PHP_EOL; + $this->error(__('Setting change rejected.'), $message); + } + $result = $this->Server->serverSettingsEditValue($cli_user, $setting, $value, $this->params['force']); + if ($result === true) { + echo 'Setting "' . $setting_name . '" changed to ' . $value . PHP_EOL; } else { - $setting = $this->Server->getSettingData($setting_name); - if (empty($setting)) { - $message = 'Invalid setting "' . $setting_name . '". Please make sure that the setting that you are attempting to change exists and if a module parameter, the modules are running.' . PHP_EOL; - $this->error(__('Setting change rejected.'), $message); - } - $result = $this->Server->serverSettingsEditValue($cli_user, $setting, $value, $this->params['force']); - if ($result === true) { - echo 'Setting "' . $setting_name . '" changed to ' . $value . PHP_EOL; - } else { - $message = __("The setting change was rejected. MISP considers the requested setting value as invalid and would lead to the following error:\n\n\"%s\"\n\nIf you still want to force this change, please supply the --force argument.\n", $result); - $this->error(__('Setting change rejected.'), $message); - } - echo PHP_EOL; + $message = __("The setting change was rejected. MISP considers the requested setting value as invalid and would lead to the following error:\n\n\"%s\"\n\nIf you still want to force this change, please supply the --force argument.\n", $result); + $this->error(__('Setting change rejected.'), $message); } } diff --git a/app/Lib/Tools/FileAccessTool.php b/app/Lib/Tools/FileAccessTool.php index 218b064ae..9a2216680 100644 --- a/app/Lib/Tools/FileAccessTool.php +++ b/app/Lib/Tools/FileAccessTool.php @@ -4,13 +4,16 @@ class FileAccessTool { /** * Creates temporary file, but you have to delete it after use. - * @param string $dir + * @param string|null $dir * @param string $prefix * @return string * @throws Exception */ - public static function createTempFile($dir, $prefix = 'MISP') + public static function createTempFile($dir = null, $prefix = 'MISP') { + if ($dir === null) { + $dir = Configure::read('MISP.tmpdir') ?: sys_get_temp_dir(); + } $tempFile = tempnam($dir, $prefix); if ($tempFile === false) { throw new Exception("An error has occurred while attempt to create a temporary file in path `$dir`."); @@ -26,8 +29,11 @@ class FileAccessTool */ public static function readFromFile($file, $fileSize = -1) { - $fileSize = $fileSize === -1 ? null : $fileSize; - $content = file_get_contents($file, false, null, 0, $fileSize); + if ($fileSize === -1) { + $content = file_get_contents($file); + } else { + $content = file_get_contents($file, false, null, 0, $fileSize); + } if ($content === false) { throw new Exception("An error has occurred while attempt to read file `$file`."); } @@ -37,15 +43,35 @@ class FileAccessTool /** * @param string $file * @param mixed $content + * @param int $flags * @throws Exception */ - public static function writeToFile($file, $content) + public static function writeToFile($file, $content, $flags = LOCK_EX) { - if (file_put_contents($file, $content, LOCK_EX) === false) { - throw new Exception("An error has occurred while attempt to write to file `$file`."); + if (file_put_contents($file, $content, $flags) === false) { + $freeSpace = disk_free_space($file); + throw new Exception("An error has occurred while attempt to write to file `$file`. Maybe not enough space? ($freeSpace bytes left)"); } } + /** + * @param mixed $content + * @param string|null $dir + * @return string Path to temp file + * @throws Exception + */ + public static function writeToTempFile($content, $dir = null) + { + $tempFile = self::createTempFile($dir); + try { + self::writeToFile($tempFile, $content, 0); // Lock is not need + } catch (Exception $e) { + self::deleteFile($tempFile); + throw $e; + } + return $tempFile; + } + /** * @param string $file * @param mixed $content diff --git a/app/Model/Server.php b/app/Model/Server.php index 697c2c190..233edf838 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -2,6 +2,7 @@ App::uses('AppModel', 'Model'); App::uses('GpgTool', 'Tools'); App::uses('ServerSyncTool', 'Tools'); +App::uses('FileAccessTool', 'Tools'); /** * @property-read array $serverSettings @@ -2141,11 +2142,10 @@ class Server extends AppModel } } $value = trim($value); - if ($setting['type'] == 'boolean') { - $value = ($value ? true : false); - } - if ($setting['type'] == 'numeric') { - $value = intval($value); + if ($setting['type'] === 'boolean') { + $value = (bool)$value; + } else if ($setting['type'] === 'numeric') { + $value = (int)($value); } if (!empty($setting['test'])) { $testResult = $this->{$setting['test']}($value); @@ -2159,27 +2159,25 @@ class Server extends AppModel $errorMessage = $testResult; } return $errorMessage; - } else { - $oldValue = Configure::read($setting['name']); - $settingSaveResult = $this->serverSettingsSaveValue($setting['name'], $value); - if ($settingSaveResult) { - $this->Log = ClassRegistry::init('Log'); - $change = array($setting['name'] => array($oldValue, $value)); - $this->Log->createLogEntry($user, 'serverSettingsEdit', 'Server', 0, 'Server setting changed', $change); + } + $oldValue = Configure::read($setting['name']); + $settingSaveResult = $this->serverSettingsSaveValue($setting['name'], $value); + if ($settingSaveResult) { + $change = array($setting['name'] => array($oldValue, $value)); + $this->loadLog()->createLogEntry($user, 'serverSettingsEdit', 'Server', 0, 'Server setting changed', $change); - // execute after hook - if (isset($setting['afterHook'])) { - $afterResult = call_user_func_array(array($this, $setting['afterHook']), array($setting['name'], $value)); - if ($afterResult !== true) { - $change = 'There was an issue after setting a new setting. The error message returned is: ' . $afterResult; - $this->Log->createLogEntry($user, 'serverSettingsEdit', 'Server', 0, 'Server setting issue', $change); - return $afterResult; - } + // execute after hook + if (isset($setting['afterHook'])) { + $afterResult = call_user_func_array(array($this, $setting['afterHook']), array($setting['name'], $value)); + if ($afterResult !== true) { + $change = 'There was an issue after setting a new setting. The error message returned is: ' . $afterResult; + $this->loadLog()->createLogEntry($user, 'serverSettingsEdit', 'Server', 0, 'Server setting issue', $change); + return $afterResult; } - return true; - } else { - return __('Something went wrong. MISP tried to save a malformed config file. Setting change reverted.'); } + return true; + } else { + return __('Something went wrong. MISP tried to save a malformed config file. Setting change reverted.'); } } @@ -2197,26 +2195,18 @@ class Server extends AppModel } // validate if current config.php is intact: - $current = file_get_contents($configFilePath); - $current = trim($current); - if (strlen($current) < 20) { - $this->Log = ClassRegistry::init('Log'); - $this->Log->create(); - $this->Log->save(array( - 'org' => 'SYSTEM', - 'model' => 'Server', - 'model_id' => 0, - 'email' => 'SYSTEM', - 'action' => 'error', - 'user_id' => 0, - 'title' => 'Error: Tried to modify server settings but current config is broken.', - )); + $current = FileAccessTool::readFromFile($configFilePath); + if (strlen(trim($current)) < 20) { + $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', 0, 'Error: Tried to modify server settings but current config is broken.'); return false; } $safeConfigChanges = empty(Configure::read('MISP.server_settings_skip_backup_rotate')); if ($safeConfigChanges) { + $backupFilePath = APP . 'Config' . DS . 'config.backup.php'; // Create current config file backup - copy($configFilePath, APP . 'Config' . DS . 'config.php.bk'); + if (!copy($configFilePath, $backupFilePath)) { + throw new Exception("Could not create config backup `$backupFilePath`."); + } } $settingObject = $this->getCurrentServerSettings(); foreach ($settingObject as $branchName => $branch) { @@ -2263,36 +2253,32 @@ class Server extends AppModel if ($safeConfigChanges) { $previous_file_perm = substr(sprintf('%o', fileperms($configFilePath)), -4); - $randomFilename = $this->generateRandomFileName(); - // To protect us from 2 admin users having a concurrent file write to the config file, solar flares and the bogeyman - if (file_put_contents(APP . 'Config' . DS . $randomFilename, $settingsString) === false) { + try { + $tmpFile = FileAccessTool::writeToTempFile($settingsString); + } catch (Exception $e) { + $this->logException('Could not create temp config file.', $e); $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', 0, 'Error: Could not create temp config file.'); return false; } - rename(APP . 'Config' . DS . $randomFilename, $configFilePath); + if (!rename($tmpFile, $configFilePath)) { + FileAccessTool::deleteFile($tmpFile); + throw new Exception("Could not rename `$tmpFile` to config file `$configFilePath`."); + } if (function_exists('opcache_reset')) { opcache_reset(); } chmod($configFilePath, octdec($previous_file_perm)); - $config_saved = file_get_contents($configFilePath); + $config_saved = FileAccessTool::readFromFile($configFilePath); // if the saved config file is empty, restore the backup. if (strlen($config_saved) < 20) { - copy(APP . 'Config' . DS . 'config.php.bk', $configFilePath); - $this->Log = ClassRegistry::init('Log'); - $this->Log->create(); - $this->Log->save(array( - 'org' => 'SYSTEM', - 'model' => 'Server', - 'model_id' => 0, - 'email' => 'SYSTEM', - 'action' => 'error', - 'user_id' => 0, - 'title' => 'Error: Something went wrong saving the config file, reverted to backup file.', - )); + rename($backupFilePath, $configFilePath); + $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', 0, 'Error: Something went wrong saving the config file, reverted to backup file.'); return false; + } else { + FileAccessTool::deleteFile($backupFilePath); } } else { - file_put_contents($configFilePath, $settingsString); + FileAccessTool::writeToFile($configFilePath, $settingsString); if (function_exists('opcache_reset')) { opcache_reset(); } From e241985b37e672e003c9b0cc09ec56355d530b98 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Fri, 8 Oct 2021 16:35:57 +0200 Subject: [PATCH 12/78] chg: [misp-stix] Update --- app/files/scripts/misp-stix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/scripts/misp-stix b/app/files/scripts/misp-stix index 83d9300ed..be3f694cd 160000 --- a/app/files/scripts/misp-stix +++ b/app/files/scripts/misp-stix @@ -1 +1 @@ -Subproject commit 83d9300edeb94ac98fad6f8feb3e2640f583bb59 +Subproject commit be3f694cd46ac26619ed8e8eaa73f45b2f62d7df From 7ce3916be9a8e410034458c1da6fe6e70587688a Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Fri, 8 Oct 2021 16:40:49 +0200 Subject: [PATCH 13/78] chg: [PyMISP] Update --- PyMISP | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyMISP b/PyMISP index e07321bfa..506410709 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit e07321bfa90d2259e1489773cf0b8769cdcf675c +Subproject commit 506410709304cc7832b364228c52572681a2c603 From b496161f5bf2a7f15ce52cf0dec62a52fc9d713e Mon Sep 17 00:00:00 2001 From: MrBoba <2mr.boba@gmail.com> Date: Fri, 8 Oct 2021 17:14:02 +0000 Subject: [PATCH 14/78] fix: [internal] withCredentials property was added into $.ajaxSetup() to get rid of 403 and 302 responses --- app/webroot/js/misp.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/webroot/js/misp.js b/app/webroot/js/misp.js index d0a0795e9..35475cb2e 100644 --- a/app/webroot/js/misp.js +++ b/app/webroot/js/misp.js @@ -5379,3 +5379,13 @@ $('td.rotate').hover(function() { var t = parseInt($(this).index()) + 1; $table.find('td:nth-child(' + t + ')').css('background-color', ''); }); + +$.ajaxSetup({ + xhrFields: { + withCredentials: true + }, + headers: { + 'X-Requested-With': 'XMLHttpRequest', + }, +}); + From ea5c81cde965e27dff3a03a0267d6aff5bb1039e Mon Sep 17 00:00:00 2001 From: Jeroen Pinoy Date: Sat, 9 Oct 2021 00:30:10 +0200 Subject: [PATCH 15/78] fix: [community-metadata] Fix typos and improve wording --- app/files/community-metadata/defaults.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/files/community-metadata/defaults.json b/app/files/community-metadata/defaults.json index 7cfa723cd..7f663c28d 100644 --- a/app/files/community-metadata/defaults.json +++ b/app/files/community-metadata/defaults.json @@ -20,7 +20,7 @@ "uuid": "6201a65a-38fa-4b4a-bea7-cf1fb7bc1563", "org_uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f", "org_name": "CIRCL", - "description": "CIRCL operates an information sharing dedicated to national/governmental CSIRTs/CERTs with national responsibilities in the EEA (European Economic Area).", + "description": "CIRCL operates an information sharing community dedicated to national/governmental CSIRTs/CERTs with national responsibilities in the EEA (European Economic Area).", "url": "https://www.circl.lu/misp/", "sector": "n/g CSIRTs in EEA", "nationality": "International", @@ -36,9 +36,9 @@ "uuid": "79e28013-747a-4430-b7ba-ed92d053b221", "org_uuid": "55f6e5ae-2c60-40e5-964f-47a8950d210f", "org_name": "CIRCL", - "description": "CIRCL operates an information sharing dedicated to the financial sector.", + "description": "CIRCL operates an information sharing community dedicated to the financial sector.", "url": "https://www.circl.lu/misp/", - "sector": "Financial sector including banks, financial institution or payement processing organisations", + "sector": "Financial sector including banks, financial institutions and payment processing organisations", "nationality": "International", "type": "Vetted Information Sharing Community", "email": "info@circl.lu", @@ -68,7 +68,7 @@ "uuid": "5e4656fe-8e34-441c-9c47-6545011fb688", "org_uuid": "569b6c1f-bd1c-49c8-9244-0484bce2ab96", "org_name": "eCrimeLabs", - "description": "The Danish MISP User group/Community operates as an informaiton sharing dedicated to Danish organisations", + "description": "The Danish MISP User group/Community operates an information sharing community dedicated to Danish organisations.", "url": "https://dk.ecrimelabs.net", "sector": "undefined", "nationality": "Danish", From 6e6faa962c82bb121d70628ac341061e074fd7f0 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 9 Oct 2021 11:04:03 +0200 Subject: [PATCH 16/78] new: [test] Build test --- .github/workflows/main.yml | 18 ++++++++++++++---- tests/build-test.sh | 26 ++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 tests/build-test.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3238d76bf..5d1d15eb1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -70,9 +70,8 @@ jobs: # Repo is missing for unknown reason LC_ALL=C.UTF-8 sudo apt-add-repository ppa:ondrej/php -y sudo apt-get -y install curl python3 python3-zmq python3-requests python3-pip python3-nose python3-redis python3-lxml apache2 libapache2-mod-php$php_version libfuzzy-dev - sudo pip3 install virtualenv # virtualenv must be instaled from pip and not from ubuntu pacckages - sudo pip3 install --upgrade -r requirements.txt - curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - + sudo pip3 install virtualenv # virtualenv must be instaled from pip and not from ubuntu packages + curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python - sudo chown $USER:www-data $HOME/.composer pushd app sudo -H -u $USER php composer.phar install --no-progress @@ -167,12 +166,16 @@ jobs: - name: Update Galaxies run: sudo -E su $USER -c 'app/Console/cake Admin updateGalaxies' + - name: Update Taxonomies run: sudo -E su $USER -c 'app/Console/cake Admin updateTaxonomies' + - name: Update Warninglists run: sudo -E su $USER -c 'app/Console/cake Admin updateWarningLists' + - name: Update Noticelists run: sudo -E su $USER -c 'app/Console/cake Admin updateNoticeLists' + - name: Update Object Templates run: sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates 1' @@ -215,6 +218,13 @@ jobs: cat tests/keys.py popd + - name: Build test + run: | + . ./venv/bin/activate + pip install git+https://github.com/kbandla/pydeep.git + pushd tests + ./build-test.sh + - name: Run PHP tests run: | ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/ @@ -222,7 +232,7 @@ jobs: - name: Run tests run: | - source $HOME/.poetry/env # enable poetry binary + export PATH=$HOME/.local/env:$PATH # enable poetry binary # Ensure the perms of config files sudo chown -R $USER:www-data `pwd`/app/Config diff --git a/tests/build-test.sh b/tests/build-test.sh new file mode 100644 index 000000000..a1bb0e540 --- /dev/null +++ b/tests/build-test.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -e +set -o xtrace + +# Check if dependencies for zmq are properly installed +python3 ./../app/files/scripts/mispzmq/mispzmqtest.py + +# Check if all attachments handlers dependencies are correctly installed +python3 ./../app/files/scripts/generate_file_objects.py -c | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if len([i for i in data.values() if i == True]) == 0 else 1)' + +# Try to extract data from file +python3 ./../app/files/scripts/generate_file_objects.py -p /bin/ls | python3 -c 'import sys, json; data = json.load(sys.stdin); sys.exit(0 if "objects" in data else 1)' + +# Test converting stix1 to MISP format +curl https://stixproject.github.io/documentation/idioms/c2-indicator/indicator-for-c2-ip-address.xml > ./../app/files/scripts/tmp/test-stix1.xml +python3 ./../app/files/scripts/stix2misp.py test-stix1.xml 1 1 ./../app/files/scripts/synonymsToTagNames.json | python3 -c 'import sys; data = sys.stdin.read().strip(); print(data); sys.exit(0 if data == "1" else 1)' +rm -f ./../app/files/scripts/tmp/{test-stix1.xml,test-stix1.xml.json} + +# Test converting stix2 to MISP format +curl https://raw.githubusercontent.com/oasis-open/cti-stix2-json-schemas/master/examples/indicator-for-c2-ip-address.json > ./../app/files/scripts/tmp/test-stix2.json +python3 ./../app/files/scripts/stix2/stix2misp.py ./../tmp/test-stix2.json 1 1 ./../app/files/scripts/synonymsToTagNames.json | python3 -c 'import sys; data = sys.stdin.read().strip(); print(data); sys.exit(0 if data == "1" else 1)' +rm -f ./../app/files/scripts/tmp/{test-stix2.json,test-stix2.json.stix2} + +# Test converting MISP to STIX2 +python3 ./../app/files/scripts/stix2/misp2stix2.py -i event.json | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if data["success"] == 1 else 1)' +rm -f event.json.out From 3562899af0fb51257ba691c59f148d5df60d825d Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 9 Oct 2021 14:43:13 +0200 Subject: [PATCH 17/78] chg: [internal] Background processing refactoring --- app/Console/Command/EventShell.php | 19 +++---- app/Lib/Tools/FileAccessTool.php | 9 ++- app/Model/Event.php | 89 ++++++++++++------------------ 3 files changed, 52 insertions(+), 65 deletions(-) diff --git a/app/Console/Command/EventShell.php b/app/Console/Command/EventShell.php index bcc9862fb..3223fc9f9 100644 --- a/app/Console/Command/EventShell.php +++ b/app/Console/Command/EventShell.php @@ -522,17 +522,15 @@ class EventShell extends AppShell public function processfreetext() { - $this->ConfigLoad->execute(); if (empty($this->args[0])) { die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Process free text'] . PHP_EOL); } $inputFile = $this->args[0]; - $tempdir = new Folder(APP . 'tmp/cache/ingest', true, 0750); - $tempFile = new File(APP . 'tmp/cache/ingest' . DS . $inputFile); - $inputData = $tempFile->read(); - $inputData = json_decode($inputData, true); - $tempFile->delete(); + $inputFile = $inputFile[0] === '/' ? $inputFile : APP . 'tmp/cache/ingest' . DS . $inputFile; + $inputData = FileAccessTool::readFromFile($inputFile); + $inputData = $this->Event->jsonDecode($inputData); + FileAccessTool::deleteFile($inputFile); Configure::write('CurrentUserId', $inputData['user']['id']); $this->Event->processFreeTextData( $inputData['user'], @@ -548,16 +546,15 @@ class EventShell extends AppShell public function processmoduleresult() { - $this->ConfigLoad->execute(); if (empty($this->args[0])) { die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Process module result'] . PHP_EOL); } $inputFile = $this->args[0]; - $tempDir = new Folder(APP . 'tmp/cache/ingest', true, 0750); - $tempFile = new File(APP . 'tmp/cache/ingest' . DS . $inputFile); - $inputData = json_decode($tempFile->read(), true); - $tempFile->delete(); + $inputFile = $inputFile[0] === '/' ? $inputFile : APP . 'tmp/cache/ingest' . DS . $inputFile; + $inputData = FileAccessTool::readFromFile($inputFile); + $inputData = $this->Event->jsonDecode($inputData); + FileAccessTool::deleteFile($inputFile); Configure::write('CurrentUserId', $inputData['user']['id']); $this->Event->processModuleResultsData( $inputData['user'], diff --git a/app/Lib/Tools/FileAccessTool.php b/app/Lib/Tools/FileAccessTool.php index 9a2216680..a5716e6a5 100644 --- a/app/Lib/Tools/FileAccessTool.php +++ b/app/Lib/Tools/FileAccessTool.php @@ -35,7 +35,14 @@ class FileAccessTool $content = file_get_contents($file, false, null, 0, $fileSize); } if ($content === false) { - throw new Exception("An error has occurred while attempt to read file `$file`."); + if (!file_exists($file)) { + $message = "file doesn't exists"; + } else if (!is_readable($file)) { + $message = "file is not readable"; + } else { + $message = 'unknown error'; + } + throw new Exception("An error has occurred while attempt to read file `$file`: $message."); } return $content; } diff --git a/app/Model/Event.php b/app/Model/Event.php index 71bb61e00..99165798a 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -6785,7 +6785,10 @@ class Event extends AppModel public function processFreeTextDataRouter($user, $attributes, $id, $default_comment = '', $proposals = false, $adhereToWarninglists = false, $returnRawResults = false) { if (Configure::read('MISP.background_jobs') && count($attributes) > 5) { // on background process just big attributes batch - list($job, $randomFileName, $tempFile) = $this->__initiateProcessJob($user, $id); + /** @var Job $job */ + $job = ClassRegistry::init('Job'); + $job->createJob($user, Job::WORKER_PRIO, "process_freetext_data", 'Event: ' . $id, 'Processing...'); + $tempData = array( 'user' => $user, 'attributes' => $attributes, @@ -6796,73 +6799,53 @@ class Event extends AppModel 'jobId' => $job->id, ); - $writeResult = $tempFile->write(json_encode($tempData)); - if (!$writeResult) { - return $this->processFreeTextData($user, $attributes, $id, $default_comment, $proposals, $adhereToWarninglists, false, $returnRawResults); - } - $tempFile->close(); - $process_id = CakeResque::enqueue( - 'prio', + try { + $filePath = FileAccessTool::writeToTempFile(json_encode($tempData)); + $process_id = CakeResque::enqueue( + Job::WORKER_PRIO, 'EventShell', - array('processfreetext', $randomFileName), + array('processfreetext', $filePath), true - ); - $job->saveField('process_id', $process_id); - return 'Freetext ingestion queued for background processing. Attributes will be added to the event as they are being processed.'; - } else { - return $this->processFreeTextData($user, $attributes, $id, $default_comment, $proposals, $adhereToWarninglists, false, $returnRawResults); + ); + $job->saveField('process_id', $process_id); + return 'Freetext ingestion queued for background processing. Attributes will be added to the event as they are being processed.'; + } catch (Exception $e) { + $this->logException("Could not process freetext in background.", $e, LOG_NOTICE); + } } + return $this->processFreeTextData($user, $attributes, $id, $default_comment, $proposals, $adhereToWarninglists, false, $returnRawResults); } - public function processModuleResultsDataRouter($user, $resolved_data, $id, $default_comment = '', $adhereToWarninglists = false) + public function processModuleResultsDataRouter($user, $resolved_data, $id, $default_comment = '') { if (Configure::read('MISP.background_jobs')) { - list($job, $randomFileName, $tempFile) = $this->__initiateProcessJob($user, $id, 'module_results'); + /** @var Job $job */ + $job = ClassRegistry::init('Job'); + $job->createJob($user, Job::WORKER_PRIO, "process_module_results_data", 'Event: ' . $id, 'Processing...'); + $tempData = array( - 'user' => $user, - 'misp_format' => $resolved_data, - 'id' => $id, - 'default_comment' => $default_comment, - 'jobId' => $job->id + 'user' => $user, + 'misp_format' => $resolved_data, + 'id' => $id, + 'default_comment' => $default_comment, + 'jobId' => $job->id ); - $writeResult = $tempFile->write(json_encode($tempData)); - if ($writeResult) { - $tempFile->close(); + + try { + $filePath = FileAccessTool::writeToTempFile(json_encode($tempData)); $process_id = CakeResque::enqueue( - 'prio', - 'EventShell', - array('processmoduleresult', $randomFileName), - true + Job::WORKER_PRIO, + 'EventShell', + array('processmoduleresult', $filePath), + true ); $job->saveField('process_id', $process_id); return 'Module results ingestion queued for background processing. Related data will be added to the event as it is being processed.'; + } catch (Exception $e) { + $this->logException("Could not process module results in background.", $e, LOG_NOTICE); } - $tempFile->delete(); } - return $this->processModuleResultsData($user, $resolved_data, $id, $default_comment = ''); - } - - private function __initiateProcessJob(array $user, $id, $format = 'freetext') - { - $job = ClassRegistry::init('Job'); - $job->create(); - $data = array( - 'worker' => 'prio', - 'job_type' => 'process_' . $format . '_data', - 'job_input' => 'Event: ' . $id, - 'status' => 0, - 'retries' => 0, - 'org_id' => $user['org_id'], - 'org' => $user['Organisation']['name'], - 'message' => 'Processing...' - ); - $job->save($data); - $randomFileName = $this->generateRandomFileName() . '.json'; - App::uses('Folder', 'Utility'); - App::uses('File', 'Utility'); - $tempdir = new Folder(APP . 'tmp/cache/ingest', true, 0755); - $tempFile = new File(APP . 'tmp/cache/ingest' . DS . $randomFileName, true, 0644); - return array($job, $randomFileName, $tempFile); + return $this->processModuleResultsData($user, $resolved_data, $id, $default_comment); } /** From 0faf986b7841c92cc190b794233a2d64e92fb51f Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 9 Oct 2021 15:44:55 +0200 Subject: [PATCH 18/78] fix: [shell] EventShell::contactemail command --- app/Console/Command/EventShell.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/Console/Command/EventShell.php b/app/Console/Command/EventShell.php index 3223fc9f9..c8d503004 100644 --- a/app/Console/Command/EventShell.php +++ b/app/Console/Command/EventShell.php @@ -8,6 +8,7 @@ require_once 'AppShell.php'; * @property Event $Event * @property Job $Job * @property Tag $Tag + * @property Server $Server */ class EventShell extends AppShell { @@ -279,8 +280,7 @@ class EventShell extends AppShell public function contactemail() { - $this->ConfigLoad->execute(); - if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || + if (empty($this->args[0]) || empty($this->args[1]) || !isset($this->args[2]) || empty($this->args[3]) || empty($this->args[4])) { die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Contact email'] . PHP_EOL); } @@ -289,11 +289,11 @@ class EventShell extends AppShell $message = $this->args[1]; $all = $this->args[2]; $userId = $this->args[3]; - $processId = $this->args[4]; + $jobId = $this->args[4]; $user = $this->getUser($userId); $result = $this->Event->sendContactEmail($id, $message, $all, $user); - $this->Job->saveStatus($processId, $result); + $this->Job->saveStatus($jobId, $result); } public function postsemail() From 2f71e0f7f4473946bfd40f291e51e8273a4ae874 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 9 Oct 2021 17:07:29 +0200 Subject: [PATCH 19/78] chg: [internal] Allow to save raw data --- app/Controller/AttributesController.php | 2 +- app/Model/Attribute.php | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 0340af5e2..3bbf5e695 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -435,7 +435,7 @@ class AttributesController extends AppController 'category' => $this->request->data['Attribute']['category'], 'type' => 'attachment', 'event_id' => $event['Event']['id'], - 'data' => base64_encode($tmpfile->read()), + 'data_raw' => $tmpfile->read(), 'comment' => $this->request->data['Attribute']['comment'], 'to_ids' => 0, 'distribution' => $this->request->data['Attribute']['distribution'], diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index db7afe75e..99c5dffa3 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -444,8 +444,13 @@ class Attribute extends AppModel } $result = true; // if the 'data' field is set on the $this->data then save the data to the correct file - if (isset($this->data['Attribute']['type']) && $this->typeIsAttachment($this->data['Attribute']['type']) && !empty($this->data['Attribute']['data'])) { - $result = $result && $this->saveBase64EncodedAttachment($this->data['Attribute']); // TODO : is this correct? + if (isset($this->data['Attribute']['type']) && $this->typeIsAttachment($this->data['Attribute']['type'])) { + if (!empty($this->data['Attribute']['data_raw'])) { + $this->data['Attribute']['data'] = $this->data['Attribute']['data_raw']; + $result = $this->saveAttachment($this->data['Attribute']); + } elseif (!empty($this->data['Attribute']['data'])) { + $result = $this->saveBase64EncodedAttachment($this->data['Attribute']); // TODO : is this correct? + } } $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_attribute_notifications_enable'); $kafkaTopic = Configure::read('Plugin.Kafka_attribute_notifications_topic'); @@ -3586,11 +3591,11 @@ class Attribute extends AppModel if (isset($v['data'])) { $attribute['data'] = $result['data']; } - if ($k == 'malware-sample') { + if ($k === 'malware-sample') { $attribute['value'] = $filename . '|' . $result['md5']; - } elseif ($k == 'size-in-bytes') { + } elseif ($k === 'size-in-bytes') { $attribute['value'] = $tmpfile->size(); - } elseif ($k == 'filename') { + } elseif ($k === 'filename') { $attribute['value'] = $filename; } else { $attribute['value'] = $result[$v['type']]; @@ -3637,7 +3642,7 @@ class Attribute extends AppModel public function captureAttribute($attribute, $eventId, $user, $objectId = false, $log = false, $parentEvent = false, &$validationErrors = false, $params = array()) { $attribute['event_id'] = $eventId; - $attribute['object_id'] = $objectId ? $objectId : 0; + $attribute['object_id'] = $objectId ?: 0; if (!isset($attribute['to_ids'])) { $attribute['to_ids'] = $this->typeDefinitions[$attribute['type']]['to_ids']; } From c89893cbabead2b429b928efbb5f0a5a327becc2 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 9 Oct 2021 17:16:49 +0200 Subject: [PATCH 20/78] chg: [internal] Use FileAccessTool in AttachmentTool --- app/Lib/Tools/AttachmentTool.php | 41 ++++++++++---------------------- app/Lib/Tools/FileAccessTool.php | 22 ++++++++++------- 2 files changed, 26 insertions(+), 37 deletions(-) diff --git a/app/Lib/Tools/AttachmentTool.php b/app/Lib/Tools/AttachmentTool.php index 5630844fe..daf43e3e2 100644 --- a/app/Lib/Tools/AttachmentTool.php +++ b/app/Lib/Tools/AttachmentTool.php @@ -207,10 +207,7 @@ class AttachmentTool } else { $path = $this->attachmentDir() . DS . $path; - $file = new File($path, true); - if (!$file->write($data)) { - throw new Exception("Could not save attachment to file '$path'."); - } + FileAccessTool::writeToFile($path, $data, true); } return true; @@ -309,46 +306,32 @@ class AttachmentTool { $tempDir = $this->tempDir(); - $contentsFile = new File($tempDir . DS . $md5, true); - if (!$contentsFile->write($content)) { - throw new Exception("Could not write content to file '{$contentsFile->path}'."); - } - $contentsFile->close(); + FileAccessTool::writeToFile($tempDir . DS . $md5, $content); + FileAccessTool::writeToFile($tempDir . DS . $md5 . '.filename.txt', $originalFilename); - $fileNameFile = new File($tempDir . DS . $md5 . '.filename.txt', true); - if (!$fileNameFile->write($originalFilename)) { - throw new Exception("Could not write original file name to file '{$fileNameFile->path}'."); - } - $fileNameFile->close(); - - $zipFile = new File($tempDir . DS . $md5 . '.zip'); + $zipFile = $tempDir . DS . $md5 . '.zip'; $exec = [ 'zip', '-j', // junk (don't record) directory names '-P', // use standard encryption self::ZIP_PASSWORD, - escapeshellarg($zipFile->path), - escapeshellarg($contentsFile->path), - escapeshellarg($fileNameFile->path), + escapeshellarg($zipFile), + escapeshellarg($tempDir . DS . $md5), + escapeshellarg($tempDir . DS . $md5 . '.filename.txt'), ]; try { $this->execute($exec); - $zipContent = $zipFile->read(); - if ($zipContent === false) { - throw new Exception("Could not read content of newly created ZIP file."); - } - - return $zipContent; + return FileAccessTool::readFromFile($zipFile); } catch (Exception $e) { - throw new Exception("Could not create encrypted ZIP file '{$zipFile->path}'.", 0, $e); + throw new Exception("Could not create encrypted ZIP file '$zipFile'.", 0, $e); } finally { - $fileNameFile->delete(); - $contentsFile->delete(); - $zipFile->delete(); + FileAccessTool::deleteFile($tempDir . DS . $md5); + FileAccessTool::deleteFile($tempDir . DS . $md5 . '.filename.txt'); + FileAccessTool::deleteFile($zipFile); } } diff --git a/app/Lib/Tools/FileAccessTool.php b/app/Lib/Tools/FileAccessTool.php index a5716e6a5..c6e8a8342 100644 --- a/app/Lib/Tools/FileAccessTool.php +++ b/app/Lib/Tools/FileAccessTool.php @@ -50,13 +50,20 @@ class FileAccessTool /** * @param string $file * @param mixed $content - * @param int $flags + * @param bool $createFolder * @throws Exception */ - public static function writeToFile($file, $content, $flags = LOCK_EX) + public static function writeToFile($file, $content, $createFolder = false) { - if (file_put_contents($file, $content, $flags) === false) { - $freeSpace = disk_free_space($file); + $dir = dirname($file); + if ($createFolder && !is_dir($dir)) { + if (!mkdir($dir, 0766, true)) { + throw new Exception("An error has occurred while attempt to create directory `$dir`."); + } + } + + if (file_put_contents($file, $content, LOCK_EX) === false) { + $freeSpace = disk_free_space($dir); throw new Exception("An error has occurred while attempt to write to file `$file`. Maybe not enough space? ($freeSpace bytes left)"); } } @@ -70,11 +77,10 @@ class FileAccessTool public static function writeToTempFile($content, $dir = null) { $tempFile = self::createTempFile($dir); - try { - self::writeToFile($tempFile, $content, 0); // Lock is not need - } catch (Exception $e) { + if (file_put_contents($tempFile, $content) === false) { self::deleteFile($tempFile); - throw $e; + $freeSpace = disk_free_space(dirname($tempFile)); + throw new Exception("An error has occurred while attempt to write to file `$tempFile`. Maybe not enough space? ($freeSpace bytes left)"); } return $tempFile; } From 5e320a1dbd8c20b8b648b16660131111a607ab78 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 9 Oct 2021 17:28:44 +0200 Subject: [PATCH 21/78] chg: [internal] Do not encode/decode base64 for simpleAddMalwareSample --- app/Model/Attribute.php | 36 +++++++++++++++++++++++++++--------- app/Model/MispObject.php | 1 + 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 99c5dffa3..46386d39b 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -3191,6 +3191,28 @@ class Attribute extends AppModel return $result; } + /** + * @param string $originalFilename + * @param string $content + * @param array $hashTypes + * @return array + */ + private function handleMaliciousRaw($originalFilename, $content, array $hashTypes) + { + $attachmentTool = $this->loadAttachmentTool(); + $hashes = $attachmentTool->computeHashes($content, $hashTypes); + try { + $encrypted = $attachmentTool->encrypt($originalFilename, $content, $hashes['md5']); + } catch (Exception $e) { + $this->logException("Could not create encrypted malware sample.", $e); + return ['success' => false]; + } + + $hashes['success'] = true; + $hashes['data_raw'] = $encrypted; + return $hashes; + } + /** * @return bool Return true if at least one advanced extraction tool is available */ @@ -3529,7 +3551,7 @@ class Attribute extends AppModel public function simpleAddMalwareSample($event_id, $attribute_settings, $filename, $tmpfile) { $attributes = array( - 'malware-sample' => array('type' => 'malware-sample', 'data' => 1, 'category' => '', 'to_ids' => 1, 'disable_correlation' => 0, 'object_relation' => 'malware-sample'), + 'malware-sample' => array('type' => 'malware-sample', 'category' => '', 'to_ids' => 1, 'disable_correlation' => 0, 'object_relation' => 'malware-sample'), 'filename' => array('type' => 'filename', 'category' => '', 'to_ids' => 0, 'disable_correlation' => 0, 'object_relation' => 'filename'), 'md5' => array('type' => 'md5', 'category' => '', 'to_ids' => 1, 'disable_correlation' => 0, 'object_relation' => 'md5'), 'sha1' => array('type' => 'sha1', 'category' => '', 'to_ids' => 1, 'disable_correlation' => 0, 'object_relation' => 'sha1'), @@ -3537,16 +3559,14 @@ class Attribute extends AppModel 'size-in-bytes' => array('type' => 'size-in-bytes', 'category' => 'Other', 'to_ids' => 0, 'disable_correlation' => 1, 'object_relation' => 'size-in-bytes') ); $hashes = array('md5', 'sha1', 'sha256'); - $this->Object = ClassRegistry::init('MispObject'); - $this->ObjectTemplate = ClassRegistry::init('ObjectTemplate'); - $current = $this->ObjectTemplate->find('first', array( + $current = $this->Object->ObjectTemplate->find('first', array( 'fields' => array('MAX(version) AS version', 'uuid'), 'conditions' => array('uuid' => '688c46fb-5edb-40a3-8273-1af7923e2215'), 'recursive' => -1, 'group' => array('uuid') )); if (!empty($current)) { - $object_template = $this->ObjectTemplate->find('first', array( + $object_template = $this->Object->ObjectTemplate->find('first', array( 'conditions' => array( 'ObjectTemplate.uuid' => '688c46fb-5edb-40a3-8273-1af7923e2215', 'ObjectTemplate.version' => $current[0]['version'] @@ -3576,7 +3596,7 @@ class Attribute extends AppModel 'event_id' => $event_id, 'comment' => !empty($attribute_settings['comment']) ? $attribute_settings['comment'] : '' ); - $result = $this->handleMaliciousBase64($event_id, $filename, base64_encode($tmpfile->read()), $hashes); + $result = $this->handleMaliciousRaw($filename, $tmpfile->read(), $hashes); foreach ($attributes as $k => $v) { $attribute = array( 'distribution' => 5, @@ -3588,11 +3608,9 @@ class Attribute extends AppModel 'event_id' => $event_id, 'object_relation' => $v['object_relation'] ); - if (isset($v['data'])) { - $attribute['data'] = $result['data']; - } if ($k === 'malware-sample') { $attribute['value'] = $filename . '|' . $result['md5']; + $attribute['data_raw'] = $result['data_raw']; } elseif ($k === 'size-in-bytes') { $attribute['value'] = $tmpfile->size(); } elseif ($k === 'filename') { diff --git a/app/Model/MispObject.php b/app/Model/MispObject.php index 807121f00..d102d2ffa 100644 --- a/app/Model/MispObject.php +++ b/app/Model/MispObject.php @@ -7,6 +7,7 @@ App::uses('TmpFileTool', 'Tools'); * @property SharingGroup $SharingGroup * @property Attribute $Attribute * @property ObjectReference $ObjectReference + * @property ObjectTemplate $ObjectTemplate */ class MispObject extends AppModel { From 6da85b4494e4d4099eea6f267b1375e910248184 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 9 Oct 2021 17:45:12 +0200 Subject: [PATCH 22/78] chg: [internal] Optimise Attribute::valueIsUnique check --- app/Model/Attribute.php | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 46386d39b..be3a03d0b 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -678,7 +678,7 @@ class Attribute extends AppModel */ public function valueIsUnique($fields) { - if (isset($this->data['Attribute']['deleted']) && $this->data['Attribute']['deleted']) { + if (!empty($this->data['Attribute']['deleted'])) { return true; } // We escape this rule for objects as we can have the same category/type/value combination in different objects @@ -686,14 +686,11 @@ class Attribute extends AppModel return true; } - $eventId = $this->data['Attribute']['event_id']; - $category = $this->data['Attribute']['category']; $type = $this->data['Attribute']['type']; - $conditions = array( - 'Attribute.event_id' => $eventId, + 'Attribute.event_id' => $this->data['Attribute']['event_id'], 'Attribute.type' => $type, - 'Attribute.category' => $category, + 'Attribute.category' => $this->data['Attribute']['category'], 'Attribute.deleted' => 0, 'Attribute.object_id' => 0, ); @@ -711,18 +708,7 @@ class Attribute extends AppModel $conditions['Attribute.id !='] = $this->data['Attribute']['id']; } - $params = array( - 'recursive' => -1, - 'fields' => array('id'), - 'conditions' => $conditions, - 'order' => false, - ); - if (!empty($this->find('first', $params))) { - // value isn't unique - return false; - } - // value is unique - return true; + return !$this->hasAny($conditions); } public function validateTypeValue($fields) From de73e4731807a2b8a1f7f2ade91eacfa5f9389ab Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 9 Oct 2021 18:21:31 +0200 Subject: [PATCH 23/78] chg: [internal] Faster saving origin file --- app/Model/Attribute.php | 22 +++++++++++++--------- app/Model/Event.php | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index be3a03d0b..8fbe311b9 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -447,10 +447,10 @@ class Attribute extends AppModel if (isset($this->data['Attribute']['type']) && $this->typeIsAttachment($this->data['Attribute']['type'])) { if (!empty($this->data['Attribute']['data_raw'])) { $this->data['Attribute']['data'] = $this->data['Attribute']['data_raw']; - $result = $this->saveAttachment($this->data['Attribute']); } elseif (!empty($this->data['Attribute']['data'])) { - $result = $this->saveBase64EncodedAttachment($this->data['Attribute']); // TODO : is this correct? + $this->data['Attribute']['data'] = base64_decode($this->data['Attribute']['data']); } + $result = $this->saveAttachment($this->data['Attribute']); } $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_attribute_notifications_enable'); $kafkaTopic = Configure::read('Plugin.Kafka_attribute_notifications_topic'); @@ -1545,8 +1545,18 @@ class Attribute extends AppModel return $this->loadAttachmentTool()->getFile($attribute['event_id'], $attribute['id'], $path_suffix); } - public function saveAttachment($attribute, $path_suffix='') + /** + * @param array $attribute + * @param string $path_suffix + * @return bool + * @throws Exception + */ + private function saveAttachment(array $attribute, $path_suffix='') { + if ($attribute['data'] === false) { + $this->log("Invalid attachment data provided for attribute with ID {$attribute['id']}."); + return false; + } $result = $this->loadAttachmentTool()->save($attribute['event_id'], $attribute['id'], $attribute['data'], $path_suffix); if ($result) { $this->loadAttachmentScan()->backgroundScan(AttachmentScan::TYPE_ATTRIBUTE, $attribute); @@ -1570,12 +1580,6 @@ class Attribute extends AppModel } } - public function saveBase64EncodedAttachment($attribute) - { - $attribute['data'] = base64_decode($attribute['data']); - return $this->saveAttachment($attribute); - } - /** * Currently, as image are considered files with JPG (JPEG), PNG or GIF extension. * @param array $attribute diff --git a/app/Model/Event.php b/app/Model/Event.php index 99165798a..b0e9af443 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -7207,7 +7207,7 @@ class Event extends AppModel 'distribution' => $distribution, 'object_relation' => 'imported-sample', 'value' => $original_filename, - 'data' => base64_encode($file), + 'data_raw' => $file, 'object_id' => $object_id, 'disable_correlation' => true ), From 982684766075f7816df5f13608aa680039cb7952 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 9 Oct 2021 18:44:33 +0200 Subject: [PATCH 24/78] chg: [internal] Delete unused method Attribute::saveAndEncryptAttribute --- app/Model/Attribute.php | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 8fbe311b9..1622a330d 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -3326,42 +3326,6 @@ class Attribute extends AppModel return $attribute; } - public function saveAndEncryptAttribute($attribute, $user = false) - { - $hashes = array('md5' => 'malware-sample', 'sha1' => 'filename|sha1', 'sha256' => 'filename|sha256'); - if ($attribute['encrypt']) { - $result = $this->handleMaliciousBase64($attribute['event_id'], $attribute['value'], $attribute['data'], array_keys($hashes)); - if (!$result['success']) { - return 'Could not handle the sample'; - } - foreach ($hashes as $hash => $typeName) { - if (!$result[$hash]) { - continue; - } - $attributeToSave = array( - 'Attribute' => array( - 'value' => $attribute['value'] . '|' . $result[$hash], - 'category' => $attribute['category'], - 'type' => $typeName, - 'event_id' => $attribute['event_id'], - 'comment' => $attribute['comment'], - 'to_ids' => 1, - 'distribution' => $attribute['distribution'], - 'sharing_group_id' => isset($attribute['sharing_group_id']) ? $attribute['sharing_group_id'] : 0, - ) - ); - if ($hash == 'md5') { - $attributeToSave['Attribute']['data'] = $result['data']; - } - $this->create(); - if (!$this->save($attributeToSave)) { - return $this->validationErrors; - } - } - } - return true; - } - private function __createTagSubQuery($tag_id, $blocked = false, $scope = 'Event', $limitAttributeHitsTo = 'event') { $conditionKey = $blocked ? array('NOT' => array('EventTag.tag_id' => $tag_id)) : array('EventTag.tag_id' => $tag_id); From 4049c1e12778181f22bb521a3b79600aa7ed7cb9 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 9 Oct 2021 18:45:51 +0200 Subject: [PATCH 25/78] chg: [internal] Attribute::onDemandEncrypt faster --- app/Model/Attribute.php | 33 +++++++++++++++---------- app/Model/Behavior/TrimBehavior.php | 37 +---------------------------- 2 files changed, 22 insertions(+), 48 deletions(-) diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 1622a330d..7a95a2096 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -445,9 +445,10 @@ class Attribute extends AppModel $result = true; // if the 'data' field is set on the $this->data then save the data to the correct file if (isset($this->data['Attribute']['type']) && $this->typeIsAttachment($this->data['Attribute']['type'])) { - if (!empty($this->data['Attribute']['data_raw'])) { + if (isset($this->data['Attribute']['data_raw'])) { $this->data['Attribute']['data'] = $this->data['Attribute']['data_raw']; - } elseif (!empty($this->data['Attribute']['data'])) { + unset($this->data['Attribute']['data_raw']); + } elseif (isset($this->data['Attribute']['data'])) { $this->data['Attribute']['data'] = base64_decode($this->data['Attribute']['data']); } $result = $this->saveAttachment($this->data['Attribute']); @@ -3295,7 +3296,7 @@ class Attribute extends AppModel } } $saveResult = true; - foreach ($attributes as $k => $attribute) { + foreach ($attributes as $attribute) { if (!empty($attribute['encrypt']) && $attribute['encrypt']) { $attribute = $this->onDemandEncrypt($attribute); } @@ -3314,14 +3315,26 @@ class Attribute extends AppModel return $saveResult; } - public function onDemandEncrypt($attribute) + /** + * @param array $attribute + * @return array + */ + public function onDemandEncrypt(array $attribute) { if (strpos($attribute['value'], '|') !== false) { $temp = explode('|', $attribute['value']); $attribute['value'] = $temp[0]; } - $result = $this->handleMaliciousBase64($attribute['event_id'], $attribute['value'], $attribute['data'], array('md5')); - $attribute['data'] = $result['data']; + + $content = base64_decode($attribute['data']); + if ($content === false) { + $this->log("Invalid attachment data provided for attribute with ID {$attribute['id']}."); + return $attribute; + } + + $result = $this->handleMaliciousRaw($attribute['value'], $content, array('md5')); + $attribute['data_raw'] = $result['data_raw']; + unset($attribute['data']); $attribute['value'] = $attribute['value'] . '|' . $result['md5']; return $attribute; } @@ -3637,9 +3650,7 @@ class Attribute extends AppModel } } if (isset($attribute['encrypt'])) { - $result = $this->handleMaliciousBase64($eventId, $attribute['value'], $attribute['data'], array('md5')); - $attribute['data'] = $result['data']; - $attribute['value'] = $attribute['value'] . '|' . $result['md5']; + $attribute = $this->onDemandEncrypt($attribute); } $this->create(); if (!isset($attribute['distribution'])) { @@ -3710,9 +3721,7 @@ class Attribute extends AppModel $attribute['event_id'] = $eventId; $attribute['object_id'] = $objectId; if (isset($attribute['encrypt'])) { - $result = $this->handleMaliciousBase64($eventId, $attribute['value'], $attribute['data'], array('md5')); - $attribute['data'] = $result['data']; - $attribute['value'] = $attribute['value'] . '|' . $result['md5']; + $attribute = $this->onDemandEncrypt($attribute); } unset($attribute['id']); if (isset($attribute['uuid'])) { diff --git a/app/Model/Behavior/TrimBehavior.php b/app/Model/Behavior/TrimBehavior.php index 7df869da5..b0573204c 100644 --- a/app/Model/Behavior/TrimBehavior.php +++ b/app/Model/Behavior/TrimBehavior.php @@ -8,45 +8,10 @@ */ class TrimBehavior extends ModelBehavior { - -/** - * - * @param Model $Model - * @param unknown_type $settings - */ - public function setup(Model $Model, $settings = array()) - { - if (!isset($this->settings[$Model->alias])) { - $this->settings[$Model->alias] = array( - 'fields' => 'all', - ); - } - $this->settings[$Model->alias] = array_merge( - $this->settings[$Model->alias], - (array)$settings - ); - } - - /** - * - * @param $options - */ public function beforeValidate(Model $Model, $options = array()) - { - $this->trimStringFields($Model); - return true; - } - - /** - * Trim String Fields - * - * @param Model $Model - * @param unknown_type $array - */ - public function trimStringFields(Model $Model) { foreach ($Model->data[$Model->name] as $key => $field) { - if (is_string($field)) { + if ($key !== 'data' && $key !== 'data_raw' && is_string($field)) { $Model->data[$Model->name][$key] = trim($field); } } From 9bfe634cacc9257f222a318da09b792ce768e1a2 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 9 Oct 2021 23:32:51 +0200 Subject: [PATCH 26/78] chg: [internal] Default distribution method --- app/Controller/AttributesController.php | 7 +--- app/Controller/EventReportsController.php | 7 +--- app/Controller/EventsController.php | 6 +-- app/Model/Attribute.php | 50 ++++++++++------------- app/Model/Event.php | 14 +------ app/Model/EventReport.php | 5 +-- 6 files changed, 28 insertions(+), 61 deletions(-) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 3bbf5e695..f3ccc0c10 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -289,12 +289,7 @@ class AttributesController extends AppController $this->loadModel('SharingGroup'); $sgs = $this->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1); $this->set('sharingGroups', $sgs); - $initialDistribution = 5; - $configuredDistribution = Configure::check('MISP.default_attribute_distribution'); - if ($configuredDistribution != null && $configuredDistribution != 'event') { - $initialDistribution = $configuredDistribution; - } - $this->set('initialDistribution', $initialDistribution); + $this->set('initialDistribution', $this->Attribute->defaultDistribution()); $fieldDesc = array(); $distributionLevels = $this->Attribute->distributionLevels; if (empty($sgs)) { diff --git a/app/Controller/EventReportsController.php b/app/Controller/EventReportsController.php index 1b6a45f6e..78e511ce3 100644 --- a/app/Controller/EventReportsController.php +++ b/app/Controller/EventReportsController.php @@ -482,12 +482,7 @@ class EventReportsController extends AppController { $distributionLevels = $this->EventReport->Event->Attribute->distributionLevels; $this->set('distributionLevels', $distributionLevels); - $initialDistribution = 5; - $configuredDistribution = Configure::check('MISP.default_attribute_distribution'); - if ($configuredDistribution != null && $configuredDistribution != 'event') { - $initialDistribution = $configuredDistribution; - } - $this->set('initialDistribution', $initialDistribution); + $this->set('initialDistribution', $this->EventReport->Event->Attribute->defaultDistribution()); } private function __injectSharingGroupsDataToViewContext() diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 8830aeb7c..bb8ee3ece 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -3693,11 +3693,7 @@ class EventsController extends AppController { if ($distribution === false) { if (Configure::read('MISP.default_attribute_distribution') != null) { - if (Configure::read('MISP.default_attribute_distribution') == 'event') { - $distribution = 5; - } else { - $distribution = Configure::read('MISP.default_attribute_distribution'); - } + $distribution = $this->Event->Attribute->defaultDistribution(); } else { $distribution = 0; } diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 7a95a2096..3c46af750 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -610,10 +610,7 @@ class Attribute extends AppModel // Set defaults for when some of the mandatory fields don't have defaults // These fields all have sane defaults either based on another field, or due to server settings if (!isset($this->data['Attribute']['distribution'])) { - $this->data['Attribute']['distribution'] = Configure::read('MISP.default_attribute_distribution'); - if ($this->data['Attribute']['distribution'] === 'event') { - $this->data['Attribute']['distribution'] = 5; - } + $this->data['Attribute']['distribution'] = $this->defaultDistribution(); } // If category is not provided, assign default category by type if (empty($this->data['Attribute']['category'])) { @@ -3287,14 +3284,7 @@ class Attribute extends AppModel public function saveAttributes($attributes, $user) { - $defaultDistribution = 5; - if (Configure::read('MISP.default_attribute_distribution') != null) { - if (Configure::read('MISP.default_attribute_distribution') === 'event') { - $defaultDistribution = 5; - } else { - $defaultDistribution = Configure::read('MISP.default_attribute_distribution'); - } - } + $defaultDistribution = $this->defaultDistribution(); $saveResult = true; foreach ($attributes as $attribute) { if (!empty($attribute['encrypt']) && $attribute['encrypt']) { @@ -3499,14 +3489,7 @@ class Attribute extends AppModel public function fetchDistributionData($user) { - $initialDistribution = 5; - if (Configure::read('MISP.default_attribute_distribution') != null) { - if (Configure::read('MISP.default_attribute_distribution') === 'event') { - $initialDistribution = 5; - } else { - $initialDistribution = Configure::read('MISP.default_attribute_distribution'); - } - } + $initialDistribution = $this->defaultDistribution(); $sgs = $this->SharingGroup->fetchAllAuthorised($user, 'name', 1); $distributionLevels = $this->distributionLevels; if (empty($sgs)) { @@ -3654,10 +3637,7 @@ class Attribute extends AppModel } $this->create(); if (!isset($attribute['distribution'])) { - $attribute['distribution'] = Configure::read('MISP.default_attribute_distribution'); - if ($attribute['distribution'] == 'event') { - $attribute['distribution'] = 5; - } + $attribute['distribution'] = $this->defaultDistribution(); } $params = array( 'fieldList' => $this->captureFields @@ -3778,10 +3758,7 @@ class Attribute extends AppModel return 'Invalid sharing group choice.'; } } else if (!isset($attribute['distribution'])) { - $attribute['distribution'] = Configure::read('MISP.default_attribute_distribution'); - if ($attribute['distribution'] === 'event') { - $attribute['distribution'] = 5; - } + $attribute['distribution'] = $this->defaultDistribution(); } $fieldList = self::EDITABLE_FIELDS; if (empty($existingAttribute)) { @@ -4249,6 +4226,23 @@ class Attribute extends AppModel return $typeCategoryMapping; } + /** + * Fetch default distribution from `MISP.default_attribute_distribution` setting. If this setting is not defined, + * default distribution is `5` (Inherit event) + * @return int + */ + public function defaultDistribution() + { + static $distribution; + if ($distribution === null) { + $distribution = Configure::read('MISP.default_attribute_distribution'); + if ($distribution === null || $distribution === 'event') { + $distribution = 5; + } + } + return $distribution; + } + public function __isset($name) { if ($name === 'typeDefinitions' || $name === 'categoryDefinitions') { diff --git a/app/Model/Event.php b/app/Model/Event.php index b0e9af443..54dfa03a9 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -5657,13 +5657,7 @@ class Event extends AppModel public function handleMispFormatFromModuleResult(&$result) { - $defaultDistribution = 5; - if (!empty(Configure::read('MISP.default_attribute_distribution'))) { - $defaultDistribution = Configure::read('MISP.default_attribute_distribution'); - if ($defaultDistribution == 'event') { - $defaultDistribution = 5; - } - } + $defaultDistribution = $this->Attribute->defaultDistribution(); $event = array(); if (!empty($result['results']['Attribute'])) { $attributes = array(); @@ -7179,11 +7173,7 @@ class Event extends AppModel */ public function add_original_file($file, $original_filename, $event_id, $format) { - if (!Configure::check('MISP.default_attribute_distribution') || Configure::read('MISP.default_attribute_distribution') === 'event') { - $distribution = 5; - } else { - $distribution = Configure::read('MISP.default_attribute_distribution'); - } + $distribution = $this->Attribute->defaultDistribution(); $this->Object->create(); $object = array( 'name' => 'original-imported-file', diff --git a/app/Model/EventReport.php b/app/Model/EventReport.php index 89eacac54..07f7bf7cd 100644 --- a/app/Model/EventReport.php +++ b/app/Model/EventReport.php @@ -82,10 +82,7 @@ class EventReport extends AppModel // Set defaults for when some of the mandatory fields don't have defaults // These fields all have sane defaults either based on another field, or due to server settings if (!isset($this->data['EventReport']['distribution'])) { - $this->data['EventReport']['distribution'] = Configure::read('MISP.default_attribute_distribution'); - if ($this->data['EventReport']['distribution'] == 'event') { - $this->data['EventReport']['distribution'] = 5; - } + $this->data['EventReport']['distribution'] = $this->Event->Attribute->defaultDistribution(); } return true; } From f9a54c3d4d4fcc87b2e9dfd98c9a842d261bc51b Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 10:23:40 +0200 Subject: [PATCH 27/78] fix: [internal] Deleting event propagation to ZMQ and Kafka --- app/Model/Event.php | 50 ++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index 54dfa03a9..353287d0e 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -397,32 +397,32 @@ class Event extends AppModel 'event_orgc' => $orgc['Orgc']['name'], 'comment' => __('Automatically blocked by deleting event'), )); + } - if (!empty($this->data['Event']['id'])) { - if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) { - $pubSubTool = $this->getPubSubTool(); - $pubSubTool->event_save(array('Event' => $this->data['Event']), 'delete'); + if (!empty($this->data['Event']['id'])) { + if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) { + $pubSubTool = $this->getPubSubTool(); + $pubSubTool->event_save(array('Event' => $this->data['Event']), 'delete'); + } + if (Configure::read('Plugin.Kafka_enable')) { + $kafkaEventTopic = Configure::read('Plugin.Kafka_event_notifications_topic'); + if(Configure::read('Plugin.Kafka_event_notifications_enable') && !empty($kafkaEventTopic)) { + $kafkaPubTool = $this->getKafkaPubTool(); + $kafkaPubTool->publishJson($kafkaEventTopic, array('Event' => $this->data['Event']), 'delete'); } - if (Configure::read('Plugin.Kafka_enable')) { - $kafkaEventTopic = Configure::read('Plugin.Kafka_event_notifications_topic'); - if(Configure::read('Plugin.Kafka_event_notifications_enable') && !empty($kafkaEventTopic)) { - $kafkaPubTool = $this->getKafkaPubTool(); - $kafkaPubTool->publishJson($kafkaEventTopic, array('Event' => $this->data['Event']), 'delete'); - } - $kafkaPubTopic = Configure::read('Plugin.Kafka_event_publish_notifications_topic'); - if (!empty($this->data['Event']['published']) && Configure::read('Plugin.Kafka_event_publish_notifications_enable') && !empty($kafkaPubTopic)) { - $hostOrg = $this->Org->find('first', array('conditions' => array('name' => Configure::read('MISP.org')), 'fields' => array('id'))); - if (!empty($hostOrg)) { - $user = array('org_id' => $hostOrg['Org']['id'], 'Role' => array('perm_sync' => 0, 'perm_audit' => 0, 'perm_site_admin' => 0), 'Organisation' => $hostOrg['Org']); - $params = array('eventid' => $this->data['Event']['id']); - if (Configure::read('Plugin.Kafka_include_attachments')) { - $params['includeAttachments'] = 1; - } - $fullEvent = $this->fetchEvent($user, $params); - if (!empty($fullEvent)) { - $kafkaPubTool = $this->getKafkaPubTool(); - $kafkaPubTool->publishJson($kafkaPubTopic, $fullEvent[0], 'delete'); - } + $kafkaPubTopic = Configure::read('Plugin.Kafka_event_publish_notifications_topic'); + if (!empty($this->data['Event']['published']) && Configure::read('Plugin.Kafka_event_publish_notifications_enable') && !empty($kafkaPubTopic)) { + $hostOrg = $this->Org->find('first', array('conditions' => array('name' => Configure::read('MISP.org')), 'fields' => array('id'))); + if (!empty($hostOrg)) { + $user = array('org_id' => $hostOrg['Org']['id'], 'Role' => array('perm_sync' => 0, 'perm_audit' => 0, 'perm_site_admin' => 0), 'Organisation' => $hostOrg['Org']); + $params = array('eventid' => $this->data['Event']['id']); + if (Configure::read('Plugin.Kafka_include_attachments')) { + $params['includeAttachments'] = 1; + } + $fullEvent = $this->fetchEvent($user, $params); + if (!empty($fullEvent)) { + $kafkaPubTool = $this->getKafkaPubTool(); + $kafkaPubTool->publishJson($kafkaPubTopic, $fullEvent[0], 'delete'); } } } @@ -460,7 +460,7 @@ class Event extends AppModel } if (!isset($this->data['Event']['threat_level_id'])) { - $this->data['Event']['threat_level_id'] = Configure::read('MISP.default_event_threat_level') ? Configure::read('MISP.default_event_threat_level') : 4; + $this->data['Event']['threat_level_id'] = Configure::read('MISP.default_event_threat_level') ?: 4; } // generate UUID if it doesn't exist From 18d38b7478545b49a596c498a46d2a557451cffb Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 11:12:23 +0200 Subject: [PATCH 28/78] fix: [internal] Remove unused Event::checkIfAuthorised method --- app/Model/Event.php | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index 353287d0e..eca2cfecb 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -4900,33 +4900,6 @@ class Event extends AppModel return true; } - // convenience method to check whether a user can see an event - public function checkIfAuthorised($user, $id) - { - if (!isset($user['id'])) { - throw new MethodNotAllowedException('Invalid user.'); - } - $this->id = $id; - if (!$this->exists()) { - return false; - } - if ($user['Role']['perm_site_admin']) { - return true; - } - $event = $this->find('first', array( - 'conditions' => array('id' => $id), - 'recursive' => -1, - 'fields' => array('id', 'sharing_group_id', 'distribution', 'org_id') - )); - if ($event['Event']['org_id'] == $user['org_id'] || ($event['Event']['distribution'] > 0 && $event['Event']['distribution'] < 4)) { - return true; - } - if ($event['Event']['distribution'] == 4 && $this->SharingGroup->checkIfAuthorised($user, $event['Event']['sharing_group_id'])) { - return true; - } - return false; - } - // expects a date string in the YYYY-MM-DD format // returns the passed string or false if the format is invalid // based on the fix provided by stevengoosensB From 3f490c0989a3ca87f09f6a653d790df101982aea Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 11:13:00 +0200 Subject: [PATCH 29/78] chg: [internal] Move Attribute::resizeImage method to AttachmentTool --- app/Lib/Tools/AttachmentTool.php | 50 ++++++++++++++++++++++++++++++ app/Model/Attribute.php | 52 +------------------------------- app/Model/ShadowAttribute.php | 2 +- 3 files changed, 52 insertions(+), 52 deletions(-) diff --git a/app/Lib/Tools/AttachmentTool.php b/app/Lib/Tools/AttachmentTool.php index daf43e3e2..a9126c355 100644 --- a/app/Lib/Tools/AttachmentTool.php +++ b/app/Lib/Tools/AttachmentTool.php @@ -380,6 +380,56 @@ class AttachmentTool return $this->executeAndParseJsonOutput([$pythonBin, self::ADVANCED_EXTRACTION_SCRIPT_PATH, '-c']); } + /** + * @param string $data + * @param int $maxWidth + * @param int $maxHeight + * @return string + * @throws Exception + */ + public function resizeImage($data, $maxWidth, $maxHeight) + { + $image = imagecreatefromstring($data); + if ($image === false) { + throw new Exception("Image is not valid."); + } + + $currentWidth = imagesx($image); + $currentHeight = imagesy($image); + + // Compute thumbnail size with keeping ratio + if ($currentWidth > $currentHeight) { + $newWidth = min($currentWidth, $maxWidth); + $divisor = $currentWidth / $newWidth; + $newHeight = floor($currentHeight / $divisor); + } else { + $newHeight = min($currentHeight, $maxHeight); + $divisor = $currentHeight / $newHeight; + $newWidth = floor($currentWidth / $divisor); + } + + $imageThumbnail = imagecreatetruecolor($newWidth, $newHeight); + + // Allow transparent background + imagealphablending($imageThumbnail, false); + imagesavealpha($imageThumbnail, true); + $transparent = imagecolorallocatealpha($imageThumbnail, 255, 255, 255, 127); + imagefilledrectangle($imageThumbnail, 0, 0, $newWidth, $newHeight, $transparent); + + // Resize image + imagecopyresampled($imageThumbnail, $image, 0, 0, 0, 0, $newWidth, $newHeight, $currentWidth, $currentHeight); + imagedestroy($image); + + // Output image to string + ob_start(); + imagepng($imageThumbnail, null, 9); + $imageData = ob_get_contents(); + ob_end_clean(); + imagedestroy($imageThumbnail); + + return $imageData; + } + private function tempFileName() { $randomName = (new RandomTool())->random_str(false, 12); diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 3c46af750..f1e5a70b5 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -1611,7 +1611,7 @@ class Attribute extends AppModel // Thumbnail doesn't exists, we need to generate it $imageData = $this->getAttachment($attribute['Attribute']); - $imageData = $this->resizeImage($imageData, $maxWidth, $maxHeight); + $imageData = $this->loadAttachmentTool()->resizeImage($imageData, $maxWidth, $maxHeight); // Save just when requested default thumbnail size if ($maxWidth == 200 && $maxHeight == 200) { @@ -1625,56 +1625,6 @@ class Attribute extends AppModel return $imageData; } - /** - * @param string $data - * @param int $maxWidth - * @param int $maxHeight - * @return string - * @throws Exception - */ - public function resizeImage($data, $maxWidth, $maxHeight) - { - $image = imagecreatefromstring($data); - if ($image === false) { - throw new Exception("Image is not valid."); - } - - $currentWidth = imagesx($image); - $currentHeight = imagesy($image); - - // Compute thumbnail size with keeping ratio - if ($currentWidth > $currentHeight) { - $newWidth = min($currentWidth, $maxWidth); - $divisor = $currentWidth / $newWidth; - $newHeight = floor($currentHeight / $divisor); - } else { - $newHeight = min($currentHeight, $maxHeight); - $divisor = $currentHeight / $newHeight; - $newWidth = floor($currentWidth / $divisor); - } - - $imageThumbnail = imagecreatetruecolor($newWidth, $newHeight); - - // Allow transparent background - imagealphablending($imageThumbnail, false); - imagesavealpha($imageThumbnail, true); - $transparent = imagecolorallocatealpha($imageThumbnail, 255, 255, 255, 127); - imagefilledrectangle($imageThumbnail, 0, 0, $newWidth, $newHeight, $transparent); - - // Resize image - imagecopyresampled($imageThumbnail, $image, 0, 0, 0, 0, $newWidth, $newHeight, $currentWidth, $currentHeight); - imagedestroy($image); - - // Output image to string - ob_start(); - imagepng($imageThumbnail, null, 9); - $imageData = ob_get_contents(); - ob_end_clean(); - imagedestroy($imageThumbnail); - - return $imageData; - } - /** * @param array $user * @param array $resultArray diff --git a/app/Model/ShadowAttribute.php b/app/Model/ShadowAttribute.php index c11e9e75b..66540162a 100644 --- a/app/Model/ShadowAttribute.php +++ b/app/Model/ShadowAttribute.php @@ -868,7 +868,7 @@ class ShadowAttribute extends AppModel // Thumbnail doesn't exists, we need to generate it $imageData = $this->getAttachment($shadowAttribute['ShadowAttribute']); - $imageData = $this->Attribute->resizeImage($imageData, $maxWidth, $maxHeight); + $imageData = $this->loadAttachmentTool()->resizeImage($imageData, $maxWidth, $maxHeight); // Save just when requested default thumbnail size if ($maxWidth == 200 && $maxHeight == 200) { From 5cfb9aade74255ab5d5b5f7e2845fb62ea8f6bf0 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 11:13:53 +0200 Subject: [PATCH 30/78] chg: [internal] Simplified Attribute::editAttribute method --- app/Model/Attribute.php | 49 ++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index f1e5a70b5..ad84ef5c8 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -3726,31 +3726,30 @@ class Attribute extends AppModel 'Validation errors: ' . json_encode($this->validationErrors) . ' Full Attribute: ' . json_encode($attribute) ); return $this->validationErrors; - } else { - if (isset($attribute['Sighting']) && !empty($attribute['Sighting'])) { - $this->Sighting->captureSightings($attribute['Sighting'], $this->id, $eventId, $user); - } - if ($user['Role']['perm_tagger']) { - /* - We should uncomment the line below in the future once we have tag soft-delete - A solution to still keep the behavior for previous instance could be to not soft-delete the Tag if the remote instance - has a version below x - */ - // $this->AttributeTag->pruneOutdatedAttributeTagsFromSync(isset($attribute['Tag']) ? $attribute['Tag'] : array(), $existingAttribute['AttributeTag']); - if (isset($attribute['Tag'])) { - foreach ($attribute['Tag'] as $tag) { - $tag_id = $this->AttributeTag->Tag->captureTag($tag, $user); - if ($tag_id) { - $tag['id'] = $tag_id; - // fix the IDs here - $this->AttributeTag->handleAttributeTag($this->id, $attribute['event_id'], $tag); - } else { - // If we couldn't attach the tag it is most likely because we couldn't create it - which could have many reasons - // However, if a tag couldn't be added, it could also be that the user is a tagger but not a tag editor - // In which case if no matching tag is found, no tag ID is returned. Logging these is pointless as it is the correct behaviour. - if ($user['Role']['perm_tag_editor']) { - $this->loadLog()->createLogEntry($user, 'edit', 'Attribute', $this->id, 'Failed create or attach Tag ' . $tag['name'] . ' to the attribute.'); - } + } + if (!empty($attribute['Sighting'])) { + $this->Sighting->captureSightings($attribute['Sighting'], $this->id, $eventId, $user); + } + if ($user['Role']['perm_tagger']) { + /* + We should uncomment the line below in the future once we have tag soft-delete + A solution to still keep the behavior for previous instance could be to not soft-delete the Tag if the remote instance + has a version below x + */ + // $this->AttributeTag->pruneOutdatedAttributeTagsFromSync(isset($attribute['Tag']) ? $attribute['Tag'] : array(), $existingAttribute['AttributeTag']); + if (isset($attribute['Tag'])) { + foreach ($attribute['Tag'] as $tag) { + $tag_id = $this->AttributeTag->Tag->captureTag($tag, $user); + if ($tag_id) { + $tag['id'] = $tag_id; + // fix the IDs here + $this->AttributeTag->handleAttributeTag($this->id, $attribute['event_id'], $tag); + } else { + // If we couldn't attach the tag it is most likely because we couldn't create it - which could have many reasons + // However, if a tag couldn't be added, it could also be that the user is a tagger but not a tag editor + // In which case if no matching tag is found, no tag ID is returned. Logging these is pointless as it is the correct behaviour. + if ($user['Role']['perm_tag_editor']) { + $this->loadLog()->createLogEntry($user, 'edit', 'Attribute', $this->id, 'Failed create or attach Tag ' . $tag['name'] . ' to the attribute.'); } } } From ee4480bc3032ace09e0cabdd6eb47147b6bd5ccf Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 11:37:36 +0200 Subject: [PATCH 31/78] chg: [internal] Use hasAny method for checkIfAuthorised methods --- app/Model/SharingGroupOrg.php | 13 ++++--------- app/Model/SharingGroupServer.php | 14 +++++--------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/app/Model/SharingGroupOrg.php b/app/Model/SharingGroupOrg.php index 7da5745be..bb381dbf7 100644 --- a/app/Model/SharingGroupOrg.php +++ b/app/Model/SharingGroupOrg.php @@ -86,14 +86,9 @@ class SharingGroupOrg extends AppModel // pass a sharing group ID and an organisation ID, returns true if it has a matching attached organisation object public function checkIfAuthorised($id, $org_id) { - $sg = $this->find('first', array( - 'conditions' => array('sharing_group_id' => $id, 'org_id' => $org_id), - 'recursive' => -1, - 'fields' => array('id'), - )); - if (!empty($sg)) { - return true; - } - return false; + return $this->hasAny([ + 'sharing_group_id' => $id, + 'org_id' => $org_id, + ]); } } diff --git a/app/Model/SharingGroupServer.php b/app/Model/SharingGroupServer.php index 7b162b428..aaa53ac9a 100644 --- a/app/Model/SharingGroupServer.php +++ b/app/Model/SharingGroupServer.php @@ -97,14 +97,10 @@ class SharingGroupServer extends AppModel // pass a sharing group ID, returns true if it has the local server object attached with "all_orgs" set public function checkIfAuthorised($id) { - $sg = $this->find('first', array( - 'conditions' => array('sharing_group_id' => $id, 'all_orgs' => 1, 'server_id' => 0), - 'recursive' => -1, - 'fields' => array('id'), - )); - if (!empty($sg)) { - return true; - } - return false; + return $this->hasAny([ + 'sharing_group_id' => $id, + 'all_orgs' => 1, + 'server_id' => 0 + ]); } } From 2d7f9e7c8515667da44a5cd76f33761be5700b62 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 11:37:54 +0200 Subject: [PATCH 32/78] chg: [internal] Use ?: operator --- app/Controller/EventsController.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index bb8ee3ece..1becd451a 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -1981,13 +1981,13 @@ class EventsController extends AppController } if (!empty($this->data)) { if (!isset($this->request->data['Event']['distribution'])) { - $this->request->data['Event']['distribution'] = Configure::read('MISP.default_event_distribution') ? Configure::read('MISP.default_event_distribution') : 0; + $this->request->data['Event']['distribution'] = Configure::read('MISP.default_event_distribution') ?: 0; } if (!isset($this->request->data['Event']['analysis'])) { $this->request->data['Event']['analysis'] = 0; } if (!isset($this->request->data['Event']['threat_level_id'])) { - $this->request->data['Event']['threat_level_id'] = Configure::read('MISP.default_event_threat_level') ? Configure::read('MISP.default_event_threat_level') : 4; + $this->request->data['Event']['threat_level_id'] = Configure::read('MISP.default_event_threat_level') ?: 4; } if (!isset($this->request->data['Event']['date'])) { $this->request->data['Event']['date'] = date('Y-m-d'); From 69b150b46abc82c2b78e89543df109ffce230f75 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 11:38:35 +0200 Subject: [PATCH 33/78] chg: [internal] Use hasAny for SG existence check --- app/Model/SharingGroup.php | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/app/Model/SharingGroup.php b/app/Model/SharingGroup.php index 3f5fbb619..bc8b2cc1d 100644 --- a/app/Model/SharingGroup.php +++ b/app/Model/SharingGroup.php @@ -99,22 +99,16 @@ class SharingGroup extends AppModel public function beforeDelete($cascade = false) { - $countEvent = $this->Event->find('count', array( - 'recursive' => -1, - 'conditions' => array('sharing_group_id' => $this->id) - )); - $countThread = $this->Thread->find('count', array( - 'recursive' => -1, - 'conditions' => array('sharing_group_id' => $this->id) - )); - $countAttribute = $this->Attribute->find('count', array( - 'recursive' => -1, - 'conditions' => array('sharing_group_id' => $this->id) - )); - if (($countEvent + $countThread + $countAttribute) == 0) { - return true; + if ($this->Event->hasAny(['sharing_group_id' => $this->id])) { + return false; } - return false; + if ($this->Thread->hasAny(['sharing_group_id' => $this->id])) { + return false; + } + if ($this->Attribute->hasAny(['sharing_group_id' => $this->id])) { + return false; + } + return true; } /** @@ -431,11 +425,7 @@ class SharingGroup extends AppModel public function checkIfExists($uuid) { - return !empty($this->SharingGroup->find('first', array( - 'conditions' => array('SharingGroup.uuid' => $uuid), - 'recursive' => -1, - 'fields' => array('SharingGroup.id') - ))); + return $this->hasAny(['SharingGroup.uuid' => $uuid]); } // returns true if the SG exists and the user is allowed to see it From aaa81bfd66e5f2ea86f248a5d3c1f44bcfc18efa Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 11:39:02 +0200 Subject: [PATCH 34/78] chg: [internal] Simplified SharingGroup::checkIfAuthorised method --- app/Model/SharingGroup.php | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/app/Model/SharingGroup.php b/app/Model/SharingGroup.php index bc8b2cc1d..e9457e09f 100644 --- a/app/Model/SharingGroup.php +++ b/app/Model/SharingGroup.php @@ -428,11 +428,18 @@ class SharingGroup extends AppModel return $this->hasAny(['SharingGroup.uuid' => $uuid]); } - // returns true if the SG exists and the user is allowed to see it + /** + * Returns true if the SG exists and the user is allowed to see it + * @param array $user + * @param int|string $id SG ID or UUID + * @param bool $adminCheck + * @return bool|mixed + */ public function checkIfAuthorised($user, $id, $adminCheck = true) { - if (isset($this->__sgAuthorisationCache['access'][boolval($adminCheck)][$id])) { - return $this->__sgAuthorisationCache['access'][boolval($adminCheck)][$id]; + $adminCheck = (bool)$adminCheck; + if (isset($this->__sgAuthorisationCache['access'][$adminCheck][$id])) { + return $this->__sgAuthorisationCache['access'][$adminCheck][$id]; } if (Validation::uuid($id)) { $sgid = $this->find('first', array( @@ -443,6 +450,7 @@ class SharingGroup extends AppModel if (empty($sgid)) { return false; } + $uuid = $id; $id = $sgid['SharingGroup']['id']; } else { if (!$this->exists($id)) { @@ -452,12 +460,15 @@ class SharingGroup extends AppModel if (!isset($user['id'])) { throw new MethodNotAllowedException('Invalid user.'); } - if (($adminCheck && $user['Role']['perm_site_admin']) || $this->SharingGroupServer->checkIfAuthorised($id) || $this->SharingGroupOrg->checkIfAuthorised($id, $user['org_id'])) { - $this->__sgAuthorisationCache['access'][boolval($adminCheck)][$id] = true; - return true; + $authorized = ($adminCheck && $user['Role']['perm_site_admin']) || + $this->SharingGroupServer->checkIfAuthorised($id) || + $this->SharingGroupOrg->checkIfAuthorised($id, $user['org_id']); + $this->__sgAuthorisationCache['access'][$adminCheck][$id] = $authorized; + if (isset($uuid)) { + // If uuid was provided, cache also result by UUID to make check faster + $this->__sgAuthorisationCache['access'][$adminCheck][$uuid] = $authorized; } - $this->__sgAuthorisationCache['access'][boolval($adminCheck)][$id] = false; - return false; + return $authorized; } /** From a7ea4bbc4d3aeda4f0c846e317b6f0e4c2f1b8b6 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 11:57:34 +0200 Subject: [PATCH 35/78] chg: [internal] Remove unused method --- app/Model/SharingGroup.php | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/app/Model/SharingGroup.php b/app/Model/SharingGroup.php index e9457e09f..1f6977282 100644 --- a/app/Model/SharingGroup.php +++ b/app/Model/SharingGroup.php @@ -1021,18 +1021,4 @@ class SharingGroup extends AppModel } return $sg[0]; } - - public function getSharingGroupIdByUuid($user, $data) - { - $sg = $this->find('first', array( - 'conditions' => array('SharingGroup.uuid' => $data['sharing_group_id']), - 'recursive' => -1, - 'fields' => array('SharingGroup.id') - )); - if (!empty($sg) && $this->checkIfAuthorised($user, $sg['SharingGroup']['id'])) { - $data['sharing_group_id'] = $sg['SharingGroup']['id']; - return $data; - } - return false; - } } From 3af50e07704ea3eb5e821dd9bd34e2647c7ea884 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 11:58:37 +0200 Subject: [PATCH 36/78] chg: [internal] Faster validating SG --- app/Model/SharingGroup.php | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/app/Model/SharingGroup.php b/app/Model/SharingGroup.php index 1f6977282..2943726a2 100644 --- a/app/Model/SharingGroup.php +++ b/app/Model/SharingGroup.php @@ -69,10 +69,8 @@ class SharingGroup extends AppModel 'access' => array() ); - public function beforeValidate($options = array()) { - parent::beforeValidate(); if (empty($this->data['SharingGroup']['uuid'])) { $this->data['SharingGroup']['uuid'] = CakeText::uuid(); } else { @@ -86,12 +84,7 @@ class SharingGroup extends AppModel $this->data['SharingGroup']['active'] = 0; } $this->data['SharingGroup']['modified'] = $date; - $sameNameSG = $this->find('first', array( - 'conditions' => array('SharingGroup.name' => $this->data['SharingGroup']['name']), - 'recursive' => -1, - 'fields' => array('SharingGroup.name') - )); - if (!empty($sameNameSG) && !isset($this->data['SharingGroup']['id'])) { + if (!isset($this->data['SharingGroup']['id']) && $this->hasAny(['SharingGroup.name' => $this->data['SharingGroup']['name']])) { $this->data['SharingGroup']['name'] = $this->data['SharingGroup']['name'] . '_' . mt_rand(0, 9999); } return true; From 4ddab29b7dd6a7fdfdcbafce6c2093d8d38d86e3 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 12:36:28 +0200 Subject: [PATCH 37/78] chg: [internal] Cache capturing tag results --- app/Model/Event.php | 92 ++++++++++++++++++++++++++---------------- app/Model/EventTag.php | 1 + 2 files changed, 59 insertions(+), 34 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index eca2cfecb..83fe59b97 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -3390,7 +3390,7 @@ class Event extends AppModel // When we receive an event via REST, we might end up with organisations, sharing groups, tags that we do not know // or which we need to update. All of that is controlled in this method. - private function __captureObjects($data, $user, $server=false) + private function __captureObjects($data, array $user, $server=false) { // First we need to check whether the event or any attributes are tied to a sharing group and whether the user is even allowed to create the sharing group / is part of it if (isset($data['Event']['distribution']) && $data['Event']['distribution'] == 4) { @@ -3442,78 +3442,102 @@ class Event extends AppModel } $event_tag_ids = array(); + $capturedTags = []; // cache captured tag + $eventTags = []; if (isset($data['Event']['EventTag'])) { if (isset($data['Event']['EventTag']['id'])) { $data['Event']['EventTag'] = array($data['Event']['EventTag']); } - $eventTags = array(); - foreach ($data['Event']['EventTag'] as $k => $tag) { - $temp = $this->EventTag->Tag->captureTag($data['Event']['EventTag'][$k]['Tag'], $user); - if ($temp && !in_array($temp, $event_tag_ids)) { - $eventTags[] = array('tag_id' => $temp); - $event_tag_ids[] = $temp; + foreach ($data['Event']['EventTag'] as $tag) { + $tagId = $this->captureTagWithCache($tag['Tag'], $user, $capturedTags); + if ($tagId && !in_array($tagId, $event_tag_ids)) { + $eventTags[] = array('tag_id' => $tagId); + $event_tag_ids[] = $tagId; } - unset($data['Event']['EventTag'][$k]); } - $data['Event']['EventTag'] = $eventTags; - } else { - $data['Event']['EventTag'] = array(); } if (isset($data['Event']['Tag'])) { if (isset($data['Event']['Tag']['name'])) { $data['Event']['Tag'] = array($data['Event']['Tag']); } foreach ($data['Event']['Tag'] as $tag) { - $tag_id = $this->EventTag->Tag->captureTag($tag, $user); + $tag_id = $this->captureTagWithCache($tag, $user, $capturedTags); if ($tag_id && !in_array($tag_id, $event_tag_ids)) { - $data['Event']['EventTag'][] = array('tag_id' => $tag_id); + $eventTags[] = array('tag_id' => $tag_id); $event_tag_ids[] = $tag_id; } } unset($data['Event']['Tag']); } + $data['Event']['EventTag'] = $eventTags; if (!empty($data['Event']['Attribute'])) { - $data['Event']['Attribute'] = $this->__captureAttributeTags($data['Event']['Attribute'], $user); + $data['Event']['Attribute'] = $this->__captureAttributeTags($data['Event']['Attribute'], $user, $capturedTags); } if (!empty($data['Event']['Object'])) { foreach ($data['Event']['Object'] as $k => $object) { - if (!empty($data['Event']['Object'][$k]['Attribute'])) { - $data['Event']['Object'][$k]['Attribute'] = $this->__captureAttributeTags($data['Event']['Object'][$k]['Attribute'], $user); + if (!empty($object['Attribute'])) { + $data['Event']['Object'][$k]['Attribute'] = $this->__captureAttributeTags($object['Attribute'], $user, $capturedTags); } } } return $data; } - private function __captureAttributeTags($attributes, $user) + /** + * @param array $tag + * @param array $user + * @param array $capturedTags + * @return false|int + * @throws Exception + */ + private function captureTagWithCache(array $tag, array $user, array &$capturedTags) + { + $tagName = $tag['name']; + if (isset($capturedTags[$tagName])) { + $tagId = $capturedTags[$tagName]; + } else { + $tagId = (int)$this->Attribute->AttributeTag->Tag->captureTag($tag, $user); + if ($tagId) { + $capturedTags[$tagName] = $tagId; + } + } + return $tagId; + } + + /** + * Capture tags for attributes and replace tags just by IDs + * @param array $attributes + * @param array $user + * @param array $capturedTags + * @return array + * @throws Exception + */ + private function __captureAttributeTags(array $attributes, array $user, array &$capturedTags) { foreach ($attributes as $k => $a) { - if (isset($attributes[$k]['AttributeTag'])) { - if (isset($attributes[$k]['AttributeTag']['id'])) { - $attributes[$k]['AttributeTag'] = array($attributes[$k]['AttributeTag']); + $attributeTags = []; + if (isset($a['AttributeTag'])) { + if (isset($a['AttributeTag']['id'])) { + $a['AttributeTag'] = array($a['AttributeTag']); } - $attributeTags = array(); - foreach ($attributes[$k]['AttributeTag'] as $tk => $tag) { - $attributeTags[] = array('tag_id' => $this->Attribute->AttributeTag->Tag->captureTag($attributes[$k]['AttributeTag'][$tk]['Tag'], $user)); - unset($attributes[$k]['AttributeTag'][$tk]); + foreach ($a['AttributeTag'] as $tag) { + $attributeTags[] = array('tag_id' => $this->captureTagWithCache($tag['Tag'], $user, $capturedTags)); } - $attributes[$k]['AttributeTag'] = $attributeTags; - } else { - $attributes[$k]['AttributeTag'] = array(); } - if (isset($attributes[$k]['Tag'])) { - if (isset($attributes[$k]['Tag']['name'])) { - $attributes[$k]['Tag'] = array($attributes[$k]['Tag']); + if (isset($a['Tag'])) { + if (isset($a['Tag']['name'])) { + $a['Tag'] = array($a['Tag']); } - foreach ($attributes[$k]['Tag'] as $tag) { - $tag_id = $this->Attribute->AttributeTag->Tag->captureTag($tag, $user); - if ($tag_id) { - $attributes[$k]['AttributeTag'][] = array('tag_id' => $tag_id); + foreach ($a['Tag'] as $tag) { + $tagId = $this->captureTagWithCache($tag, $user, $capturedTags); + if ($tagId) { + $attributeTags[] = array('tag_id' => $tagId); } } unset($attributes[$k]['Tag']); } + $attributes[$k]['AttributeTag'] = $attributeTags; } return $attributes; } diff --git a/app/Model/EventTag.php b/app/Model/EventTag.php index 4174cbd7b..013d29126 100644 --- a/app/Model/EventTag.php +++ b/app/Model/EventTag.php @@ -3,6 +3,7 @@ App::uses('AppModel', 'Model'); /** * @property Event $Event + * @property Tag $Tag */ class EventTag extends AppModel { From 962929dcbd1b4eb04951320952023ae4754a1555 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 12:41:39 +0200 Subject: [PATCH 38/78] chg: [internal] Remove unused method Tag::findEventTags --- app/Model/Tag.php | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/app/Model/Tag.php b/app/Model/Tag.php index d64136ad7..0edb295b3 100644 --- a/app/Model/Tag.php +++ b/app/Model/Tag.php @@ -386,25 +386,6 @@ class Tag extends AppModel return $existingTag['Tag']['id']; } - // find all tags that belong to a given eventId - public function findEventTags($eventId) - { - $tags = array(); - $params = array( - 'recursive' => 1, - 'contain' => 'EventTag', - ); - $result = $this->find('all', $params); - foreach ($result as $tag) { - foreach ($tag['EventTag'] as $eventTag) { - if ($eventTag['event_id'] == $eventId) { - $tags[] = $tag['Tag']; - } - } - } - return $tags; - } - public function random_color() { $colour = '#'; From c43d1c18b0acd9db460b3cff287c7c6ca75b5dfd Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 13:24:06 +0200 Subject: [PATCH 39/78] chg: [internal] Simplified SharingGroup::appendOrgsAndServers --- app/Model/Event.php | 16 ++++++++++++---- app/Model/SharingGroup.php | 8 ++------ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index 83fe59b97..32399e0bd 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -3388,9 +3388,16 @@ class Event extends AppModel return $element; } - // When we receive an event via REST, we might end up with organisations, sharing groups, tags that we do not know - // or which we need to update. All of that is controlled in this method. - private function __captureObjects($data, array $user, $server=false) + /** + * When we receive an event via REST, we might end up with organisations, sharing groups, tags that we do not know + * or which we need to update. All of that is controlled in this method. + * @param array $data + * @param array $user + * @param array|false $server + * @return array + * @throws Exception + */ + private function __captureObjects(array $data, array $user, $server=false) { // First we need to check whether the event or any attributes are tied to a sharing group and whether the user is even allowed to create the sharing group / is part of it if (isset($data['Event']['distribution']) && $data['Event']['distribution'] == 4) { @@ -3497,8 +3504,9 @@ class Event extends AppModel if (isset($capturedTags[$tagName])) { $tagId = $capturedTags[$tagName]; } else { - $tagId = (int)$this->Attribute->AttributeTag->Tag->captureTag($tag, $user); + $tagId = $this->Attribute->AttributeTag->Tag->captureTag($tag, $user); if ($tagId) { + $tagId = (int)$tagId; $capturedTags[$tagName] = $tagId; } } diff --git a/app/Model/SharingGroup.php b/app/Model/SharingGroup.php index 2943726a2..aab99cfaa 100644 --- a/app/Model/SharingGroup.php +++ b/app/Model/SharingGroup.php @@ -260,9 +260,7 @@ class SharingGroup extends AppModel 'fields' => $orgFields, 'conditions' => ['id' => array_keys($orgsToFetch)], ]); - foreach ($orgs as $org) { - $orgsById[$org['Organisation']['id']] = $org['Organisation']; - } + $orgsById = array_column(array_column($orgs, 'Organisation'), null, 'id'); } $serversById = []; @@ -272,9 +270,7 @@ class SharingGroup extends AppModel 'fields' => $serverFields, 'conditions' => ['id' => array_keys($serverToFetch)], ]); - foreach ($servers as $server) { - $serversById[$server['Server']['id']] = $server['Server']; - } + $serversById = array_column(array_column($servers, 'Server'), null, 'id'); } foreach ($sharingGroups as &$sg) { From 26badb4e3e3a57c93b0f739a6d98075af898f3a3 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 14:26:33 +0200 Subject: [PATCH 40/78] chg: [internal] Save multiple tags in one call --- app/Model/Attribute.php | 7 ++++--- app/Model/AttributeTag.php | 27 ++++++++++++++------------- app/Model/Event.php | 7 ++++--- app/Model/EventTag.php | 18 ++++++++---------- 4 files changed, 30 insertions(+), 29 deletions(-) diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index ad84ef5c8..24407a37f 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -3609,14 +3609,15 @@ class Attribute extends AppModel 'Validation errors: ' . json_encode($this->validationErrors) . ' Full Attribute: ' . json_encode($attribute) ); } else { - if (isset($attribute['AttributeTag'])) { + if (!empty($attribute['AttributeTag'])) { + $toSave = []; foreach ($attribute['AttributeTag'] as $at) { unset($at['id']); - $this->AttributeTag->create(); $at['attribute_id'] = $this->id; $at['event_id'] = $eventId; - $this->AttributeTag->save($at); + $toSave[] = $at; } + $this->AttributeTag->saveMany($toSave, ['validate' => true]); } if (isset($attribute['Tag'])) { if (!empty($attribute['Tag']['name'])) { diff --git a/app/Model/AttributeTag.php b/app/Model/AttributeTag.php index bd8e7b4ec..a72f8e0b9 100644 --- a/app/Model/AttributeTag.php +++ b/app/Model/AttributeTag.php @@ -9,18 +9,20 @@ class AttributeTag extends AppModel { public $actsAs = array('AuditLog', 'Containable'); - public $validate = array( - 'attribute_id' => array( - 'valueNotEmpty' => array( - 'rule' => array('valueNotEmpty'), - ), - ), - 'tag_id' => array( - 'valueNotEmpty' => array( - 'rule' => array('valueNotEmpty'), - ), - ), - ); + public $validate = [ + 'attribute_id' => [ + 'rule' => 'numeric', + 'required' => true, + ], + 'event_id' => [ + 'rule' => 'numeric', + 'required' => true, + ], + 'tag_id' => [ + 'rule' => 'numeric', + 'required' => true, + ], + ]; public $belongsTo = array( 'Attribute' => array( @@ -381,5 +383,4 @@ class AttributeTag extends AppModel $attribute_tags_name['tags'] = array_diff_key($attribute_tags_name['tags'], $attribute_tags_name['clusters']); // de-dup if needed. return $attribute_tags_name; } - } diff --git a/app/Model/Event.php b/app/Model/Event.php index 32399e0bd..4e0054787 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -3850,12 +3850,13 @@ class Event extends AppModel 'change' => '' )); } - if (isset($data['Event']['EventTag'])) { + if (!empty($data['Event']['EventTag'])) { + $toSave = []; foreach ($data['Event']['EventTag'] as $et) { - $this->EventTag->create(); $et['event_id'] = $this->id; - $this->EventTag->save($et); + $toSave[] = $et; } + $this->EventTag->saveMany($toSave, ['validate' => true]); } $parentEvent = $this->find('first', array( 'conditions' => array('Event.id' => $this->id), diff --git a/app/Model/EventTag.php b/app/Model/EventTag.php index 013d29126..bf40429dd 100644 --- a/app/Model/EventTag.php +++ b/app/Model/EventTag.php @@ -10,16 +10,14 @@ class EventTag extends AppModel public $actsAs = array('AuditLog', 'Containable'); public $validate = array( - 'event_id' => array( - 'valueNotEmpty' => array( - 'rule' => array('valueNotEmpty'), - ), - ), - 'tag_id' => array( - 'valueNotEmpty' => array( - 'rule' => array('valueNotEmpty'), - ), - ), + 'event_id' => [ + 'rule' => 'numeric', + 'required' => true, + ], + 'tag_id' => [ + 'rule' => 'numeric', + 'required' => true, + ], ); public $belongsTo = array( From d8a5441a21b60a2e103075cfbfb952382838a311 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 14:48:29 +0200 Subject: [PATCH 41/78] chg: [internal] Faster saving attributes --- app/Controller/AttributesController.php | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index f3ccc0c10..cf8923030 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -111,9 +111,6 @@ class AttributesController extends AppController if ($eventId === false) { throw new MethodNotAllowedException(__('No event ID set.')); } - if (!$this->userRole['perm_add']) { - throw new MethodNotAllowedException(__('You do not have permissions to create attributes')); - } $event = $this->Attribute->Event->fetchSimpleEvent($this->Auth->user(), $eventId, ['contain' => ['Orgc']]); if (!$event) { throw new NotFoundException(__('Invalid event')); @@ -169,19 +166,19 @@ class AttributesController extends AppController $inserted_ids = array(); foreach ($attributes as $k => $attribute) { $validationErrors = array(); - $this->Attribute->captureAttribute($attribute, $event['Event']['id'], $this->Auth->user(), false, false, false, $validationErrors, $this->params['named']); + $this->Attribute->captureAttribute($attribute, $event['Event']['id'], $this->Auth->user(), false, false, $event, $validationErrors, $this->params['named']); if (empty($validationErrors)) { $inserted_ids[] = $this->Attribute->id; - $successes +=1; + $successes++; } else { $fails["attribute_" . $k] = $validationErrors; } } - if (!empty($successes)) { + if ($successes !== 0) { $this->Attribute->Event->unpublishEvent($event['Event']['id']); } if ($this->_isRest()) { - if (!empty($successes)) { + if ($successes !== 0) { $attributes = $this->Attribute->find('all', array( 'recursive' => -1, 'conditions' => array('Attribute.id' => $inserted_ids), @@ -191,7 +188,7 @@ class AttributesController extends AppController ) ) )); - if (count($attributes) == 1) { + if (count($attributes) === 1) { $attributes = $attributes[0]; } else { $result = array('Attribute' => array()); @@ -286,8 +283,7 @@ class AttributesController extends AppController $categories = $this->_arrayToValuesIndexArray($categories); $this->set('categories', $categories); - $this->loadModel('SharingGroup'); - $sgs = $this->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1); + $sgs = $this->Attribute->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', true); $this->set('sharingGroups', $sgs); $this->set('initialDistribution', $this->Attribute->defaultDistribution()); $fieldDesc = array(); @@ -312,7 +308,7 @@ class AttributesController extends AppController $this->set('typeDefinitions', $this->Attribute->typeDefinitions); $this->set('categoryDefinitions', $this->Attribute->categoryDefinitions); $this->set('event', $event); - $this->set('action', $this->action); + $this->set('action', $this->request->action); } public function download($id = null) From 6986d8498761e62b87b724046f8750468b99cebf Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 15:40:04 +0200 Subject: [PATCH 42/78] chg: [internal] Simplified code for MispObject::captureObject --- app/Model/MispObject.php | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/app/Model/MispObject.php b/app/Model/MispObject.php index d102d2ffa..3923c8365 100644 --- a/app/Model/MispObject.php +++ b/app/Model/MispObject.php @@ -991,28 +991,25 @@ class MispObject extends AppModel return true; } } - if (isset($object['Object']['id'])) { - unset($object['Object']['id']); - } + unset($object['Object']['id']); $object['Object']['event_id'] = $eventId; - if ($this->save($object)) { - if ($unpublish) { - $this->Event->unpublishEvent($eventId); - } - $objectId = $this->id; - if (!empty($object['Object']['Attribute'])) { - foreach ($object['Object']['Attribute'] as $attribute) { - $this->Attribute->captureAttribute($attribute, $eventId, $user, $objectId, false, $parentEvent); - } - } - return true; - } else { + if (!$this->save($object)) { $this->loadLog()->createLogEntry($user, 'add', 'Object', 0, 'Object dropped due to validation for Event ' . $eventId . ' failed: ' . $object['Object']['name'], 'Validation errors: ' . json_encode($this->validationErrors) . ' Full Object: ' . json_encode($object) ); + return 'fail'; } - return 'fail'; + if ($unpublish) { + $this->Event->unpublishEvent($eventId); + } + $objectId = $this->id; + if (!empty($object['Object']['Attribute'])) { + foreach ($object['Object']['Attribute'] as $attribute) { + $this->Attribute->captureAttribute($attribute, $eventId, $user, $objectId, false, $parentEvent); + } + } + return true; } public function editObject($object, array $event, $user, $log, $force = false, &$nothingToChange = false) From b3ca92a0ec51485a09abb3b7bccedf195a630e9c Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 15:40:41 +0200 Subject: [PATCH 43/78] chg: [internal] Remove unused code when saving attributes for event --- app/Model/Event.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index 4e0054787..e2f6c44c1 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -3864,11 +3864,11 @@ class Event extends AppModel )); if (!empty($data['Event']['Attribute'])) { $attributeHashes = []; - foreach ($data['Event']['Attribute'] as $k => $attribute) { + foreach ($data['Event']['Attribute'] as $attribute) { $attributeHash = sha1($attribute['value'] . '|' . $attribute['type'] . '|' . $attribute['category'], true); if (!isset($attributeHashes[$attributeHash])) { // do not save duplicate values $attributeHashes[$attributeHash] = true; - $data['Event']['Attribute'][$k] = $this->Attribute->captureAttribute($attribute, $this->id, $user, 0, null, $parentEvent); + $this->Attribute->captureAttribute($attribute, $this->id, $user, 0, null, $parentEvent); } } unset($attributeHashes); @@ -3910,9 +3910,9 @@ class Event extends AppModel if (!empty($data['Event']['published']) && 1 == $data['Event']['published']) { // do the necessary actions to publish the event (email, upload,...) if (('true' != Configure::read('MISP.disablerestalert')) && (empty($server) || empty($server['Server']['publish_without_email']))) { - $this->sendAlertEmailRouter($this->getID(), $user); + $this->sendAlertEmailRouter($this->id, $user); } - $this->publish($this->getID(), $passAlong); + $this->publish($this->id, $passAlong); } if (empty($data['Event']['locked']) && !empty(Configure::read('MISP.default_event_tag_collection'))) { $this->TagCollection = ClassRegistry::init('TagCollection'); From 0427ee33b906b0d009e8fc9cb97aec84254bac44 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 15:42:37 +0200 Subject: [PATCH 44/78] chg: [internal] Remove unused attribute from MispObject::captureObject method --- app/Model/Event.php | 2 +- app/Model/MispObject.php | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index e2f6c44c1..dcb509a50 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -3877,7 +3877,7 @@ class Event extends AppModel if (!empty($data['Event']['Object'])) { $referencesToCapture = []; foreach ($data['Event']['Object'] as $object) { - $result = $this->Object->captureObject($object, $this->id, $user, null, false, $breakOnDuplicate, $parentEvent); + $result = $this->Object->captureObject($object, $this->id, $user, false, $breakOnDuplicate, $parentEvent); if (isset($object['ObjectReference'])) { foreach ($object['ObjectReference'] as $objectRef) { $objectRef['source_uuid'] = $object['uuid']; diff --git a/app/Model/MispObject.php b/app/Model/MispObject.php index 3923c8365..cdbad4b24 100644 --- a/app/Model/MispObject.php +++ b/app/Model/MispObject.php @@ -967,14 +967,13 @@ class MispObject extends AppModel * @param array $object * @param int $eventId * @param array $user - * @param false $log - Not used anymore * @param bool $unpublish * @param false $breakOnDuplicate * @param array|false $parentEvent * @return bool|string * @throws Exception */ - public function captureObject($object, $eventId, $user, $log = false, $unpublish = true, $breakOnDuplicate = false, $parentEvent = false) + public function captureObject($object, $eventId, $user, $unpublish = true, $breakOnDuplicate = false, $parentEvent = false) { $this->create(); if (!isset($object['Object'])) { From ae08f5d38299398a4e1c2afa37fe09eacad192af Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 15:54:42 +0200 Subject: [PATCH 45/78] chg: [internal] No need to initialize Sighting model --- app/Model/Event.php | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index dcb509a50..deb3089e3 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -15,6 +15,7 @@ App::uses('SendEmailTemplate', 'Tools'); * @property EventTag $EventTag * @property SharingGroup $SharingGroup * @property ThreatLevel $ThreatLevel + * @property Sighting $Sighting */ class Event extends AppModel { @@ -585,7 +586,6 @@ class Event extends AppModel public function attachSightingsCountToEvents(array $user, array $events) { $eventIds = array_column(array_column($events, 'Event'), 'id'); - $this->Sighting = ClassRegistry::init('Sighting'); $this->Sighting->virtualFields['count'] = 'count(Sighting.id)'; $sightings = $this->Sighting->find('list', array( 'fields' => array('Sighting.event_id', 'Sighting.count'), @@ -2113,9 +2113,6 @@ class Event extends AppModel $sharingGroupData = $options['sgReferenceOnly'] ? [] : $this->__cacheSharingGroupData($user, $useCache); // Initialize classes that will be necessary during event fetching - if ((empty($options['metadata']) && empty($options['noSightings'])) && !isset($this->Sighting)) { - $this->Sighting = ClassRegistry::init('Sighting'); - } if (!empty($options['includeDecayScore']) && !isset($this->DecayingModel)) { $this->DecayingModel = ClassRegistry::init('DecayingModel'); } @@ -3901,7 +3898,6 @@ class Event extends AppModel } // zeroq: check if sightings are attached and add to event if (isset($data['Sighting']) && !empty($data['Sighting'])) { - $this->Sighting = ClassRegistry::init('Sighting'); $this->Sighting->captureSightings($data['Sighting'], null, $this->id, $user); } if ($fromXml) { @@ -4160,7 +4156,6 @@ class Event extends AppModel } // zeroq: if sightings then attach to event if (isset($data['Sighting']) && !empty($data['Sighting'])) { - $this->Sighting = ClassRegistry::init('Sighting'); $this->Sighting->captureSightings($data['Sighting'], null, $this->id, $user); } // if published -> do the actual publishing @@ -4393,16 +4388,13 @@ class Event extends AppModel * @param array $server * @param array $event * @param array $sightingsUuidsToPush - * @throws HttpClientJsonException + * @throws HttpSocketJsonException * @throws JsonException * @throws Exception */ private function pushSightingsToServer(array $server, array $event, array $sightingsUuidsToPush = []) { App::uses('ServerSyncTool', 'Tools'); - if (!isset($this->Sighting)) { - $this->Sighting = ClassRegistry::init('Sighting'); - } $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); try { if ($serverSync->eventExists($event) === false) { From d646a685658eb02887c5b3285c9bef93eed92a56 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 16:04:58 +0200 Subject: [PATCH 46/78] chg: [internal] Remove dead code --- app/Model/Event.php | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index deb3089e3..5187c099b 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -885,7 +885,7 @@ class Event extends AppModel * This function receives the reference of the variable, so no return is required as it directly * modifies the original data. */ - public function cleanupEventArrayFromXML(&$data) + private function cleanupEventArrayFromXML(&$data) { $objects = array('Attribute', 'ShadowAttribute', 'Object'); foreach ($objects as $object) { @@ -3775,32 +3775,11 @@ class Event extends AppModel if ($fromXml) { $data = $this->__captureObjects($data, $user, $server); } - if ($data === false) { - $failedCapture = true; - } } } else { if ($fromXml) { $data = $this->__captureObjects($data, $user, $server); } - if ($data === false) { - $failedCapture = true; - } - } - if (!empty($failedCapture)) { - $this->Log->create(); - $this->Log->save(array( - 'org' => $user['Organisation']['name'], - 'model' => 'Event', - 'model_id' => 0, - 'email' => $user['email'], - 'action' => 'add', - 'user_id' => $user['id'], - 'title' => 'Event could not be saved due to a failed sharing group capture.', - 'change' => '' - )); - $validationErrors['Event'] = 'Issues saving a Sharing Group.'; - return json_encode($validationErrors); } $fieldList = array( 'org_id', From 3d0e678231949586bdd605019c56e29507680c0b Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 16:07:57 +0200 Subject: [PATCH 47/78] chg: [internal] Simplify Event::__captureObjects code --- app/Model/Event.php | 97 ++++++++++++++++++++------------------------- 1 file changed, 44 insertions(+), 53 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index 5187c099b..018e2d01a 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -3367,6 +3367,12 @@ class Event extends AppModel return $template; } + /** + * @param array $element + * @param array $user + * @param bool|false $server + * @return array + */ public function captureSGForElement($element, $user, $server=false) { if (isset($element['SharingGroup'])) { @@ -3388,45 +3394,35 @@ class Event extends AppModel /** * When we receive an event via REST, we might end up with organisations, sharing groups, tags that we do not know * or which we need to update. All of that is controlled in this method. - * @param array $data + * @param array $event * @param array $user * @param array|false $server * @return array * @throws Exception */ - private function __captureObjects(array $data, array $user, $server=false) + private function __captureObjects(array $event, array $user, $server=false) { // First we need to check whether the event or any attributes are tied to a sharing group and whether the user is even allowed to create the sharing group / is part of it - if (isset($data['Event']['distribution']) && $data['Event']['distribution'] == 4) { - $data['Event'] = $this->captureSGForElement($data['Event'], $user, $server); + if (isset($event['distribution']) && $event['distribution'] == 4) { + $event = $this->captureSGForElement($event, $user, $server); } - if (!empty($data['Event']['Attribute'])) { - foreach ($data['Event']['Attribute'] as $k => $a) { - unset($data['Event']['Attribute']['id']); + if (!empty($event['Attribute'])) { + foreach ($event['Attribute'] as $k => $a) { + unset($event['Attribute']['id']); if (isset($a['distribution']) && $a['distribution'] == 4) { - $data['Event']['Attribute'][$k] = $this->captureSGForElement($a, $user, $server); - if ($data['Event']['Attribute'][$k] === false) { - unset($data['Event']['Attribute']); - } + $event['Attribute'][$k] = $this->captureSGForElement($a, $user, $server); } } } - if (!empty($data['Event']['Object'])) { - foreach ($data['Event']['Object'] as $k => $o) { + if (!empty($event['Object'])) { + foreach ($event['Object'] as $k => $o) { if (isset($o['distribution']) && $o['distribution'] == 4) { - $data['Event']['Object'][$k] = $this->captureSGForElement($o, $user, $server); - if ($data['Event']['Object'][$k] === false) { - unset($data['Event']['Object'][$k]); - continue; - } + $event['Object'][$k] = $this->captureSGForElement($o, $user, $server); } if (!empty($o['Attribute'])) { foreach ($o['Attribute'] as $k2 => $a) { if (isset($a['distribution']) && $a['distribution'] == 4) { - $data['Event']['Object'][$k]['Attribute'][$k2] = $this->captureSGForElement($a, $user, $server); - if ($data['Event']['Object'][$k]['Attribute'][$k2] === false) { - unset($data['Event']['Object'][$k]['Attribute'][$k2]); - } + $event['Object'][$k]['Attribute'][$k2] = $this->captureSGForElement($a, $user, $server); } } } @@ -3435,24 +3431,24 @@ class Event extends AppModel // first we want to see how the creator organisation is encoded // The options here are either by passing an organisation object along or simply passing a string along - if (isset($data['Event']['Orgc'])) { - $data['Event']['orgc_id'] = $this->Orgc->captureOrg($data['Event']['Orgc'], $user); - unset($data['Event']['Orgc']); - } elseif (isset($data['Event']['orgc'])) { - $data['Event']['orgc_id'] = $this->Orgc->captureOrg($data['Event']['orgc'], $user); - unset($data['Event']['orgc']); + if (isset($event['Orgc'])) { + $event['orgc_id'] = $this->Orgc->captureOrg($event['Orgc'], $user); + unset($event['Orgc']); + } elseif (isset($event['orgc'])) { + $event['orgc_id'] = $this->Orgc->captureOrg($event['orgc'], $user); + unset($event['orgc']); } else { - $data['Event']['orgc_id'] = $user['org_id']; + $event['orgc_id'] = $user['org_id']; } $event_tag_ids = array(); $capturedTags = []; // cache captured tag $eventTags = []; - if (isset($data['Event']['EventTag'])) { - if (isset($data['Event']['EventTag']['id'])) { - $data['Event']['EventTag'] = array($data['Event']['EventTag']); + if (isset($event['EventTag'])) { + if (isset($event['EventTag']['id'])) { + $event['EventTag'] = array($event['EventTag']); } - foreach ($data['Event']['EventTag'] as $tag) { + foreach ($event['EventTag'] as $tag) { $tagId = $this->captureTagWithCache($tag['Tag'], $user, $capturedTags); if ($tagId && !in_array($tagId, $event_tag_ids)) { $eventTags[] = array('tag_id' => $tagId); @@ -3460,32 +3456,32 @@ class Event extends AppModel } } } - if (isset($data['Event']['Tag'])) { - if (isset($data['Event']['Tag']['name'])) { - $data['Event']['Tag'] = array($data['Event']['Tag']); + if (isset($event['Tag'])) { + if (isset($event['Tag']['name'])) { + $event['Tag'] = array($event['Tag']); } - foreach ($data['Event']['Tag'] as $tag) { + foreach ($event['Tag'] as $tag) { $tag_id = $this->captureTagWithCache($tag, $user, $capturedTags); if ($tag_id && !in_array($tag_id, $event_tag_ids)) { $eventTags[] = array('tag_id' => $tag_id); $event_tag_ids[] = $tag_id; } } - unset($data['Event']['Tag']); + unset($event['Tag']); } - $data['Event']['EventTag'] = $eventTags; + $event['EventTag'] = $eventTags; - if (!empty($data['Event']['Attribute'])) { - $data['Event']['Attribute'] = $this->__captureAttributeTags($data['Event']['Attribute'], $user, $capturedTags); + if (!empty($event['Attribute'])) { + $event['Attribute'] = $this->__captureAttributeTags($event['Attribute'], $user, $capturedTags); } - if (!empty($data['Event']['Object'])) { - foreach ($data['Event']['Object'] as $k => $object) { + if (!empty($event['Object'])) { + foreach ($event['Object'] as $k => $object) { if (!empty($object['Attribute'])) { - $data['Event']['Object'][$k]['Attribute'] = $this->__captureAttributeTags($object['Attribute'], $user, $capturedTags); + $event['Object'][$k]['Attribute'] = $this->__captureAttributeTags($object['Attribute'], $user, $capturedTags); } } } - return $data; + return $event; } /** @@ -3771,16 +3767,11 @@ class Event extends AppModel $created_id = $existingEvent['Event']['id']; } return $existingEvent['Event']['id']; - } else { - if ($fromXml) { - $data = $this->__captureObjects($data, $user, $server); - } - } - } else { - if ($fromXml) { - $data = $this->__captureObjects($data, $user, $server); } } + if ($fromXml) { + $data['Event'] = $this->__captureObjects($data['Event'], $user, $server); + } $fieldList = array( 'org_id', 'orgc_id', From a33bf233422dcfe7cf06d2cf72410d2044516976 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 18:08:55 +0200 Subject: [PATCH 48/78] chg: [internal] Simplify SharingGroup::checkIfAuthorisedToSave --- app/Model/SharingGroup.php | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/app/Model/SharingGroup.php b/app/Model/SharingGroup.php index aab99cfaa..26fd0585b 100644 --- a/app/Model/SharingGroup.php +++ b/app/Model/SharingGroup.php @@ -304,7 +304,6 @@ class SharingGroup extends AppModel // 3. Sync users // a. as long as they are at least users of the SG (they can circumvent the extend rule to // avoid situations where no one can create / edit an SG on an instance after a push) - public function checkIfAuthorisedToSave($user, $sg) { if (isset($sg[0])) { @@ -318,8 +317,9 @@ class SharingGroup extends AppModel } // First let us find out if we already have the SG $local = $this->find('first', array( - 'recursive' => -1, - 'conditions' => array('uuid' => $sg['uuid']) + 'recursive' => -1, + 'conditions' => array('uuid' => $sg['uuid']), + 'fields' => ['id'], )); if (empty($local)) { $orgCheck = false; @@ -332,6 +332,7 @@ class SharingGroup extends AppModel if ($org['Organisation']['uuid'] == $user['Organisation']['uuid']) { if ($user['Role']['perm_sync'] || $org['extend'] == 1) { $orgCheck = true; + break; } } } @@ -396,20 +397,12 @@ class SharingGroup extends AppModel return true; } } - $sgo = $this->SharingGroupOrg->find('first', array( - 'conditions' => array( - 'sharing_group_id' => $id, - 'org_id' => $user['org_id'], - 'extend' => 1, - ), - 'recursive' => -1, - 'fields' => array('id', 'org_id', 'extend') + + return $this->SharingGroupOrg->hasAny(array( + 'sharing_group_id' => $id, + 'org_id' => $user['org_id'], + 'extend' => 1, )); - if (empty($sgo)) { - return false; - } else { - return true; - } } public function checkIfExists($uuid) From d9e89955bcebbf91b47d3c29974232a85dab640b Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 18:41:44 +0200 Subject: [PATCH 49/78] chg: [internal] Simplify fetching Kafka topic --- app/Model/AppModel.php | 26 +++++++++++++++++++++++--- app/Model/Attribute.php | 11 +++++------ app/Model/AttributeTag.php | 15 ++++++--------- app/Model/EventTag.php | 14 ++++++-------- app/Model/MispObject.php | 7 ++----- app/Model/ObjectReference.php | 10 ++++------ app/Model/Sighting.php | 15 ++++++--------- app/Model/Tag.php | 15 ++++++--------- app/Model/User.php | 7 +++---- 9 files changed, 61 insertions(+), 59 deletions(-) diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index b026687b0..81b4835e0 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -2577,9 +2577,10 @@ class AppModel extends Model return true; } - public function publishKafkaNotification($topicName, $data, $action = false) { - $kafkaTopic = Configure::read('Plugin.Kafka_' . $topicName . '_notifications_topic'); - if (Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_' . $topicName . '_notifications_enable') && !empty($kafkaTopic)) { + public function publishKafkaNotification($topicName, $data, $action = false) + { + $kafkaTopic = $this->kafkaTopic($topicName); + if ($kafkaTopic) { $this->getKafkaPubTool()->publishJson($kafkaTopic, $data, $action); } } @@ -3275,4 +3276,23 @@ class AppModel extends Model } return $this->Log; } + + /** + * @param string $name + * @return string|null Null when Kafka is not enabled, topic is not enabled or topic is not defined + */ + protected function kafkaTopic($name) + { + static $kafkaEnabled; + if ($kafkaEnabled === null) { + $kafkaEnabled = (bool)Configure::read('Plugin.Kafka_enable'); + } + if ($kafkaEnabled) { + if (!Configure::read("Plugin.Kafka_{$name}_notifications_enable")) { + return null; + } + return Configure::read("Plugin.Kafka_{$name}_notifications_topic") ?: null; + } + return null; + } } diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 24407a37f..420787544 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -454,9 +454,8 @@ class Attribute extends AppModel $result = $this->saveAttachment($this->data['Attribute']); } $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_attribute_notifications_enable'); - $kafkaTopic = Configure::read('Plugin.Kafka_attribute_notifications_topic'); - $pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_attribute_notifications_enable') && !empty($kafkaTopic); - if ($pubToZmq || $pubToKafka) { + $kafkaTopic = $this->kafkaTopic('attribute'); + if ($pubToZmq || $kafkaTopic) { $attribute = $this->fetchAttribute($this->id); if (!empty($attribute)) { $user = array( @@ -481,7 +480,7 @@ class Attribute extends AppModel $pubSubTool->attribute_save($attribute, $action); unset($attribute['Attribute']['data']); } - if ($pubToKafka) { + if ($kafkaTopic) { if (Configure::read('Plugin.Kafka_include_attachments') && $this->typeIsAttachment($attribute['Attribute']['type'])) { $attribute['Attribute']['data'] = $this->base64EncodeAttachment($attribute['Attribute']); } @@ -513,8 +512,8 @@ class Attribute extends AppModel $pubSubTool = $this->getPubSubTool(); $pubSubTool->attribute_save($this->data, 'delete'); } - $kafkaTopic = Configure::read('Plugin.Kafka_attribute_notifications_topic'); - if (Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_attribute_notifications_enable') && !empty($kafkaTopic)) { + $kafkaTopic = $this->kafkaTopic('attribute'); + if ($kafkaTopic) { $kafkaPubTool = $this->getKafkaPubTool(); $kafkaPubTool->publishJson($kafkaTopic, $this->data, 'delete'); } diff --git a/app/Model/AttributeTag.php b/app/Model/AttributeTag.php index a72f8e0b9..63f8f0a9c 100644 --- a/app/Model/AttributeTag.php +++ b/app/Model/AttributeTag.php @@ -35,11 +35,9 @@ class AttributeTag extends AppModel public function afterSave($created, $options = array()) { - parent::afterSave($created, $options); $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_tag_notifications_enable'); - $kafkaTopic = Configure::read('Plugin.Kafka_tag_notifications_topic'); - $pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_tag_notifications_enable') && !empty($kafkaTopic); - if ($pubToZmq || $pubToKafka) { + $kafkaTopic = $this->kafkaTopic('tag'); + if ($pubToZmq || $kafkaTopic) { $tag = $this->find('first', array( 'recursive' => -1, 'conditions' => array('AttributeTag.id' => $this->id), @@ -52,7 +50,7 @@ class AttributeTag extends AppModel $pubSubTool = $this->getPubSubTool(); $pubSubTool->tag_save($tag, 'attached to attribute'); } - if ($pubToKafka) { + if ($kafkaTopic) { $kafkaPubTool = $this->getKafkaPubTool(); $kafkaPubTool->publishJson($kafkaTopic, $tag, 'attached to attribute'); } @@ -62,9 +60,8 @@ class AttributeTag extends AppModel public function beforeDelete($cascade = true) { $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_tag_notifications_enable'); - $kafkaTopic = Configure::read('Plugin.Kafka_tag_notifications_topic'); - $pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_tag_notifications_enable') && !empty($kafkaTopic); - if ($pubToZmq || $pubToKafka) { + $kafkaTopic = $this->kafkaTopic('tag'); + if ($pubToZmq || $kafkaTopic) { if (!empty($this->id)) { $tag = $this->find('first', array( 'recursive' => -1, @@ -78,7 +75,7 @@ class AttributeTag extends AppModel $pubSubTool = $this->getPubSubTool(); $pubSubTool->tag_save($tag, 'detached from attribute'); } - if ($pubToKafka) { + if ($kafkaTopic) { $kafkaPubTool = $this->getKafkaPubTool(); $kafkaPubTool->publishJson($kafkaTopic, $tag, 'detached from attribute'); } diff --git a/app/Model/EventTag.php b/app/Model/EventTag.php index bf40429dd..99b00ebfc 100644 --- a/app/Model/EventTag.php +++ b/app/Model/EventTag.php @@ -29,9 +29,8 @@ class EventTag extends AppModel { parent::afterSave($created, $options); $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_tag_notifications_enable'); - $kafkaTopic = Configure::read('Plugin.Kafka_tag_notifications_topic'); - $pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_tag_notifications_enable') && !empty($kafkaTopic); - if ($pubToZmq || $pubToKafka) { + $kafkaTopic = $this->kafkaTopic('tag'); + if ($pubToZmq || $kafkaTopic) { $tag = $this->find('first', array( 'recursive' => -1, 'conditions' => array('EventTag.id' => $this->id), @@ -43,7 +42,7 @@ class EventTag extends AppModel $pubSubTool = $this->getPubSubTool(); $pubSubTool->tag_save($tag, 'attached to event'); } - if ($pubToKafka) { + if ($kafkaTopic) { $kafkaPubTool = $this->getKafkaPubTool(); $kafkaPubTool->publishJson($kafkaTopic, $tag, 'attached to event'); } @@ -53,9 +52,8 @@ class EventTag extends AppModel public function beforeDelete($cascade = true) { $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_tag_notifications_enable'); - $kafkaTopic = Configure::read('Plugin.Kafka_tag_notifications_topic'); - $pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_tag_notifications_enable') && !empty($kafkaTopic); - if ($pubToZmq || $pubToKafka) { + $kafkaTopic = $this->kafkaTopic('tag'); + if ($pubToZmq || $kafkaTopic) { if (!empty($this->id)) { $tag = $this->find('first', array( 'recursive' => -1, @@ -68,7 +66,7 @@ class EventTag extends AppModel $pubSubTool = $this->getPubSubTool(); $pubSubTool->tag_save($tag, 'detached from event'); } - if ($pubToKafka) { + if ($kafkaTopic) { $kafkaPubTool = $this->getKafkaPubTool(); $kafkaPubTool->publishJson($kafkaTopic, $tag, 'detached from event'); } diff --git a/app/Model/MispObject.php b/app/Model/MispObject.php index cdbad4b24..bd7856bd5 100644 --- a/app/Model/MispObject.php +++ b/app/Model/MispObject.php @@ -300,11 +300,8 @@ class MispObject extends AppModel $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_object_notifications_enable') && empty($this->data['Object']['skip_zmq']); - $kafkaTopic = Configure::read('Plugin.Kafka_object_notifications_topic'); - $pubToKafka = Configure::read('Plugin.Kafka_enable') && - Configure::read('Plugin.Kafka_object_notifications_enable') && - !empty($kafkaTopic) && - empty($this->data['Object']['skip_kafka']); + $kafkaTopic = $this->kafkaTopic('object'); + $pubToKafka = $kafkaTopic && empty($this->data['Object']['skip_kafka']); if ($pubToZmq || $pubToKafka) { $object = $this->find('first', array( 'conditions' => array('Object.id' => $this->id), diff --git a/app/Model/ObjectReference.php b/app/Model/ObjectReference.php index 5d1016438..c80034d62 100644 --- a/app/Model/ObjectReference.php +++ b/app/Model/ObjectReference.php @@ -45,8 +45,7 @@ class ObjectReference extends AppModel $this->data['ObjectReference']['uuid'] = CakeText::uuid(); } if (empty($this->data['ObjectReference']['timestamp'])) { - $date = new DateTime(); - $this->data['ObjectReference']['timestamp'] = $date->getTimestamp(); + $this->data['ObjectReference']['timestamp'] = time(); } if (!isset($this->data['ObjectReference']['comment'])) { $this->data['ObjectReference']['comment'] = ''; @@ -57,9 +56,8 @@ class ObjectReference extends AppModel public function afterSave($created, $options = array()) { $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_object_reference_notifications_enable'); - $kafkaTopic = Configure::read('Plugin.Kafka_object_reference_notifications_topic'); - $pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_object_reference_notifications_enable') && !empty($kafkaTopic); - if ($pubToZmq || $pubToKafka) { + $kafkaTopic = $this->kafkaTopic('object_reference'); + if ($pubToZmq || $kafkaTopic) { $object_reference = $this->find('first', array( 'conditions' => array('ObjectReference.id' => $this->id), 'recursive' => -1 @@ -72,7 +70,7 @@ class ObjectReference extends AppModel $pubSubTool = $this->getPubSubTool(); $pubSubTool->object_reference_save($object_reference, $action); } - if ($pubToKafka) { + if ($kafkaTopic) { $kafkaPubTool = $this->getKafkaPubTool(); $kafkaPubTool->publishJson($kafkaTopic, $object_reference, $action); } diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php index 95b28ff8b..2947588f9 100644 --- a/app/Model/Sighting.php +++ b/app/Model/Sighting.php @@ -77,11 +77,9 @@ class Sighting extends AppModel public function afterSave($created, $options = array()) { - parent::afterSave($created, $options = array()); $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_sighting_notifications_enable'); - $kafkaTopic = Configure::read('Plugin.Kafka_sighting_notifications_topic'); - $pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_sighting_notifications_enable') && !empty($kafkaTopic); - if ($pubToZmq || $pubToKafka) { + $kafkaTopic = $this->kafkaTopic('sighting'); + if ($pubToZmq || $kafkaTopic) { $user = array( 'org_id' => -1, 'Role' => array( @@ -93,7 +91,7 @@ class Sighting extends AppModel $pubSubTool = $this->getPubSubTool(); $pubSubTool->sighting_save($sighting, 'add'); } - if ($pubToKafka) { + if ($kafkaTopic) { $kafkaPubTool = $this->getKafkaPubTool(); $kafkaPubTool->publishJson($kafkaTopic, $sighting, 'add'); } @@ -105,9 +103,8 @@ class Sighting extends AppModel { parent::beforeDelete(); $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_sighting_notifications_enable'); - $kafkaTopic = Configure::read('Plugin.Kafka_sighting_notifications_topic'); - $pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_sighting_notifications_enable') && !empty($kafkaTopic); - if ($pubToZmq || $pubToKafka) { + $kafkaTopic = $this->kafkaTopic('sighting'); + if ($pubToZmq || $kafkaTopic) { $user = array( 'org_id' => -1, 'Role' => array( @@ -119,7 +116,7 @@ class Sighting extends AppModel $pubSubTool = $this->getPubSubTool(); $pubSubTool->sighting_save($sighting, 'delete'); } - if ($pubToKafka) { + if ($kafkaTopic) { $kafkaPubTool = $this->getKafkaPubTool(); $kafkaPubTool->publishJson($kafkaTopic, $sighting, 'delete'); } diff --git a/app/Model/Tag.php b/app/Model/Tag.php index 0edb295b3..f6b2bc0ea 100644 --- a/app/Model/Tag.php +++ b/app/Model/Tag.php @@ -106,11 +106,9 @@ class Tag extends AppModel public function afterSave($created, $options = array()) { - parent::afterSave($created, $options); $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_tag_notifications_enable'); - $kafkaTopic = Configure::read('Plugin.Kafka_tag_notifications_topic'); - $pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_tag_notifications_enable') && !empty($kafkaTopic); - if ($pubToZmq || $pubToKafka) { + $kafkaTopic = $this->kafkaTopic('tag'); + if ($pubToZmq || $kafkaTopic) { $tag = $this->find('first', array( 'recursive' => -1, 'conditions' => array('Tag.id' => $this->id) @@ -120,7 +118,7 @@ class Tag extends AppModel $pubSubTool = $this->getPubSubTool(); $pubSubTool->tag_save($tag, $action); } - if ($pubToKafka) { + if ($kafkaTopic) { $kafkaPubTool = $this->getKafkaPubTool(); $kafkaPubTool->publishJson($kafkaTopic, $tag, $action); } @@ -130,9 +128,8 @@ class Tag extends AppModel public function beforeDelete($cascade = true) { $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_tag_notifications_enable'); - $kafkaTopic = Configure::read('Plugin.Kafka_tag_notifications_topic'); - $pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_tag_notifications_enable') && !empty($kafkaTopic); - if ($pubToZmq || $pubToKafka) { + $kafkaTopic = $this->kafkaTopic('tag'); + if ($pubToZmq || $kafkaTopic) { if (!empty($this->id)) { $tag = $this->find('first', array( 'recursive' => -1, @@ -142,7 +139,7 @@ class Tag extends AppModel $pubSubTool = $this->getPubSubTool(); $pubSubTool->tag_save($tag, 'delete'); } - if ($pubToKafka) { + if ($kafkaTopic) { $kafkaPubTool = $this->getKafkaPubTool(); $kafkaPubTool->publishJson($kafkaTopic, $tag, 'delete'); } diff --git a/app/Model/User.php b/app/Model/User.php index 88cb104e0..81f9c130f 100644 --- a/app/Model/User.php +++ b/app/Model/User.php @@ -279,9 +279,8 @@ class User extends AppModel public function afterSave($created, $options = array()) { $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_user_notifications_enable'); - $kafkaTopic = Configure::read('Plugin.Kafka_user_notifications_topic'); - $pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_user_notifications_enable') && !empty($kafkaTopic); - if ($pubToZmq || $pubToKafka) { + $kafkaTopic = $this->kafkaTopic('user'); + if ($pubToZmq || $kafkaTopic) { if (!empty($this->data)) { $user = $this->data; if (!isset($user['User'])) { @@ -311,7 +310,7 @@ class User extends AppModel $pubSubTool = $this->getPubSubTool(); $pubSubTool->modified($user, 'user', $action); } - if ($pubToKafka) { + if ($kafkaTopic) { $kafkaPubTool = $this->getKafkaPubTool(); $kafkaPubTool->publishJson($kafkaTopic, $user, $action); } From 34914d44deffbf79b038f82287279671a55d0af3 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 21:34:50 +0200 Subject: [PATCH 50/78] fix: [internal] Remove unused SharingGroup::getSGSyncRules method --- app/Model/SharingGroup.php | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/app/Model/SharingGroup.php b/app/Model/SharingGroup.php index 26fd0585b..8b7d68531 100644 --- a/app/Model/SharingGroup.php +++ b/app/Model/SharingGroup.php @@ -535,36 +535,6 @@ class SharingGroup extends AppModel return false; } - public function getSGSyncRules($sg) - { - $results = array( - 'conditional' => array(), - 'full' => array(), - 'orgs' => array(), - 'no_server_settings' => false - ); - if (isset($sg['SharingGroupServer'])) { - foreach ($sg['SharingGroupServer'] as $server) { - if ($server['server_id'] != 0) { - if ($server['all_orgs']) { - $results['full'][] = $server['id']; - } else { - $results['conditional'][] = $server['id']; - } - } - } - if (empty($results['full']) && empty($results['conditional'])) { - return false; - } - } else { - $results['no_server_settings'] = true; - } - foreach ($sg['SharingGroupOrg'] as $org) { - $results['orgs'][] = $org['Organisation']['uuid']; - } - return $results; - } - /* * Capture a sharing group * Return false if something goes wrong From 99db1b7d7ceacb646ec98a8fd970b6d2cd442f9e Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 10 Oct 2021 21:42:58 +0200 Subject: [PATCH 51/78] chg: [internal] Better logging when saving SharingGroup --- app/Model/SharingGroup.php | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/app/Model/SharingGroup.php b/app/Model/SharingGroup.php index 8b7d68531..aa4db6076 100644 --- a/app/Model/SharingGroup.php +++ b/app/Model/SharingGroup.php @@ -551,7 +551,6 @@ class SharingGroup extends AppModel if (!empty($server) && !empty($server['Server']['local'])) { $syncLocal = true; } - $this->Log = ClassRegistry::init('Log'); $existingSG = !isset($sg['uuid']) ? null : $this->find('first', array( 'recursive' => -1, 'conditions' => array('SharingGroup.uuid' => $sg['uuid']), @@ -562,7 +561,6 @@ class SharingGroup extends AppModel ) )); $forceUpdate = false; - $sg_id = 0; if (empty($existingSG)) { if (!$user['Role']['perm_sharing_group']) { return false; @@ -572,7 +570,7 @@ class SharingGroup extends AppModel return false; } } else { - $existingCaptureResult = $this->captureSGExisting($user, $existingSG, $sg, $syncLocal); + $existingCaptureResult = $this->captureSGExisting($user, $existingSG, $sg); if ($existingCaptureResult !== true) { return $existingCaptureResult; } @@ -593,17 +591,16 @@ class SharingGroup extends AppModel /* * Capture updates for an existing sharing group - * Return true if updates are occuring + * Return true if updates are occurring * Return false if something goes wrong * Return an integer if no update is done but the sharing group can be attached * * @param array $user * @param array $existingSG * @param array $sg - * @param boolean syncLocal * @return int || false || true */ - public function captureSGExisting($user, $existingSG, $sg, $syncLocal) + private function captureSGExisting($user, $existingSG, $sg) { if (!$this->checkIfAuthorised($user, $existingSG['SharingGroup']['id']) && !$user['Role']['perm_sync']) { return false; @@ -640,7 +637,7 @@ class SharingGroup extends AppModel * @param boolean syncLocal * @return int || false */ - public function captureSGNew($user, $sg, $syncLocal) + private function captureSGNew($user, $sg, $syncLocal) { // check if current user is contained in the SG and we are in a local sync setup if (!empty($sg['uuid'])) { @@ -657,25 +654,14 @@ class SharingGroup extends AppModel !($user['Role']['perm_sync'] && $syncLocal ) && !$authorisedToSave ) { - $this->Log->create(); - $entry = array( - 'org' => $user['Organisation']['name'], - 'model' => 'SharingGroup', - 'model_id' => 0, - 'email' => $user['email'], - 'action' => 'error', - 'user_id' => $user['id'], - 'title' => "Tried to save a sharing group with UUID '{$sg['uuid']}' but the user does not belong to it." - ); - $this->Log->save($entry); + $this->loadLog()->createLogEntry($user, 'error', 'SharingGroup', 0, "Tried to save a sharing group with UUID '{$sg['uuid']}' but the user does not belong to it."); return false; } - $this->create(); - $newSG = array(); - $date = date('Y-m-d H:i:s'); if (empty($sg['name'])) { return false; } + $this->create(); + $date = date('Y-m-d H:i:s'); $newSG = [ 'name' => $sg['name'], 'releasability' => !isset($sg['releasability']) ? '' : $sg['releasability'], From dc5451682964aada709c523bfa22cf2c47f12cd1 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 11 Oct 2021 09:40:15 +0200 Subject: [PATCH 52/78] chg: [internal] Use FileAccessTool to read country galaxy cluster --- app/Model/Organisation.php | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/app/Model/Organisation.php b/app/Model/Organisation.php index be9634806..0291b9abd 100644 --- a/app/Model/Organisation.php +++ b/app/Model/Organisation.php @@ -1,6 +1,7 @@ exists()) { - $list = $this->jsonDecode($file->read())['values']; - $file->close(); - } else { - $this->log("MISP Galaxy are not updated, countries will not be available.", LOG_WARNING); + try { + $content = FileAccessTool::readFromFile(APP . '/files/misp-galaxy/clusters/country.json'); + $list = $this->jsonDecode($content)['values']; + } catch (Exception $e) { + $this->logException("MISP Galaxy are not updated, countries will not be available.", $e, LOG_WARNING); $list = []; } } From aedadd070bdd670c20c2929c2fb2810d0dee4032 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 11 Oct 2021 09:41:09 +0200 Subject: [PATCH 53/78] chg: [internal] Use findColumn for Org::getOrgIdsFromMeta method --- app/Model/Organisation.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Model/Organisation.php b/app/Model/Organisation.php index 0291b9abd..db52513ea 100644 --- a/app/Model/Organisation.php +++ b/app/Model/Organisation.php @@ -460,7 +460,7 @@ class Organisation extends AppModel public function getOrgIdsFromMeta($metaConditions) { - $orgIds = $this->find('list', array( + $orgIds = $this->find('column', array( 'conditions' => $metaConditions, 'fields' => array('id'), 'recursive' => -1 @@ -468,7 +468,7 @@ class Organisation extends AppModel if (empty($orgIds)) { return array(-1); } - return array_values($orgIds); + return $orgIds; } public function checkDesiredOrg($suggestedOrg, $registration) From 618c5cf52a91c1bb336eff63bef96729be309b3a Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 11 Oct 2021 09:41:35 +0200 Subject: [PATCH 54/78] chg: [internal] Use hasAny for Org::canSee method --- app/Model/Organisation.php | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/app/Model/Organisation.php b/app/Model/Organisation.php index db52513ea..1f97788b3 100644 --- a/app/Model/Organisation.php +++ b/app/Model/Organisation.php @@ -430,31 +430,33 @@ class Organisation extends AppModel } /** - * @param array $event + * Attach organisations to evnet + * @param array $data * @param array $fields * @return array */ - public function attachOrgs($data, $fields, $scope = 'Event') + public function attachOrgs($data, $fields) { + $event = $data['Event']; $toFetch = []; - if (!isset($this->__orgCache[$data[$scope]['orgc_id']])) { - $toFetch[] = $data[$scope]['orgc_id']; + if (!isset($this->__orgCache[$event['orgc_id']])) { + $toFetch[] = $event['orgc_id']; } - if (!isset($this->__orgCache[$data[$scope]['org_id']]) && $data[$scope]['org_id'] != $data[$scope]['orgc_id']) { - $toFetch[] = $data[$scope]['org_id']; + if (!isset($this->__orgCache[$event['org_id']]) && $event['org_id'] != $event['orgc_id']) { + $toFetch[] = $event['org_id']; } if (!empty($toFetch)) { $orgs = $this->find('all', array( 'conditions' => array('id' => $toFetch), 'recursive' => -1, - 'fields' => $fields + 'fields' => $fields, )); foreach ($orgs as $org) { $this->__orgCache[$org[$this->alias]['id']] = $org[$this->alias]; } } - $data['Orgc'] = $this->__orgCache[$data[$scope]['orgc_id']]; - $data['Org'] = $this->__orgCache[$data[$scope]['org_id']]; + $data['Orgc'] = $this->__orgCache[$event['orgc_id']]; + $data['Org'] = $this->__orgCache[$event['org_id']]; return $data; } @@ -516,12 +518,8 @@ class Organisation extends AppModel // Check if there is event from given org that can current user see $eventConditions = $this->Event->createEventConditions($user); $eventConditions['AND']['Event.orgc_id'] = $orgId; - $event = $this->Event->find('first', array( - 'fields' => array('Event.id'), - 'recursive' => -1, - 'conditions' => $eventConditions, - )); - if (empty($event)) { + $event = $this->Event->hasAny($eventConditions); + if (!$event) { $proposalConditions = $this->Event->ShadowAttribute->buildConditions($user); $proposalConditions['AND']['ShadowAttribute.org_id'] = $orgId; $proposal = $this->Event->ShadowAttribute->find('first', array( From bf89ef2231852cf81d6cd32bdc237ee2e369d5fb Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 3 Oct 2021 17:03:24 +0200 Subject: [PATCH 55/78] chg: [feed] Check ETag when fetching freetext feed --- app/Lib/Tools/SyncTool.php | 2 +- app/Model/Feed.php | 117 ++++++++++++++++++++++++------------- 2 files changed, 77 insertions(+), 42 deletions(-) diff --git a/app/Lib/Tools/SyncTool.php b/app/Lib/Tools/SyncTool.php index b59a192b6..6609d7ff1 100644 --- a/app/Lib/Tools/SyncTool.php +++ b/app/Lib/Tools/SyncTool.php @@ -38,7 +38,7 @@ class SyncTool return $this->createHttpSocket($params); } - public function setupHttpSocketFeed($feed = null) + public function setupHttpSocketFeed() { return $this->createHttpSocket(['compress' => true]); } diff --git a/app/Model/Feed.php b/app/Model/Feed.php index ae272d11f..6b1586f04 100644 --- a/app/Model/Feed.php +++ b/app/Model/Feed.php @@ -209,7 +209,7 @@ class Feed extends AppModel private function downloadManifest($feed, HttpSocket $HttpSocket = null) { $manifestUrl = $feed['Feed']['url'] . '/manifest.json'; - $data = $this->feedGetUri($feed, $manifestUrl, $HttpSocket, true); + $data = $this->feedGetUri($feed, $manifestUrl, $HttpSocket); try { return $this->jsonDecode($data); @@ -231,6 +231,54 @@ class Feed extends AppModel return $events; } + /** + * Load remote file with cache support and etag checking. + * @param array $feed + * @param HttpSocket $HttpSocket + * @return string + * @throws HttpSocketHttpException + */ + private function getFreetextFeedRemote(array $feed, HttpSocket $HttpSocket) + { + $feedCache = APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . (int)$feed['Feed']['id'] . '.cache'; + $feedCacheEtag = APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . (int)$feed['Feed']['id'] . '.etag'; + + $etag = null; + if (file_exists($feedCache)) { + if (time() - filemtime($feedCache) < 600) { + $data = file_get_contents($feedCache); + if ($data !== false) { + return $data; + } + } else if (file_exists($feedCacheEtag)) { + $etag = file_get_contents($feedCacheEtag); + } + } + + try { + $response = $this->feedGetUriRemote($feed, $feed['Feed']['url'], $HttpSocket, $etag); + } catch (HttpSocketHttpException $e) { + if ($e->getCode() === 304) { // not modified + $data = file_get_contents($feedCache); + if ($data === false) { + return $this->feedGetUriRemote($feed, $feed['Feed']['url'], $HttpSocket); // cache file is not readable, fetch without etag + } + return $data; + } else { + throw $e; + } + } + + $savedToCache = file_put_contents($feedCache, $response->body, LOCK_EX); // Save to cache + if ($savedToCache !== false && $response->getHeader('ETag')) { + file_put_contents($feedCacheEtag, $response->getHeader('ETag'), LOCK_EX); // Save etag to file + } else { + unlink($feedCacheEtag); + } + + return $response->body; + } + /** * @param array $feed * @param HttpSocket|null $HttpSocket Null can be for local feed @@ -243,29 +291,11 @@ class Feed extends AppModel */ public function getFreetextFeed($feed, HttpSocket $HttpSocket = null, $type = 'freetext', $page = 1, $limit = 60, &$params = array()) { - $isLocal = $this->isFeedLocal($feed); - $data = false; - - if (!$isLocal) { - $feedCache = APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . intval($feed['Feed']['id']) . '.cache'; - if (file_exists($feedCache)) { - $file = new File($feedCache); - if (time() - $file->lastChange() < 600) { - $data = $file->read(); - if ($data === false) { - throw new Exception("Could not read feed cache file '$feedCache'."); - } - } - } - } - - if ($data === false) { + if ($this->isFeedLocal($feed)) { $feedUrl = $feed['Feed']['url']; - $data = $this->feedGetUri($feed, $feedUrl, $HttpSocket, true); - - if (!$isLocal) { - file_put_contents($feedCache, $data); // save to cache - } + $data = $this->feedGetUri($feed, $feedUrl, $HttpSocket); + } else { + $data = $this->getFreetextFeedRemote($feed, $HttpSocket); } App::uses('ComplexTypeTool', 'Tools'); @@ -891,7 +921,7 @@ class Feed extends AppModel { App::uses('SyncTool', 'Tools'); $syncTool = new SyncTool(); - return $syncTool->setupHttpSocketFeed($feed); + return $syncTool->setupHttpSocketFeed(); } /** @@ -1794,11 +1824,10 @@ class Feed extends AppModel * @param array $feed * @param string $uri * @param HttpSocket|null $HttpSocket Null can be for local feed - * @param bool $followRedirect * @return string * @throws Exception */ - private function feedGetUri($feed, $uri, HttpSocket $HttpSocket = null, $followRedirect = false) + private function feedGetUri($feed, $uri, HttpSocket $HttpSocket = null) { if ($this->isFeedLocal($feed)) { if (file_exists($uri)) { @@ -1812,36 +1841,42 @@ class Feed extends AppModel } } - if ($HttpSocket === null) { - throw new Exception("Feed {$feed['Feed']['name']} is not local, but HttpSocket is not initialized."); + return $this->feedGetUriRemote($feed, $uri, $HttpSocket)->body; + } + + /** + * @param array $feed + * @param string $uri + * @param HttpSocket $HttpSocket + * @param string|null $etag + * @return false|HttpSocketResponse + * @throws HttpSocketHttpException + */ + private function feedGetUriRemote(array $feed, $uri, HttpSocket $HttpSocket, $etag = null) + { + $request = $this->__createFeedRequest($feed['Feed']['headers']); + if ($etag) { + $request['header']['If-None-Match'] = $etag; } - $request = $this->__createFeedRequest($feed['Feed']['headers']); - try { - if ($followRedirect) { - $response = $this->getFollowRedirect($HttpSocket, $uri, $request); - } else { - $response = $HttpSocket->get($uri, array(), $request); - } + $response = $this->getFollowRedirect($HttpSocket, $uri, $request); } catch (Exception $e) { throw new Exception("Fetching the '$uri' failed with exception: {$e->getMessage()}", 0, $e); } if ($response->code != 200) { // intentionally != - throw new Exception("Fetching the '$uri' failed with HTTP error {$response->code}: {$response->reasonPhrase}"); + throw new HttpSocketHttpException($response, $uri); } - $data = $response->body; - $contentType = $response->getHeader('Content-Type'); if ($contentType === 'application/zip') { $zipFile = new File($this->tempFileName()); - $zipFile->write($data); + $zipFile->write($response->body); $zipFile->close(); try { - $data = $this->unzipFirstFile($zipFile); + $response->body = $this->unzipFirstFile($zipFile); } catch (Exception $e) { throw new Exception("Fetching the '$uri' failed: {$e->getMessage()}"); } finally { @@ -1849,7 +1884,7 @@ class Feed extends AppModel } } - return $data; + return $response; } /** From b26f0e435b12726b28dfec9d9bd4a6013049fed9 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 3 Oct 2021 17:21:28 +0200 Subject: [PATCH 56/78] chg: [feed] Clean cache after feed modification --- app/Model/Feed.php | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/app/Model/Feed.php b/app/Model/Feed.php index 6b1586f04..5e2aae019 100644 --- a/app/Model/Feed.php +++ b/app/Model/Feed.php @@ -92,6 +92,18 @@ class Feed extends AppModel return $results; } + public function afterSave($created, $options = array()) + { + if (!$created) { + if (file_exists(APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . (int)$this->data['Feed']['id'] . '.cache')) { + unlink(APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . (int)$this->data['Feed']['id'] . '.cache'); + } + if (file_exists(APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . (int)$this->data['Feed']['id'] . '.etag')) { + unlink(APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . (int)$this->data['Feed']['id'] . '.etag'); + } + } + } + public function validateInputSource($fields) { if (!empty($this->data['Feed']['input_source'])) { @@ -273,7 +285,9 @@ class Feed extends AppModel if ($savedToCache !== false && $response->getHeader('ETag')) { file_put_contents($feedCacheEtag, $response->getHeader('ETag'), LOCK_EX); // Save etag to file } else { - unlink($feedCacheEtag); + if (file_exists($feedCacheEtag)) { + unlink($feedCacheEtag); + } } return $response->body; From 8a8f5095ee2e75038fc61c9c48af3eda7e26401a Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 3 Oct 2021 17:56:30 +0200 Subject: [PATCH 57/78] chg: [feed] Faster saving freetext attributes --- app/Model/Feed.php | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/app/Model/Feed.php b/app/Model/Feed.php index 5e2aae019..f1679b18d 100644 --- a/app/Model/Feed.php +++ b/app/Model/Feed.php @@ -1186,13 +1186,10 @@ class Feed extends AppModel $data[$key]['to_ids'] = $feed['Feed']['override_ids'] ? 0 : $value['to_ids']; $uniqueValues[$value['value']] = true; } - $data = array_values($data); - foreach ($data as $k => $chunk) { - $this->Event->Attribute->create(); - $this->Event->Attribute->save($chunk); - if ($k % 100 === 0) { - $this->jobProgress($jobId, null, 50 + round(($k + 1) / count($data) * 50)); - } + $chunks = array_chunk($data, 100); + foreach ($chunks as $k => $chunk) { + $this->Event->Attribute->saveMany($chunk, ['validate' => true, 'parentEvent' => $event]); + $this->jobProgress($jobId, null, 50 + round(($k * 100 + 1) / count($data) * 50)); } if (!empty($data) || !empty($attributesToDelete)) { unset($event['Event']['timestamp']); From 629b711524693991f1efae0575ded0661b3760aa Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 3 Oct 2021 18:54:37 +0200 Subject: [PATCH 58/78] chf: [feed] Cache MISP feed manifest file --- app/Model/Feed.php | 78 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 70 insertions(+), 8 deletions(-) diff --git a/app/Model/Feed.php b/app/Model/Feed.php index f1679b18d..90d542736 100644 --- a/app/Model/Feed.php +++ b/app/Model/Feed.php @@ -95,12 +95,7 @@ class Feed extends AppModel public function afterSave($created, $options = array()) { if (!$created) { - if (file_exists(APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . (int)$this->data['Feed']['id'] . '.cache')) { - unlink(APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . (int)$this->data['Feed']['id'] . '.cache'); - } - if (file_exists(APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . (int)$this->data['Feed']['id'] . '.etag')) { - unlink(APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . (int)$this->data['Feed']['id'] . '.etag'); - } + $this->cleanFileCache((int)$this->data['Feed']['id']); } } @@ -165,7 +160,7 @@ class Feed extends AppModel */ public function getNewEventUuids($feed, HttpSocket $HttpSocket = null) { - $manifest = $this->downloadManifest($feed, $HttpSocket); + $manifest = $this->isFeedLocal($feed) ? $this->downloadManifest($feed) : $this->getRemoteManifest($feed, $HttpSocket); $this->Event = ClassRegistry::init('Event'); $events = $this->Event->find('all', array( 'conditions' => array( @@ -230,6 +225,73 @@ class Feed extends AppModel } } + /** + * @param int $feedId + */ + private function cleanFileCache($feedId) + { + $cacheDir = APP . 'tmp' . DS . 'cache' . DS; + foreach (["misp_feed_{$feedId}_manifest.cache.gz", "misp_feed_{$feedId}_manifest.etag", "misp_feed_$feedId.cache", "misp_feed_$feedId.etag"] as $fileName) { + if (file_exists($cacheDir . $fileName)) { + unlink($cacheDir . $fileName); + } + } + } + + /** + * Get remote manifest for feed with etag checking. + * @param array $feed + * @param HttpSocketExtended $HttpSocket + * @return array + * @throws HttpSocketHttpException + * @throws JsonException + */ + private function getRemoteManifest(array $feed, HttpSocketExtended $HttpSocket) + { + $feedCache = APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . (int)$feed['Feed']['id'] . '_manifest.cache.gz'; + $feedCacheEtag = APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . (int)$feed['Feed']['id'] . '_manifest.etag'; + + $etag = null; + if (file_exists($feedCache) && file_exists($feedCacheEtag)) { + $etag = file_get_contents($feedCacheEtag); + } + + $manifestUrl = $feed['Feed']['url'] . '/manifest.json'; + + try { + $response = $this->feedGetUriRemote($feed, $manifestUrl, $HttpSocket, $etag); + } catch (HttpSocketHttpException $e) { + if ($e->getCode() === 304) { // not modified + $data = file_get_contents("compress.zlib://$feedCache"); + if ($data === false) { + return $this->feedGetUriRemote($feed, $manifestUrl, $HttpSocket)->json(); // cache file is not readable, fetch without etag + } + return $this->jsonDecode($data); + } else { + throw $e; + } + } + + if ($response->getHeader('ETag')) { + $file = gzopen($feedCache, 'wb1'); + $savedToCache = gzwrite($file, $response->body); + gzclose($file); + if ($savedToCache !== false) { + file_put_contents($feedCacheEtag, $response->getHeader('ETag'), LOCK_EX); // Save etag to file + } else { + if (file_exists($feedCacheEtag)) { + unlink($feedCacheEtag); + } + } + } else { + if (file_exists($feedCacheEtag)) { + unlink($feedCacheEtag); + } + } + + return $response->json(); + } + /** * @param array $feed * @param HttpSocket|null $HttpSocket Null can be for local feed @@ -238,7 +300,7 @@ class Feed extends AppModel */ public function getManifest(array $feed, HttpSocket $HttpSocket = null) { - $events = $this->downloadManifest($feed, $HttpSocket); + $events = $this->isFeedLocal($feed) ? $this->downloadManifest($feed) : $this->getRemoteManifest($feed, $HttpSocket); $events = $this->__filterEventsIndex($events, $feed); return $events; } From 8a4628c7e251cae964672b3955084775c8313a57 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 3 Oct 2021 19:42:40 +0200 Subject: [PATCH 59/78] chg: [feed] Support unicode for feed preview search --- app/Controller/FeedsController.php | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/app/Controller/FeedsController.php b/app/Controller/FeedsController.php index 59e7fac52..e340aad7e 100644 --- a/app/Controller/FeedsController.php +++ b/app/Controller/FeedsController.php @@ -673,7 +673,7 @@ class FeedsController extends AppController $passedArgs = array(); App::uses('SyncTool', 'Tools'); $syncTool = new SyncTool(); - $HttpSocket = $syncTool->setupHttpSocketFeed($feed); + $HttpSocket = $syncTool->setupHttpSocketFeed(); try { $events = $this->Feed->getManifest($feed, $HttpSocket); } catch (Exception $e) { @@ -682,28 +682,25 @@ class FeedsController extends AppController } if (!empty($this->params['named']['searchall'])) { - $searchAll = trim(strtolower($this->params['named']['searchall'])); + $searchAll = trim(mb_strtolower($this->params['named']['searchall'])); foreach ($events as $uuid => $event) { - $found = false; if ($uuid === $searchAll) { - $found = true; + continue; } - if (!$found && strpos(strtolower($event['info']), $searchAll) !== false) { - $found = true; + if (strpos(mb_strtolower($event['info']), $searchAll) !== false) { + continue; } - if (!$found && strpos(strtolower($event['Orgc']['name']), $searchAll) !== false) { - $found = true; + if (strpos(mb_strtolower($event['Orgc']['name']), $searchAll) !== false) { + continue; } - if (!$found && !empty($event['Tag'])) { + if (!empty($event['Tag'])) { foreach ($event['Tag'] as $tag) { - if (strpos(strtolower($tag['name']), $searchAll) !== false) { - $found = true; + if (strpos(mb_strtolower($tag['name']), $searchAll) !== false) { + continue 2; } } } - if (!$found) { - unset($events[$uuid]); - } + unset($events[$uuid]); } } foreach ($filterParams as $k => $filter) { @@ -722,9 +719,7 @@ class FeedsController extends AppController $params = $customPagination->createPaginationRules($events, $this->passedArgs, $this->alias); $this->params->params['paging'] = array($this->modelClass => $params); $events = $customPagination->sortArray($events, $params, true); - if (is_array($events)) { - $customPagination->truncateByPagination($events, $params); - } + $customPagination->truncateByPagination($events, $params); if ($this->_isRest()) { return $this->RestResponse->viewData($events, $this->response->type()); } From 72f180c0af190917b5c2972b138cbcc0692dd6fe Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 3 Oct 2021 19:56:23 +0200 Subject: [PATCH 60/78] chg: [feed] Simplified code for updating events from MISP feed --- app/Model/Feed.php | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/app/Model/Feed.php b/app/Model/Feed.php index 90d542736..efc444d2e 100644 --- a/app/Model/Feed.php +++ b/app/Model/Feed.php @@ -167,13 +167,13 @@ class Feed extends AppModel 'Event.uuid' => array_keys($manifest), ), 'recursive' => -1, - 'fields' => array('Event.id', 'Event.uuid', 'Event.timestamp') + 'fields' => array('Event.uuid', 'Event.timestamp') )); $result = array('add' => array(), 'edit' => array()); foreach ($events as $event) { $eventUuid = $event['Event']['uuid']; if ($event['Event']['timestamp'] < $manifest[$eventUuid]['timestamp']) { - $result['edit'][] = array('uuid' => $eventUuid, 'id' => $event['Event']['id']); + $result['edit'][] = $eventUuid; } else { $this->__cleanupFile($feed, '/' . $eventUuid . '.json'); } @@ -705,10 +705,9 @@ class Feed extends AppModel $currentItem++; } - foreach ($actions['edit'] as $editTarget) { - $uuid = $editTarget['uuid']; + foreach ($actions['edit'] as $uuid) { try { - $result = $this->__updateEventFromFeed($HttpSocket, $feed, $uuid, $editTarget['id'], $user, $filterRules); + $result = $this->__updateEventFromFeed($HttpSocket, $feed, $uuid, $user, $filterRules); if ($result === true) { $results['add']['success'] = $uuid; } else if ($result !== 'blocked') { @@ -1024,13 +1023,12 @@ class Feed extends AppModel * @param HttpSocket|null $HttpSocket Null can be for local feed * @param array $feed * @param string $uuid - * @param int $eventId * @param $user * @param array|bool $filterRules * @return mixed * @throws Exception */ - private function __updateEventFromFeed(HttpSocket $HttpSocket = null, $feed, $uuid, $eventId, $user, $filterRules) + private function __updateEventFromFeed(HttpSocket $HttpSocket = null, $feed, $uuid, $user, $filterRules) { $event = $this->downloadAndParseEventFromFeed($feed, $uuid, $HttpSocket); $event = $this->__prepareEvent($event, $feed, $filterRules); From c1c179dcff41b7e0e0f6e6da3c11cb4bfbf1040f Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 4 Oct 2021 09:45:58 +0200 Subject: [PATCH 61/78] chg: [feed] Use FileAccessTool --- app/Model/Feed.php | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/app/Model/Feed.php b/app/Model/Feed.php index efc444d2e..2494119bc 100644 --- a/app/Model/Feed.php +++ b/app/Model/Feed.php @@ -2,6 +2,7 @@ App::uses('AppModel', 'Model'); App::uses('RandomTool', 'Tools'); App::uses('TmpFileTool', 'Tools'); +App::uses('FileAccessTool', 'Tools'); class Feed extends AppModel { @@ -232,9 +233,7 @@ class Feed extends AppModel { $cacheDir = APP . 'tmp' . DS . 'cache' . DS; foreach (["misp_feed_{$feedId}_manifest.cache.gz", "misp_feed_{$feedId}_manifest.etag", "misp_feed_$feedId.cache", "misp_feed_$feedId.etag"] as $fileName) { - if (file_exists($cacheDir . $fileName)) { - unlink($cacheDir . $fileName); - } + FileAccessTool::deleteFileIfExists($cacheDir . $fileName); } } @@ -273,20 +272,15 @@ class Feed extends AppModel } if ($response->getHeader('ETag')) { - $file = gzopen($feedCache, 'wb1'); - $savedToCache = gzwrite($file, $response->body); - gzclose($file); - if ($savedToCache !== false) { - file_put_contents($feedCacheEtag, $response->getHeader('ETag'), LOCK_EX); // Save etag to file - } else { - if (file_exists($feedCacheEtag)) { - unlink($feedCacheEtag); - } + try { + FileAccessTool::writeCompressedFile($feedCache, $response->body); + FileAccessTool::writeToFile($feedCacheEtag, $response->getHeader('ETag')); + } catch (Exception $e) { + FileAccessTool::deleteFileIfExists($feedCacheEtag); + $this->logException("Could not save file `$feedCache` to cache.", $e, LOG_NOTICE); } } else { - if (file_exists($feedCacheEtag)) { - unlink($feedCacheEtag); - } + FileAccessTool::deleteFileIfExists($feedCacheEtag); } return $response->json(); @@ -343,13 +337,14 @@ class Feed extends AppModel } } - $savedToCache = file_put_contents($feedCache, $response->body, LOCK_EX); // Save to cache - if ($savedToCache !== false && $response->getHeader('ETag')) { - file_put_contents($feedCacheEtag, $response->getHeader('ETag'), LOCK_EX); // Save etag to file - } else { - if (file_exists($feedCacheEtag)) { - unlink($feedCacheEtag); + try { + FileAccessTool::writeToFile($feedCache, $response->body); + if ($response->getHeader('ETag')) { + FileAccessTool::writeToFile($feedCacheEtag, $response->getHeader('ETag')); } + } catch (Exception $e) { + FileAccessTool::deleteFileIfExists($feedCacheEtag); + $this->logException("Could not save file `$feedCache` to cache.", $e, LOG_NOTICE); } return $response->body; @@ -1128,9 +1123,7 @@ class Feed extends AppModel { if ($this->isFeedLocal($feed)) { if (isset($feed['Feed']['delete_local_file']) && $feed['Feed']['delete_local_file']) { - if (file_exists($feed['Feed']['url'] . $file)) { - unlink($feed['Feed']['url'] . $file); - } + FileAccessTool::deleteFileIfExists($feed['Feed']['url'] . $file); } } return true; From af15ea493448255d9db97eb87d96dd8085fafee5 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 9 Oct 2021 15:30:30 +0200 Subject: [PATCH 62/78] chg: [feed] Move feed cache to proper folder --- app/Model/Feed.php | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/app/Model/Feed.php b/app/Model/Feed.php index 2494119bc..87fd4aa81 100644 --- a/app/Model/Feed.php +++ b/app/Model/Feed.php @@ -74,6 +74,8 @@ class Feed extends AppModel 'url_params' => '' ]; + const CACHE_DIR = APP . 'tmp' . DS . 'cache' . DS . 'feeds' . DS; + /* * Cleanup of empty belongsto relationships */ @@ -231,9 +233,8 @@ class Feed extends AppModel */ private function cleanFileCache($feedId) { - $cacheDir = APP . 'tmp' . DS . 'cache' . DS; foreach (["misp_feed_{$feedId}_manifest.cache.gz", "misp_feed_{$feedId}_manifest.etag", "misp_feed_$feedId.cache", "misp_feed_$feedId.etag"] as $fileName) { - FileAccessTool::deleteFileIfExists($cacheDir . $fileName); + FileAccessTool::deleteFileIfExists(self::CACHE_DIR . $fileName); } } @@ -247,8 +248,8 @@ class Feed extends AppModel */ private function getRemoteManifest(array $feed, HttpSocketExtended $HttpSocket) { - $feedCache = APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . (int)$feed['Feed']['id'] . '_manifest.cache.gz'; - $feedCacheEtag = APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . (int)$feed['Feed']['id'] . '_manifest.etag'; + $feedCache = self::CACHE_DIR . 'misp_feed_' . (int)$feed['Feed']['id'] . '_manifest.cache.gz'; + $feedCacheEtag = self::CACHE_DIR . 'misp_feed_' . (int)$feed['Feed']['id'] . '_manifest.etag'; $etag = null; if (file_exists($feedCache) && file_exists($feedCacheEtag)) { @@ -308,8 +309,8 @@ class Feed extends AppModel */ private function getFreetextFeedRemote(array $feed, HttpSocket $HttpSocket) { - $feedCache = APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . (int)$feed['Feed']['id'] . '.cache'; - $feedCacheEtag = APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . (int)$feed['Feed']['id'] . '.etag'; + $feedCache = self::CACHE_DIR . 'misp_feed_' . (int)$feed['Feed']['id'] . '.cache'; + $feedCacheEtag = self::CACHE_DIR . 'misp_feed_' . (int)$feed['Feed']['id'] . '.etag'; $etag = null; if (file_exists($feedCache)) { @@ -872,7 +873,7 @@ class Feed extends AppModel public function downloadEventFromFeed(array $feed, $uuid) { $filerRules = $this->__prepareFilterRules($feed); - $HttpSocket = $this->isFeedLocal($feed) ? null : $this->__setupHttpSocket($feed); + $HttpSocket = $this->isFeedLocal($feed) ? null : $this->__setupHttpSocket(); $event = $this->downloadAndParseEventFromFeed($feed, $uuid, $HttpSocket); return $this->__prepareEvent($event, $feed, $filerRules); } @@ -987,7 +988,7 @@ class Feed extends AppModel return $filterRules; } - private function __setupHttpSocket($feed) + private function __setupHttpSocket() { App::uses('SyncTool', 'Tools'); $syncTool = new SyncTool(); @@ -1062,7 +1063,7 @@ class Feed extends AppModel $this->data['Feed']['settings'] = json_decode($this->data['Feed']['settings'], true); } - $HttpSocket = $this->isFeedLocal($this->data) ? null : $this->__setupHttpSocket($this->data); + $HttpSocket = $this->isFeedLocal($this->data) ? null : $this->__setupHttpSocket(); if ($this->data['Feed']['source_format'] === 'misp') { $this->jobProgress($jobId, 'Fetching event manifest.'); try { @@ -1330,7 +1331,7 @@ class Feed extends AppModel private function __cacheFeed($feed, $redis, $jobId = false) { - $HttpSocket = $this->isFeedLocal($feed) ? null : $this->__setupHttpSocket($feed); + $HttpSocket = $this->isFeedLocal($feed) ? null : $this->__setupHttpSocket(); if ($feed['Feed']['source_format'] === 'misp') { return $this->__cacheMISPFeed($feed, $redis, $HttpSocket, $jobId); } else { @@ -1895,11 +1896,7 @@ class Feed extends AppModel { if ($this->isFeedLocal($feed)) { if (file_exists($uri)) { - $data = file_get_contents($uri); - if ($data === false) { - throw new Exception("Could not read local file '$uri'."); - } - return $data; + return FileAccessTool::readFromFile($uri); } else { throw new Exception("Local file '$uri' doesn't exists."); } @@ -1967,7 +1964,7 @@ class Feed extends AppModel for ($i = 0; $i < $iterations; $i++) { $response = $HttpSocket->get($url, array(), $request); if ($response->isRedirect()) { - $HttpSocket = $this->__setupHttpSocket(null); // Replace $HttpSocket with fresh instance + $HttpSocket = $this->__setupHttpSocket(); // Replace $HttpSocket with fresh instance $url = trim($response->getHeader('Location'), '='); } else { return $response; From 7fd45b4b111b9974dbd236c9476b25f7e17955ce Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 11 Oct 2021 11:52:42 +0200 Subject: [PATCH 63/78] chg: [internal] Add new submodules to diagnostics page --- app/Model/Server.php | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/app/Model/Server.php b/app/Model/Server.php index 3453dd5eb..52ccc5e72 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -3823,36 +3823,47 @@ class Server extends AppModel $status = array(); foreach ($submodules_names as $submodule_name_info) { $submodule_name_info = explode(' ', $submodule_name_info); - $superproject_submodule_commit_id = $submodule_name_info[0]; - $submodule_name = $submodule_name_info[1]; + list($superproject_submodule_commit_id, $submodule_name) = $submodule_name_info; $temp = $this->getSubmoduleGitStatus($submodule_name, $superproject_submodule_commit_id); - if ( !empty($temp) ) { + if (!empty($temp) ) { $status[$submodule_name] = $temp; } } return $status; } - private function _isAcceptedSubmodule($submodule) { - $accepted_submodules_names = array('PyMISP', + private function _isAcceptedSubmodule($submodule) + { + $accepted_submodules_names = array( + 'PyMISP', 'app/files/misp-galaxy', 'app/files/taxonomies', 'app/files/misp-objects', 'app/files/noticelists', 'app/files/warninglists', 'app/files/misp-decaying-models', - 'cti-python-stix2' + 'app/files/scripts/cti-python-stix2', + 'app/files/scripts/misp-opendata', + 'app/files/scripts/python-maec', + 'app/files/scripts/python-stix', + ); - return in_array($submodule, $accepted_submodules_names); + return in_array($submodule, $accepted_submodules_names, true); } - public function getSubmoduleGitStatus($submodule_name, $superproject_submodule_commit_id) { + /** + * @param string $submodule_name + * @param string $superproject_submodule_commit_id + * @return array + */ + private function getSubmoduleGitStatus($submodule_name, $superproject_submodule_commit_id) + { $status = array(); if ($this->_isAcceptedSubmodule($submodule_name)) { $path = APP . '../' . $submodule_name; $submodule_name=(strpos($submodule_name, '/') >= 0 ? explode('/', $submodule_name) : $submodule_name); $submodule_name=end($submodule_name); - $submoduleRemote=exec('cd ' . $path . '; git config --get remote.origin.url'); + //$submoduleRemote=exec('cd ' . $path . '; git config --get remote.origin.url'); exec(sprintf('cd %s; git rev-parse HEAD', $path), $submodule_current_commit_id); if (!empty($submodule_current_commit_id[0])) { $submodule_current_commit_id = $submodule_current_commit_id[0]; From 6aee82112bf279b0e2628e9beb6894eb4e9dc37b Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 11 Oct 2021 12:08:48 +0200 Subject: [PATCH 64/78] chg: [UI] Show proper error when uploading event that already exists --- app/Model/Event.php | 50 ++++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index 018e2d01a..ed8dc498d 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -3651,7 +3651,21 @@ class Event extends AppModel return $results; } - // Low level function to add an Event based on an Event $data array + /** + * Low level function to add an Event based on an Event $data array. + * + * @param array $data + * @param bool $fromXml + * @param array $user + * @param int $org_id + * @param int|null $passAlong Server ID or null + * @param bool $fromPull + * @param int|null $jobId + * @param int $created_id + * @param array $validationErrors + * @return bool|int|string True when new event was created, int when event with the same uuid already exists, string when validation errors + * @throws Exception + */ public function _add(array &$data, $fromXml, array $user, $org_id = 0, $passAlong = null, $fromPull = false, $jobId = null, &$created_id = 0, &$validationErrors = array()) { if (Configure::read('MISP.enableEventBlocklisting') !== false && isset($data['Event']['uuid'])) { @@ -5878,10 +5892,10 @@ class Event extends AppModel $shell_command .= ' 2>' . APP . 'tmp/logs/exec-errors.log'; $result = shell_exec($shell_command); $result = preg_split("/\r\n|\n|\r/", trim($result)); - $result = end($result); + $result = trim(end($result)); $tempFile = file_get_contents($tempFilePath); unlink($tempFilePath); - if (trim($result) == '1') { + if ($result === '1') { $data = file_get_contents($output_path); if ($data === false) { throw new Exception("Could not get content of `$output_path` file."); @@ -5902,24 +5916,26 @@ class Event extends AppModel $this->publish($created_id); } return $created_id; + } else if (is_numeric($result)) { + return __('Event with the same UUID already exists.'); + } else if (is_string($result)) { + return $result; } return $validationIssues; + } else if ($result === '2') { + $response = __('Issues while loading the stix file. '); + } elseif ($result === '3') { + $response = __('Issues with the maec library. '); } else { - if (trim($result) == '2') { - $response = __('Issues while loading the stix file. '); - } elseif (trim($result) == '3') { - $response = __('Issues with the maec library. '); - } else { - $response = __('Issues executing the ingestion script or invalid input. '); - } - if (!$user['Role']['perm_site_admin']) { - $response .= __('Please ask your administrator to '); - } else { - $response .= __('Please '); - } - $response .= ' ' . __('check whether the dependencies for STIX are met via the diagnostic tool.'); - return $response; + $response = __('Issues executing the ingestion script or invalid input. '); } + if (!$user['Role']['perm_site_admin']) { + $response .= __('Please ask your administrator to '); + } else { + $response .= __('Please '); + } + $response .= ' ' . __('check whether the dependencies for STIX are met via the diagnostic tool.'); + return $response; } private function __getTagNamesFromSynonyms($scriptDir) From da6d5aaa127bbde63f2d18bd4f6b5ba2bf3b97e5 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 12 Oct 2021 07:18:33 +0200 Subject: [PATCH 65/78] chg: [MISP/cakephp] updated - to get latest CA bundle --- app/Lib/cakephp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Lib/cakephp b/app/Lib/cakephp index cf14e6546..c31bb4b4b 160000 --- a/app/Lib/cakephp +++ b/app/Lib/cakephp @@ -1 +1 @@ -Subproject commit cf14e6546ec44e3369e3531add11fdb946656280 +Subproject commit c31bb4b4be00d2a0db22c9a038f9fad8a5950efe From 60ac21e252abcdc08f8baec79c73eb739e0ddae6 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Tue, 12 Oct 2021 09:14:28 +0200 Subject: [PATCH 66/78] fix: [log] Undefined index local --- app/Model/Behavior/AuditLogBehavior.php | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/Model/Behavior/AuditLogBehavior.php b/app/Model/Behavior/AuditLogBehavior.php index 698706205..6e14c97d4 100644 --- a/app/Model/Behavior/AuditLogBehavior.php +++ b/app/Model/Behavior/AuditLogBehavior.php @@ -150,12 +150,13 @@ class AuditLogBehavior extends ModelBehavior $modelName = $model->name === 'MispObject' ? 'Object' : $model->name; if ($modelName === 'AttributeTag' || $modelName === 'EventTag') { - $action = $model->data[$model->alias]['local'] ? AuditLog::ACTION_TAG_LOCAL : AuditLog::ACTION_TAG; + $isLocal = isset($model->data[$model->alias]['local']) ? $model->data[$model->alias]['local'] : false; + $action = $isLocal ? AuditLog::ACTION_TAG_LOCAL : AuditLog::ACTION_TAG; $tagInfo = $this->getTagInfo($model, $model->data[$model->alias]['tag_id']); if ($tagInfo) { $modelTitle = $tagInfo['tag_name']; if ($tagInfo['is_galaxy']) { - $action = $model->data[$model->alias]['local'] ? AuditLog::ACTION_GALAXY_LOCAL : AuditLog::ACTION_GALAXY; + $action = $isLocal ? AuditLog::ACTION_GALAXY_LOCAL : AuditLog::ACTION_GALAXY; if ($tagInfo['galaxy_cluster_name']) { $modelTitle = $tagInfo['galaxy_cluster_name']; } @@ -225,12 +226,13 @@ class AuditLogBehavior extends ModelBehavior $id = $model->id; if ($modelName === 'AttributeTag' || $modelName === 'EventTag') { - $action = $model->data[$model->alias]['local'] ? AuditLog::ACTION_REMOVE_TAG_LOCAL : AuditLog::ACTION_REMOVE_TAG; + $isLocal = isset($model->data[$model->alias]['local']) ? $model->data[$model->alias]['local'] : false; + $action = $isLocal ? AuditLog::ACTION_REMOVE_TAG_LOCAL : AuditLog::ACTION_REMOVE_TAG; $tagInfo = $this->getTagInfo($model, $model->data[$model->alias]['tag_id']); if ($tagInfo) { $modelTitle = $tagInfo['tag_name']; if ($tagInfo['is_galaxy']) { - $action = $model->data[$model->alias]['local'] ? AuditLog::ACTION_REMOVE_GALAXY_LOCAL : AuditLog::ACTION_REMOVE_GALAXY; + $action = $isLocal ? AuditLog::ACTION_REMOVE_GALAXY_LOCAL : AuditLog::ACTION_REMOVE_GALAXY; if ($tagInfo['galaxy_cluster_name']) { $modelTitle = $tagInfo['galaxy_cluster_name']; } From b5856d6f73b5d9d6705555cdae21691029ffaa2d Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Tue, 12 Oct 2021 09:25:15 +0200 Subject: [PATCH 67/78] chg: [log] Log when saving tags fails for attribute or event --- app/Model/Attribute.php | 6 +++++- app/Model/Event.php | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 420787544..c9bd4d990 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -3616,7 +3616,11 @@ class Attribute extends AppModel $at['event_id'] = $eventId; $toSave[] = $at; } - $this->AttributeTag->saveMany($toSave, ['validate' => true]); + if (!$this->AttributeTag->saveMany($toSave, ['validate' => true])) { + $this->log("Could not save tags when capturing attribute with ID {$this->id}.", LOG_WARNING); + } else if (!empty($this->AttributeTag->validationErrors)) { + $this->log("Could not save some tags when capturing attribute with ID {$this->id}: " . json_encode($this->AttributeTag->validationErrors), LOG_WARNING); + } } if (isset($attribute['Tag'])) { if (!empty($attribute['Tag']['name'])) { diff --git a/app/Model/Event.php b/app/Model/Event.php index 018e2d01a..31c5b3b1a 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -3823,7 +3823,11 @@ class Event extends AppModel $et['event_id'] = $this->id; $toSave[] = $et; } - $this->EventTag->saveMany($toSave, ['validate' => true]); + if (!$this->EventTag->saveMany($toSave, ['validate' => true])) { + $this->log("Could not save tags when capturing event with ID {$this->id}.", LOG_WARNING); + } else if (!empty($this->EventTag->validationErrors)) { + $this->log("Could not save some tags when capturing event with ID {$this->id}: " . json_encode($this->EventTag->validationErrors), LOG_WARNING); + } } $parentEvent = $this->find('first', array( 'conditions' => array('Event.id' => $this->id), From d25c8e7b2f24be28021a7bae96124277e454aaa0 Mon Sep 17 00:00:00 2001 From: misp-test Date: Tue, 12 Oct 2021 11:05:44 +0200 Subject: [PATCH 68/78] keep tag local state when importing from json or sync from internal Fixes MISP#7810 When importing an Event via JSON, local tags inside the json should stay local after import too, and not be attached as global ones. Same applies for Sync-Operations from internal instances (for any other instance local tags get stripped anyway) --- app/Model/Event.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index 1c74ebdc9..51080a945 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -3463,7 +3463,7 @@ class Event extends AppModel foreach ($event['Tag'] as $tag) { $tag_id = $this->captureTagWithCache($tag, $user, $capturedTags); if ($tag_id && !in_array($tag_id, $event_tag_ids)) { - $eventTags[] = array('tag_id' => $tag_id); + $eventTags[] = array('tag_id' => $tag_id, 'local' => $tag['local']); $event_tag_ids[] = $tag_id; } } From 742f01a8a900db0417c29c971dd75fcda7d86e4e Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Tue, 12 Oct 2021 12:01:06 +0200 Subject: [PATCH 69/78] fix: [internal] Fix saving tags --- app/Model/Event.php | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index 51080a945..07f74a4a1 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -3463,7 +3463,10 @@ class Event extends AppModel foreach ($event['Tag'] as $tag) { $tag_id = $this->captureTagWithCache($tag, $user, $capturedTags); if ($tag_id && !in_array($tag_id, $event_tag_ids)) { - $eventTags[] = array('tag_id' => $tag_id, 'local' => $tag['local']); + $eventTags[] = [ + 'tag_id' => $tag_id, + 'local' => isset($tag['local']) ? $tag['local'] : 0, + ]; $event_tag_ids[] = $tag_id; } } @@ -3533,7 +3536,10 @@ class Event extends AppModel foreach ($a['Tag'] as $tag) { $tagId = $this->captureTagWithCache($tag, $user, $capturedTags); if ($tagId) { - $attributeTags[] = array('tag_id' => $tagId); + $attributeTags[] = [ + 'tag_id' => $tagId, + 'local' => isset($tag['local']) ? $tag['local'] : 0, + ]; } } unset($attributes[$k]['Tag']); From 0ae53cc1f16dccce8c60a8e77bbba9d7d50854fe Mon Sep 17 00:00:00 2001 From: iglocska Date: Tue, 12 Oct 2021 15:02:34 +0200 Subject: [PATCH 70/78] chg: [queryversion] bump --- app/Controller/AppController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index 98da6bc50..f8c7fdcce 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -33,7 +33,7 @@ class AppController extends Controller public $helpers = array('OrgImg', 'FontAwesome', 'UserName', 'DataPathCollector'); - private $__queryVersion = '130'; + private $__queryVersion = '131'; public $pyMispVersion = '2.4.148'; public $phpmin = '7.2'; public $phprec = '7.4'; From 4822aad82c724c28ca7a1d62b36b0462f5b85877 Mon Sep 17 00:00:00 2001 From: iglocska Date: Tue, 12 Oct 2021 15:10:45 +0200 Subject: [PATCH 71/78] Revert "fix: [internal] withCredentials property was added into $.ajaxSetup() to get rid of 403 and 302 responses" This reverts commit b496161f5bf2a7f15ce52cf0dec62a52fc9d713e. --- app/webroot/js/misp.js | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/app/webroot/js/misp.js b/app/webroot/js/misp.js index 35475cb2e..d0a0795e9 100644 --- a/app/webroot/js/misp.js +++ b/app/webroot/js/misp.js @@ -5379,13 +5379,3 @@ $('td.rotate').hover(function() { var t = parseInt($(this).index()) + 1; $table.find('td:nth-child(' + t + ')').css('background-color', ''); }); - -$.ajaxSetup({ - xhrFields: { - withCredentials: true - }, - headers: { - 'X-Requested-With': 'XMLHttpRequest', - }, -}); - From d93a683f15ce4cbbd90233ca719e2b22d28b27b9 Mon Sep 17 00:00:00 2001 From: iglocska Date: Tue, 12 Oct 2021 15:34:59 +0200 Subject: [PATCH 72/78] fix: [attribute index] action ACL fixed --- app/View/Attributes/index.ctp | 77 +++++++++++++++++++++++++++-------- 1 file changed, 60 insertions(+), 17 deletions(-) diff --git a/app/View/Attributes/index.ctp b/app/View/Attributes/index.ctp index 837118849..b9040104e 100755 --- a/app/View/Attributes/index.ctp +++ b/app/View/Attributes/index.ctp @@ -133,14 +133,23 @@ echo $this->element('/genericElements/IndexTable/index_table', [ 'url_params_data_paths' => [ 'Attribute.id' ], - 'icon' => 'comment' + 'icon' => 'comment', + 'complex_requirement' => [ + 'function' => function ($object) use ($isSiteAdmin, $me) { + return $isSiteAdmin || ($object['Event']['orgc_id'] !== $me['org_id']); + } + ] ], [ 'onclick' => "deleteObject('shadow_attributes', 'delete', '[onclick_params_data_path]');", 'onclick_params_data_path' => 'Attribute.id', 'icon' => 'trash', 'title' => __('Propose deletion'), - 'requirement' => $isSiteAdmin, + 'complex_requirement' => [ + 'function' => function ($object) use ($isSiteAdmin, $me) { + return $isSiteAdmin || ($object['Event']['orgc_id'] !== $me['org_id']); + } + ] ], [ 'title' => __('Propose enrichment'), @@ -148,8 +157,12 @@ echo $this->element('/genericElements/IndexTable/index_table', [ 'onclick' => 'simplePopup(\'' . $baseurl . '/events/queryEnrichment/[onclick_params_data_path]/ShadowAttribute\');', 'onclick_params_data_path' => 'Attribute.id', 'complex_requirement' => [ - 'function' => function ($object) use ($modules) { - return isset($modules) && isset($object['Attribute']['type']); + 'function' => function ($object) use ($modules, $isSiteAdmin, $me) { + return ( + ($isSiteAdmin || ($object['Event']['orgc_id'] !== $me['org_id'])) && + isset($cortex_modules) && + isset($cortex_modules['types'][$object['type']]) + ); }, 'options' => [ 'datapath' => [ @@ -164,8 +177,12 @@ echo $this->element('/genericElements/IndexTable/index_table', [ 'onclick' => 'simplePopup(\'' . $baseurl . '/events/queryEnrichment/[onclick_params_data_path]/ShadowAttribute/Cortex\');', 'onclick_params_data_path' => 'Attribute.id', 'complex_requirement' => [ - 'function' => function ($object) use ($cortex_modules) { - return isset($cortex_modules) && isset($cortex_modules['types'][$object['type']]); + 'function' => function ($object) use ($cortex_modules, $isSiteAdmin, $me) { + return ( + ($isSiteAdmin || ($object['Event']['orgc_id'] !== $me['org_id'])) && + isset($cortex_modules) && + isset($cortex_modules['types'][$object['type']]) + ); }, 'options' => [ 'datapath' => [ @@ -175,7 +192,8 @@ echo $this->element('/genericElements/IndexTable/index_table', [ ], ], [ - 'icon' => 'grip-lines-vertical' + 'icon' => 'grip-lines-vertical', + 'requirement' => $isSiteAdmin ], [ 'title' => __('Add enrichment'), @@ -183,8 +201,12 @@ echo $this->element('/genericElements/IndexTable/index_table', [ 'onclick' => 'simplePopup(\'' . $baseurl . '/events/queryEnrichment/[onclick_params_data_path]/Attribute\');', 'onclick_params_data_path' => 'Attribute.id', 'complex_requirement' => [ - 'function' => function ($object) use ($modules) { - return isset($modules) && isset($object['Attribute']['type']); + 'function' => function ($object) use ($modules, $isSiteAdmin, $me) { + return ( + ($isSiteAdmin || ($object['Event']['orgc_id'] === $me['org_id'])) && + isset($cortex_modules) && + isset($cortex_modules['types'][$object['type']]) + ); }, 'options' => [ 'datapath' => [ @@ -199,8 +221,12 @@ echo $this->element('/genericElements/IndexTable/index_table', [ 'onclick' => 'simplePopup(\'' . $baseurl . '/events/queryEnrichment/[onclick_params_data_path]/Attribute/Cortex\');', 'onclick_params_data_path' => 'Attribute.id', 'complex_requirement' => [ - 'function' => function ($object) use ($cortex_modules) { - return isset($cortex_modules) && isset($cortex_modules['types'][$object['type']]); + 'function' => function ($object) use ($cortex_modules, $isSiteAdmin, $me) { + return ( + ($isSiteAdmin || ($object['Event']['orgc_id'] === $me['org_id'])) && + isset($cortex_modules) && + isset($cortex_modules['types'][$object['type']]) + ); }, 'options' => [ 'datapath' => [ @@ -214,7 +240,12 @@ echo $this->element('/genericElements/IndexTable/index_table', [ 'url_params_data_paths' => [ 'Attribute.id' ], - 'icon' => 'edit' + 'icon' => 'edit', + 'complex_requirement' => [ + 'function' => function ($object) use ($isSiteAdmin, $me) { + return $isSiteAdmin || ($object['Event']['orgc_id'] === $me['org_id']); + } + ] ], [ 'onclick' => "deleteObject('attributes', 'delete', '[onclick_params_data_path]');", @@ -223,8 +254,14 @@ echo $this->element('/genericElements/IndexTable/index_table', [ 'title' => __('Soft delete attribute'), 'requirement' => $isSiteAdmin, 'complex_requirement' => [ - 'function' => function ($object) { - return !empty($object['Event']['publish_timestamp']); + 'function' => function ($object, $isSiteAdmin, $me) { + return ( + ( + $isSiteAdmin || + $object['Event']['orgc_id'] !== $me['org_id']) + ) && + !empty($object['Event']['publish_timestamp'] + ); }, ] ], @@ -235,8 +272,14 @@ echo $this->element('/genericElements/IndexTable/index_table', [ 'title' => __('Permanently delete attribute'), 'requirement' => $isSiteAdmin, 'complex_requirement' => [ - 'function' => function ($object) { - return empty($object['Event']['publish_timestamp']); + 'function' => function ($object, $isSiteAdmin, $me) { + return ( + ( + $isSiteAdmin || + $object['Event']['orgc_id'] !== $me['org_id']) + ) && + empty($object['Event']['publish_timestamp'] + ); }, ] ] @@ -312,4 +355,4 @@ echo $this->element('/genericElements/SideMenu/side_menu', ['menuList' => 'event } }); }); - \ No newline at end of file + From c7493a5e0d1931938dbb67b507d6351887923bf9 Mon Sep 17 00:00:00 2001 From: iglocska Date: Tue, 12 Oct 2021 15:57:53 +0200 Subject: [PATCH 73/78] fix: [attribute index] fix galaxy widget for the attribute index - notice errors when logged in as a user --- .../genericElements/IndexTable/Fields/attributeGalaxies.ctp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/View/Elements/genericElements/IndexTable/Fields/attributeGalaxies.ctp b/app/View/Elements/genericElements/IndexTable/Fields/attributeGalaxies.ctp index a68099fad..b6129e73d 100644 --- a/app/View/Elements/genericElements/IndexTable/Fields/attributeGalaxies.ctp +++ b/app/View/Elements/genericElements/IndexTable/Fields/attributeGalaxies.ctp @@ -2,13 +2,12 @@ $attribute = Hash::extract($row, 'Attribute'); $event = Hash::extract($row, 'Event'); -$mayModify = ($isSiteAdmin || ($isAclModify && $event['user_id'] == $me['id'] && $event['orgc_id'] == $me['org_id']) || ($isAclModifyOrg && $event['orgc_id'] == $me['org_id'])); - +$mayModify = ($isSiteAdmin || ($isAclModify && $event['user_id'] == $me['id'] && $event['org_id'] == $me['org_id']) || ($isAclModifyOrg && $event['orgc_id'] == $me['org_id'])); echo $this->element('galaxyQuickViewNew', array( 'mayModify' => $mayModify, 'isAclTagger' => $isAclTagger, 'data' => (!empty($attribute['Galaxy']) ? $attribute['Galaxy'] : array()), - 'event' => $event, + 'event' => ['Event' => $event], 'target_id' => $attribute['id'], 'target_type' => 'attribute', )); From cb20a9e1d9d8ebd1d2ad140594cc96f5659161e0 Mon Sep 17 00:00:00 2001 From: iglocska Date: Tue, 12 Oct 2021 16:12:15 +0200 Subject: [PATCH 74/78] fix: [attribute index] fixed attribute tag widget - notice errors due to missing variables in the closure --- app/View/Attributes/index.ctp | 4 ++-- .../genericElements/IndexTable/Fields/attributeTags.ctp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/View/Attributes/index.ctp b/app/View/Attributes/index.ctp index b9040104e..952fd3866 100755 --- a/app/View/Attributes/index.ctp +++ b/app/View/Attributes/index.ctp @@ -254,7 +254,7 @@ echo $this->element('/genericElements/IndexTable/index_table', [ 'title' => __('Soft delete attribute'), 'requirement' => $isSiteAdmin, 'complex_requirement' => [ - 'function' => function ($object, $isSiteAdmin, $me) { + 'function' => function ($object) use ($isSiteAdmin, $me) { return ( ( $isSiteAdmin || @@ -272,7 +272,7 @@ echo $this->element('/genericElements/IndexTable/index_table', [ 'title' => __('Permanently delete attribute'), 'requirement' => $isSiteAdmin, 'complex_requirement' => [ - 'function' => function ($object, $isSiteAdmin, $me) { + 'function' => function ($object) use ($isSiteAdmin, $me) { return ( ( $isSiteAdmin || diff --git a/app/View/Elements/genericElements/IndexTable/Fields/attributeTags.ctp b/app/View/Elements/genericElements/IndexTable/Fields/attributeTags.ctp index c12f24afc..57f31d6c7 100644 --- a/app/View/Elements/genericElements/IndexTable/Fields/attributeTags.ctp +++ b/app/View/Elements/genericElements/IndexTable/Fields/attributeTags.ctp @@ -14,7 +14,7 @@ $objectId = intval($attribute['id']); 'attributeId' => $attribute['id'], 'tags' => $attribute['AttributeTag'], 'tagAccess' => ($isSiteAdmin || $mayModify), - 'localTagAccess' => ($isSiteAdmin || $mayModify || $me['org_id'] == $event['Event']['org_id'] || (int)$me['org_id'] === Configure::read('MISP.host_org_id')), + 'localTagAccess' => ($isSiteAdmin || $mayModify || $me['org_id'] == $event['org_id'] || (int)$me['org_id'] === Configure::read('MISP.host_org_id')), 'context' => 'event', 'scope' => 'attribute', 'tagConflicts' => isset($attribute['tagConflicts']) ? $attribute['tagConflicts'] : array() From fad8ee23a1591f3242833afe7a0926681c4f4b70 Mon Sep 17 00:00:00 2001 From: iglocska Date: Tue, 12 Oct 2021 16:17:46 +0200 Subject: [PATCH 75/78] chg: [version] bump --- VERSION.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.json b/VERSION.json index f15ace265..5e20a70a4 100644 --- a/VERSION.json +++ b/VERSION.json @@ -1 +1 @@ -{"major":2, "minor":4, "hotfix":149} +{"major":2, "minor":4, "hotfix":150} From 932a94f623f100796c275cff61ced090fb416fec Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 14 Oct 2021 14:20:26 +0200 Subject: [PATCH 76/78] fix: [event index] search by org fixed when using string names, fixes MISP/PyMISP#799 --- app/Controller/EventsController.php | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 1becd451a..63e2fdbdf 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -383,14 +383,18 @@ class EventsController extends AppController continue 2; } + $this->Event->Org->virtualFields = [ + 'upper_name' => 'UPPER(name)', + 'lower_uuid' => 'LOWER(name)', + ]; $orgs = array_column($this->Event->Org->find('all', [ - 'fields' => ['Org.id', 'UPPER(Org.name)', 'LOWER(Org.uuid)'], + 'fields' => ['Org.id', 'Org.upper_name', 'Org.lower_uuid'], 'recursive' => -1, ]), 'Org'); - - $orgByName = array_column($orgs, null, 'name'); - $orgByUuid = array_column($orgs, null, 'uuid'); - + unset($this->Event->Org->virtualFields['upper_name']); + unset($this->Event->Org->virtualFields['lower_uuid']); + $orgByName = array_column($orgs, null, 'upper_name'); + $orgByUuid = array_column($orgs, null, 'lower_uuid'); // if the first character is '!', search for NOT LIKE the rest of the string (excluding the '!' itself of course) $pieces = is_array($v) ? $v : explode('|', $v); $test = array(); From 4be25f0c14256f3967a81ead9354e3b741b50e49 Mon Sep 17 00:00:00 2001 From: Andreas Muehlemann Date: Thu, 14 Oct 2021 17:32:40 +0200 Subject: [PATCH 77/78] added 'git submodule sync' before 'git submodule update' --- docs/INSTALL.rhel7.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index e8d314f53..243564feb 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -219,6 +219,7 @@ installCoreRHEL7 () { cd $PATH_TO_MISP # Fetch submodules + $SUDO_WWW git submodule sync $SUDO_WWW git submodule update --init --recursive # Make git ignore filesystem permission differences for submodules $SUDO_WWW git submodule foreach --recursive git config core.filemode false From 4f653bb6d8aeef75342ac02823e9a62712b43518 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Fri, 15 Oct 2021 08:02:58 +0200 Subject: [PATCH 78/78] chg: [doc] Update to OpenBSD 7.0 --- docs/xINSTALL.OpenBSD.md | 13 ++++--------- mkdocs.yml | 2 +- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/docs/xINSTALL.OpenBSD.md b/docs/xINSTALL.OpenBSD.md index 88b16ba78..fcfd4eee8 100644 --- a/docs/xINSTALL.OpenBSD.md +++ b/docs/xINSTALL.OpenBSD.md @@ -1,5 +1,5 @@ # INSTALLATION INSTRUCTIONS -## for OpenBSD 6.9-amd64 +## for OpenBSD 7.0-amd64 !!! warning This is not fully working yet. Mostly it is a template for our ongoing documentation efforts :spider: @@ -13,11 +13,6 @@ !!! notice This guide attempts to offer native httpd or apache2/nginx. -!!! warning - As of 20181018 the native httpd server is NOT useable with MISP on OpenBSD 6.3. - Thus ONLY Apache 2.x available. - NO *rewrite* available, just yet. It will be in [the next release](https://marc.info/?l=openbsd-tech&m=152761257806283&w=2) - !!! notice As of OpenBSD 6.4 the native httpd has rewrite rules and php 5.6 is gone too. @@ -86,7 +81,7 @@ doas pkg_add -v mariadb-server #### Install misc dependencies ```bash -doas pkg_add -v curl git python--%3.8 redis libmagic autoconf--%2.71 automake--%1.16 libtool unzip--iconv rust +doas pkg_add -v curl git sqlite3 python--%3.9 redis libmagic autoconf--%2.71 automake--%1.16 libtool unzip--iconv rust ``` ```bash @@ -229,8 +224,8 @@ doas rcctl enable httpd #### Install Python virtualenv ```bash doas pkg_add -v py3-virtualenv py3-pip -doas ln -sf /usr/local/bin/pip3.8 /usr/local/bin/pip -doas ln -s /usr/local/bin/python3.8 /usr/local/bin/python +doas ln -sf /usr/local/bin/pip3.9 /usr/local/bin/pip +doas ln -s /usr/local/bin/python3.9 /usr/local/bin/python doas mkdir /usr/local/virtualenvs doas /usr/local/bin/virtualenv /usr/local/virtualenvs/MISP ``` diff --git a/mkdocs.yml b/mkdocs.yml index a735a9fec..7b4ffd98b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -76,7 +76,7 @@ nav: - 'Warning': 'xINSTALL.md' - 'Debian 10': 'xINSTALL.debian10.md' - 'Tsurugi Linux': 'xINSTALL.tsurugi.md' - - 'OpenBSD 6.8': 'xINSTALL.OpenBSD.md' + - 'OpenBSD 7.0': 'xINSTALL.OpenBSD.md' - Config Guides: - 'Elastic Search Logging': 'CONFIG.elasticsearch-logging.md' - 'Amazon S3 attachments': 'CONFIG.s3-attachments.md'