From 3dbc6c0221d4f2bc3130c7cf4f75d9869bac4c0f Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Thu, 3 Oct 2019 19:58:15 +0200 Subject: [PATCH 01/79] chg: [sync] Code cleanup --- app/Model/Server.php | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/app/Model/Server.php b/app/Model/Server.php index 2a6fd79bc..ef4e724c6 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -2671,11 +2671,11 @@ class Server extends AppModel } elseif ("incremental" == $technique) { $eventid_conditions_key = 'Event.id >'; $eventid_conditions_value = $this->data['Server']['lastpushedid']; - } elseif (true == $technique) { + } elseif (intval($technique) !== 0) { $eventid_conditions_key = 'Event.id'; $eventid_conditions_value = intval($technique); } else { - $this->redirect(array('action' => 'index')); + throw new InvalidArgumentException("Technique parameter must be 'full', 'incremental' or event ID."); } $sgs = $this->Event->SharingGroup->find('all', array( 'recursive' => -1, @@ -2683,13 +2683,11 @@ class Server extends AppModel )); $sgIds = array(); foreach ($sgs as $k => $sg) { - if (!$this->Event->SharingGroup->checkIfServerInSG($sg, $this->data)) { - unset($sgs[$k]); - continue; + if ($this->Event->SharingGroup->checkIfServerInSG($sg, $this->data)) { + $sgIds[] = $sg['SharingGroup']['id']; } - $sgIds[] = $sg['SharingGroup']['id']; } - if (!isset($sgIds) || empty($sgIds)) { + if (empty($sgIds)) { $sgIds = array(-1); } $findParams = array( From c3e3958ea7f3e4e1e2fdb5d55ca7104801f09fe6 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 8 Oct 2019 20:45:46 +0200 Subject: [PATCH 02/79] chg: [misp-galaxy] updated to the latest version --- app/files/misp-galaxy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy index ac8236d16..945ed77b9 160000 --- a/app/files/misp-galaxy +++ b/app/files/misp-galaxy @@ -1 +1 @@ -Subproject commit ac8236d16dca06076150fc9d6e7ec544645a676f +Subproject commit 945ed77b9d420e350e7dc18b9d87d967ea2a0633 From 7417f9d6a0559f282ae5c203e06c9b03a20f4134 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Wed, 9 Oct 2019 02:22:18 +0200 Subject: [PATCH 03/79] Check if Organisation index exists, fixes #4809 --- app/Model/SharingGroup.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/Model/SharingGroup.php b/app/Model/SharingGroup.php index 6796b3f92..113740579 100644 --- a/app/Model/SharingGroup.php +++ b/app/Model/SharingGroup.php @@ -189,7 +189,11 @@ class SharingGroup extends AppModel 'conditions' => array('id' => $sg['SharingGroup']['org_id']) )); } - $sg['Organisation'] = $this->__sgoCache[$sg['SharingGroup']['org_id']]['Organisation']; + if(isset($this->__sgoCache[$sg['SharingGroup']['org_id']]['Organisation'])) { + $sg['Organisation'] = $this->__sgoCache[$sg['SharingGroup']['org_id']]['Organisation']; + } else { + $sg['Organisation'] = ''; + } if (!empty($sg['SharingGroupOrg'])) { foreach ($sg['SharingGroupOrg'] as &$sgo) { if (!isset($this->__sgoCache[$sgo['org_id']])) { From 1c1bb2bc75cc477779c1349d8626e081c5e10822 Mon Sep 17 00:00:00 2001 From: Pierre-Jean Grenier Date: Wed, 9 Oct 2019 09:29:04 +0200 Subject: [PATCH 04/79] fix: check if variables are defined --- app/Controller/TagCollectionsController.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/Controller/TagCollectionsController.php b/app/Controller/TagCollectionsController.php index ea15fc20b..ee3060b39 100644 --- a/app/Controller/TagCollectionsController.php +++ b/app/Controller/TagCollectionsController.php @@ -237,6 +237,9 @@ class TagCollectionsController extends AppController $RearrangeTool = new RequestRearrangeTool(); $this->request->data = $RearrangeTool->rearrangeArray($this->request->data, $rearrangeRules); if ($id === false) { + if (!isset($this->request->data['tag_collection'])) { + throw new NotFoundException(__('Invalid tag collection')); + } $id = $this->request->data['tag_collection']; } if (!$this->request->is('post')) { @@ -247,6 +250,9 @@ class TagCollectionsController extends AppController $this->render('/Events/add_tag'); } else { if ($tag_id === false) { + if (!isset($this->request->data['tag'])) { + throw new NotFoundException(__('Invalid tag')); + } $tag_id = $this->request->data['tag']; } $conditions = array(); From 84879df14c2d521abb21b5c454116405ffeae63d Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 9 Oct 2019 10:13:41 +0200 Subject: [PATCH 05/79] chg: [misp-galaxy] updated to the latest version --- app/files/misp-galaxy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy index 945ed77b9..078a9f576 160000 --- a/app/files/misp-galaxy +++ b/app/files/misp-galaxy @@ -1 +1 @@ -Subproject commit 945ed77b9d420e350e7dc18b9d87d967ea2a0633 +Subproject commit 078a9f5763ec65aab6acbe6b51eafc153947ef00 From 1282cdb558935ab68fe5ee5519346fe05ea39425 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Wed, 9 Oct 2019 13:05:33 +0200 Subject: [PATCH 06/79] fix: [UI] Showing whitespaces for 'text' field --- app/View/Elements/Events/View/value_field.ctp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/View/Elements/Events/View/value_field.ctp b/app/View/Elements/Events/View/value_field.ctp index 177cab5e7..0b5900c0a 100644 --- a/app/View/Elements/Events/View/value_field.ctp +++ b/app/View/Elements/Events/View/value_field.ctp @@ -58,7 +58,7 @@ switch ($object['type']) { } else { $sigDisplay = str_replace("\r", '', h($sigDisplay)); $sigDisplay = str_replace(" ", ' ', $sigDisplay); - echo h($sigDisplay); + echo $sigDisplay; } break; From 09646bf91350abf1ebd831ee97d5dcb81c563e7f Mon Sep 17 00:00:00 2001 From: mokaddem Date: Wed, 9 Oct 2019 15:59:16 +0200 Subject: [PATCH 07/79] chg: [eventGraph] Renamed `rotation key` to `pivot key` and do not collaspe when adding/removing edges. Fix #3683 --- app/Lib/Tools/EventGraphTool.php | 2 +- app/webroot/js/action_table.js | 2 +- app/webroot/js/event-graph.js | 36 ++++++++++++++++++-------------- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/app/Lib/Tools/EventGraphTool.php b/app/Lib/Tools/EventGraphTool.php index 9ec0f35e3..24307d5a6 100644 --- a/app/Lib/Tools/EventGraphTool.php +++ b/app/Lib/Tools/EventGraphTool.php @@ -40,7 +40,7 @@ private function __get_event($id) { - $this->__json['available_rotation_key'] = $this->__authorized_JSON_key; + $this->__json['available_pivot_key'] = $this->__authorized_JSON_key; $fullevent = $this->__eventModel->fetchEvent($this->__user, array('eventid' => $id, 'flatten' => 0, 'includeTagRelations' => 1, 'extended' => $this->__extended_view)); $event = array(); diff --git a/app/webroot/js/action_table.js b/app/webroot/js/action_table.js index d3fe6aecd..96a45b5bb 100644 --- a/app/webroot/js/action_table.js +++ b/app/webroot/js/action_table.js @@ -247,7 +247,7 @@ class ActionTable { var td = document.createElement('td'); var btn = document.createElement('button'); btn.classList.add("btn", "btn-danger"); - btn.innerHTML = ''; + btn.innerHTML = ''; btn.type = "button"; btn.setAttribute('rowID', tr.id); if (that.row_action_button_style.tooltip !== undefined) { diff --git a/app/webroot/js/event-graph.js b/app/webroot/js/event-graph.js index 80228639c..6f746ce3e 100644 --- a/app/webroot/js/event-graph.js +++ b/app/webroot/js/event-graph.js @@ -155,7 +155,7 @@ class EventGraph { $("#select_graph_scope").val(value); } - if (value == "Rotation key") { + if (value == "Pivot key") { $("#network-scope-badge").text(value + ": " + eventGraph.scope_keyType); } else { $("#network-scope-badge").text(value); @@ -176,30 +176,30 @@ class EventGraph { label: "Scope", tooltip: "The scope represented by the network", event: function(value) { - if (value == "Rotation key" && $('#input_graph_scope_jsonkey').val() == "") { // no key selected for Rotation key scope + if (value == "Pivot key" && $('#input_graph_scope_jsonkey').val() == "") { // no key selected for Pivot key scope return; } else { eventGraph.update_scope(value); dataHandler.fetch_data_and_update(); } }, - options: ["Reference", "Tag", "Rotation key"], + options: ["Reference", "Tag", "Pivot key"], default: "Reference" }); menu_scope.add_select({ id: "input_graph_scope_jsonkey", - label: "Rotation key", + label: "Pivot key", tooltip: "The key around which the network will be constructed", event: function(value) { - if (value == "Rotation key" && $('#input_graph_scope_jsonkey').val() == "") { // no key selected for Rotation key scope + if (value == "Pivot key" && $('#input_graph_scope_jsonkey').val() == "") { // no key selected for Pivot key scope return; } else { eventGraph.scope_keyType = value; - eventGraph.update_scope("Rotation key"); + eventGraph.update_scope("Pivot key"); dataHandler.fetch_data_and_update(); } }, - options: dataHandler.available_rotation_key ? dataHandler.available_rotation_key : [], + options: dataHandler.available_pivot_key ? dataHandler.available_pivot_key : [], default: "" }); return menu_scope; @@ -298,7 +298,8 @@ class EventGraph { for(var nodeId of objectIds) { eventGraph.expand_node(nodeId); } - } + }, + title: "Expanding all nodes may takes some time" }); menu_display.add_button({ label: "Collapse all nodes", @@ -310,7 +311,8 @@ class EventGraph { for(var nodeId of objectIds) { eventGraph.collapse_node(nodeId); } - } + }, + title: "Collapsing all nodes may takes some time" }); menu_display.add_slider({ id: 'slider_display_max_char_num', @@ -1399,7 +1401,7 @@ class DataHandler { eventGraph.menu_filter.items["table_attr_value"].add_options("table_control_select_attr_value", available_object_references); } - fetch_data_and_update(stabilize, callback) { + fetch_data_and_update(stabilize, updateOnly, callback) { eventGraph.network_loading(true, loadingText_fetching); $.when(this.fetch_objects_template()).done(function() { var filtering_rules = eventGraph.get_filtering_rules(); @@ -1417,7 +1419,9 @@ class DataHandler { data: JSON.stringify( payload ), processData: false, success: function( data, textStatus, jQxhr ){ - eventGraph.reset_graphs(true); + if (updateOnly === undefined || updateOnly === false) { + eventGraph.reset_graphs(true); + } eventGraph.is_filtered = (filtering_rules.presence.length > 0 || filtering_rules.value.length > 0); eventGraph.first_draw = true; // update object state @@ -1427,8 +1431,8 @@ class DataHandler { return [[index, value]]; }); dataHandler.update_filtering_selectors(available_object_references, available_tags); - dataHandler.available_rotation_key = data.available_rotation_key; - eventGraph.menu_scope.add_options("input_graph_scope_jsonkey", dataHandler.available_rotation_key); + dataHandler.available_pivot_key = data.available_pivot_key; + eventGraph.menu_scope.add_options("input_graph_scope_jsonkey", dataHandler.available_pivot_key); if (data.items.length < nodes_ask_threshold) { eventGraph.update_graph(data); } else if (data.items.length > nodes_ask_threshold && confirm("The network contains a lot of nodes, displaying it may slow down your browser. Continue?")) { @@ -1861,7 +1865,7 @@ function genericPopupCallback(result) { // sucess and eventgraph is enabled if (result == "success" && dataHandler !== undefined) { mispInteraction.apply_callback(); - dataHandler.fetch_data_and_update(false); + dataHandler.fetch_data_and_update(false, true); } } @@ -1958,7 +1962,7 @@ function import_graph_from_json(data) { $('#checkbox_physics_enable').prop('checked', data.physics.enabled); // update data - dataHandler.fetch_data_and_update(false, function() { + dataHandler.fetch_data_and_update(false, false, function() { eventGraph.nodes.update(data.nodes); eventGraph.expand_previous_expansion(data.nodes); eventGraph.hiddenNode.clear(); @@ -2180,7 +2184,7 @@ $(document).on("keyup", function(evt) { }); eventGraph.update_scope(); -dataHandler.fetch_data_and_update(true, function() { +dataHandler.fetch_data_and_update(true, false, function() { var $select = $('#network-typeahead'); dataHandler.get_typeaheadData_search().forEach(function(element) { var $option = $(''); From cd68ea018f236aaa023a154f60dbb87ac9e85898 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Wed, 9 Oct 2019 16:27:59 +0200 Subject: [PATCH 08/79] fix: [eventGraph] Fixed UI issue with duplicated ID (attributes and objects) Fix #5181 --- app/Lib/Tools/EventGraphTool.php | 14 +++++++------- app/webroot/js/event-graph.js | 5 ++++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/app/Lib/Tools/EventGraphTool.php b/app/Lib/Tools/EventGraphTool.php index 24307d5a6..b8414d564 100644 --- a/app/Lib/Tools/EventGraphTool.php +++ b/app/Lib/Tools/EventGraphTool.php @@ -265,7 +265,7 @@ foreach ($object as $obj) { $toPush = array( - 'id' => $obj['id'], + 'id' => sprintf('o-%s', $obj['id']), 'uuid' => $obj['uuid'], 'type' => $obj['name'], 'label' => '', @@ -290,8 +290,8 @@ $toPush = array( 'id' => $rel['id'], 'uuid' => $rel['uuid'], - 'from' => $obj['id'], - 'to' => $rel['referenced_id'], + 'from' => sprintf('o-%s', $obj['id']), + 'to' => $rel['referenced_type'] == 1 ? sprintf('o-%s', $rel['referenced_id']) : $rel['referenced_id'], 'type' => $rel['relationship_type'], 'comment' => $rel['comment'], 'event_id' => $rel['event_id'], @@ -356,7 +356,7 @@ foreach ($object as $obj) { $toPush = array( - 'id' => $obj['id'], + 'id' => sprintf('o-%s', $obj['id']), 'uuid' => $obj['uuid'], 'type' => $obj['name'], 'Attribute' => $obj['Attribute'], @@ -382,7 +382,7 @@ if (!in_array($tag['name'], $added_value)) { $toPush = array( 'id' => "tag_edge_id_" . $i, - 'from' => $obj['id'], + 'from' => sprintf('o-%s', $obj['id']), 'to' => $tag['name'], ); $tagSet[$tag['name']] = $tag; @@ -466,7 +466,7 @@ foreach ($object as $obj) { $toPush = array( - 'id' => $obj['id'], + 'id' => sprintf('o-%s', $obj['id']), 'uuid' => $obj['uuid'], 'type' => $obj['name'], 'Attribute' => $obj['Attribute'], @@ -491,7 +491,7 @@ if (!in_array($keyVal, $added_value)) { $toPush = array( 'id' => "keyType_edge_id_" . $i, - 'from' => $obj['id'], + 'from' => sprintf('o-%s', $obj['id']), 'to' => "keyType_" . $keyVal, ); array_push($added_value, $keyVal); diff --git a/app/webroot/js/event-graph.js b/app/webroot/js/event-graph.js index 6f746ce3e..c9c0399ed 100644 --- a/app/webroot/js/event-graph.js +++ b/app/webroot/js/event-graph.js @@ -1576,7 +1576,8 @@ class MispInteraction { if (!that.can_create_reference(edgeData.from) || !that.can_be_referenced(edgeData.to)) { return; } - genericPopup('/objectReferences/add/'+edgeData.from, '#popover_form', function() { + var edgeFromId = edgeData.from.startsWith('o-') ? edgeData.from.substr(2) : edgeData.from; + genericPopup('/objectReferences/add/'+edgeFromId, '#popover_form', function() { $('#ObjectReferenceReferencedUuid').val(uuid); objectReferenceInput(); }); @@ -1641,6 +1642,7 @@ class MispInteraction { var selected_nodes = nodeData.nodes; for (var nodeID of selected_nodes) { var node = this.nodes.get(nodeID) + nodeID = nodeID.startsWith('o-') ? nodeID.substr(2) : nodeID; if (node.group.slice(0, 9) == "attribute") { deleteObject('attributes', 'delete', nodeID, scope_id); } else if (node.group == "object") { @@ -1653,6 +1655,7 @@ class MispInteraction { var that = mispInteraction; var id = nodeData.id var group = nodes.get(id).group; + id = id.startsWith('o-') ? id.substr(2) : id; if (group.slice(0, 9) == 'attribute') { simplePopup('/attributes/edit/'+id); } else if (group == 'object') { From d41951cf897b022d3e199fa664d7ee60d8db54d1 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Wed, 9 Oct 2019 19:34:54 +0200 Subject: [PATCH 09/79] fix: [internal] Remove closing PHP tag --- app/Model/DecayingModelsFormulas/Base.php | 2 -- app/Model/DecayingModelsFormulas/Polynomial.php | 1 - app/Model/DecayingModelsFormulas/PolynomialExtended.php | 1 - app/View/Helper/FontAwesomeHelper.php | 1 - app/View/Helper/GenericPickerHelper.php | 1 - app/View/Helper/HighlightHelper.php | 1 - app/View/Helper/OrgImgHelper.php | 1 - app/View/Helper/PivotHelper.php | 2 -- app/View/Helper/TextColourHelper.php | 1 - app/View/Helper/UserNameHelper.php | 1 - app/View/Helper/UtilityHelper.php | 2 +- app/View/Helper/XmlOutputHelper.php | 1 - 12 files changed, 1 insertion(+), 14 deletions(-) diff --git a/app/Model/DecayingModelsFormulas/Base.php b/app/Model/DecayingModelsFormulas/Base.php index 3ba47c53a..73f491037 100644 --- a/app/Model/DecayingModelsFormulas/Base.php +++ b/app/Model/DecayingModelsFormulas/Base.php @@ -124,5 +124,3 @@ abstract class DecayingModelBase // Return a True if the attribute should be marked as decayed abstract public function isDecayed($model, $attribute, $score); } - -?> diff --git a/app/Model/DecayingModelsFormulas/Polynomial.php b/app/Model/DecayingModelsFormulas/Polynomial.php index dcb75d8da..a981122e8 100644 --- a/app/Model/DecayingModelsFormulas/Polynomial.php +++ b/app/Model/DecayingModelsFormulas/Polynomial.php @@ -24,4 +24,3 @@ class Polynomial extends DecayingModelBase return $threshold > $score; } } -?> diff --git a/app/Model/DecayingModelsFormulas/PolynomialExtended.php b/app/Model/DecayingModelsFormulas/PolynomialExtended.php index 4d8e54db8..4fa8ecfbb 100644 --- a/app/Model/DecayingModelsFormulas/PolynomialExtended.php +++ b/app/Model/DecayingModelsFormulas/PolynomialExtended.php @@ -51,4 +51,3 @@ class PolynomialExtended extends Polynomial return parent::isDecayed($model, $attribute, $score); } } -?> diff --git a/app/View/Helper/FontAwesomeHelper.php b/app/View/Helper/FontAwesomeHelper.php index 34e806956..521f7f56a 100644 --- a/app/View/Helper/FontAwesomeHelper.php +++ b/app/View/Helper/FontAwesomeHelper.php @@ -446,4 +446,3 @@ App::uses('AppHelper', 'View/Helper'); } } } -?> diff --git a/app/View/Helper/GenericPickerHelper.php b/app/View/Helper/GenericPickerHelper.php index 36b30b210..9f9fe14f1 100644 --- a/app/View/Helper/GenericPickerHelper.php +++ b/app/View/Helper/GenericPickerHelper.php @@ -113,4 +113,3 @@ class GenericPickerHelper extends AppHelper { return $template; } } -?> diff --git a/app/View/Helper/HighlightHelper.php b/app/View/Helper/HighlightHelper.php index d47d17be3..571cb460e 100644 --- a/app/View/Helper/HighlightHelper.php +++ b/app/View/Helper/HighlightHelper.php @@ -40,4 +40,3 @@ App::uses('AppHelper', 'View/Helper'); } } -?> diff --git a/app/View/Helper/OrgImgHelper.php b/app/View/Helper/OrgImgHelper.php index 2a6cac1f5..5482287f5 100644 --- a/app/View/Helper/OrgImgHelper.php +++ b/app/View/Helper/OrgImgHelper.php @@ -58,4 +58,3 @@ App::uses('AppHelper', 'View/Helper'); } } } -?> diff --git a/app/View/Helper/PivotHelper.php b/app/View/Helper/PivotHelper.php index 56227c672..8918f747d 100644 --- a/app/View/Helper/PivotHelper.php +++ b/app/View/Helper/PivotHelper.php @@ -80,5 +80,3 @@ App::uses('AppHelper', 'View/Helper'); return $height + $heightToAdd; } } - -?> diff --git a/app/View/Helper/TextColourHelper.php b/app/View/Helper/TextColourHelper.php index 1a80ab2c0..aacfb0e91 100644 --- a/app/View/Helper/TextColourHelper.php +++ b/app/View/Helper/TextColourHelper.php @@ -17,4 +17,3 @@ App::uses('AppHelper', 'View/Helper'); } } } -?> diff --git a/app/View/Helper/UserNameHelper.php b/app/View/Helper/UserNameHelper.php index f21f04f08..3ba628bb3 100644 --- a/app/View/Helper/UserNameHelper.php +++ b/app/View/Helper/UserNameHelper.php @@ -21,4 +21,3 @@ App::uses('AppHelper', 'View/Helper'); return ''; } } -?> diff --git a/app/View/Helper/UtilityHelper.php b/app/View/Helper/UtilityHelper.php index b4194abb6..7ddd7490e 100644 --- a/app/View/Helper/UtilityHelper.php +++ b/app/View/Helper/UtilityHelper.php @@ -9,4 +9,4 @@ App::uses('AppHelper', 'View/Helper'); return $string; } } -?> + diff --git a/app/View/Helper/XmlOutputHelper.php b/app/View/Helper/XmlOutputHelper.php index b34169b6d..2f80d237a 100644 --- a/app/View/Helper/XmlOutputHelper.php +++ b/app/View/Helper/XmlOutputHelper.php @@ -22,4 +22,3 @@ App::uses('AppHelper', 'View/Helper'); } } } -?> From 5671c02f0b5505d5ae5225e7989fe1226cde5376 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 9 Oct 2019 23:35:50 +0200 Subject: [PATCH 10/79] fix: [stix2 import] Removed unused variable in dictionary loop --- app/files/scripts/stix2/stix2misp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/scripts/stix2/stix2misp.py b/app/files/scripts/stix2/stix2misp.py index fc7e1e2cb..6d53bc6eb 100644 --- a/app/files/scripts/stix2/stix2misp.py +++ b/app/files/scripts/stix2/stix2misp.py @@ -115,7 +115,7 @@ class StixParser(): def build_from_STIX_without_report(self): for object_type, objects in self.event.items(): - for _, _object in objects.items(): + for _object in objects.values(): self.parsing_process(_object, object_type) self.misp_event.info = "Imported with MISP import script for {}.".format(self.stix_version) From 0ab8c5b48e07df26bf419ebb9687f620d9619ca1 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Wed, 9 Oct 2019 21:28:09 +0200 Subject: [PATCH 11/79] chg: [internal] Remove unused code from AttributesController::index --- app/Controller/AttributesController.php | 8 -------- 1 file changed, 8 deletions(-) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index fae053c5b..ceede10c6 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -80,18 +80,10 @@ class AttributesController extends AppController return $this->RestResponse->viewData($attributes, $this->response->type()); } $org_ids = array(); - $tag_ids = array(); $orgs = $this->Attribute->Event->Orgc->find('list', array( 'conditions' => array('Orgc.id' => $org_ids), 'fields' => array('Orgc.id', 'Orgc.name') )); - if (!empty($tag_ids)) { - $tags = $this->Attribute->AttributeTag->Tag->find('all', array( - 'conditions' => array('Tag.id' => $tag_ids), - 'recursive' => -1, - 'fields' => array('Tag.id', 'Tag.name', 'Tag.colour') - )); - } if (!$this->_isRest()) { $temp = $this->__searchUI($attributes); $this->loadModel('Galaxy'); From 8476f667c2d698feae12d7da32b74772122012e5 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 10 Oct 2019 09:44:19 +0200 Subject: [PATCH 12/79] chg: Bumped queryversion --- app/Controller/AppController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index 3793362e1..d3d4b8e98 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -46,7 +46,7 @@ class AppController extends Controller public $helpers = array('Utility', 'OrgImg', 'FontAwesome', 'UserName'); - private $__queryVersion = '89'; + private $__queryVersion = '90'; public $pyMispVersion = '2.4.114'; public $phpmin = '7.0'; public $phprec = '7.2'; From 1e08eece84a63eddcccfc3427c7fce3e13892d15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 10 Oct 2019 11:21:28 +0200 Subject: [PATCH 13/79] chg: [PyMISP] Bump --- PyMISP | 2 +- app/files/scripts/generate_file_objects.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/PyMISP b/PyMISP index 605cdc21e..c80d35fa7 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit 605cdc21efa590209d192313b071ca0f80afba27 +Subproject commit c80d35fa75dba79d226cb4586397337df9acb7f6 diff --git a/app/files/scripts/generate_file_objects.py b/app/files/scripts/generate_file_objects.py index 73ecfd4b6..5aa74715b 100644 --- a/app/files/scripts/generate_file_objects.py +++ b/app/files/scripts/generate_file_objects.py @@ -5,7 +5,7 @@ import argparse import json try: - from pymisp import MISPEncode + from pymisp import pymisp_json_default from pymisp.tools import make_binary_objects except ImportError: pass @@ -51,7 +51,7 @@ def make_objects(path): to_return['objects'].append(fo) if fo.ObjectReference: to_return['references'] += fo.ObjectReference - return json.dumps(to_return, cls=MISPEncode) + return json.dumps(to_return, default=pymisp_json_default) if __name__ == '__main__': From be509b3833b68382a3d91319a1dae0044e807afd Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 10 Oct 2019 11:55:33 +0200 Subject: [PATCH 14/79] fix: [API] rate limit should only run on the API --- app/Controller/AppController.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index 3793362e1..3b57310dc 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -471,7 +471,9 @@ class AppController extends Controller } $this->set('notifications', $notifications); $this->ACL->checkAccess($this->Auth->user(), Inflector::variable($this->request->params['controller']), $this->action); - $this->__rateLimitCheck(); + if ($this->_isRest()) { + $this->__rateLimitCheck(); + } } private function __rateLimitCheck() From 9609ff1822bbbe9cb0baa1a7c39fdfe6b3e2c663 Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 10 Oct 2019 11:56:45 +0200 Subject: [PATCH 15/79] chg: [UI] updated the generic element index fields - json should accept single values without embedding them in a list - generic field should automatically cast a list to a comma separated string --- .../IndexTable/Fields/generic_field.ctp | 10 +++++++++- .../genericElements/IndexTable/Fields/json.ctp | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/app/View/Elements/genericElements/IndexTable/Fields/generic_field.ctp b/app/View/Elements/genericElements/IndexTable/Fields/generic_field.ctp index d6431dca6..db78fbfbf 100644 --- a/app/View/Elements/genericElements/IndexTable/Fields/generic_field.ctp +++ b/app/View/Elements/genericElements/IndexTable/Fields/generic_field.ctp @@ -1,3 +1,11 @@ 1) { + $data = implode(', ', $data); + } else { + $data = $data[0]; + } + } + echo h($data); ?> diff --git a/app/View/Elements/genericElements/IndexTable/Fields/json.ctp b/app/View/Elements/genericElements/IndexTable/Fields/json.ctp index 3683a959f..9fe805c9d 100644 --- a/app/View/Elements/genericElements/IndexTable/Fields/json.ctp +++ b/app/View/Elements/genericElements/IndexTable/Fields/json.ctp @@ -1,5 +1,9 @@ ', h($k) From 15b10bbcf7b5a2a524742a4ec0d48142729c6f78 Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 10 Oct 2019 11:58:26 +0200 Subject: [PATCH 16/79] new: [user settings] Added restrictions for certain settings - based on role permissions - enforce the checks on set/delete - add it to the UI elements - /users/view /admin/users/view now include the user settings in a simplified format --- app/Controller/UserSettingsController.php | 46 ++++++++++++++++++----- app/Controller/UsersController.php | 44 ++++++++++++++++++---- app/Model/UserSetting.php | 33 +++++++++++++++- app/View/UserSettings/index.ctp | 5 +++ app/View/UserSettings/set_setting.ctp | 4 +- 5 files changed, 113 insertions(+), 19 deletions(-) diff --git a/app/Controller/UserSettingsController.php b/app/Controller/UserSettingsController.php index 47b728c72..ed4faacf4 100644 --- a/app/Controller/UserSettingsController.php +++ b/app/Controller/UserSettingsController.php @@ -103,7 +103,15 @@ class UserSettingsController extends AppController return $this->RestResponse->viewData($userSettings, $this->response->type()); } else { $this->paginate['conditions'] = $conditions; - $this->set('data', $this->paginate()); + $data = $this->paginate(); + foreach ($data as $k => $v) { + if (!empty($this->UserSetting->validSettings[$v['UserSetting']['setting']])) { + $data[$k]['UserSetting']['restricted'] = empty($this->UserSetting->validSettings[$v['UserSetting']['setting']]['restricted']) ? '' : $this->UserSetting->validSettings[$v['UserSetting']['setting']]['restricted']; + } else { + $data[$k]['UserSetting']['restricted'] = array(); + } + } + $this->set('data', $data); $this->set('context', empty($context) ? 'null' : $context); } } @@ -138,6 +146,15 @@ class UserSettingsController extends AppController public function setSetting($user_id = false, $setting = false) { + if (!empty($setting)) { + if (!$this->UserSetting->checkSettingValidity($setting)) { + throw new MethodNotAllowedException(__('Invalid setting.')); + } + $settingPermCheck = $this->UserSetting->checkSettingAccess($this->Auth->user(), $setting); + if ($settingPermCheck !== true) { + throw new MethodNotAllowedException(__('This setting is restricted and requires the following permission(s): %s', $settingPermCheck)); + } + } // handle POST requests if ($this->request->is('post')) { // massage the request to allow for unencapsulated POST requests via the API @@ -168,16 +185,22 @@ class UserSettingsController extends AppController $userSetting['user_id'] = $this->request->data['UserSetting']['user_id']; } } - if (empty($this->request->data['UserSetting']['setting'])) { + if (empty($this->request->data['UserSetting']['setting']) || !isset($this->request->data['UserSetting']['setting'])) { throw new MethodNotAllowedException(__('This endpoint expects both a setting and a value to be set.')); - } else { - if (!$this->UserSetting->checkSettingValidity($this->request->data['UserSetting']['setting'])) { - throw new MethodNotAllowedException(__('Invalid setting.')); - } - $userSetting['setting'] = $this->request->data['UserSetting']['setting']; } - $userSetting['value'] = empty($this->request->data['UserSetting']['value']) ? '' : - json_encode(json_decode($this->request->data['UserSetting']['value'], true)); + if (!$this->UserSetting->checkSettingValidity($this->request->data['UserSetting']['setting'])) { + throw new MethodNotAllowedException(__('Invalid setting.')); + } + $settingPermCheck = $this->UserSetting->checkSettingAccess($this->Auth->user(), $this->request->data['UserSetting']['setting']); + if ($settingPermCheck !== true) { + throw new MethodNotAllowedException(__('This setting is restricted and requires the following permission(s): %s', $settingPermCheck)); + } + $userSetting['setting'] = $this->request->data['UserSetting']['setting']; + if ($this->request->data['UserSetting']['value'] !== '') { + $userSetting['value'] = json_encode(json_decode($this->request->data['UserSetting']['value'], true)); + } else { + $userSetting['value'] = ''; + } $existingSetting = $this->UserSetting->find('first', array( 'recursive' => -1, 'conditions' => array( @@ -243,6 +266,7 @@ class UserSettingsController extends AppController if (!empty($user_id) && $this->request->is('get')) { $this->request->data['UserSetting']['user_id'] = $user_id; } + $this->set('setting', $setting); $this->set('users', $users); $this->set('validSettings', $validSettings); } @@ -301,6 +325,10 @@ class UserSettingsController extends AppController if (!$checkAccess) { throw new NotFoundException(__('Invalid user setting.')); } + $settingPermCheck = $this->UserSetting->checkSettingAccess($this->Auth->user(), $userSetting['UserSetting']['setting']); + if ($settingPermCheck !== true) { + throw new MethodNotAllowedException(__('This setting is restricted and requires the following permission(s): %s', $settingPermCheck)); + } if ($this->request->is('post') || $this->request->is('delete')) { // Delete the setting that we were after. $result = $this->UserSetting->delete($userSetting['UserSetting']['id']); diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php index 5a63db404..00ed01eab 100644 --- a/app/Controller/UsersController.php +++ b/app/Controller/UsersController.php @@ -48,12 +48,18 @@ class UsersController extends AppController )); $id = $userid['User']['id']; } - $this->User->id = $id; - $this->User->recursive = 0; - if (!$this->User->exists()) { + $user = $this->User->read(null, $id); + $user = $this->User->find('first', array( + 'recursive' => -1, + 'conditions' => array('User.id' => $id), + 'contain' => array( + 'UserSetting', + 'Role' + ) + )); + if (empty($user)) { throw new NotFoundException(__('Invalid user')); } - $user = $this->User->read(null, $id); if (!empty($user['User']['gpgkey'])) { $pgpDetails = $this->User->verifySingleGPG($user); $user['User']['pgp_status'] = isset($pgpDetails[2]) ? $pgpDetails[2] : 'OK'; @@ -62,9 +68,15 @@ class UsersController extends AppController if ($this->_isRest()) { unset($user['User']['server_id']); $user['User']['password'] = '*****'; + $temp = array(); + foreach ($user['UserSetting'] as $k => $v) { + $temp[$v['setting']] = $v['value']; + } + $user['UserSetting'] = $temp; return $this->RestResponse->viewData(array( 'User' => $user['User'], - 'Role' => $user['Role'] + 'Role' => $user['Role'], + 'UserSetting' => $user['UserSetting'] ), $this->response->type()); } else { $this->set('user', $user); @@ -465,11 +477,17 @@ class UsersController extends AppController public function admin_view($id = null) { - $this->User->id = $id; - if (!$this->User->exists()) { + $user = $this->User->find('first', array( + 'recursive' => -1, + 'conditions' => array('User.id' => $id), + 'contain' => array( + 'UserSetting', + 'Role' + ) + )); + if (empty($user)) { throw new NotFoundException(__('Invalid user')); } - $user = $this->User->read(null, $id); if (!empty($user['User']['gpgkey'])) { $pgpDetails = $this->User->verifySingleGPG($user); $user['User']['pgp_status'] = isset($pgpDetails[2]) ? $pgpDetails[2] : 'OK'; @@ -485,6 +503,16 @@ class UsersController extends AppController } if ($this->_isRest()) { $user['User']['password'] = '*****'; + $temp = array(); + foreach ($user['UserSetting'] as $k => $v) { + $temp[$v['setting']] = $v['value']; + } + $user['UserSetting'] = $temp; + return $this->RestResponse->viewData(array( + 'User' => $user['User'], + 'Role' => $user['Role'], + 'UserSetting' => $user['UserSetting'] + ), $this->response->type()); return $this->RestResponse->viewData(array('User' => $user['User']), $this->response->type()); } else { $temp = $this->User->data['User']['invited_by']; diff --git a/app/Model/UserSetting.php b/app/Model/UserSetting.php index 9d4de0497..c472185aa 100644 --- a/app/Model/UserSetting.php +++ b/app/Model/UserSetting.php @@ -45,6 +45,10 @@ class UserSetting extends AppModel ) ) ) + ), + 'dashboard_access' => array( + 'placeholder' => 1, + 'restricted' => 'perm_site_admin' ) ); @@ -56,7 +60,11 @@ class UserSetting extends AppModel if (empty($this->data['UserSetting']['timestamp'])) { $this->data['UserSetting']['timestamp'] = time(); } - if (!empty($this->data['UserSetting']['value']) && $this->data['UserSetting']['value'] !== 'null') { + if ( + isset($this->data['UserSetting']['value']) && + $this->data['UserSetting']['value'] !== '' && + $this->data['UserSetting']['value'] !== 'null' + ) { if (is_array($this->data['UserSetting']['value'])) { $this->data['UserSetting']['value'] = json_encode($this->data['UserSetting']['value']); } @@ -80,6 +88,29 @@ class UserSetting extends AppModel return isset($this->validSettings[$setting]); } + public function checkSettingAccess($user, $setting) + { + if (!empty($this->validSettings[$setting]['restricted'])) { + $role_check = $this->validSettings[$setting]['restricted']; + if (!is_array($role_check)) { + $role_check = array($role_check); + } + $userHasValidRole = false; + foreach ($role_check as $role) { + if (!empty($user['Role'][$role])) { + return true; + } + } + if (!$userHasValidRole) { + foreach ($role_check as &$role) { + $role = substr($role, 5); + } + return implode(', ', $role_check); + } + } + return true; + } + /* * canModify expects an auth user object or a user ID and a loaded setting as input parameters * check if the user can modify/remove the given entry diff --git a/app/View/UserSettings/index.ctp b/app/View/UserSettings/index.ctp index 9035f7c6c..6881781d8 100644 --- a/app/View/UserSettings/index.ctp +++ b/app/View/UserSettings/index.ctp @@ -69,6 +69,11 @@ 'sort' => 'type', 'element' => 'json', 'data_path' => 'UserSetting.value' + ), + array( + 'name' => __('Restricted to'), + 'sort' => 'type', + 'data_path' => 'UserSetting.restricted' ) ), 'title' => __('User settings management'), diff --git a/app/View/UserSettings/set_setting.ctp b/app/View/UserSettings/set_setting.ctp index 633786201..dffa2fd8f 100644 --- a/app/View/UserSettings/set_setting.ctp +++ b/app/View/UserSettings/set_setting.ctp @@ -19,7 +19,9 @@ array( 'div' => 'clear', 'class' => 'input input-xxlarge', - 'options' => array_combine(array_keys($validSettings), array_keys($validSettings)) + 'options' => array_combine(array_keys($validSettings), array_keys($validSettings)), + 'default' => $setting, + 'disabled' => (boolean)$setting ) ), $this->Form->input( From c0ec14630396194f3ba4a4153e391424e0cb9b9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 10 Oct 2019 13:11:46 +0200 Subject: [PATCH 17/79] fix: [PyMISP] Travis tests --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8a823eda2..75f7fcd92 100644 --- a/.travis.yml +++ b/.travis.yml @@ -160,10 +160,9 @@ script: - ./curl_tests.sh $AUTH - popd - pushd PyMISP + - git submodule init + - git submodule update - pipenv install -d - - pushd tests - - git clone https://github.com/viper-framework/viper-test-files.git - - popd - pipenv run python tests/testlive_comprehensive.py - pipenv run python tests/test.py - pipenv run python tests/test_mispevent.py From 306ec539fb81bd1f6f53404d97d4118a50b76a7c Mon Sep 17 00:00:00 2001 From: "Marcelo H. P. C. Chaves" <7375233+mhpchaves@users.noreply.github.com> Date: Thu, 10 Oct 2019 09:10:34 -0300 Subject: [PATCH 18/79] chg: [doc] ssdeep can be installed via pkg_add --- docs/xINSTALL.OpenBSD.md | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/docs/xINSTALL.OpenBSD.md b/docs/xINSTALL.OpenBSD.md index 704114331..8be5a5dfb 100644 --- a/docs/xINSTALL.OpenBSD.md +++ b/docs/xINSTALL.OpenBSD.md @@ -250,18 +250,8 @@ doas virtualenv -ppython3 /usr/local/virtualenvs/MISP ``` #### Install ssdeep -``` -doas mkdir /usr/local/src -doas chown misp:misp /usr/local/src -cd /usr/local/src -doas -u misp git clone https://github.com/ssdeep-project/ssdeep.git -cd ssdeep -export AUTOMAKE_VERSION=1.16 -export AUTOCONF_VERSION=2.69 -doas -u misp ./bootstrap -doas -u misp ./configure --prefix=/usr -doas -u misp make -doas make install +```bash +doas pkg_add -v ssdeep ``` #### Apache2 only From b44063e7d197e5f889829f349ce0ea4652688838 Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 10 Oct 2019 15:13:34 +0200 Subject: [PATCH 19/79] fix: [internal] missing org object for users/view --- app/Controller/UsersController.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php index 00ed01eab..d97a89ba1 100644 --- a/app/Controller/UsersController.php +++ b/app/Controller/UsersController.php @@ -54,7 +54,8 @@ class UsersController extends AppController 'conditions' => array('User.id' => $id), 'contain' => array( 'UserSetting', - 'Role' + 'Role', + 'Organisation' ) )); if (empty($user)) { From 6519d12ac403c775e35dd93d509c2cb9434feddf Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 10 Oct 2019 23:47:06 +0200 Subject: [PATCH 20/79] chg: version bump --- VERSION.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.json b/VERSION.json index 4f4c7e5f5..f0d1fb40b 100644 --- a/VERSION.json +++ b/VERSION.json @@ -1 +1 @@ -{"major":2, "minor":4, "hotfix":116} +{"major":2, "minor":4, "hotfix":117} From e849b036942b004abb52871ec64bfb2a0814b860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 10 Oct 2019 23:55:39 +0200 Subject: [PATCH 21/79] chg: [PyMISP] Bump --- PyMISP | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyMISP b/PyMISP index c80d35fa7..f73571f30 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit c80d35fa75dba79d226cb4586397337df9acb7f6 +Subproject commit f73571f30d39db8f80b5bc1c4a22c8eb9f54fcd4 From e05c3b90927b03cf00fa47c4ea84cf585d7041db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 10 Oct 2019 23:59:11 +0200 Subject: [PATCH 22/79] chg: Bump recommended PYMISP version --- app/Controller/AppController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index 74e6e7f91..962552f6e 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -47,7 +47,7 @@ class AppController extends Controller public $helpers = array('Utility', 'OrgImg', 'FontAwesome', 'UserName'); private $__queryVersion = '90'; - public $pyMispVersion = '2.4.114'; + public $pyMispVersion = '2.4.117'; public $phpmin = '7.0'; public $phprec = '7.2'; public $isApiAuthed = false; From f68ae4265e0fbb91ece154e70a2f28eff6dc35d0 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Fri, 11 Oct 2019 08:38:16 +0200 Subject: [PATCH 23/79] Use python3 from base repo --- docs/INSTALL.rhel7.md | 25 +++++++++---------------- docs/xINSTALL.centos7.md | 21 ++++++++------------- 2 files changed, 17 insertions(+), 29 deletions(-) diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index cf69402fa..f31e7c982 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -169,10 +169,8 @@ yumInstallCoreDeps () { rh-php72-php-opcache \ rh-php72-php-gd -y - # Install Python 3.6 from SCL, see - # https://www.softwarecollections.org/en/scls/rhscl/rh-python36/ - RUN_PYTHON='/usr/bin/scl enable rh-python36' - sudo yum install rh-python36 -y + # Python 3.6 is now available in RHEL 7.7 base + sudo yum install python3 python3-devel -y sudo systemctl enable --now rh-php72-php-fpm.service } @@ -217,7 +215,8 @@ installCoreRHEL () { $SUDO_WWW git config core.filemode false # Create a python3 virtualenv - $SUDO_WWW $RUN_PYTHON -- virtualenv -p python3 $PATH_TO_MISP/venv + sudo pip3 install virtualenv + $SUDO_WWW python3 -- virtualenv -p python3 $PATH_TO_MISP/venv sudo mkdir /usr/share/httpd/.cache sudo chown $WWW_USER:$WWW_USER /usr/share/httpd/.cache $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U pip setuptools @@ -259,7 +258,7 @@ installCoreRHEL () { cd $PATH_TO_MISP/app/files/scripts/lief $SUDO_WWW mkdir build cd build - $SUDO_WWW scl enable devtoolset-7 rh-python36 "bash -c 'cmake3 \ + $SUDO_WWW scl enable devtoolset-7 "bash -c 'cmake3 \ -DLIEF_PYTHON_API=on \ -DPYTHON_VERSION=3.6 \ -DPYTHON_EXECUTABLE=$PATH_TO_MISP/venv/bin/python \ @@ -291,19 +290,13 @@ installCoreRHEL () { cd $PATH_TO_MISP/PyMISP $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U . - # Enable python3 for php-fpm - echo 'source scl_source enable rh-python36' | sudo tee -a /etc/opt/rh/rh-php72/sysconfig/php-fpm - sudo sed -i.org -e 's/^;\(clear_env = no\)/\1/' /etc/opt/rh/rh-php72/php-fpm.d/www.conf - sudo systemctl restart rh-php72-php-fpm.service - - umask $UMASK - # Enable dependencies detection in the diagnostics page # This allows MISP to detect GnuPG, the Python modules' versions and to read the PHP settings. - # The LD_LIBRARY_PATH setting is needed for rh-git218 to work, one might think to install httpd24 and not just httpd ... - echo "env[PATH] = /opt/rh/rh-git218/root/usr/bin:/opt/rh/rh-redis32/root/usr/bin:/opt/rh/rh-python36/root/usr/bin:/opt/rh/rh-php72/root/usr/bin:/usr/local/bin:/usr/bin:/bin" |sudo tee -a /etc/opt/rh/rh-php72/php-fpm.d/www.conf - echo "env[LD_LIBRARY_PATH] = /opt/rh/httpd24/root/usr/lib64/" |sudo tee -a /etc/opt/rh/rh-php72/php-fpm.d/www.conf + # The LD_LIBRARY_PATH setting is needed for rh-git218 to work + echo "env[PATH] = /opt/rh/rh-git218/root/usr/bin:/opt/rh/rh-redis32/root/usr/bin:/opt/rh/rh-php72/root/usr/bin:/usr/local/bin:/usr/bin:/bin" |sudo tee -a /etc/opt/rh/rh-php72/php-fpm.d/www.conf + sudo sed -i.org -e 's/^;\(clear_env = no\)/\1/' /etc/opt/rh/rh-php72/php-fpm.d/www.conf sudo systemctl restart rh-php72-php-fpm.service + umask $UMASK } # ``` diff --git a/docs/xINSTALL.centos7.md b/docs/xINSTALL.centos7.md index 51c2c06fe..5d9e53dd9 100644 --- a/docs/xINSTALL.centos7.md +++ b/docs/xINSTALL.centos7.md @@ -43,7 +43,6 @@ Make sure you are reading the parsed version of this Document. When in doubt [cl # # RHEL/CentOS Specific RUN_PHP='/usr/bin/scl enable rh-php72' -RUN_PYTHON='/usr/bin/scl enable rh-python36' SUDO_WWW='sudo -H -u apache' WWW_USER='apache' @@ -96,9 +95,8 @@ sudo yum install gcc git zip \ # Install PHP 7.2 from SCL, see https://www.softwarecollections.org/en/scls/rhscl/rh-php72/ sudo yum install rh-php72 rh-php72-php-fpm rh-php72-php-devel rh-php72-php-mysqlnd rh-php72-php-mbstring rh-php72-php-xml rh-php72-php-bcmath rh-php72-php-opcache rh-php72-php-gd -y -# Install Python 3.6 from SCL, see -# https://www.softwarecollections.org/en/scls/rhscl/rh-python36/ -sudo yum install rh-python36 -y +# Python 3.6 in now available in CentOS 7.7 base +sudo yum install python3 python3-devel -y sudo systemctl enable --now rh-php72-php-fpm.service ``` @@ -138,7 +136,8 @@ $SUDO_WWW git submodule foreach --recursive git config core.filemode false $SUDO_WWW git config core.filemode false # Create a python3 virtualenv -$SUDO_WWW $RUN_PYTHON "virtualenv -p python3 $PATH_TO_MISP/venv" +sudo pip3 install virtualenv +$SUDO_WWW python3 "virtualenv -p python3 $PATH_TO_MISP/venv" sudo mkdir /usr/share/httpd/.cache sudo chown ${WWW_USER}:${WWW_USER} /usr/share/httpd/.cache $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U pip setuptools @@ -176,7 +175,7 @@ sudo yum install devtoolset-7 cmake3 -y cd $PATH_TO_MISP/app/files/scripts/lief $SUDO_WWW mkdir build cd build -$SUDO_WWW scl enable devtoolset-7 rh-python36 'bash -c "cmake3 \ +$SUDO_WWW scl enable devtoolset-7 'bash -c "cmake3 \ -DLIEF_PYTHON_API=on \ -DLIEF_DOC=off \ -DCMAKE_INSTALL_PREFIX=$LIEF_INSTALL \ @@ -204,17 +203,13 @@ $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install . cd $PATH_TO_MISP/PyMISP $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install . -# Enable python3 for php-fpm -echo 'source scl_source enable rh-python36' | sudo tee -a /etc/opt/rh/rh-php72/sysconfig/php-fpm +# Enable dependencies detection in the diagnostics page +# This allows MISP to detect GnuPG, the Python modules' versions and to read the PHP settings. +echo "env[PATH] =/opt/rh/rh-php72/root/usr/bin:/usr/local/bin:/usr/bin:/bin" |sudo tee -a /etc/opt/rh/rh-php72/php-fpm.d/www.conf sudo sed -i.org -e 's/^;\(clear_env = no\)/\1/' /etc/opt/rh/rh-php72/php-fpm.d/www.conf sudo systemctl restart rh-php72-php-fpm.service umask $UMASK - -# Enable dependencies detection in the diagnostics page -# This allows MISP to detect GnuPG, the Python modules' versions and to read the PHP settings. -echo "env[PATH] =/opt/rh/rh-python36/root/usr/bin:/opt/rh/rh-php72/root/usr/bin:/usr/local/bin:/usr/bin:/bin" |sudo tee -a /etc/opt/rh/rh-php72/php-fpm.d/www.conf -sudo systemctl restart rh-php72-php-fpm.service ``` ### 4/ CakePHP From 175debb6ed23ceaeee1d8b02ea0bc85c158c2ac2 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Fri, 11 Oct 2019 08:41:45 +0200 Subject: [PATCH 24/79] Keep ini files out of php-fpm.d --- docs/INSTALL.rhel7.md | 9 +++------ docs/xINSTALL.centos7.md | 3 +-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index f31e7c982..9b6557ff2 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -325,19 +325,16 @@ installCake_RHEL () ## sudo yum install php-redis -y sudo scl enable rh-php72 'pecl channel-update pecl.php.net' sudo scl enable rh-php72 'yes no|pecl install redis' - echo "extension=redis.so" |sudo tee /etc/opt/rh/rh-php72/php-fpm.d/redis.ini - sudo ln -s /etc/opt/rh/rh-php72/php-fpm.d/redis.ini /etc/opt/rh/rh-php72/php.d/99-redis.ini + echo "extension=redis.so" |sudo tee /etc/opt/rh/rh-php72/php.d/99-redis.ini # Install gnupg extension sudo yum install gpgme-devel -y sudo scl enable rh-php72 'pecl install gnupg' - echo "extension=gnupg.so" |sudo tee /etc/opt/rh/rh-php72/php-fpm.d/gnupg.ini - sudo ln -s /etc/opt/rh/rh-php72/php-fpm.d/gnupg.ini /etc/opt/rh/rh-php72/php.d/99-gnupg.ini + echo "extension=gnupg.so" |sudo tee /etc/opt/rh/rh-php72/php.d/99-gnupg.ini sudo systemctl restart rh-php72-php-fpm.service # If you have not yet set a timezone in php.ini - echo 'date.timezone = "Asia/Tokyo"' |sudo tee /etc/opt/rh/rh-php72/php-fpm.d/timezone.ini - sudo ln -s ../php-fpm.d/timezone.ini /etc/opt/rh/rh-php72/php.d/99-timezone.ini + echo 'date.timezone = "Asia/Tokyo"' |sudo tee /etc/opt/rh/rh-php72/php.d/timezone.ini # Recommended: Change some PHP settings in /etc/opt/rh/rh-php72/php.ini # max_execution_time = 300 diff --git a/docs/xINSTALL.centos7.md b/docs/xINSTALL.centos7.md index 5d9e53dd9..6102599ca 100644 --- a/docs/xINSTALL.centos7.md +++ b/docs/xINSTALL.centos7.md @@ -232,8 +232,7 @@ sudo yum install php-redis -y sudo systemctl restart rh-php72-php-fpm.service # If you have not yet set a timezone in php.ini -echo 'date.timezone = "Europe/Luxembourg"' |sudo tee /etc/opt/rh/rh-php72/php-fpm.d/timezone.ini -sudo ln -s ../php-fpm.d/timezone.ini /etc/opt/rh/rh-php72/php.d/99-timezone.ini +echo 'date.timezone = "Europe/Luxembourg"' |sudo tee /etc/opt/rh/rh-php72/php.d/99-timezone.ini # Recommended: Change some PHP settings in /etc/opt/rh/rh-php72/php.ini # max_execution_time = 300 From 53ac73a30f8485812cd48c37009d4096ad13ef84 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Fri, 11 Oct 2019 08:49:05 +0200 Subject: [PATCH 25/79] Use $WWW_USER everywhere --- docs/INSTALL.rhel7.md | 8 +++++--- docs/xINSTALL.centos7.md | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index 9b6557ff2..2f015d2b6 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -158,6 +158,8 @@ yumInstallCoreDeps () { # Enable and start redis sudo systemctl enable --now rh-redis32-redis.service + WWW_USER="apache" + SUDO_WWW="sudo -H -u $WWW_USER" RUN_PHP="/usr/bin/scl enable rh-php72" PHP_INI="/etc/opt/rh/rh-php72/php.ini" # Install PHP 7.2 from SCL, see https://www.softwarecollections.org/en/scls/rhscl/rh-php72/ @@ -359,7 +361,7 @@ installCake_RHEL () # Main function to fix permissions to something sane permissions_RHEL () { sudo chown -R $WWW_USER:$WWW_USER $PATH_TO_MISP - ## ? chown -R root:apache /var/www/MISP + ## ? chown -R root:$WWW_USER /var/www/MISP sudo find $PATH_TO_MISP -type d -exec chmod g=rx {} \; sudo chmod -R g+r,o= $PATH_TO_MISP ## **Note :** For updates through the web interface to work, apache must own the /var/www/MISP folder and its subfolders as shown above, which can lead to security issues. If you do not require updates through the web interface to work, you can use the following more restrictive permissions : @@ -665,8 +667,8 @@ configWorkersRHEL () { [Service] Type=forking - User=apache - Group=apache + User=$WWW_USER + Group=$WWW_USER ExecStart=/usr/bin/scl enable rh-php72 rh-redis32 rh-mariadb102 /var/www/MISP/app/Console/worker/start.sh Restart=always RestartSec=10 diff --git a/docs/xINSTALL.centos7.md b/docs/xINSTALL.centos7.md index 6102599ca..a5de5fe5d 100644 --- a/docs/xINSTALL.centos7.md +++ b/docs/xINSTALL.centos7.md @@ -42,10 +42,10 @@ Make sure you are reading the parsed version of this Document. When in doubt [cl ```bash # # RHEL/CentOS Specific -RUN_PHP='/usr/bin/scl enable rh-php72' -SUDO_WWW='sudo -H -u apache' -WWW_USER='apache' +WWW_USER="apache" +SUDO_WWW="sudo -H -u $WWW_USER" +RUN_PHP='/usr/bin/scl enable rh-php72' PHP_INI=/etc/opt/rh/rh-php72/php.ini # ``` From d4c6aa5e885022482c120b0776e0d5b4415252bd Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Fri, 11 Oct 2019 09:27:05 +0200 Subject: [PATCH 26/79] Make chcon's more generic --- docs/INSTALL.rhel7.md | 11 ++++++----- docs/xINSTALL.centos7.md | 16 +++++++--------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index 2f015d2b6..e2c731be1 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -487,12 +487,13 @@ apacheConfig_RHEL () { sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/files/scripts/tmp sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/Plugin/CakeResque/tmp sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/Console/cake - sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/Console/worker/start.sh - sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/mispzmq/mispzmq.py - sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/mispzmq/mispzmqtest.py + sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/Console/worker/*.sh + sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/*.py + sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/*/*.py sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/lief/build/api/python/lief.so - sudo chcon -t httpd_sys_rw_content_t /tmp - sudo chcon -R -t usr_t $PATH_TO_MISP/venv + sudo chcon -R -t bin_t $PATH_TO_MISP/venv/bin/* + find $PATH_TO_MISP/venv -type f -name "*.so*" -or -name "*.so.*" | xargs sudo chcon -t lib_t + # Only run these if you want to be able to update MISP from the web interface sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/.git sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/tmp sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/Lib diff --git a/docs/xINSTALL.centos7.md b/docs/xINSTALL.centos7.md index a5de5fe5d..bbb6e7d1b 100644 --- a/docs/xINSTALL.centos7.md +++ b/docs/xINSTALL.centos7.md @@ -378,20 +378,18 @@ cat /etc/pki/tls/certs/dhparam.pem |sudo tee -a /etc/pki/tls/certs/misp.local.cr sudo systemctl restart httpd.service # Since SELinux is enabled, we need to allow httpd to write to certain directories -sudo chcon -t usr_t $PATH_TO_MISP/venv +sudo chcon -t bin_t $PATH_TO_MISP/venv/bin/* +find $PATH_TO_MISP/venv -type f -name "*.so*" -or -name "*.so.*" | xargs sudo chcon -t lib_t sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/files sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/files/terms sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/files/scripts/tmp sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/Plugin/CakeResque/tmp sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/Console/cake -sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/Console/worker/start.sh -sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/mispzmq/mispzmq.py -sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/mispzmq/mispzmqtest.py -sudo chcon -t httpd_sys_script_exec_t /usr/bin/ps -sudo chcon -t httpd_sys_script_exec_t /usr/bin/grep -sudo chcon -t httpd_sys_script_exec_t /usr/bin/awk -sudo chcon -t httpd_sys_script_exec_t /usr/bin/gpg -sudo chcon -R -t usr_t $PATH_TO_MISP/venv +sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/Console/worker/*.sh +sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/*.py +sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/*/*.py +sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/lief/build/api/python/lief.so +# Only run these if you want to be able to update MISP from the web interface sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/.git sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/tmp sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/Lib From 86f774d24089c419159d164c369ca45da002576b Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Fri, 11 Oct 2019 10:05:52 +0200 Subject: [PATCH 27/79] Use PATH_TO_MISP consistently --- docs/INSTALL.rhel7.md | 25 +++++++++++++------------ docs/xINSTALL.centos7.md | 13 +++++++------ 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index e2c731be1..68c6cc6ee 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -197,10 +197,11 @@ sudo systemctl enable --now haveged.service ```bash # installCoreRHEL () { - # Download MISP using git in the /var/www/ directory. - sudo mkdir $PATH_TO_MISP - sudo chown $WWW_USER:$WWW_USER $PATH_TO_MISP - cd /var/www + # Download MISP using git in the $PATH_TO_MISP directory. + PATH_TO_MISP="/var/www/MISP" + sudo mkdir -p $(dirname $PATH_TO_MISP) + sudo chown $WWW_USER:$WWW_USER $(dirname $PATH_TO_MISP) + cd $(dirname $PATH_TO_MISP) $SUDO_WWW git clone https://github.com/MISP/MISP.git cd $PATH_TO_MISP ##$SUDO_WWW git checkout tags/$(git describe --tags `git rev-list --tags --max-count=1`) @@ -283,7 +284,7 @@ installCoreRHEL () { fi # The following adds a PYTHONPATH to where the pyLIEF module has been compiled - echo /var/www/MISP/app/files/scripts/lief/build/api/python |$SUDO_WWW tee /var/www/MISP/venv/lib/python3.6/site-packages/lief.pth + echo $PATH_TO_MISP/app/files/scripts/lief/build/api/python |$SUDO_WWW tee $PATH_TO_MISP/venv/lib/python3.6/site-packages/lief.pth # install magic, pydeep $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U python-magic git+https://github.com/kbandla/pydeep.git plyara @@ -361,10 +362,10 @@ installCake_RHEL () # Main function to fix permissions to something sane permissions_RHEL () { sudo chown -R $WWW_USER:$WWW_USER $PATH_TO_MISP - ## ? chown -R root:$WWW_USER /var/www/MISP + ## ? chown -R root:$WWW_USER $PATH_TO_MISP sudo find $PATH_TO_MISP -type d -exec chmod g=rx {} \; sudo chmod -R g+r,o= $PATH_TO_MISP - ## **Note :** For updates through the web interface to work, apache must own the /var/www/MISP folder and its subfolders as shown above, which can lead to security issues. If you do not require updates through the web interface to work, you can use the following more restrictive permissions : + ## **Note :** For updates through the web interface to work, apache must own the $PATH_TO_MISP folder and its subfolders as shown above, which can lead to security issues. If you do not require updates through the web interface to work, you can use the following more restrictive permissions : sudo chmod -R 750 $PATH_TO_MISP sudo chmod -R g+xws $PATH_TO_MISP/app/tmp sudo chmod -R g+ws $PATH_TO_MISP/app/files @@ -530,7 +531,7 @@ firewall_RHEL () { ### 8/ Log Rotation ## 8.01/ Enable log rotation -MISP saves the stdout and stderr of its workers in /var/www/MISP/app/tmp/logs +MISP saves the stdout and stderr of its workers in $PATH_TO_MISP/app/tmp/logs To rotate these logs install the supplied logrotate script: FIXME: The below does not work @@ -546,12 +547,12 @@ logRotation_RHEL () { # Now make logrotate work under SELinux as well # Allow logrotate to modify the log files - sudo semanage fcontext -a -t httpd_sys_rw_content_t "/var/www/MISP(/.*)?" + sudo semanage fcontext -a -t httpd_sys_rw_content_t "$PATH_TO_MISP(/.*)?" sudo semanage fcontext -a -t httpd_log_t "$PATH_TO_MISP/app/tmp/logs(/.*)?" sudo chcon -R -t httpd_log_t $PATH_TO_MISP/app/tmp/logs sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/tmp/logs # Impact of the following: ?!?!?!!?111 - ##sudo restorecon -R /var/www/MISP/ + ##sudo restorecon -R $PATH_TO_MISP # Allow logrotate to read /var/www sudo checkmodule -M -m -o /tmp/misplogrotate.mod $PATH_TO_MISP/INSTALL/misplogrotate.te @@ -670,14 +671,14 @@ configWorkersRHEL () { Type=forking User=$WWW_USER Group=$WWW_USER - ExecStart=/usr/bin/scl enable rh-php72 rh-redis32 rh-mariadb102 /var/www/MISP/app/Console/worker/start.sh + ExecStart=/usr/bin/scl enable rh-php72 rh-redis32 rh-mariadb102 $PATH_TO_MISP/app/Console/worker/start.sh Restart=always RestartSec=10 [Install] WantedBy=multi-user.target" |sudo tee /etc/systemd/system/misp-workers.service - sudo chmod +x /var/www/MISP/app/Console/worker/start.sh + sudo chmod +x $PATH_TO_MISP/app/Console/worker/start.sh sudo systemctl daemon-reload sudo systemctl enable --now misp-workers.service diff --git a/docs/xINSTALL.centos7.md b/docs/xINSTALL.centos7.md index bbb6e7d1b..7da14b3a1 100644 --- a/docs/xINSTALL.centos7.md +++ b/docs/xINSTALL.centos7.md @@ -117,9 +117,10 @@ sudo systemctl enable --now redis.service ------------ ```bash # Download MISP using git in the /var/www/ directory. -sudo mkdir $PATH_TO_MISP -sudo chown ${WWW_USER}:${WWW_USER} $PATH_TO_MISP -cd /var/www +PATH_TO_MISP="/var/www/MISP" +sudo mkdir -p $(dirname $PATH_TO_MISP) +sudo chown ${WWW_USER}:${WWW_USER} ($dirname $PATH_TO_MISP) +cd $(dirname $PATH_TO_MISP) $SUDO_WWW git clone https://github.com/MISP/MISP.git cd $PATH_TO_MISP ##$SUDO_WWW git checkout tags/$(git describe --tags `git rev-list --tags --max-count=1`) @@ -181,7 +182,7 @@ $SUDO_WWW scl enable devtoolset-7 'bash -c "cmake3 \ -DCMAKE_INSTALL_PREFIX=$LIEF_INSTALL \ -DCMAKE_BUILD_TYPE=Release \ -DPYTHON_VERSION=3.6 \ --DPYTHON_EXECUTABLE=/var/www/MISP/venv/bin/python \ +-DPYTHON_EXECUTABLE=$PATH_TO_MISP/venv/bin/python \ .."' $SUDO_WWW make -j3 sudo make install @@ -535,7 +536,7 @@ then fi # TODO: Fix static path with PATH_TO_MISP -sudo sed -i -e '$i \su -s /bin/bash apache -c "scl enable rh-php72 /var/www/MISP/app/Console/worker/start.sh" > /tmp/worker_start_rc.local.log\n' /etc/rc.local +sudo sed -i -e '$i \su -s /bin/bash apache -c "scl enable rh-php72 $PATH_TO_MISP/app/Console/worker/start.sh" > /tmp/worker_start_rc.local.log\n' /etc/rc.local # Make sure it will execute sudo chmod +x /etc/rc.local @@ -566,7 +567,7 @@ $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/py $SUDO_WWW ${PATH_TO_MISP}/venv/bin/misp-modules -l 0.0.0.0 -s & # TODO: Fix static path with PATH_TO_MISP -sudo sed -i -e '$i \sudo -u apache /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s &\n' /etc/rc.local +sudo sed -i -e '$i \sudo -u apache $PATH_TO_MISP/venv/bin/misp-modules -l 127.0.0.1 -s &\n' /etc/rc.local ``` {!generic/misp-dashboard-centos.md!} From 5f6b4aef457c85f4d46eef5fb3c4ede1dc7bb514 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Fri, 11 Oct 2019 11:21:05 +0200 Subject: [PATCH 28/79] Fix logrotate module for RHEL 7/CentOS 7 --- INSTALL/misplogrotate.te | 11 +++++++++-- docs/INSTALL.rhel7.md | 1 - docs/xINSTALL.centos7.md | 1 - 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/INSTALL/misplogrotate.te b/INSTALL/misplogrotate.te index 9d8d28ebc..4e3f35a7b 100644 --- a/INSTALL/misplogrotate.te +++ b/INSTALL/misplogrotate.te @@ -1,8 +1,15 @@ -module misplogrotate 1.0; +module misplogrotate 1.1; require { + type httpd_t; type logrotate_t; + type httpd_log_t; type httpd_sys_content_t; - class dir { ioctl read getattr lock search open }; + type httpd_sys_rw_content_t; + class dir { ioctl read getattr lock search open remove_name }; + class file { unlink write }; } #============= logrotate_t ============== allow logrotate_t httpd_sys_content_t:dir { ioctl read getattr lock search open }; +allow logrotate_t httpd_sys_rw_content_t:dir { ioctl read getattr lock search open }; +allow httpd_t httpd_log_t:dir remove_name; +allow httpd_t httpd_log_t:file { unlink write }; diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index 68c6cc6ee..7cf341224 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -550,7 +550,6 @@ logRotation_RHEL () { sudo semanage fcontext -a -t httpd_sys_rw_content_t "$PATH_TO_MISP(/.*)?" sudo semanage fcontext -a -t httpd_log_t "$PATH_TO_MISP/app/tmp/logs(/.*)?" sudo chcon -R -t httpd_log_t $PATH_TO_MISP/app/tmp/logs - sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/tmp/logs # Impact of the following: ?!?!?!!?111 ##sudo restorecon -R $PATH_TO_MISP diff --git a/docs/xINSTALL.centos7.md b/docs/xINSTALL.centos7.md index 7da14b3a1..2d0714e4b 100644 --- a/docs/xINSTALL.centos7.md +++ b/docs/xINSTALL.centos7.md @@ -437,7 +437,6 @@ sudo chmod 0640 /etc/logrotate.d/misp # Allow logrotate to modify the log files sudo semanage fcontext -a -t httpd_log_t "$PATH_TO_MISP/app/tmp/logs(/.*)?" sudo chcon -R -t httpd_log_t $PATH_TO_MISP/app/tmp/logs -sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/tmp/logs # Allow logrotate to read /var/www sudo checkmodule -M -m -o /tmp/misplogrotate.mod $PATH_TO_MISP/INSTALL/misplogrotate.te From 18ed9fde3b8e2401bbbe97d14ad32214e36cf5ba Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 11 Oct 2019 17:29:58 +0200 Subject: [PATCH 29/79] fix: [stix2 import] Avoid errors with report object refs not actually present in the bundle --- app/files/scripts/stix2/stix2misp.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/files/scripts/stix2/stix2misp.py b/app/files/scripts/stix2/stix2misp.py index 6d53bc6eb..fa754e58d 100644 --- a/app/files/scripts/stix2/stix2misp.py +++ b/app/files/scripts/stix2/stix2misp.py @@ -99,8 +99,11 @@ class StixParser(): for ref in report.object_refs: object_type, uuid = ref.split('--') if object_type not in special_parsing and object_type not in galaxy_types: - object2parse = self.event[object_type][uuid] - self.parsing_process(object2parse, object_type) + try: + object2parse = self.event[object_type][uuid] + self.parsing_process(object2parse, object_type) + except KeyError: + continue if len(report_attributes['orgs']) == 1: identity = self.event['identity'][report_attributes['orgs'].pop()] self.misp_event['Org'] = {'name': identity['name']} From c29e1bad08bbc2b00c40facc2fe5917fa6400518 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 11 Oct 2019 17:31:23 +0200 Subject: [PATCH 30/79] fix: [stix2 import] Better pattern parsing for values containing an '=' --- app/files/scripts/stix2/stix2misp.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/files/scripts/stix2/stix2misp.py b/app/files/scripts/stix2/stix2misp.py index fa754e58d..b8696fd77 100644 --- a/app/files/scripts/stix2/stix2misp.py +++ b/app/files/scripts/stix2/stix2misp.py @@ -1442,7 +1442,10 @@ class ExternalStixParser(StixParser): try: type_, value = p.split('=') except ValueError: - type_, value = p.split(' = ') + try: + type_, value = p.split(' = ') + except ValueError: + type_, value = p.split(' = \'') types.append(type_.strip()) values.append(value.strip().strip('\'')) return types, values From 745512e71e09580ae533a1e14dfc3026cd9f9cd2 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Sat, 12 Oct 2019 23:45:58 +0900 Subject: [PATCH 31/79] chg: [installer] Updated installer to support latest Kali Linux chg: [doc] Amended Centos7 mirror foo --- INSTALL/INSTALL.sh | 1 + INSTALL/INSTALL.sh.sfv | 6 +- INSTALL/INSTALL.sh.sha1 | 2 +- INSTALL/INSTALL.sh.sha256 | 2 +- INSTALL/INSTALL.sh.sha384 | 2 +- INSTALL/INSTALL.sh.sha512 | 2 +- INSTALL/INSTALL.tpl.sh | 1 + docs/INSTALL.rhel7.md | 2 +- docs/xINSTALL.centos7.md | 588 -------------------------------------- mkdocs.yml | 2 +- 10 files changed, 11 insertions(+), 597 deletions(-) delete mode 100644 docs/xINSTALL.centos7.md diff --git a/INSTALL/INSTALL.sh b/INSTALL/INSTALL.sh index 8366d3119..266bc1a79 100644 --- a/INSTALL/INSTALL.sh +++ b/INSTALL/INSTALL.sh @@ -2464,6 +2464,7 @@ x86_64-debian-stretch x86_64-debian-buster x86_64-ubuntu-bionic x86_64-kali-2019.2 +x86_64-kali-2019.3 armv6l-raspbian-stretch armv7l-raspbian-stretch armv7l-debian-jessie diff --git a/INSTALL/INSTALL.sh.sfv b/INSTALL/INSTALL.sh.sfv index f86f05911..11c375fb1 100644 --- a/INSTALL/INSTALL.sh.sfv +++ b/INSTALL/INSTALL.sh.sfv @@ -1,5 +1,5 @@ -; Generated by RHash v1.3.8 on 2019-09-19 at 11:45.46 +; Generated by RHash v1.3.8 on 2019-10-12 at 23:45.03 ; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/ ; -; 98996 11:45.46 2019-09-19 INSTALL.sh -INSTALL.sh 590EBA6FCA2E8F2B5044843DF22FEED524ED3C7E 2B3E6DC0191D5D3AFD685E6ACF3E02D15261770E8E17A21F882EC2F07E908D55 13EB5B9EF3AEE3C96558DE068A0210F938491322F0AAD10DB958D9DCFA93B5313044013084E56647586AFCF8DE07C7D1 B169AA7096FAAF21E8CB8B7B7C9296C4CDB25E4B540637C15C040722EE52A2A8D2C4E8122CCEC25AFAD99F9CEC065DE14899E7C1DD571DE2B6127F08C6B79229 +; 99015 23:45.03 2019-10-12 INSTALL.sh +INSTALL.sh 0A4A963E90CD83550F59CBC270CEF3DB3E25DBDF D292D42E306A3E19FCD9A6C10B1A28F56584810D49420592A9A6644BB3EFB2E6 7453482ADF587B9760B23A0563FADE99568E497CF810BAE9CD7E44F0025D8D6BFE1B17DD2C790A6CCD72130A72CD59C6 E6BEACA555D8752D210757BE8A372AF7BA8675CA293810B8A5ADEE905FC44D41CDAD60225203D8E93B037D520703996E03E62B501EB766290C452716BB125C4F diff --git a/INSTALL/INSTALL.sh.sha1 b/INSTALL/INSTALL.sh.sha1 index 9ccef1a18..f1411624f 100644 --- a/INSTALL/INSTALL.sh.sha1 +++ b/INSTALL/INSTALL.sh.sha1 @@ -1 +1 @@ -590eba6fca2e8f2b5044843df22feed524ed3c7e INSTALL.sh +0a4a963e90cd83550f59cbc270cef3db3e25dbdf INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha256 b/INSTALL/INSTALL.sh.sha256 index fa10e2b37..def068cd8 100644 --- a/INSTALL/INSTALL.sh.sha256 +++ b/INSTALL/INSTALL.sh.sha256 @@ -1 +1 @@ -2b3e6dc0191d5d3afd685e6acf3e02d15261770e8e17a21f882ec2f07e908d55 INSTALL.sh +d292d42e306a3e19fcd9a6c10b1a28f56584810d49420592a9a6644bb3efb2e6 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha384 b/INSTALL/INSTALL.sh.sha384 index 3172f0686..46f95f6c1 100644 --- a/INSTALL/INSTALL.sh.sha384 +++ b/INSTALL/INSTALL.sh.sha384 @@ -1 +1 @@ -13eb5b9ef3aee3c96558de068a0210f938491322f0aad10db958d9dcfa93b5313044013084e56647586afcf8de07c7d1 INSTALL.sh +7453482adf587b9760b23a0563fade99568e497cf810bae9cd7e44f0025d8d6bfe1b17dd2c790a6ccd72130a72cd59c6 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha512 b/INSTALL/INSTALL.sh.sha512 index cf4f17fd2..68b123cbd 100644 --- a/INSTALL/INSTALL.sh.sha512 +++ b/INSTALL/INSTALL.sh.sha512 @@ -1 +1 @@ -b169aa7096faaf21e8cb8b7b7c9296c4cdb25e4b540637c15c040722ee52a2a8d2c4e8122ccec25afad99f9cec065de14899e7c1dd571de2b6127f08c6b79229 INSTALL.sh +e6beaca555d8752d210757be8a372af7ba8675ca293810b8a5adee905fc44d41cdad60225203d8e93b037d520703996e03e62b501eb766290c452716bb125c4f INSTALL.sh diff --git a/INSTALL/INSTALL.tpl.sh b/INSTALL/INSTALL.tpl.sh index 4a183f1cd..9b461b70c 100755 --- a/INSTALL/INSTALL.tpl.sh +++ b/INSTALL/INSTALL.tpl.sh @@ -678,6 +678,7 @@ x86_64-debian-stretch x86_64-debian-buster x86_64-ubuntu-bionic x86_64-kali-2019.2 +x86_64-kali-2019.3 armv6l-raspbian-stretch armv7l-raspbian-stretch armv7l-debian-jessie diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index cf69402fa..dcae173d5 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -29,7 +29,7 @@ Make sure you are reading the parsed version of this Document. When in doubt [cl !!! notice Maintenance for CentOS 7 will end on: June 30th, 2024 [Source[0]](https://wiki.centos.org/About/Product) [Source[1]](https://linuxlifecycle.com/) - CentOS 7.6-1810 [NetInstallURL](http://mirror.centos.org/centos/7.6.1810/os/x86_64/) + CentOS 7-1908 [NetInstallURL](http://mirror.centos.org/centos/7/os/x86_64/) This document details the steps to install MISP on Red Hat Enterprise Linux 7.x (RHEL 7.x) and CentOS 7.x. At time of this writing it was tested on versions 7.6 for both. diff --git a/docs/xINSTALL.centos7.md b/docs/xINSTALL.centos7.md deleted file mode 100644 index 51c2c06fe..000000000 --- a/docs/xINSTALL.centos7.md +++ /dev/null @@ -1,588 +0,0 @@ -# **deprecated** INSTALLATION INSTRUCTIONS -## for CentOS 7.x - - -Please use the Red Hat Enterprise Linux 7 Instructions for a CentOS 7 install. [click here](https://misp.github.io/MISP/INSTALL.rhel7/). - -### -1/ Installer and Manual install instructions - -Make sure you are reading the parsed version of this Document. When in doubt [click here](https://misp.github.io/MISP/INSTALL.rhel7/). - -!!! warning - In the **future**, to install MISP on a fresh CentOS 7 install all you need to do is: - - ```bash - # Please check the installer options first to make the best choice for your install - wget -O /tmp/INSTALL.sh https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/INSTALL.sh - bash /tmp/INSTALL.sh - - # This will install MISP Core - wget -O /tmp/INSTALL.sh https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/INSTALL.sh - bash /tmp/INSTALL.sh -c - ``` - **The above does NOT work yet** - -### 0/ MISP CentOS 7 Minimal NetInstall - Status --------------------------------------------- - -{!generic/community.md!} - -{!generic/rhelVScentos.md!} - -!!! notice - Semi-maintained and tested by @SteveClement, CentOS 7.6-1804 on 20190410
- It is still considered experimental as not everything works seemlessly. - -!!! notice - Maintenance for CentOS 7 will end on: June 30th, 2024 [Source[0]](https://wiki.centos.org/About/Product) [Source[1]](https://linuxlifecycle.com/) - CentOS 7.6-1810 [NetInstallURL](http://mirror.centos.org/centos/7.6.1810/os/x86_64/) - -{!generic/globalVariables.md!} - -```bash -# -# RHEL/CentOS Specific -RUN_PHP='/usr/bin/scl enable rh-php72' -RUN_PYTHON='/usr/bin/scl enable rh-python36' -SUDO_WWW='sudo -H -u apache' -WWW_USER='apache' - -PHP_INI=/etc/opt/rh/rh-php72/php.ini -# -``` - -### 1/ Minimal CentOS install -------------------------- - -Install a minimal CentOS 7.x system with the software: - -- OpenSSH server -- LAMP server (actually, this is done below) -- Mail server - - -```bash -# Make sure you set your hostname CORRECTLY vs. like an brute (manually in /etc/hostname) -sudo hostnamectl set-hostname misp.local # Your choice, in a production environment, it's best to use a FQDN - -# Make sure your system is up2date: -sudo yum update -y -``` - -### 2/ Dependencies * ----------------- -Once the system is installed you can perform the following steps as root or with sudo. - -```bash -# We need some packages from the Extra Packages for Enterprise Linux repository -sudo yum install epel-release -y - -# Since MISP 2.4 PHP 5.5 is a minimal requirement, so we need a newer version than CentOS base provides -# Software Collections is a way do to this, see https://wiki.centos.org/AdditionalResources/Repositories/SCL -sudo yum install centos-release-scl -y - -# Because (neo)vim is just so practical -sudo yum install neovim -y - -# Install the dependencies: -sudo yum install gcc git zip \ - httpd \ - mod_ssl \ - redis \ - mariadb mariadb-server \ - python-devel python-pip python-zmq \ - libxslt-devel zlib-devel ssdeep-devel -y - -# Install PHP 7.2 from SCL, see https://www.softwarecollections.org/en/scls/rhscl/rh-php72/ -sudo yum install rh-php72 rh-php72-php-fpm rh-php72-php-devel rh-php72-php-mysqlnd rh-php72-php-mbstring rh-php72-php-xml rh-php72-php-bcmath rh-php72-php-opcache rh-php72-php-gd -y - -# Install Python 3.6 from SCL, see -# https://www.softwarecollections.org/en/scls/rhscl/rh-python36/ -sudo yum install rh-python36 -y - -sudo systemctl enable --now rh-php72-php-fpm.service -``` - -!!! notice - $RUN_PHP makes php available for you if using rh-php72. e.g: sudo $RUN_PHP "pear list | grep Crypt_GPG" - -```bash -# GPG needs lots of entropy, haveged provides entropy -sudo yum install haveged -y -sudo systemctl enable --now haveged.service - -# Enable and start redis -sudo systemctl enable --now redis.service -``` - -### 3/ MISP code ------------- -```bash -# Download MISP using git in the /var/www/ directory. -sudo mkdir $PATH_TO_MISP -sudo chown ${WWW_USER}:${WWW_USER} $PATH_TO_MISP -cd /var/www -$SUDO_WWW git clone https://github.com/MISP/MISP.git -cd $PATH_TO_MISP -##$SUDO_WWW git checkout tags/$(git describe --tags `git rev-list --tags --max-count=1`) -# if the last shortcut doesn't work, specify the latest version manually -# example: git checkout tags/v2.4.XY -# the message regarding a "detached HEAD state" is expected behaviour -# (you only have to create a new branch, if you want to change stuff and do a pull request for example) - -# Fetch submodules -$SUDO_WWW git submodule update --init --recursive -# Make git ignore filesystem permission differences for submodules -$SUDO_WWW git submodule foreach --recursive git config core.filemode false -# Make git ignore filesystem permission differences -$SUDO_WWW git config core.filemode false - -# Create a python3 virtualenv -$SUDO_WWW $RUN_PYTHON "virtualenv -p python3 $PATH_TO_MISP/venv" -sudo mkdir /usr/share/httpd/.cache -sudo chown ${WWW_USER}:${WWW_USER} /usr/share/httpd/.cache -$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U pip setuptools - -# install Mitre's STIX and its dependencies by running the following commands: -##sudo yum install python-importlib python-lxml python-dateutil python-six -y - -cd $PATH_TO_MISP/app/files/scripts -$SUDO_WWW git clone https://github.com/CybOXProject/python-cybox.git -$SUDO_WWW git clone https://github.com/STIXProject/python-stix.git -$SUDO_WWW git clone --branch master --single-branch https://github.com/lief-project/LIEF.git lief -$SUDO_WWW git clone https://github.com/CybOXProject/mixbox.git - -cd $PATH_TO_MISP/app/files/scripts/python-cybox -# If you umask is has been changed from the default, it is a good idea to reset it to 0022 before installing python modules -UMASK=$(umask) -umask 0022 -cd $PATH_TO_MISP/app/files/scripts/python-stix -$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install . - -# install maec -$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U maec - -# install zmq -$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U zmq - -# install redis -$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U redis - -# lief needs manual compilation -sudo yum install devtoolset-7 cmake3 -y - - -# TODO: Fix static path with PATH_TO_MISP -cd $PATH_TO_MISP/app/files/scripts/lief -$SUDO_WWW mkdir build -cd build -$SUDO_WWW scl enable devtoolset-7 rh-python36 'bash -c "cmake3 \ --DLIEF_PYTHON_API=on \ --DLIEF_DOC=off \ --DCMAKE_INSTALL_PREFIX=$LIEF_INSTALL \ --DCMAKE_BUILD_TYPE=Release \ --DPYTHON_VERSION=3.6 \ --DPYTHON_EXECUTABLE=/var/www/MISP/venv/bin/python \ -.."' -$SUDO_WWW make -j3 -sudo make install -cd api/python/lief_pybind11-prefix/src/lief_pybind11 -$SUDO_WWW $PATH_TO_MISP/venv/bin/python setup.py install -$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip - -# install magic, pydeep -$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U python-magic git+https://github.com/kbandla/pydeep.git - -cd $PATH_TO_MISP/app/files/scripts/mixbox -$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install . - -# install STIX2.0 library to support STIX 2.0 export: -cd $PATH_TO_MISP/cti-python-stix2 -$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install . - -# install PyMISP -cd $PATH_TO_MISP/PyMISP -$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install . - -# Enable python3 for php-fpm -echo 'source scl_source enable rh-python36' | sudo tee -a /etc/opt/rh/rh-php72/sysconfig/php-fpm -sudo sed -i.org -e 's/^;\(clear_env = no\)/\1/' /etc/opt/rh/rh-php72/php-fpm.d/www.conf -sudo systemctl restart rh-php72-php-fpm.service - -umask $UMASK - -# Enable dependencies detection in the diagnostics page -# This allows MISP to detect GnuPG, the Python modules' versions and to read the PHP settings. -echo "env[PATH] =/opt/rh/rh-python36/root/usr/bin:/opt/rh/rh-php72/root/usr/bin:/usr/local/bin:/usr/bin:/bin" |sudo tee -a /etc/opt/rh/rh-php72/php-fpm.d/www.conf -sudo systemctl restart rh-php72-php-fpm.service -``` - -### 4/ CakePHP ------------ -#### CakePHP is now included as a submodule of MISP and has been fetch by a previous step. -#### Install CakeResque along with its dependencies if you intend to use the built in background jobs. -```bash -sudo chown -R ${WWW_USER}:${WWW_USER} $PATH_TO_MISP -sudo mkdir /usr/share/httpd/.composer -sudo chown ${WWW_USER}:${WWW_USER} /usr/share/httpd/.composer -cd $PATH_TO_MISP/app -# Update composer.phar (optional) -#$SUDO_WWW $RUN_PHP -- php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" -#$SUDO_WWW $RUN_PHP -- php -r "if (hash_file('SHA384', 'composer-setup.php') === 'a5c698ffe4b8e849a443b120cd5ba38043260d5c4023dbf93e1558871f1f07f58274fc6f4c93bcfd858c6bd0775cd8d1') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;" -#$SUDO_WWW $RUN_PHP "php composer-setup.php" -#$SUDO_WWW $RUN_PHP -- php -r "unlink('composer-setup.php');" -$SUDO_WWW $RUN_PHP "php composer.phar install" - -sudo yum install php-redis -y -sudo systemctl restart rh-php72-php-fpm.service - -# If you have not yet set a timezone in php.ini -echo 'date.timezone = "Europe/Luxembourg"' |sudo tee /etc/opt/rh/rh-php72/php-fpm.d/timezone.ini -sudo ln -s ../php-fpm.d/timezone.ini /etc/opt/rh/rh-php72/php.d/99-timezone.ini - -# Recommended: Change some PHP settings in /etc/opt/rh/rh-php72/php.ini -# max_execution_time = 300 -# memory_limit = 2048M -# upload_max_filesize = 50M -# post_max_size = 50M -for key in upload_max_filesize post_max_size max_execution_time max_input_time memory_limit -do - sudo sed -i "s/^\($key\).*/\1 = $(eval echo \${$key})/" $PHP_INI -done -sudo systemctl restart rh-php72-php-fpm.service - -# To use the scheduler worker for scheduled tasks, do the following: -sudo cp -fa $PATH_TO_MISP/INSTALL/setup/config.php $PATH_TO_MISP/app/Plugin/CakeResque/Config/config.php -``` - -### 5/ Set the permissions ----------------------- -```bash -# Make sure the permissions are set correctly using the following commands as root: -sudo chown -R ${WWW_USER}:${WWW_USER} $PATH_TO_MISP -sudo find $PATH_TO_MISP -type d -exec chmod g=rx {} \; -sudo chmod -R g+r,o= $PATH_TO_MISP -sudo chmod -R 750 $PATH_TO_MISP -sudo chmod -R g+xws $PATH_TO_MISP/app/tmp -sudo chmod -R g+ws $PATH_TO_MISP/app/files -sudo chmod -R g+ws $PATH_TO_MISP/app/files/scripts/tmp -sudo chmod -R g+rw $PATH_TO_MISP/venv -sudo chmod -R g+rw $PATH_TO_MISP/.git -sudo chown ${WWW_USER}:${WWW_USER} $PATH_TO_MISP/app/files -sudo chown ${WWW_USER}:${WWW_USER} $PATH_TO_MISP/app/files/terms -sudo chown ${WWW_USER}:${WWW_USER} $PATH_TO_MISP/app/files/scripts/tmp -sudo chown ${WWW_USER}:${WWW_USER} $PATH_TO_MISP/app/Plugin/CakeResque/tmp -sudo chown -R ${WWW_USER}:${WWW_USER} $PATH_TO_MISP/app/Config -sudo chown -R ${WWW_USER}:${WWW_USER} $PATH_TO_MISP/app/tmp -sudo chown -R ${WWW_USER}:${WWW_USER} $PATH_TO_MISP/app/webroot/img/orgs -sudo chown -R ${WWW_USER}:${WWW_USER} $PATH_TO_MISP/app/webroot/img/custom -``` - -### 6/ Create a database and user ------------------------------ -```bash -# Enable, start and secure your mysql database server -sudo systemctl enable --now mariadb.service - -sudo yum install expect -y - -# Add your credentials if needed, if sudo has NOPASS, comment out the relevant lines -pw="Password1234" - -expect -f - <<-EOF - set timeout 10 - - spawn sudo mysql_secure_installation - expect "*?assword*" - send -- "$pw\r" - expect "Enter current password for root (enter for none):" - send -- "\r" - expect "Set root password?" - send -- "y\r" - expect "New password:" - send -- "${DBPASSWORD_ADMIN}\r" - expect "Re-enter new password:" - send -- "${DBPASSWORD_ADMIN}\r" - expect "Remove anonymous users?" - send -- "y\r" - expect "Disallow root login remotely?" - send -- "y\r" - expect "Remove test database and access to it?" - send -- "y\r" - expect "Reload privilege tables now?" - send -- "y\r" - expect eof -EOF - -sudo yum remove tcl expect -y - -# Additionally, it is probably a good idea to make the database server listen on localhost only -echo [mysqld] |sudo tee /etc/my.cnf.d/bind-address.cnf -echo bind-address=127.0.0.1 |sudo tee -a /etc/my.cnf.d/bind-address.cnf -sudo systemctl restart mariadb.service -``` - -#### Manual procedure: -```bash -# Enter the mysql shell -mysql -u root -p -``` - -``` -MariaDB [(none)]> create database misp; -MariaDB [(none)]> grant usage on *.* to misp@localhost identified by 'XXXXXXXXX'; -MariaDB [(none)]> grant all privileges on misp.* to misp@localhost ; -MariaDB [(none)]> exit -``` - -#### Same as Manual but for copy/paste foo: -```bash -sudo mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "create database $DBNAME;" -sudo mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "grant usage on *.* to $DBNAME@localhost identified by '$DBPASSWORD_MISP';" -sudo mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "grant all privileges on $DBNAME.* to '$DBUSER_MISP'@'localhost';" -sudo mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "flush privileges;" -``` - -#### Import the empty MySQL database from MYSQL.sql -```bash -$SUDO_WWW cat $PATH_TO_MISP/INSTALL/MYSQL.sql | mysql -u $DBUSER_MISP -p$DBPASSWORD_MISP $DBNAME -``` - - -### 7/ Apache configuration ------------------------ - -!!! notice - SELinux note, to check if it is running: - ```bash - $ sestatus - SELinux status: disabled - ``` - If it is disabled, you can ignore the **chcon/setsebool/semanage/checkmodule/semodule*** commands. - -```bash -# Now configure your apache server with the DocumentRoot $PATH_TO_MISP/app/webroot/ -# A sample vhost can be found in $PATH_TO_MISP/INSTALL/apache.misp.centos7 - -sudo cp $PATH_TO_MISP/INSTALL/apache.misp.centos7.ssl /etc/httpd/conf.d/misp.ssl.conf -sudo rm /etc/httpd/conf.d/ssl.conf -sudo chmod 644 /etc/httpd/conf.d/misp.ssl.conf -sudo sed -i '/Listen 80/a Listen 443' /etc/httpd/conf/httpd.conf - -# If a valid SSL certificate is not already created for the server, create a self-signed certificate: -echo "The Common Name used below will be: ${OPENSSL_CN}" -# This will take a rather long time, be ready. (13min on a VM, 8GB Ram, 1 core) -sudo openssl dhparam -out /etc/pki/tls/certs/dhparam.pem 4096 -sudo openssl genrsa -des3 -passout pass:x -out /tmp/misp.local.key 4096 -sudo openssl rsa -passin pass:x -in /tmp/misp.local.key -out /etc/pki/tls/private/misp.local.key -sudo rm /tmp/misp.local.key -sudo openssl req -new -subj "/C=${OPENSSL_C}/ST=${OPENSSL_ST}/L=${OPENSSL_L}/O=${OPENSSL_O}/OU=${OPENSSL_OU}/CN=${OPENSSL_CN}/emailAddress=${OPENSSL_EMAILADDRESS}" -key /etc/pki/tls/private/misp.local.key -out /etc/pki/tls/certs/misp.local.csr -sudo openssl x509 -req -days 365 -in /etc/pki/tls/certs/misp.local.csr -signkey /etc/pki/tls/private/misp.local.key -out /etc/pki/tls/certs/misp.local.crt -sudo ln -s /etc/pki/tls/certs/misp.local.csr /etc/pki/tls/certs/misp-chain.crt -cat /etc/pki/tls/certs/dhparam.pem |sudo tee -a /etc/pki/tls/certs/misp.local.crt - -sudo systemctl restart httpd.service - -# Since SELinux is enabled, we need to allow httpd to write to certain directories -sudo chcon -t usr_t $PATH_TO_MISP/venv -sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/files -sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/files/terms -sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/files/scripts/tmp -sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/Plugin/CakeResque/tmp -sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/Console/cake -sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/Console/worker/start.sh -sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/mispzmq/mispzmq.py -sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/mispzmq/mispzmqtest.py -sudo chcon -t httpd_sys_script_exec_t /usr/bin/ps -sudo chcon -t httpd_sys_script_exec_t /usr/bin/grep -sudo chcon -t httpd_sys_script_exec_t /usr/bin/awk -sudo chcon -t httpd_sys_script_exec_t /usr/bin/gpg -sudo chcon -R -t usr_t $PATH_TO_MISP/venv -sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/.git -sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/tmp -sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/Lib -sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/Config -sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/tmp -sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/webroot/img/orgs -sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/webroot/img/custom -sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/files/scripts/mispzmq -``` - -!!! warning - Todo: Revise all permissions so update in Web UI works. - -```bash -# Allow httpd to connect to the redis server and php-fpm over tcp/ip -sudo setsebool -P httpd_can_network_connect on - -# Allow httpd to send emails from php -sudo setsebool -P httpd_can_sendmail on - -# Enable and start the httpd service -sudo systemctl enable --now httpd.service - -# Open a hole in the iptables firewall -sudo firewall-cmd --zone=public --add-port=80/tcp --permanent -sudo firewall-cmd --zone=public --add-port=443/tcp --permanent -sudo firewall-cmd --reload - -# We seriously recommend using only HTTPS / SSL ! -# Add SSL support by running: sudo yum install mod_ssl -# Check out the apache.misp.ssl file for an example -``` - -### 8/ Log rotation ---------------- -```bash -# MISP saves the stdout and stderr of its workers in $PATH_TO_MISP/app/tmp/logs -# To rotate these logs install the supplied logrotate script: - -sudo cp $PATH_TO_MISP/INSTALL/misp.logrotate /etc/logrotate.d/misp -sudo chmod 0640 /etc/logrotate.d/misp - -# Now make logrotate work under SELinux as well -# Allow logrotate to modify the log files -sudo semanage fcontext -a -t httpd_log_t "$PATH_TO_MISP/app/tmp/logs(/.*)?" -sudo chcon -R -t httpd_log_t $PATH_TO_MISP/app/tmp/logs -sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/tmp/logs - -# Allow logrotate to read /var/www -sudo checkmodule -M -m -o /tmp/misplogrotate.mod $PATH_TO_MISP/INSTALL/misplogrotate.te -sudo semodule_package -o /tmp/misplogrotate.pp -m /tmp/misplogrotate.mod -sudo semodule -i /tmp/misplogrotate.pp -``` - -### 9/ MISP configuration ---------------------- -```bash -# There are 4 sample configuration files in $PATH_TO_MISP/app/Config that need to be copied -$SUDO_WWW cp -a $PATH_TO_MISP/app/Config/bootstrap.default.php $PATH_TO_MISP/app/Config/bootstrap.php -$SUDO_WWW cp -a $PATH_TO_MISP/app/Config/database.default.php $PATH_TO_MISP/app/Config/database.php -$SUDO_WWW cp -a $PATH_TO_MISP/app/Config/core.default.php $PATH_TO_MISP/app/Config/core.php -$SUDO_WWW cp -a $PATH_TO_MISP/app/Config/config.default.php $PATH_TO_MISP/app/Config/config.php - -echo " 'Database/Mysql', - //'datasource' => 'Database/Postgres', - 'persistent' => false, - 'host' => '$DBHOST', - 'login' => '$DBUSER_MISP', - 'port' => 3306, // MySQL & MariaDB - //'port' => 5432, // PostgreSQL - 'password' => '$DBPASSWORD_MISP', - 'database' => '$DBNAME', - 'prefix' => '', - 'encoding' => 'utf8', - ); -}" | $SUDO_WWW tee $PATH_TO_MISP/app/Config/database.php - -# Configure the fields in the newly created files: -# config.php : baseurl (example: 'baseurl' => 'http://misp',) - don't use "localhost" it causes issues when browsing externally -# core.php : Uncomment and set the timezone: `// date_default_timezone_set('UTC');` -# database.php : login, port, password, database -# DATABASE_CONFIG has to be filled -# With the default values provided in section 6, this would look like: -# class DATABASE_CONFIG { -# public $default = array( -# 'datasource' => 'Database/Mysql', -# 'persistent' => false, -# 'host' => 'localhost', -# 'login' => 'misp', // grant usage on *.* to misp@localhost -# 'port' => 3306, -# 'password' => 'XXXXdbpasswordhereXXXXX', // identified by 'XXXXdbpasswordhereXXXXX'; -# 'database' => 'misp', // create database misp; -# 'prefix' => '', -# 'encoding' => 'utf8', -# ); -#} - -# Important! Change the salt key in $PATH_TO_MISP/app/Config/config.php -# The admin user account will be generated on the first login, make sure that the salt is changed before you create that user -# If you forget to do this step, and you are still dealing with a fresh installation, just alter the salt, -# delete the user from mysql and log in again using the default admin credentials (admin@admin.test / admin) - -# If you want to be able to change configuration parameters from the webinterface: -sudo chown ${WWW_USER}:${WWW_USER} $PATH_TO_MISP/app/Config/config.php -sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/Config/config.php - -# Generate a GPG encryption key. -cat >/tmp/gen-key-script < /tmp/worker_start_rc.local.log\n' /etc/rc.local -# Make sure it will execute -sudo chmod +x /etc/rc.local - -echo "Admin (root) DB Password: $DBPASSWORD_ADMIN" -echo "User (misp) DB Password: $DBPASSWORD_MISP" -``` - -```bash -# some misp-modules dependencies -sudo yum install openjpeg-devel -y - -sudo chmod 2777 /usr/local/src -sudo chown root:users /usr/local/src -cd /usr/local/src/ -$SUDO_WWW git clone https://github.com/MISP/misp-modules.git -cd misp-modules -# pip install -$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I -r REQUIREMENTS -$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install . -sudo yum install rubygem-rouge rubygem-asciidoctor -y -##sudo gem install asciidoctor-pdf --pre - -# install additional dependencies for extended object generation and extraction -$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install maec python-magic pathlib -$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/pydeep.git - -# Start misp-modules -$SUDO_WWW ${PATH_TO_MISP}/venv/bin/misp-modules -l 0.0.0.0 -s & - -# TODO: Fix static path with PATH_TO_MISP -sudo sed -i -e '$i \sudo -u apache /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s &\n' /etc/rc.local -``` - -{!generic/misp-dashboard-centos.md!} - -{!generic/MISP_CAKE_init.md!} - -{!generic/INSTALL.done.md!} - -{!generic/recommended.actions.md!} - -{!generic/hardening.md!} diff --git a/mkdocs.yml b/mkdocs.yml index 0dd2a45e0..a1b54cb75 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -71,7 +71,7 @@ nav: - 'Ubuntu 18.04': 'INSTALL.ubuntu1804.md' - 'Kali Linux': 'INSTALL.kali.md' - 'RHEL7/CentOS7': 'INSTALL.rhel7.md' - - 'RHEL8': 'INSTALL.rhel8.md' + - 'RHEL8/CentOS8': 'INSTALL.rhel8.md' - xInstall Guides: - 'Warning': 'xINSTALL.md' - 'Centos 6': 'xINSTALL.centos6.md' From 121084a73775278f8cf2e42e858d982c01002ef5 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 13 Oct 2019 13:10:06 +0200 Subject: [PATCH 32/79] fix: [UI] Show image attachment for previewing event --- app/View/Elements/Events/View/value_field.ctp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/View/Elements/Events/View/value_field.ctp b/app/View/Elements/Events/View/value_field.ctp index 0b5900c0a..d8b9ebe10 100644 --- a/app/View/Elements/Events/View/value_field.ctp +++ b/app/View/Elements/Events/View/value_field.ctp @@ -5,7 +5,7 @@ switch ($object['type']) { case 'attachment': case 'malware-sample': if ($object['type'] === 'attachment' && isset($object['image'])) { - if (extension_loaded('gd')) { + if ($object['image'] === true) { $img = ''; $img .= ''; echo $img; From 35ffd4a87938298430b84feb4505da89ccd6459a Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 13 Oct 2019 14:21:30 +0200 Subject: [PATCH 33/79] fix: [UI] Remove unused code from Communities view template --- app/View/Communities/view.ctp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/app/View/Communities/view.ctp b/app/View/Communities/view.ctp index 7c5648a1e..1e21550a7 100644 --- a/app/View/Communities/view.ctp +++ b/app/View/Communities/view.ctp @@ -56,12 +56,3 @@ element('/genericElements/SideMenu/side_menu', array('menuList' => 'sync', 'menuItem' => 'view_community')); ?> - From fcb3f85795c45bbe9389a1413746187dc1b809b4 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 13 Oct 2019 16:29:14 +0200 Subject: [PATCH 34/79] fix: [UI] Removed unused JS from galaxy view --- app/View/Elements/galaxyQuickView.ctp | 4 ---- app/View/Elements/galaxyQuickViewMini.ctp | 4 ---- 2 files changed, 8 deletions(-) diff --git a/app/View/Elements/galaxyQuickView.ctp b/app/View/Elements/galaxyQuickView.ctp index 2114f7864..3b8d403b0 100755 --- a/app/View/Elements/galaxyQuickView.ctp +++ b/app/View/Elements/galaxyQuickView.ctp @@ -151,9 +151,5 @@ $(this).parent().children('.collapse-status-container').children('.collapse-status').html('+'); } }); - $('.delete-cluster').click(function() { - var tagName = $(this).data('tag-name'); - removeTag($id = false, $tag_id = false, $galaxy = false); - }); }); diff --git a/app/View/Elements/galaxyQuickViewMini.ctp b/app/View/Elements/galaxyQuickViewMini.ctp index b5203b499..7328cc009 100644 --- a/app/View/Elements/galaxyQuickViewMini.ctp +++ b/app/View/Elements/galaxyQuickViewMini.ctp @@ -146,9 +146,5 @@ $(document).ready(function () { $(this).children('span').html('+'); } }); - $('.delete-cluster').click(function() { - var tagName = $(this).data('tag-name'); - removeTag($id = false, $tag_id = false, $galaxy = false); - }); }); From ed498aa80484b5b1b5d715134288eafe2f134ade Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 13 Oct 2019 19:49:59 +0200 Subject: [PATCH 35/79] fix: [UI] Input is not pair element --- .../Attributes/ajax/attributeViewFieldForm.ctp | 4 ++-- app/View/DecayingModel/decaying_tool.ctp | 18 +++++++++--------- .../DecayingModel/decaying_tool_basescore.ctp | 12 ++++++------ .../Events/View/eventFilteringQueryBuilder.ctp | 2 +- .../Elements/Events/View/row_attribute.ctp | 2 +- .../Elements/Objects/object_similarities.ctp | 2 +- app/View/Elements/serverRuleElements/pull.ctp | 4 ++-- app/View/Elements/view_galaxy_matrix.ctp | 2 +- app/View/Events/upload_analysis_file.ctp | 2 +- app/View/Feeds/add.ctp | 4 ++-- app/View/Feeds/edit.ctp | 4 ++-- app/View/GalaxyClusters/ajax/index.ctp | 2 +- .../Organisations/ajax/sg_org_row_empty.ctp | 2 +- app/View/SharingGroups/add.ctp | 8 ++++---- app/View/SharingGroups/edit.ctp | 8 ++++---- app/View/Tags/index.ctp | 2 +- app/View/Users/statistics_galaxymatrix.ctp | 4 ++-- app/webroot/js/decayingTool.js | 2 +- app/webroot/js/misp.js | 4 ++-- app/webroot/js/network-distribution-graph.js | 2 +- 20 files changed, 45 insertions(+), 45 deletions(-) diff --git a/app/View/Attributes/ajax/attributeViewFieldForm.ctp b/app/View/Attributes/ajax/attributeViewFieldForm.ctp index a00889a23..d40382a09 100644 --- a/app/View/Attributes/ajax/attributeViewFieldForm.ctp +++ b/app/View/Attributes/ajax/attributeViewFieldForm.ctp @@ -1,8 +1,8 @@ '; + echo ''; } else if ($value === 'Yes') { - echo ''; + echo ''; } else { echo nl2br(h($value)) . ' '; } diff --git a/app/View/DecayingModel/decaying_tool.ctp b/app/View/DecayingModel/decaying_tool.ctp index 9ee6531c1..8098f55e7 100644 --- a/app/View/DecayingModel/decaying_tool.ctp +++ b/app/View/DecayingModel/decaying_tool.ctp @@ -5,21 +5,21 @@
- +
- + @@ -41,7 +41,7 @@ } ?> - +
@@ -85,17 +85,17 @@ ' . h($config['greek']).'' : ''); ?> - value= max= oninput="refreshGraph(this);" > - step= value= oninput="$('#input_').val(this.value).trigger('input');"> + value= max= oninput="refreshGraph(this);"> + step= value= oninput="$('#input_').val(this.value).trigger('input');"> - +
- + diff --git a/app/View/DecayingModel/decaying_tool_basescore.ctp b/app/View/DecayingModel/decaying_tool_basescore.ctp index e878967b7..0febdd1f0 100644 --- a/app/View/DecayingModel/decaying_tool_basescore.ctp +++ b/app/View/DecayingModel/decaying_tool_basescore.ctp @@ -1,11 +1,11 @@
- +
- +
@@ -49,8 +49,8 @@ @@ -77,8 +77,8 @@ diff --git a/app/View/Elements/Events/View/eventFilteringQueryBuilder.ctp b/app/View/Elements/Events/View/eventFilteringQueryBuilder.ctp index 1946389c3..ecf07c81d 100644 --- a/app/View/Elements/Events/View/eventFilteringQueryBuilder.ctp +++ b/app/View/Elements/Events/View/eventFilteringQueryBuilder.ctp @@ -1,7 +1,7 @@ ') - .append($('
- - + +
- - + +
- > + >
diff --git a/app/View/Elements/Objects/object_similarities.ctp b/app/View/Elements/Objects/object_similarities.ctp index 7c824c39d..1b089ff43 100644 --- a/app/View/Elements/Objects/object_similarities.ctp +++ b/app/View/Elements/Objects/object_similarities.ctp @@ -86,7 +86,7 @@ if (!isset($simple_flattened_attribute_noval) || !isset($simple_flattened_attrib
- > + > diff --git a/app/View/Elements/serverRuleElements/pull.ctp b/app/View/Elements/serverRuleElements/pull.ctp index a38cd9ba4..bb478418f 100755 --- a/app/View/Elements/serverRuleElements/pull.ctp +++ b/app/View/Elements/serverRuleElements/pull.ctp @@ -15,7 +15,7 @@ >>
- + << @@ -44,7 +44,7 @@ >> - + << diff --git a/app/View/Elements/view_galaxy_matrix.ctp b/app/View/Elements/view_galaxy_matrix.ctp index e7ee13a9f..d3cb7baee 100644 --- a/app/View/Elements/view_galaxy_matrix.ctp +++ b/app/View/Elements/view_galaxy_matrix.ctp @@ -88,7 +88,7 @@ foreach($tabs as $tabName => $column): - + diff --git a/app/View/Events/upload_analysis_file.ctp b/app/View/Events/upload_analysis_file.ctp index d6268f651..eafd597cd 100644 --- a/app/View/Events/upload_analysis_file.ctp +++ b/app/View/Events/upload_analysis_file.ctp @@ -229,7 +229,7 @@ function processString(text) } $("#individualLines").find('tbody') .append($('
').html('')) + .append($('').html('')) .append($('').text(filepath)) .append($('').text(size)) .append($('').text(activity)) diff --git a/app/View/Feeds/add.ctp b/app/View/Feeds/add.ctp index d824386c3..a023fde15 100755 --- a/app/View/Feeds/add.ctp +++ b/app/View/Feeds/add.ctp @@ -67,11 +67,11 @@
-
+
-
+
diff --git a/app/View/Feeds/edit.ctp b/app/View/Feeds/edit.ctp index f81cbf6e2..7767a77e6 100755 --- a/app/View/Feeds/edit.ctp +++ b/app/View/Feeds/edit.ctp @@ -48,11 +48,11 @@
-
+
-
+
diff --git a/app/View/GalaxyClusters/ajax/index.ctp b/app/View/GalaxyClusters/ajax/index.ctp index 8e5c53fb1..c0a378955 100644 --- a/app/View/GalaxyClusters/ajax/index.ctp +++ b/app/View/GalaxyClusters/ajax/index.ctp @@ -38,7 +38,7 @@ - + diff --git a/app/View/Organisations/ajax/sg_org_row_empty.ctp b/app/View/Organisations/ajax/sg_org_row_empty.ctp index 51aab6194..ea915dfe0 100644 --- a/app/View/Organisations/ajax/sg_org_row_empty.ctp +++ b/app/View/Organisations/ajax/sg_org_row_empty.ctp @@ -7,7 +7,7 @@ $checked = ''; if ($extend) $checked = 'checked'; ?> - + diff --git a/app/View/SharingGroups/add.ctp b/app/View/SharingGroups/add.ctp index 54f3871e9..b3ec48e3d 100644 --- a/app/View/SharingGroups/add.ctp +++ b/app/View/SharingGroups/add.ctp @@ -49,13 +49,13 @@ ?>
- + - +
- +
@@ -79,7 +79,7 @@
>>
diff --git a/app/View/Users/statistics_galaxymatrix.ctp b/app/View/Users/statistics_galaxymatrix.ctp index 8612e5c1c..248cc0aa5 100644 --- a/app/View/Users/statistics_galaxymatrix.ctp +++ b/app/View/Users/statistics_galaxymatrix.ctp @@ -24,9 +24,9 @@
- + - +
diff --git a/app/webroot/js/decayingTool.js b/app/webroot/js/decayingTool.js index d61f0ea06..58fa7b0e2 100644 --- a/app/webroot/js/decayingTool.js +++ b/app/webroot/js/decayingTool.js @@ -1017,7 +1017,7 @@ ModelTable.prototype = { } var is_row_selected = $('#saveForm #save-model-button').data('modelid') == model.DecayingModel.id; return cells_html = [ - this._gen_td('', 'DMCheckbox'), + this._gen_td('', 'DMCheckbox'), this._gen_td_link('/decayingModel/view/'+model.DecayingModel.id, this._h(model.DecayingModel.id), 'DMId'), this._gen_td( this._h(model.DecayingModel.name) + (model.DecayingModel.default ? '' : '') , diff --git a/app/webroot/js/misp.js b/app/webroot/js/misp.js index cde9b2da6..1df2c4ffb 100644 --- a/app/webroot/js/misp.js +++ b/app/webroot/js/misp.js @@ -2812,7 +2812,7 @@ function sharingGroupPopulateOrganisations() { if (org.removable == 1) { html += ''; diff --git a/app/webroot/js/network-distribution-graph.js b/app/webroot/js/network-distribution-graph.js index ac43f3d39..3c13c92cc 100644 --- a/app/webroot/js/network-distribution-graph.js +++ b/app/webroot/js/network-distribution-graph.js @@ -258,7 +258,7 @@ var $div = '%s', $tagData ); + $tagConflictData = ''; + if (!empty($tagConflicts)) { + foreach ($tagConflicts as $tagConflict) { + $tagConflictData .= sprintf( + '
%s | %s
', + h(implode(', ', $tagConflict['tags'])), + h($tagConflict['conflict']) + ); + } + } + $tagConflictData .= ''; + echo $tagConflictData; ?> diff --git a/app/View/Events/ajax/ajaxTags.ctp b/app/View/Events/ajax/ajaxTags.ctp index 6001997d0..e56be1370 100644 --- a/app/View/Events/ajax/ajaxTags.ctp +++ b/app/View/Events/ajax/ajaxTags.ctp @@ -6,6 +6,7 @@ echo $this->element('ajaxTags', array( 'event' => $event, 'tags' => $tags, - 'tagAccess' => ($isSiteAdmin || $mayModify) + 'tagAccess' => ($isSiteAdmin || $mayModify), + 'tagConflicts' => $tagConflicts )); ?> diff --git a/app/View/Events/view.ctp b/app/View/Events/view.ctp index d3c4615f7..aebb01d62 100644 --- a/app/View/Events/view.ctp +++ b/app/View/Events/view.ctp @@ -133,7 +133,8 @@ 'event' => $event, 'tags' => $event['EventTag'], 'tagAccess' => ($isSiteAdmin || $mayModify || $me['org_id'] == $event['Event']['orgc_id']), - 'required_taxonomies' => $required_taxonomies + 'required_taxonomies' => $required_taxonomies, + 'tagConflicts' => $tagConflicts ) ) ) diff --git a/app/webroot/css/main.css b/app/webroot/css/main.css index caf510376..c8243c893 100644 --- a/app/webroot/css/main.css +++ b/app/webroot/css/main.css @@ -2394,3 +2394,12 @@ table tr:hover .down-expand-button { #notice_message { margin: 10px; } + +.tag-conflict-notice { + margin: 5px 0px; +} + +.attributeTagContainer .tag-conflict-notice { + max-width: 350px; + overflow: auto; +} \ No newline at end of file From 983a58afbac7fa5f27e3c3bc3fad1cfea7367425 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 4 Nov 2019 12:41:52 +0100 Subject: [PATCH 72/79] chg: [default] old default 'TLP Amber' is now 'tlp:amber' to be consistent and use MISP taxonomy naming --- app/Config/config.default.php | 2 +- app/Model/Event.php | 2 +- app/Model/Post.php | 2 +- app/Model/Server.php | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/Config/config.default.php b/app/Config/config.default.php index d41ceb911..dc55fb86e 100644 --- a/app/Config/config.default.php +++ b/app/Config/config.default.php @@ -17,7 +17,7 @@ $config = array( 'org' => 'ORGNAME', 'showorg' => true, 'threatlevel_in_email_subject' => true, - 'email_subject_TLP_string' => 'TLP Amber', + 'email_subject_TLP_string' => 'tlp:amber', 'email_subject_tag' => 'tlp', 'email_subject_include_tag_name' => true, 'background_jobs' => true, diff --git a/app/Model/Event.php b/app/Model/Event.php index fec7a2b2a..d951b582a 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -3173,7 +3173,7 @@ class Event extends AppModel $bodyevent = $temp[0]; $body = $temp[1]; $result = true; - $tplColorString = !empty(Configure::read('MISP.email_subject_TLP_string')) ? Configure::read('MISP.email_subject_TLP_string') : "TLP Amber"; + $tplColorString = !empty(Configure::read('MISP.email_subject_TLP_string')) ? Configure::read('MISP.email_subject_TLP_string') : "tlp:amber"; $subject = "[" . Configure::read('MISP.org') . " MISP] Need info about event " . $id . " - ".$tplColorString; $result = $this->User->sendEmail($reporter, $bodyevent, $body, $subject, $user) && $result; } diff --git a/app/Model/Post.php b/app/Model/Post.php index eac2a6b14..199450eec 100644 --- a/app/Model/Post.php +++ b/app/Model/Post.php @@ -118,7 +118,7 @@ class Post extends AppModel $bodyDetail .= "The following message was added: \n"; $bodyDetail .= "\n"; $bodyDetail .= $message . "\n"; - $tplColorString = !empty(Configure::read('MISP.email_subject_TLP_string')) ? Configure::read('MISP.email_subject_TLP_string') : "TLP Amber"; + $tplColorString = !empty(Configure::read('MISP.email_subject_TLP_string')) ? Configure::read('MISP.email_subject_TLP_string') : "tlp:amber"; $subject = "[" . Configure::read('MISP.org') . " MISP] New post in discussion " . $post['Post']['thread_id'] . " - ".$tplColorString; foreach ($orgMembers as $recipient) { $this->User->sendEmail($recipient, $bodyDetail, $body, $subject); diff --git a/app/Model/Server.php b/app/Model/Server.php index 8bf27755b..701d4bd75 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -430,7 +430,7 @@ class Server extends AppModel 'email_subject_TLP_string' => array( 'level' => 2, 'description' => __('This is the TLP string for e-mails when email_subject_tag is not found.'), - 'value' => 'TLP Amber', + 'value' => 'tlp:amber', 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', From 1aaa590e3001904c2dc6d8e0e822dcd032357775 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 4 Nov 2019 13:41:53 +0100 Subject: [PATCH 73/79] chg: [event:view] Added notice and improved inconsistency text message --- app/Controller/EventsController.php | 11 +++++++++ app/Model/Taxonomy.php | 2 ++ app/View/Elements/ajaxTags.ctp | 11 +++++---- app/View/Events/view.ctp | 35 +++++++++++++++++++++++++++++ app/webroot/css/main.css | 5 +++++ 5 files changed, 60 insertions(+), 4 deletions(-) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 6bb5cbed2..be5987a9e 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -1270,6 +1270,7 @@ class EventsController extends AppController 'named_params' => $this->params['named'] ); $exception = false; + $warningTagConflict = array(); $filters = $this->_harvestParameters($filterData, $exception); $this->loadModel('GalaxyCluster'); @@ -1372,6 +1373,9 @@ class EventsController extends AppController } $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies(Hash::extract($event['EventTag'], '{n}.Tag.name')); + foreach ($tagConflicts as $tagConflict) { + $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } $this->set('tagConflicts', $tagConflicts); $startDate = null; @@ -1391,6 +1395,9 @@ class EventsController extends AppController } } $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies(Hash::extract($attribute['AttributeTag'], '{n}.Tag.name')); + foreach ($tagConflicts as $tagConflict) { + $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } $event['Attribute'][$k]['tagConflicts'] = $tagConflicts; } $attributeTagsName = $this->Event->Attribute->AttributeTag->extractAttributeTagsNameFromEvent($event, 'both'); @@ -1418,10 +1425,14 @@ class EventsController extends AppController } } $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies(Hash::extract($attribute['AttributeTag'], '{n}.Tag.name')); + foreach ($tagConflicts as $tagConflict) { + $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } $event['Object'][$k]['Attribute'][$k2]['tagConflicts'] = $tagConflicts; } } } + $this->set('warningTagConflict', $warningTagConflict); $filters['sort'] = 'timestamp'; $filters['direction'] = 'desc'; if (isset($filters['distribution'])) { diff --git a/app/Model/Taxonomy.php b/app/Model/Taxonomy.php index f640e040e..34e5a71f9 100644 --- a/app/Model/Taxonomy.php +++ b/app/Model/Taxonomy.php @@ -595,11 +595,13 @@ class Taxonomy extends AppModel if (isset($taxonomy['Taxonomy']['exclusive']) && $taxonomy['Taxonomy']['exclusive']) { $conflictingTaxonomy[] = array( 'tags' => $potTaxonomy['tagNames'], + 'taxonomy' => $taxonomy, 'conflict' => sprintf(__('Taxonomy `%s` is an exclusive Taxonomy'), $taxonomy['Taxonomy']['namespace']) ); } elseif (isset($taxonomy['TaxonomyPredicate'][0]['exclusive']) && $taxonomy['TaxonomyPredicate'][0]['exclusive']) { $conflictingTaxonomy[] = array( 'tags' => $potTaxonomy['tagNames'], + 'taxonomy' => $taxonomy, 'conflict' => sprintf( __('Predicate `%s` is exclusive from Taxonomy `%s` is exclusive'), $taxonomy['TaxonomyPredicate'][0]['value'], diff --git a/app/View/Elements/ajaxTags.ctp b/app/View/Elements/ajaxTags.ctp index b538ebbc0..bf00904fe 100644 --- a/app/View/Elements/ajaxTags.ctp +++ b/app/View/Elements/ajaxTags.ctp @@ -162,16 +162,19 @@ '%s', $tagData ); - $tagConflictData = ''; + $tagConflictData = ''; if (!empty($tagConflicts)) { + $tagConflictData .= '
'; foreach ($tagConflicts as $tagConflict) { $tagConflictData .= sprintf( - '
%s | %s
', - h(implode(', ', $tagConflict['tags'])), + '%s
', h($tagConflict['conflict']) ); + foreach ($tagConflict['tags'] as $tag) { + $tagConflictData .= sprintf('%s
', h($tag)); + } } + $tagConflictData .= '
'; } - $tagConflictData .= ''; echo $tagConflictData; ?> diff --git a/app/View/Events/view.ctp b/app/View/Events/view.ctp index aebb01d62..eb70b1b9f 100644 --- a/app/View/Events/view.ctp +++ b/app/View/Events/view.ctp @@ -331,6 +331,41 @@ element('genericElements/viewMetaTable', array('table_data' => $table_data)); ?>
+ + +
+

+ '; ?> + %s', $baseurl, h($taxonomy['Taxonomy']['id']), h($taxonomy['Taxonomy']['namespace']), h($taxonomy['Taxonomy']['description'])); + echo '
    '; + if ($taxonomy['Taxonomy']['exclusive']) { + echo sprintf( + '
  • %s
    • ', + sprintf( + ('%s is an exclusive taxonomy. Only one Tag of this taxonomy is allowed on an element.'), + sprintf('%s', h($taxonomy['Taxonomy']['namespace'])) + ) + ); + } else { + foreach ($taxonomy['TaxonomyPredicate'] as $predicate) { + echo sprintf( + '
  • %s
    • ', + sprintf( + ('%s is an exclusive taxonomy predicate. Only one Tag of this predicate is allowed on an element'), + sprintf('%s', h($predicate['value'])) + ) + ); + } + } + echo '
'; + } + ?> + ' ?> +
+ + diff --git a/app/webroot/css/main.css b/app/webroot/css/main.css index c8243c893..524c20716 100644 --- a/app/webroot/css/main.css +++ b/app/webroot/css/main.css @@ -2402,4 +2402,9 @@ table tr:hover .down-expand-button { .attributeTagContainer .tag-conflict-notice { max-width: 350px; overflow: auto; +} + +.tag-conflict-notice .apply_css_arrow { + display: inline-block; + padding: 0px 6px; } \ No newline at end of file From 0222b8c0904f20a87e6cdd12eb82a1a781e59764 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 4 Nov 2019 14:55:43 +0100 Subject: [PATCH 74/79] chg: [tag:exclusive] Added support of local while checking for exclusivity --- app/Controller/EventsController.php | 45 +++++++++++++++++++++-------- app/Model/Taxonomy.php | 21 +++++++++++++- app/View/Elements/ajaxTags.ctp | 23 +++++++++++++-- app/webroot/css/main.css | 16 ++++++++++ 4 files changed, 89 insertions(+), 16 deletions(-) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index be5987a9e..a2b127dac 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -1160,7 +1160,13 @@ class EventsController extends AppController unset($event['Object'][$k]['Attribute'][$k2]['AttributeTag'][$k3]); } } - $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies(Hash::extract($attribute['AttributeTag'], '{n}.Tag.name')); + $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); + foreach ($tagConflicts['global'] as $tagConflict) { + $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } + foreach ($tagConflicts['local'] as $tagConflict) { + $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } $event['Object'][$k]['Attribute'][$k2]['tagConflicts'] = $tagConflicts; } } @@ -1171,7 +1177,13 @@ class EventsController extends AppController unset($event['Attribute'][$k]['AttributeTag'][$k2]); } } - $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies(Hash::extract($attribute['AttributeTag'], '{n}.Tag.name')); + $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); + foreach ($tagConflicts['global'] as $tagConflict) { + $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } + foreach ($tagConflicts['local'] as $tagConflict) { + $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } $event['Attribute'][$k]['tagConflicts'] = $tagConflicts; } if (empty($this->passedArgs['sort'])) { @@ -1372,8 +1384,11 @@ class EventsController extends AppController } } - $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies(Hash::extract($event['EventTag'], '{n}.Tag.name')); - foreach ($tagConflicts as $tagConflict) { + $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($event['EventTag']); + foreach ($tagConflicts['global'] as $tagConflict) { + $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } + foreach ($tagConflicts['local'] as $tagConflict) { $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; } $this->set('tagConflicts', $tagConflicts); @@ -1394,8 +1409,11 @@ class EventsController extends AppController unset($event['Attribute'][$k]['AttributeTag'][$k2]); } } - $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies(Hash::extract($attribute['AttributeTag'], '{n}.Tag.name')); - foreach ($tagConflicts as $tagConflict) { + $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); + foreach ($tagConflicts['global'] as $tagConflict) { + $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } + foreach ($tagConflicts['local'] as $tagConflict) { $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; } $event['Attribute'][$k]['tagConflicts'] = $tagConflicts; @@ -1424,8 +1442,11 @@ class EventsController extends AppController unset($event['Object'][$k]['Attribute'][$k2]['AttributeTag'][$k3]); } } - $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies(Hash::extract($attribute['AttributeTag'], '{n}.Tag.name')); - foreach ($tagConflicts as $tagConflict) { + $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); + foreach ($tagConflicts['global'] as $tagConflict) { + $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } + foreach ($tagConflicts['local'] as $tagConflict) { $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; } $event['Object'][$k]['Attribute'][$k2]['tagConflicts'] = $tagConflicts; @@ -3857,15 +3878,15 @@ class EventsController extends AppController } $tagsOnEvent = $this->Event->EventTag->find('all', array( 'conditions' => array( - 'Tag.id' => $tag_id, - 'EventTag.event_id' => $id + 'EventTag.event_id' => $id, + 'EventTag.local' => $local ), 'contain' => 'Tag', 'fields' => array('Tag.name'), 'recursive' => -1 )); - $exclusive_test_passed = $this->Taxonomy->checkIfNewTagIsAllowedByTaxonomy($tag['Tag']['name'], $tagsOnEvent); - if (!empty($exclusive_test_passed)) { + $exclusive_test_passed = $this->Taxonomy->checkIfNewTagIsAllowedByTaxonomy($tag['Tag']['name'], Hash::extract($tagsOnEvent, '{n}.Tag.name')); + if (!$exclusive_test_passed) { $fail = __('Tag is not allowed due to taxonomy exclusivity settings'); continue; } diff --git a/app/Model/Taxonomy.php b/app/Model/Taxonomy.php index 34e5a71f9..741324868 100644 --- a/app/Model/Taxonomy.php +++ b/app/Model/Taxonomy.php @@ -573,7 +573,26 @@ class Taxonomy extends AppModel return true; } - public function checkIfTagInconsistencies($tagNameList) + public function checkIfTagInconsistencies($tagList) + { + $eventTags = array(); + $localEventTags = array(); + foreach($tagList as $tag) { + if ($tag['local'] == 0) { + $eventTags[] = $tag['Tag']['name']; + } else { + $localEventTags[] = $tag['Tag']['name']; + } + } + $tagConflicts = $this->getTagConflicts($eventTags); + $localTagConflicts = $this->getTagConflicts($localEventTags); + return array( + 'global' => $tagConflicts, + 'local' => $localTagConflicts + ); + } + + public function getTagConflicts($tagNameList) { $potentiallyConflictingTaxonomy = array(); $conflictingTaxonomy = array(); diff --git a/app/View/Elements/ajaxTags.ctp b/app/View/Elements/ajaxTags.ctp index bf00904fe..6a0c15255 100644 --- a/app/View/Elements/ajaxTags.ctp +++ b/app/View/Elements/ajaxTags.ctp @@ -163,9 +163,11 @@ $tagData ); $tagConflictData = ''; - if (!empty($tagConflicts)) { + if (!empty($tagConflicts['global'])) { $tagConflictData .= '
'; - foreach ($tagConflicts as $tagConflict) { + $tagConflictData .= ''; + $tagConflictData .= '
'; + foreach ($tagConflicts['global'] as $tagConflict) { $tagConflictData .= sprintf( '%s
', h($tagConflict['conflict']) @@ -174,7 +176,22 @@ $tagConflictData .= sprintf('%s
', h($tag)); } } - $tagConflictData .= '
'; + $tagConflictData .= '
'; + } + if (!empty($tagConflicts['local'])) { + $tagConflictData .= '
'; + $tagConflictData .= ''; + $tagConflictData .= '
'; + foreach ($tagConflicts['local'] as $tagConflict) { + $tagConflictData .= sprintf( + '%s
', + h($tagConflict['conflict']) + ); + foreach ($tagConflict['tags'] as $tag) { + $tagConflictData .= sprintf('%s
', h($tag)); + } + } + $tagConflictData .= '
'; } echo $tagConflictData; ?> diff --git a/app/webroot/css/main.css b/app/webroot/css/main.css index 524c20716..550d79ad6 100644 --- a/app/webroot/css/main.css +++ b/app/webroot/css/main.css @@ -2397,6 +2397,12 @@ table tr:hover .down-expand-button { .tag-conflict-notice { margin: 5px 0px; + padding-right: 14px; + position: relative; +} + +.tag-conflict-notice div.text-container { + overflow: auto; } .attributeTagContainer .tag-conflict-notice { @@ -2407,4 +2413,14 @@ table tr:hover .down-expand-button { .tag-conflict-notice .apply_css_arrow { display: inline-block; padding: 0px 6px; +} + +.tag-conflict-notice i.icon { + position: absolute; + padding: 3px; + right: 0px; + top: 0px; + background-color: #3b3b3b; + color: white; + border-radius: 3px; } \ No newline at end of file From c291ebfdee041830488e0a0fd014d40e05262bee Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 4 Nov 2019 15:12:55 +0100 Subject: [PATCH 75/79] chg: [attribute:addTag] Added support of `exclusive` at Attribute level --- app/Controller/AttributesController.php | 15 +++++++++++++++ app/Controller/TagsController.php | 5 ++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 114908215..b777155db 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -2959,6 +2959,7 @@ class AttributesController extends AppController public function addTag($id = false, $tag_id = false) { + $this->Taxonomy = $log = ClassRegistry::init('Taxonomy'); $rearrangeRules = array( 'request' => false, 'Attribute' => false, @@ -3099,6 +3100,20 @@ class AttributesController extends AppController $fails++; continue; } + $tagsOnAttribute = $this->Attribute->AttributeTag->find('all', array( + 'conditions' => array( + 'AttributeTag.attribute_id' => $id, + 'AttributeTag.local' => $local + ), + 'contain' => 'Tag', + 'fields' => array('Tag.name'), + 'recursive' => -1 + )); + $exclusive_test_passed = $this->Taxonomy->checkIfNewTagIsAllowedByTaxonomy($tag['Tag']['name'], Hash::extract($tagsOnAttribute, '{n}.Tag.name')); + if (!$exclusive_test_passed) { + $fails++; + continue; + } $this->Attribute->AttributeTag->create(); if ($this->Attribute->AttributeTag->save(array('attribute_id' => $id, 'tag_id' => $tag_id, 'event_id' => $eventId, 'local' => $local))) { if (!$local) { diff --git a/app/Controller/TagsController.php b/app/Controller/TagsController.php index 3d0675e63..c5c1e9778 100644 --- a/app/Controller/TagsController.php +++ b/app/Controller/TagsController.php @@ -488,7 +488,7 @@ class TagsController extends AppController 'conditions' => array('Event.id' => $id) )); $this->set('required_taxonomies', $this->EventTag->Event->getRequiredTaxonomies()); - $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies(Hash::extract($tags, '{n}.Tag.name')); + $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($tags); $this->set('tagConflicts', $tagConflicts); $this->set('event', $event); $this->layout = 'ajax'; @@ -499,6 +499,7 @@ class TagsController extends AppController { $this->helpers[] = 'TextColour'; $this->loadModel('AttributeTag'); + $this->loadModel('Taxonomy'); $this->Tag->AttributeTag->Attribute->id = $id; if (!$this->Tag->AttributeTag->Attribute->exists()) { @@ -531,6 +532,8 @@ class TagsController extends AppController $this->set('event', $event); $this->set('attributeTags', $attributeTags); $this->set('attributeId', $id); + $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attributeTags); + $this->set('tagConflicts', $tagConflicts); $this->layout = 'ajax'; $this->render('/Attributes/ajax/ajaxAttributeTags'); } From 21fac1d165aa234eb1ca73864edf342fef318bdb Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 4 Nov 2019 15:19:15 +0100 Subject: [PATCH 76/79] chg: [event:view] Exclusive notice UI improvements --- app/Model/Taxonomy.php | 5 ++--- app/View/Elements/Events/View/row_attribute.ctp | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/app/Model/Taxonomy.php b/app/Model/Taxonomy.php index 741324868..add3d363f 100644 --- a/app/Model/Taxonomy.php +++ b/app/Model/Taxonomy.php @@ -622,9 +622,8 @@ class Taxonomy extends AppModel 'tags' => $potTaxonomy['tagNames'], 'taxonomy' => $taxonomy, 'conflict' => sprintf( - __('Predicate `%s` is exclusive from Taxonomy `%s` is exclusive'), - $taxonomy['TaxonomyPredicate'][0]['value'], - $taxonomy['Taxonomy']['namespace'] + __('Predicate `%s` is exclusive'), + $taxonomy['TaxonomyPredicate'][0]['value'] ) ); } diff --git a/app/View/Elements/Events/View/row_attribute.ctp b/app/View/Elements/Events/View/row_attribute.ctp index 26f64b478..2fd3227bd 100644 --- a/app/View/Elements/Events/View/row_attribute.ctp +++ b/app/View/Elements/Events/View/row_attribute.ctp @@ -156,7 +156,7 @@
Date: Mon, 4 Nov 2019 15:46:53 +0100 Subject: [PATCH 77/79] fix: [Tag:exclusive] Code clean up --- app/Controller/AttributesController.php | 4 ++-- app/Controller/EventsController.php | 28 ++++++++++++------------- app/Controller/TaxonomiesController.php | 19 ----------------- app/Model/Event.php | 7 ------- app/Model/Taxonomy.php | 4 ++-- app/View/Events/view.ctp | 4 ++-- 6 files changed, 20 insertions(+), 46 deletions(-) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index b777155db..91ecf4da3 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -3109,8 +3109,8 @@ class AttributesController extends AppController 'fields' => array('Tag.name'), 'recursive' => -1 )); - $exclusive_test_passed = $this->Taxonomy->checkIfNewTagIsAllowedByTaxonomy($tag['Tag']['name'], Hash::extract($tagsOnAttribute, '{n}.Tag.name')); - if (!$exclusive_test_passed) { + $exclusiveTestPassed = $this->Taxonomy->checkIfNewTagIsAllowedByTaxonomy($tag['Tag']['name'], Hash::extract($tagsOnAttribute, '{n}.Tag.name')); + if (!$exclusiveTestPassed) { $fails++; continue; } diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index a2b127dac..3aa3971c2 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -1162,10 +1162,10 @@ class EventsController extends AppController } $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); foreach ($tagConflicts['global'] as $tagConflict) { - $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; } foreach ($tagConflicts['local'] as $tagConflict) { - $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; } $event['Object'][$k]['Attribute'][$k2]['tagConflicts'] = $tagConflicts; } @@ -1179,10 +1179,10 @@ class EventsController extends AppController } $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); foreach ($tagConflicts['global'] as $tagConflict) { - $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; } foreach ($tagConflicts['local'] as $tagConflict) { - $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; } $event['Attribute'][$k]['tagConflicts'] = $tagConflicts; } @@ -1282,7 +1282,7 @@ class EventsController extends AppController 'named_params' => $this->params['named'] ); $exception = false; - $warningTagConflict = array(); + $warningTagConflicts = array(); $filters = $this->_harvestParameters($filterData, $exception); $this->loadModel('GalaxyCluster'); @@ -1386,10 +1386,10 @@ class EventsController extends AppController $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($event['EventTag']); foreach ($tagConflicts['global'] as $tagConflict) { - $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; } foreach ($tagConflicts['local'] as $tagConflict) { - $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; } $this->set('tagConflicts', $tagConflicts); @@ -1411,10 +1411,10 @@ class EventsController extends AppController } $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); foreach ($tagConflicts['global'] as $tagConflict) { - $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; } foreach ($tagConflicts['local'] as $tagConflict) { - $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; } $event['Attribute'][$k]['tagConflicts'] = $tagConflicts; } @@ -1444,16 +1444,16 @@ class EventsController extends AppController } $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); foreach ($tagConflicts['global'] as $tagConflict) { - $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; } foreach ($tagConflicts['local'] as $tagConflict) { - $warningTagConflict[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; } $event['Object'][$k]['Attribute'][$k2]['tagConflicts'] = $tagConflicts; } } } - $this->set('warningTagConflict', $warningTagConflict); + $this->set('warningTagConflicts', $warningTagConflicts); $filters['sort'] = 'timestamp'; $filters['direction'] = 'desc'; if (isset($filters['distribution'])) { @@ -3885,8 +3885,8 @@ class EventsController extends AppController 'fields' => array('Tag.name'), 'recursive' => -1 )); - $exclusive_test_passed = $this->Taxonomy->checkIfNewTagIsAllowedByTaxonomy($tag['Tag']['name'], Hash::extract($tagsOnEvent, '{n}.Tag.name')); - if (!$exclusive_test_passed) { + $exclusiveTestPassed = $this->Taxonomy->checkIfNewTagIsAllowedByTaxonomy($tag['Tag']['name'], Hash::extract($tagsOnEvent, '{n}.Tag.name')); + if (!$exclusiveTestPassed) { $fail = __('Tag is not allowed due to taxonomy exclusivity settings'); continue; } diff --git a/app/Controller/TaxonomiesController.php b/app/Controller/TaxonomiesController.php index fdf18517b..21ae042ab 100644 --- a/app/Controller/TaxonomiesController.php +++ b/app/Controller/TaxonomiesController.php @@ -239,25 +239,6 @@ class TaxonomiesController extends AppController } } - public function testExclusive() - { - $res = $this->Taxonomy->checkIfNewTagIsAllowedByTaxonomy('tlp:green', array('tlp:white1')); - $res2 = $this->Taxonomy->checkIfNewTagIsAllowedByTaxonomy('tlp:green', array('tlp:white')); - $res3 = $this->Taxonomy->checkIfNewTagIsAllowedByTaxonomy('estimative-language:confidence-in-analytic-judgment="high"', array('estimative-language:confidence-in-analytic-judgment="low"')); - return $this->RestResponse->viewData([$res, $res2, $res3], 'html'); - } - public function testInconsistencies() - { - $res = $this->Taxonomy->checkIfTagInconsistencies(array('tlp:green', 'tlp:white1')); - $res2 = $this->Taxonomy->checkIfTagInconsistencies(array('estimative-language:confidence-in-analytic-judgment="low"', 'estimative-language:confidence-in-analytic-judgment="high"')); - $res3 = $this->Taxonomy->checkIfTagInconsistencies(array( - 'fr-classif:classifiees-defense="CONFIDENTIEL_DEFENSE"', - 'fr-classif:classifiees-defense="TRES_SECRET_DEFENSE"', - 'fr-classif:non-classifiees-defense="CONFIDENTIEL"' - )); - return $this->RestResponse->viewData([$res, $res2, $res3], 'html'); - } - public function addTag($taxonomy_id = false) { if ((!$this->_isSiteAdmin() && !$this->userRole['perm_tagger']) || !$this->request->is('post')) { diff --git a/app/Model/Event.php b/app/Model/Event.php index 72d0f2b0c..fec7a2b2a 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -6944,11 +6944,4 @@ class Event extends AppModel } return true; } - - public function checkIfTagAllowedByTaxonomy($tagName) - { - $this->Taxonomy = ClassRegistry::init('Taxonomy'); - // $taxonomy - $taxonomy = $this->Taxonomy->getTaxonomyForTag($tag['Tag']['name']); - } } diff --git a/app/Model/Taxonomy.php b/app/Model/Taxonomy.php index add3d363f..c987b176a 100644 --- a/app/Model/Taxonomy.php +++ b/app/Model/Taxonomy.php @@ -564,9 +564,9 @@ class Taxonomy extends AppModel if (!$prefixIsFree) { // at this point, we have a duplicated namespace(-predicate) $taxonomy = $this->getTaxonomyForTag($newTagName); - if (isset($taxonomy['Taxonomy']['exclusive']) && $taxonomy['Taxonomy']['exclusive']) { + if (!empty($taxonomy['Taxonomy']['exclusive'])) { return false; // only one tag of this taxonomy is allowed - } elseif (isset($taxonomy['TaxonomyPredicate'][0]['exclusive']) && $taxonomy['TaxonomyPredicate'][0]['exclusive']) { + } elseif (!empty($taxonomy['TaxonomyPredicate'][0]['exclusive'])) { return false; // only one tag belonging to this predicate is allowed } } diff --git a/app/View/Events/view.ctp b/app/View/Events/view.ctp index eb70b1b9f..fbd8ecb9d 100644 --- a/app/View/Events/view.ctp +++ b/app/View/Events/view.ctp @@ -332,12 +332,12 @@
- +

'; ?> %s', $baseurl, h($taxonomy['Taxonomy']['id']), h($taxonomy['Taxonomy']['namespace']), h($taxonomy['Taxonomy']['description'])); echo '
    '; if ($taxonomy['Taxonomy']['exclusive']) { From 9e2e64f2c1f228b466f6483b256e677bbf90fea5 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 5 Nov 2019 10:50:45 +0100 Subject: [PATCH 78/79] chg: [PyMISP] bump PyMISP to the latest version --- PyMISP | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyMISP b/PyMISP index 21c16c8c7..87fd06a88 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit 21c16c8c75c61a00a0f01d1848687f7fe6455325 +Subproject commit 87fd06a8893feafaffd461d6d611be4d02e5a4a2 From c92d0bdf2353206d39233a313682589f891de9c3 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 5 Nov 2019 10:55:37 +0100 Subject: [PATCH 79/79] chg: [PyMISP] updated to the latest version --- PyMISP | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyMISP b/PyMISP index 87fd06a88..204fd6ba8 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit 87fd06a8893feafaffd461d6d611be4d02e5a4a2 +Subproject commit 204fd6ba8cc916844156c1819c8375f6bbbca995
- element('ajaxTags', array('attributeId' => $object['id'], 'tags' => $object['AttributeTag'], 'tagAccess' => ($isSiteAdmin || $mayModify || $me['org_id'] == $event['Event']['org_id']), 'context' => $context, 'scope' => 'attribute')); ?> + element('ajaxTags', array('attributeId' => $object['id'], 'tags' => $object['AttributeTag'], 'tagAccess' => ($isSiteAdmin || $mayModify || $me['org_id'] == $event['Event']['org_id']), 'context' => $context, 'scope' => 'attribute', 'tagConflicts' => $object['tagConflicts'])); ?>
- element('ajaxTags', array('attributeId' => $object['id'], 'tags' => $object['AttributeTag'], 'tagAccess' => ($isSiteAdmin || $mayModify || $me['org_id'] == $event['Event']['org_id']), 'context' => $context, 'scope' => 'attribute', 'tagConflicts' => $object['tagConflicts'])); ?> + element('ajaxTags', array('attributeId' => $object['id'], 'tags' => $object['AttributeTag'], 'tagAccess' => ($isSiteAdmin || $mayModify || $me['org_id'] == $event['Event']['org_id']), 'context' => $context, 'scope' => 'attribute', 'tagConflicts' => isset($object['tagConflicts']) ? $object['tagConflicts'] : array())); ?>