From 8afe4b5ed1532b181d71892c19552cf7aa2db95f Mon Sep 17 00:00:00 2001 From: "A. Cristallo" Date: Tue, 16 Oct 2018 17:37:00 +0200 Subject: [PATCH 01/15] Update INSTALL.rhel7.txt Added instruction (at line 109) and updated line 8, minor change. Tested on RHEL 7.5 and CentOS 7.5 --- INSTALL/INSTALL.rhel7.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/INSTALL/INSTALL.rhel7.txt b/INSTALL/INSTALL.rhel7.txt index 3eff62555..082346c6b 100644 --- a/INSTALL/INSTALL.rhel7.txt +++ b/INSTALL/INSTALL.rhel7.txt @@ -5,7 +5,7 @@ INSTALLATION INSTRUCTIONS for RHEL 7.x | 0/ Overview and Assumptions | +----------------------------------------+ This document details the steps to install MISP on Red Hat Enterprise Linux 7.x (RHEL 7.x). At time of this writing it -was tested on version 7.4. +was tested on version 7.5. The following assumptions with regard to this installation have been made. @@ -106,6 +106,7 @@ yum install rh-python36 3.01/ Download MISP code using git in /var/www/ directory cd /var/www git clone https://github.com/MISP/MISP.git +cd MISP git checkout tags/$(git describe --tags `git rev-list --tags --max-count=1`) # if the last shortcut doesn't work, specify the latest version manually # example: git checkout tags/v2.4.XY From c96be93e3cd6d351790c725dbe030292c921dcab Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 18 Oct 2018 09:51:14 +0200 Subject: [PATCH 02/15] fix: [internal] Fix of wonky model function calls across the application for getting default attachment directories --- app/Controller/EventsController.php | 3 +-- app/Controller/ShadowAttributesController.php | 6 ++---- app/Model/AppModel.php | 10 ++++++++++ app/Model/Attribute.php | 15 +++++---------- app/Model/Server.php | 12 +----------- 5 files changed, 19 insertions(+), 27 deletions(-) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 2ac68ec8e..df2b884a9 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -2770,8 +2770,7 @@ class EventsController extends AppController // write $attachments_dir = Configure::read('MISP.attachments_dir'); if (empty($attachments_dir)) { - $this->loadModel('Server'); - $attachments_dir = $this->Server->getDefaultAttachments_dir(); + $attachments_dir = $this->Event->getDefaultAttachments_dir(); } $rootDir = $attachments_dir . DS . $id . DS; App::uses('Folder', 'Utility'); diff --git a/app/Controller/ShadowAttributesController.php b/app/Controller/ShadowAttributesController.php index 773087d8f..176424599 100644 --- a/app/Controller/ShadowAttributesController.php +++ b/app/Controller/ShadowAttributesController.php @@ -201,8 +201,7 @@ class ShadowAttributesController extends AppController { $attachments_dir = Configure::read('MISP.attachments_dir'); if (empty($attachments_dir)) { - $my_server = ClassRegistry::init('Server'); - $attachments_dir = $my_server->getDefaultAttachments_dir(); + $attachments_dir = $this->ShadowAttribute->getDefaultAttachments_dir(); } $pathOld = $attachments_dir . DS . 'shadow' . DS . $shadowId; $pathNew = $attachments_dir . DS . $newId; @@ -506,8 +505,7 @@ class ShadowAttributesController extends AppController { $attachments_dir = Configure::read('MISP.attachments_dir'); if (empty($attachments_dir)) { - $my_server = ClassRegistry::init('Server'); - $attachments_dir = $my_server->getDefaultAttachments_dir(); + $attachments_dir = $this->ShadowAttribute->getDefaultAttachments_dir(); } $path = $attachments_dir . DS . 'shadow' . DS . $shadowAttribute['event_id'] . DS; $file = $shadowAttribute['id']; diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 009245a5f..198e12e5d 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -1793,4 +1793,14 @@ class AppModel extends Model } return $val / (1024 * 1024); } + + public function getDefaultAttachments_dir() + { + return APP . 'files'; + } + + public function getDefaultTmp_dir() + { + return sys_get_temp_dir(); + } } diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index b37444a48..50185771a 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -675,8 +675,7 @@ class Attribute extends AppModel // only delete the file if it exists $attachments_dir = Configure::read('MISP.attachments_dir'); if (empty($attachments_dir)) { - $my_server = ClassRegistry::init('Server'); - $attachments_dir = $my_server->getDefaultAttachments_dir(); + $attachments_dir = $this->getDefaultAttachments_dir(); } // Special case - If using S3, we have to delete from there @@ -1531,8 +1530,7 @@ class Attribute extends AppModel { $attachments_dir = Configure::read('MISP.attachments_dir'); if (empty($attachments_dir)) { - $my_server = ClassRegistry::init('Server'); - $attachments_dir = $my_server->getDefaultAttachments_dir(); + $attachments_dir = $this->getDefaultAttachments_dir(); } if ($this->attachmentDirIsS3()) { @@ -1557,8 +1555,7 @@ class Attribute extends AppModel { $attachments_dir = Configure::read('MISP.attachments_dir'); if (empty($attachments_dir)) { - $my_server = ClassRegistry::init('Server'); - $attachments_dir = $my_server->getDefaultAttachments_dir(); + $attachments_dir = $this->getDefaultAttachments_dir(); } if ($this->attachmentDirIsS3()) { @@ -2993,8 +2990,7 @@ class Attribute extends AppModel } $attachments_dir = Configure::read('MISP.attachments_dir'); if (empty($attachments_dir)) { - $my_server = ClassRegistry::init('Server'); - $attachments_dir = $my_server->getDefaultAttachments_dir(); + $attachments_dir = $this->getDefaultAttachments_dir(); } // If we've set attachments to S3, we can't write there @@ -3003,8 +2999,7 @@ class Attribute extends AppModel // Sometimes it's not set? if (empty($attachments_dir)) { // Get a default tmpdir - $my_server = ClassRegistry::init('Server'); - $attachments_dir = $my_server->getDefaultTmp_dir(); + $attachments_dir = $this->getDefaultTmp_dir(); } } diff --git a/app/Model/Server.php b/app/Model/Server.php index 2c822d0cb..ae96380bf 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -2051,7 +2051,7 @@ class Server extends AppModel 'action' => 'pull', 'user_id' => $user['id'], 'title' => 'Failed to pull event #' . $eventid . '.', - 'change' => 'Reason:' . $message + 'change' => 'Reason:' . $message )); } } @@ -4142,16 +4142,6 @@ class Server extends AppModel return $final; } - public function getDefaultAttachments_dir() - { - return APP . 'files'; - } - - public function getDefaultTmp_dir() - { - return sys_get_temp_dir(); - } - public function fetchServer($id) { if (empty($id)) { From a40aa1748e7fc971998d84fe44695a3af5ccc564 Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 18 Oct 2018 15:01:04 +0200 Subject: [PATCH 03/15] fix: [internal] getPythonVersion woes --- app/Model/Event.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index f11ff7d60..d4b225a24 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -4993,12 +4993,12 @@ class Event extends AppModel if ($stix_version == '2') { $scriptFile = APP . 'files/scripts/stix2/stix2misp.py'; $tempFilePath = APP . 'files/scripts/tmp/' . $filename; - $shell_command = $this->Server->getPythonVersion() . ' ' . $scriptFile . ' ' . $tempFilePath; + $shell_command = $this->getPythonVersion() . ' ' . $scriptFile . ' ' . $tempFilePath; $output_path = $tempFilePath . '.stix2'; } elseif ($stix_version == '1' || $stix_version == '1.1' || $stix_version == '1.2') { $scriptFile = APP . 'files/scripts/stix2misp.py'; $tempFilePath = APP . 'files/scripts/tmp/' . $filename; - $shell_command = $this->Server->getPythonVersion() . ' ' . $scriptFile . ' ' . $filename; + $shell_command = $this->getPythonVersion() . ' ' . $scriptFile . ' ' . $filename; $output_path = $tempFilePath . '.json'; } else { throw new MethodNotAllowedException('Invalid STIX version'); From f20b4f70cdcd2d30d9f676c208135101f4a45163 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 18 Oct 2018 15:12:21 +0200 Subject: [PATCH 04/15] fix: [stix2 import] Fixed json dict monkey syntax error --- app/files/scripts/stix2/stix2misp_mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/scripts/stix2/stix2misp_mapping.py b/app/files/scripts/stix2/stix2misp_mapping.py index 3a26654c1..624026f3d 100644 --- a/app/files/scripts/stix2/stix2misp_mapping.py +++ b/app/files/scripts/stix2/stix2misp_mapping.py @@ -299,7 +299,7 @@ domain_pattern_mapping = {'value': {'type': 'domain'}} ip_pattern_mapping = {'value': {'type': 'ip-dst'}} external_pattern_mapping = {'domain-name': domain_pattern_mapping, - 'email-addr': {'value': {'type': 'email-dst'}} + 'email-addr': {'value': {'type': 'email-dst'}}, 'file': file_mapping, 'ipv4-addr': ip_pattern_mapping, 'ipv6-addr': ip_pattern_mapping, From 91807f5052835f5a03180db504657d2373803787 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 18 Oct 2018 15:13:19 +0200 Subject: [PATCH 05/15] fix: [stix2 import] Fixed MISP event info field when importing STIX2 without report object --- app/files/scripts/stix2/stix2misp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/files/scripts/stix2/stix2misp.py b/app/files/scripts/stix2/stix2misp.py index 38a46749b..06a6b3b7b 100644 --- a/app/files/scripts/stix2/stix2misp.py +++ b/app/files/scripts/stix2/stix2misp.py @@ -108,6 +108,7 @@ class StixParser(): if object_type not in ('relationship', 'report'): for _, _object in objects.items(): self.parsing_process(_object, object_type) + self.misp_event.info = "Imported with MISP import script for {}.".format(self.stix_version) def set_distribution(self): for attribute in self.misp_event.attributes: From 6f9551e202aeca5245d6e5bd3576c49c97ba8e3e Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 18 Oct 2018 20:10:11 +0200 Subject: [PATCH 06/15] fix: [internal] Unneeded model initialisation for getDefaultAttachments_dir() --- app/Controller/AttributesController.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 5ca19003c..afa1ba9fc 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -430,8 +430,7 @@ class AttributesController extends AppController { $attachments_dir = Configure::read('MISP.attachments_dir'); if (empty($attachments_dir)) { - $this->loadModel('Server'); - $attachments_dir = $this->Server->getDefaultAttachments_dir(); + $attachments_dir = $this->Attribute->getDefaultAttachments_dir(); } $is_s3 = substr($attachments_dir, 0, 2) === "s3"; From e419c80e9e0625beade1d1ab3b9cb1325c9249da Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 18 Oct 2018 20:12:24 +0200 Subject: [PATCH 07/15] fix: [internal] Sharing group capturing fixed, fixes #3573 - As reported by @eCrimeLabs --- app/Model/Event.php | 14 +++++++++++++- app/Model/ObjectReference.php | 16 ++++++++-------- app/Model/SharingGroup.php | 3 ++- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index d4b225a24..5b655641f 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -902,6 +902,8 @@ class Event extends AppModel } $updated = null; $newLocation = $newTextBody = ''; + debug($event); + throw new Exception(); $result = $this->__executeRestfulEventToServer($event, $server, null, $newLocation, $newTextBody, $HttpSocket); if ($result !== true) { return $result; @@ -3202,6 +3204,7 @@ class Event extends AppModel } $data['Event']['Attribute'] = array_values($data['Event']['Attribute']); } + $referencesToCapture = array(); if (!empty($data['Event']['Object'])) { foreach ($data['Event']['Object'] as $object) { $result = $this->Object->captureObject($object, $this->id, $user, $this->Log); @@ -3209,11 +3212,20 @@ class Event extends AppModel foreach ($data['Event']['Object'] as $object) { if (isset($object['ObjectReference'])) { foreach ($object['ObjectReference'] as $objectRef) { - $result = $this->Object->ObjectReference->captureReference($objectRef, $this->id, $user, $this->Log); + $objectRef['source_uuid'] = $object['uuid']; + $referencesToCapture[] = $objectRef; } } } } + foreach ($referencesToCapture as $referenceToCapture) { + $result = $this->Object->ObjectReference->captureReference( + $referenceToCapture, + $this->id, + $user, + $this->Log + ); + } // zeroq: check if sightings are attached and add to event if (isset($data['Sighting']) && !empty($data['Sighting'])) { $this->Sighting = ClassRegistry::init('Sighting'); diff --git a/app/Model/ObjectReference.php b/app/Model/ObjectReference.php index 43a701b6f..13a616a52 100644 --- a/app/Model/ObjectReference.php +++ b/app/Model/ObjectReference.php @@ -191,8 +191,8 @@ class ObjectReference extends AppModel } } } - if (isset($reference['object_uuid'])) { - $conditions = array('Object.uuid' => $reference['object_uuid']); + if (isset($reference['source_uuid'])) { + $conditions = array('Object.uuid' => $reference['source_uuid']); } elseif (isset($reference['object_id'])) { $conditions = array('Object.id' => $reference['object_id']); } else { @@ -229,26 +229,26 @@ class ObjectReference extends AppModel if (empty($referencedObject)) { return true; } - $referenced_type = 0; + $referenced_type = 'Attribute'; } else { - $referenced_type = 1; + $referenced_type = 'Object'; } $objectTypes = array('Attribute', 'Object'); if (!isset($sourceObject['Object']) || $sourceObject['Object']['event_id'] != $eventId) { return true; } - if ($referencedObject[$objectTypes[$referenced_type]]['event_id'] != $eventId) { + if ($referencedObject[$referenced_type]['event_id'] != $eventId) { return true; } $this->create(); unset($reference['id']); $reference['referenced_type'] = $referenced_type; $reference['object_id'] = $sourceObject['Object']['id']; - $reference['referenced_id'] = $referencedObject[$objectTypes[$referenced_type]]['id']; - $reference['referenced_uuid'] = $referencedObject[$objectTypes[$referenced_type]]['uuid']; + $reference['referenced_id'] = $referencedObject[$referenced_type]['id']; + $reference['referenced_uuid'] = $referencedObject[$referenced_type]['uuid']; $reference['object_uuid'] = $sourceObject['Object']['uuid']; $reference['event_id'] = $eventId; - $this->save(array('ObjectReference' => $reference)); + $result = $this->save(array('ObjectReference' => $reference)); return true; } } diff --git a/app/Model/SharingGroup.php b/app/Model/SharingGroup.php index 1aafb1e45..c6dd26a55 100644 --- a/app/Model/SharingGroup.php +++ b/app/Model/SharingGroup.php @@ -158,7 +158,8 @@ class SharingGroup extends AppModel 'SharingGroup.id', 'SharingGroup.name', 'SharingGroup.releasability', - 'SharingGroup.description' + 'SharingGroup.description', + 'SharingGroup.org_id' ), 'contain' => array() ), From 0b0432536afed6409c52563f3007564d9211af12 Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 18 Oct 2018 20:18:06 +0200 Subject: [PATCH 08/15] chg: [warninglist] warninglists updated, fixes #3775 --- app/files/warninglists | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/warninglists b/app/files/warninglists index a205adba1..0bd8f8d16 160000 --- a/app/files/warninglists +++ b/app/files/warninglists @@ -1 +1 @@ -Subproject commit a205adba1fe251f9f14c3073994bb3f6f0ab35b9 +Subproject commit 0bd8f8d1615f830163cc15bb0d71a6daef6db931 From cebca8bcb796cbb15ed19a2e34d2bd88801052cf Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Fri, 19 Oct 2018 08:53:10 +0900 Subject: [PATCH 09/15] chg: [tools] Added x-sharedlib clause in testForBinExec if on OpenBSD. --- app/Model/Server.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Model/Server.php b/app/Model/Server.php index ae96380bf..ebc65aaad 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -2713,7 +2713,7 @@ class Server extends AppModel return true; } if (is_executable($value)) { - if (finfo_file($finfo, $value) == "application/x-executable") { + if (finfo_file($finfo, $value) == "application/x-executable" || (finfo_file($finfo, $value) == "application/x-sharedlib") && PHP_OS == "OpenBSD") { finfo_close($finfo); return true; } else { From bc06f07848940eeb2b7a0c2d5c988e184443a568 Mon Sep 17 00:00:00 2001 From: iglocska Date: Fri, 19 Oct 2018 10:10:34 +0200 Subject: [PATCH 10/15] fix: [cleanup] Removed debug from the bug fixing session --- app/Model/Event.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index 5b655641f..62944070c 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -902,8 +902,6 @@ class Event extends AppModel } $updated = null; $newLocation = $newTextBody = ''; - debug($event); - throw new Exception(); $result = $this->__executeRestfulEventToServer($event, $server, null, $newLocation, $newTextBody, $HttpSocket); if ($result !== true) { return $result; From 8925e885811671b62bee1648aa5852537d42ecf9 Mon Sep 17 00:00:00 2001 From: Tom King Date: Fri, 19 Oct 2018 09:54:51 +0100 Subject: [PATCH 11/15] fix: Check if the format is xml or application/xml on __sendResponse --- app/Controller/Component/RestResponseComponent.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Controller/Component/RestResponseComponent.php b/app/Controller/Component/RestResponseComponent.php index 50d0f9657..77eefe48a 100644 --- a/app/Controller/Component/RestResponseComponent.php +++ b/app/Controller/Component/RestResponseComponent.php @@ -342,7 +342,7 @@ class RestResponseComponent extends Component private function __sendResponse($response, $code, $format = false, $raw = false, $download = false) { - if (strtolower($format) === 'application/xml') { + if (strtolower($format) === 'application/xml' || strtolower($format) === 'xml') { if (!$raw) { if (isset($response[0])) { if (count(array_keys($response[0])) == 1) { From 4f4737d22a9c0e58c63d78e2349fc653a762ba6e Mon Sep 17 00:00:00 2001 From: iglocska Date: Fri, 19 Oct 2018 11:25:55 +0200 Subject: [PATCH 12/15] new: [upgrade] Preparing the data for recovery after the object reference sync fix - update the timestamps of all events / objcts that are affected and are locked = 0 --- app/Model/AppModel.php | 76 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 198e12e5d..181188f63 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -70,7 +70,7 @@ class AppModel extends Model 1 => false, 2 => false, 3 => false, 4 => true, 5 => false, 6 => false, 7 => false, 8 => false, 9 => false, 10 => false, 11 => false, 12 => false, 13 => false, 14 => false, 15 => false, 18 => false, 19 => false, 20 => false, - 21 => false, 22 => false + 21 => false, 22 => false, 23 => false ); public function afterSave($created, $options = array()) @@ -163,6 +163,9 @@ class AppModel extends Model case 12: $this->__forceSettings(); break; + case 23: + $this->__bumpReferences(); + break; default: $this->updateDatabase($command); break; @@ -1803,4 +1806,75 @@ class AppModel extends Model { return sys_get_temp_dir(); } + + private function __bumpReferences() + { + $this->Event = ClassRegistry::init('Event'); + $this->AdminSetting = ClassRegistry::init('AdminSetting'); + $existingSetting = $this->AdminSetting->find('first', array( + 'conditions' => array('AdminSetting.setting' => 'update_23') + )); + if (empty($existingSetting)) { + $this->AdminSetting->create(); + $data = array( + 'setting' => 'update_23', + 'value' => 1 + ); + $this->AdminSetting->save($data); + $references = $this->Event->Object->ObjectReference->find('list', array( + 'recursive' => -1, + 'fields' => array('ObjectReference.event_id', 'ObjectReference.event_id'), + 'group' => array('ObjectReference.event_id') + )); + $event_ids = array(); + $object_ids = array(); + foreach ($references as $reference) { + $event = $this->Event->find('first', array( + 'conditions' => array( + 'Event.id' => $reference, + 'Event.locked' => 0 + ), + 'recursive' => -1, + 'fields' => array('Event.id', 'Event.locked') + )); + if (!empty($event)) { + $event_ids[] = $event['Event']['id']; + $event_references = $this->Event->Object->ObjectReference->find('list', array( + 'conditions' => array('ObjectReference.event_id' => $reference), + 'recursive' => -1, + 'fields' => array('ObjectReference.object_id', 'ObjectReference.object_id') + )); + $object_ids = array_merge($object_ids, array_values($event_references)); + } + } + if (!empty($object_ids)) { + $this->Event->Object->updateAll( + array( + 'Object.timestamp' => 'Object.timestamp + 1' + ), + array('Object.id' => $object_ids) + ); + $this->Event->updateAll( + array( + 'Event.timestamp' => 'Event.timestamp + 1' + ), + array('Event.id' => $event_ids) + ); + } + $this->Log = ClassRegistry::init('Log'); + $this->Log->create(); + $entry = array( + 'org' => 'SYSTEM', + 'model' => 'Server', + 'model_id' => 0, + 'email' => 'SYSTEM', + 'action' => 'update_database', + 'user_id' => 0, + 'title' => 'Bumped the timestamps of locked events containing object references.', + 'change' => sprintf('Event timestamps updated: %s; Object timestamps updated: %s', count($event_ids), count($object_ids)) + ); + $this->Log->save($entry); + } + return true; + } } From 8ce06f73513af3a60f91090cccfc2f0b7d92ace9 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 19 Oct 2018 12:28:59 +0200 Subject: [PATCH 13/15] fix: #3769 Att&ck matrix now render multiple kill_chain by column. --- app/Model/Galaxy.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Model/Galaxy.php b/app/Model/Galaxy.php index b4dd282af..d209f5924 100644 --- a/app/Model/Galaxy.php +++ b/app/Model/Galaxy.php @@ -345,6 +345,7 @@ class Galaxy extends AppModel foreach ($galaxyElements as $element) { if ($element['key'] == 'kill_chain') { $kc = explode(":", $element['value'])[2]; + $attackClusters[$kc][] = $cluster; $toBeAdded = true; } if ($element['key'] == 'external_id') { @@ -352,7 +353,6 @@ class Galaxy extends AppModel } } if ($toBeAdded) { - $attackClusters[$kc][] = $cluster; array_push($attackTactic['attackTags'], $cluster['tag_name']); } } From 572b0d13a4532585f8bb1fddc73779c77d8e2421 Mon Sep 17 00:00:00 2001 From: iglocska Date: Sat, 20 Oct 2018 15:58:43 +0200 Subject: [PATCH 14/15] new: [API description] Describe how to run diagnostics on MISP via the API --- app/Controller/Component/RestResponseComponent.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/Controller/Component/RestResponseComponent.php b/app/Controller/Component/RestResponseComponent.php index 50d0f9657..10d813752 100644 --- a/app/Controller/Component/RestResponseComponent.php +++ b/app/Controller/Component/RestResponseComponent.php @@ -170,7 +170,10 @@ class RestResponseComponent extends Component 'edit' => array( 'description' => "POST an Server object in JSON format to this API to edit a server.", 'optional' => array('url', 'name', 'authkey', 'json', 'push', 'pull', 'push_rules', 'pull_rules', 'submitted_cert', 'submitted_client_cert', 'remote_org_id') - ) + ), + 'serverSettings' => array( + 'description' => "Send a GET request to this endpoint to get a full diagnostic along with all currently set settings of the current instance." + ) ), 'Sighting' => array( 'add' => array( From 1187fb2a27233bc15559971610931c772b8c6763 Mon Sep 17 00:00:00 2001 From: iglocska Date: Sun, 21 Oct 2018 22:47:22 +0200 Subject: [PATCH 15/15] new: [API] Added CSV as return format for event index --- app/Controller/AppController.php | 11 +++++++++- app/Controller/EventsController.php | 5 +++++ app/Controller/ServersController.php | 6 ++++-- app/Lib/Export/CsvExport.php | 30 ++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 3 deletions(-) diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index a33c99245..4b16bf919 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -505,10 +505,19 @@ class AppController extends Controller return $this->request->header('Accept') === 'application/json' || $this->RequestHandler->prefers() === 'json'; } + protected function _isCsv($data=false) + { + if ($this->params['ext'] === 'csv' || $this->request->header('Accept') === 'application/csv' || $this->RequestHandler->prefers() === 'csv') { + return true; + } else { + return false; + } + } + protected function _isRest() { $api = $this->__isApiFunction($this->request->params['controller'], $this->request->params['action']); - if (isset($this->RequestHandler) && ($api || $this->RequestHandler->isXml() || $this->_isJson())) { + if (isset($this->RequestHandler) && ($api || $this->RequestHandler->isXml() || $this->_isJson() || $this->_isCsv())) { if ($this->_isJson()) { if (!empty($this->request->input()) && empty($this->request->input('json_decode'))) { throw new MethodNotAllowedException('Invalid JSON input. Make sure that the JSON input is a correctly formatted JSON string. This request has been blocked to avoid an unfiltered request.'); diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index df2b884a9..9eda663e9 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -818,6 +818,11 @@ class EventsController extends AppController $this->set('analysisLevels', $this->Event->analysisLevels); $this->set('distributionLevels', $this->Event->distributionLevels); $this->set('shortDist', $this->Event->shortDist); + if ($this->params['ext'] === 'csv') { + App::uses('CsvExport', 'Export'); + $export = new CsvExport(); + return $this->RestResponse->viewData($export->eventIndex($events), 'csv'); + } if ($this->request->is('ajax')) { $this->autoRender = false; $this->layout = false; diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index ae7421760..d552761e7 100644 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -918,7 +918,7 @@ class ServersController extends AppController $this->set('files', $files); } // Only run this check on the diagnostics tab - if ($tab == 'diagnostics' || $tab == 'download') { + if ($tab == 'diagnostics' || $tab == 'download' || $this->_isRest()) { $php_ini = php_ini_loaded_file(); $this->set('php_ini', $php_ini); $advanced_attachments = shell_exec($this->Server->getPythonVersion() . ' ' . APP . 'files/scripts/generate_file_objects.py -c'); @@ -1022,7 +1022,7 @@ class ServersController extends AppController $worker_array = $this->Server->workerDiagnostics($workerIssueCount); } $this->set('worker_array', $worker_array); - if ($tab == 'download') { + if ($tab == 'download' || $this->_isRest()) { foreach ($dumpResults as $key => $dr) { unset($dumpResults[$key]['description']); } @@ -1806,4 +1806,6 @@ misp.direct_call(relative_path, body) $this->render('ajax/get_api_info'); } } + + } diff --git a/app/Lib/Export/CsvExport.php b/app/Lib/Export/CsvExport.php index 88ecf94dc..fda9a3cf6 100644 --- a/app/Lib/Export/CsvExport.php +++ b/app/Lib/Export/CsvExport.php @@ -207,4 +207,34 @@ class CsvExport return ''; } + public function eventIndex($events) + { + $fields = array( + 'id', 'date', 'info', 'tags', 'uuid', 'published', 'analysis', 'attribute_count', 'orgc_id', 'orgc_name', 'orgc_uuid', 'timestamp', 'distribution', 'sharing_group_id', 'threat_level_id', + 'publish_timestamp', 'extends_uuid' + ); + $result = implode(',', $fields) . PHP_EOL; + foreach ($events as $key => $event) { + $event['tags'] = ''; + if (!empty($event['EventTag'])) { + $tags = array(); + foreach ($event['EventTag'] as $et) { + $tags[] = $et['Tag']['name']; + } + $tags = implode(', ', $tags); + } else { + $tags = ''; + } + $event['Event']['tags'] = $tags; + $event['Event']['orgc_name'] = $event['Orgc']['name']; + $event['Event']['orgc_uuid'] = $event['Orgc']['uuid']; + $current = array(); + foreach ($fields as $field) { + $current[] = $this->__escapeCSVField($event['Event'][$field]); + } + $result .= implode(', ', $current) . PHP_EOL; + } + return $result; + } + }