From c67b575ba63df03bc84eec2ec3f6bc0cd0b1d67c Mon Sep 17 00:00:00 2001 From: iglocska Date: Mon, 2 Jul 2018 10:38:24 +0200 Subject: [PATCH 1/3] fix: [bug] Potential fix for SQL return size limit reached when fetching a list of attributes --- app/Model/Attribute.php | 66 ++++++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 4302356c3..850db1c26 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -2605,39 +2605,57 @@ class Attribute extends AppModel { )); return $results; } - $results = $this->find('all', $params); - // return false if we're paginating - if (isset($options['limit']) && empty($results)) return false; + if ($options['enforceWarninglist']) { $this->Warninglist = ClassRegistry::init('Warninglist'); $warninglists = $this->Warninglist->fetchForEventView(); } - $results = array_values($results); - $proposals_block_attributes = Configure::read('MISP.proposals_block_attributes'); - foreach ($results as $key => $attribute) { - if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttributes($warninglists, $attribute['Attribute'])) { - unset($results[$key]); - continue; + + if (empty($params['limit'])) { + $pagesToFetch = $this->find('count', array('conditions' => $params['conditions'])); + $loopLimit = 100000; + $pagesToFetch = ceil($pagesToFetch / $loopLimit); + $loop = true; + } else { + $loop = false; + $pagesToFetch = 1; + } + + $attributes = array(); + for ($i = 0; $i < $pagesToFetch; $i++) { + if ($loop) { + $params['limit'] = $loopLimit; + $params['page'] = $i+1; } - if (!empty($options['includeAttributeUuid']) || !empty($options['includeEventUuid'])) { - $results[$key]['Attribute']['event_uuid'] = $results[$key]['Event']['uuid']; - } - if ($proposals_block_attributes) { - if (!empty($attribute['ShadowAttribute'])) { - unset($results[$key]); - } else { - unset($results[$key]['ShadowAttribute']); + $results = $this->find('all', $params); + // return false if we're paginating + if (isset($options['limit']) && empty($results)) return false; + $results = array_values($results); + $proposals_block_attributes = Configure::read('MISP.proposals_block_attributes'); + foreach ($results as $key => $attribute) { + if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttributes($warninglists, $attribute['Attribute'])) { + continue; } - } - if ($options['withAttachments']) { - if ($this->typeIsAttachment($attribute['Attribute']['type'])) { - $encodedFile = $this->base64EncodeAttachment($attribute['Attribute']); - $results[$key]['Attribute']['data'] = $encodedFile; + if (!empty($options['includeAttributeUuid']) || !empty($options['includeEventUuid'])) { + $results[$key]['Attribute']['event_uuid'] = $results[$key]['Event']['uuid']; } + if ($proposals_block_attributes) { + if (!empty($attribute['ShadowAttribute'])) { + continue; + } else { + unset($results[$key]['ShadowAttribute']); + } + } + if ($options['withAttachments']) { + if ($this->typeIsAttachment($attribute['Attribute']['type'])) { + $encodedFile = $this->base64EncodeAttachment($attribute['Attribute']); + $results[$key]['Attribute']['data'] = $encodedFile; + } + } + $attributes[] = $results[$key]; } } - $results = array_values($results); - return $results; + return $attributes; } // Method gets and converts the contents of a file passed along as a base64 encoded string with the original filename into a zip archive From 59b17b5af6ca564b2887f6131c32be08200af128 Mon Sep 17 00:00:00 2001 From: iglocska Date: Mon, 2 Jul 2018 16:56:50 +0200 Subject: [PATCH 2/3] new: [sync] Added flag to avoid using the proxy - in some cases you have internal sync between instances in which case going through the proxy is silly --- app/Controller/ServersController.php | 2 +- app/Lib/Tools/SyncTool.php | 7 ++++--- app/Model/AppModel.php | 6 +++++- app/View/Servers/add.ctp | 4 ++++ app/View/Servers/edit.ctp | 4 ++++ app/View/Servers/index.ctp | 2 ++ 6 files changed, 20 insertions(+), 5 deletions(-) diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index 91154c647..80559c834 100644 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -377,7 +377,7 @@ class ServersController extends AppController { } if (!$fail) { // say what fields are to be updated - $fieldList = array('id', 'url', 'push', 'pull', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal'); + $fieldList = array('id', 'url', 'push', 'pull', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy'); $this->request->data['Server']['id'] = $id; if (isset($this->request->data['Server']['authkey']) && "" != $this->request->data['Server']['authkey']) $fieldList[] = 'authkey'; if(isset($this->request->data['Server']['organisation_type']) && isset($json)) { diff --git a/app/Lib/Tools/SyncTool.php b/app/Lib/Tools/SyncTool.php index 80d1a5dc0..d3b7d0e7a 100644 --- a/app/Lib/Tools/SyncTool.php +++ b/app/Lib/Tools/SyncTool.php @@ -16,9 +16,10 @@ class SyncTool { } } $HttpSocket = new HttpSocket($params); - - $proxy = Configure::read('Proxy'); - if (isset($proxy['host']) && !empty($proxy['host'])) $HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']); + if (empty($server['Server']['skip_proxy'])) { + $proxy = Configure::read('Proxy'); + if (isset($proxy['host']) && !empty($proxy['host'])) $HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']); + } return $HttpSocket; } diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index df53b65b7..72398e460 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -63,7 +63,8 @@ class AppModel extends Model { public $db_changes = array( 1 => false, 2 => false, 3 => false, 4 => true, 5 => false, 6 => false, - 7 => false, 8 => false, 9 => false, 10 => false, 11 => false, 12 => false + 7 => false, 8 => false, 9 => false, 10 => false, 11 => false, 12 => false, + 13 => false ); function afterSave($created, $options = array()) { @@ -957,6 +958,9 @@ class AppModel extends Model { INDEX `timestamp` (`timestamp`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8;"; break; + case 13: + $sqlArray[] = "ALTER TABLE `servers` ADD `skip_proxy` tinyint(1) NOT NULL DEFAULT 0;"; + break; case 'fixNonEmptySharingGroupID': $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; diff --git a/app/View/Servers/add.ctp b/app/View/Servers/add.ctp index 4b60eaf84..34672ee37 100644 --- a/app/View/Servers/add.ctp +++ b/app/View/Servers/add.ctp @@ -88,6 +88,10 @@ echo $this->Form->input('self_signed', array( 'type' => 'checkbox', )); + ?> +
+ Form->input('skip_proxy', array('type' => 'checkbox', 'label' => 'Skip proxy (if applicable)')); echo $this->Form->input('Server.submitted_cert', array( 'label' => '' . __('Server certificate file') . '', diff --git a/app/View/Servers/edit.ctp b/app/View/Servers/edit.ctp index 8c96edbdb..96753fd63 100644 --- a/app/View/Servers/edit.ctp +++ b/app/View/Servers/edit.ctp @@ -94,6 +94,10 @@ echo $this->Form->input('self_signed', array( 'type' => 'checkbox', )); + ?> +
+ Form->input('skip_proxy', array('type' => 'checkbox', 'label' => 'Skip proxy (if applicable)')); ?>

diff --git a/app/View/Servers/index.ctp b/app/View/Servers/index.ctp index 7d5a4a8e9..b5f009c58 100644 --- a/app/View/Servers/index.ctp +++ b/app/View/Servers/index.ctp @@ -31,6 +31,7 @@ Paginator->sort('cert_file');?> Paginator->sort('client_cert_file');?> Paginator->sort('self_signed');?> + Paginator->sort('skip_proxy');?> Paginator->sort('org');?> @@ -78,6 +79,7 @@ foreach ($servers as $server):     + Date: Mon, 2 Jul 2018 17:29:53 +0200 Subject: [PATCH 3/3] new: [edit strategy API] To support a smoother integration with the Hive, new API that describes what the edit strategy is for an event - GET on /events/getEditStrategy/[id] - where id can be either a local ID or a UUID - returns a JSON dictionary with the following fields: - strategy: edit | extend (edit if it's an own event, extend otherwise) - extensions: list of dictionaries with existing extensions created by the user's org (containing the id, uuid, info fields) - The algorithms implementing this should prioritise as such: 1. Check if user can edit the event (strategy == edit) - if yes, edit 2. If no, check if extensions exist - if yes, edit one of those 3. If no, create a new extension to the original event --- app/Controller/Component/ACLComponent.php | 1 + app/Controller/EventsController.php | 41 +++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php index f4cbcd697..890743396 100644 --- a/app/Controller/Component/ACLComponent.php +++ b/app/Controller/Component/ACLComponent.php @@ -106,6 +106,7 @@ class ACLComponent extends Component { 'filterEventIdsForPush' => array('perm_sync'), 'filterEventIndex' => array('*'), 'freeTextImport' => array('perm_add'), + 'getEditStrategy' => array('perm_add'), 'getEventInfoById' => array('*'), 'getEventGraphReferences' => array('*'), 'getEventGraphTags' => array('*'), diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index cd83e6a71..f96135197 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -5098,4 +5098,45 @@ class EventsController extends AppController { return $this->RestResponse->viewData(array(), $this->response->type()); } } + + public function getEditStrategy($id) { + // find the id of the event, change $id to it and proceed to read the event as if the ID was entered. + if (Validation::uuid($id)) { + $this->Event->recursive = -1; + $event = $this->Event->find('first', array( + 'recursive' => -1, + 'conditions' => array('Event.uuid' => $id), + 'fields' => array('Event.id', 'Event.uuid', 'Event.orgc_id') + )); + if ($event == null) throw new NotFoundException('Invalid event'); + $id = $event['Event']['id']; + } else if (!is_numeric($id)) { + throw new NotFoundException(__('Invalid event')); + } else { + $event = $this->Event->find('first', array( + 'recursive' => -1, + 'conditions' => array('Event.id' => $id), + 'fields' => array('Event.id', 'Event.uuid', 'Event.orgc_id') + )); + } + if (empty($event)) throw new NotFoundException(__('Invalid event')); + $response = array('extensions' => array()); + if ($event['Event']['orgc_id'] === $this->Auth->user('org_id')) { + $response['strategy'] = 'edit'; + } else { + $response['strategy'] = 'extend'; + } + $extendedEvents = $this->Event->find('all', array( + 'recursive' => -1, + 'fields' => array('Event.id', 'Event.info', 'Event.uuid'), + 'conditions' => array( + 'Event.extends_uuid' => $event['Event']['uuid'], + 'Event.orgc_id' => $this->Auth->user('org_id') + ) + )); + foreach ($extendedEvents as $extendedEvent) { + $response['extensions'][] = $extendedEvent['Event']; + } + return $this->RestResponse->viewData($response, $this->response->type()); + } }