diff --git a/app/Console/Command/ServerShell.php b/app/Console/Command/ServerShell.php index a82b9d97d..ad2d8df72 100644 --- a/app/Console/Command/ServerShell.php +++ b/app/Console/Command/ServerShell.php @@ -85,6 +85,7 @@ class ServerShell extends AppShell $userId = $this->args[0]; $user = $this->getUser($userId); + Configure::write('CurrentUserId', $userId); if (!empty($this->args[1])) { $technique = $this->args[1]; @@ -129,6 +130,7 @@ class ServerShell extends AppShell $user = $this->getUser($userId); $serverId = $this->args[1]; $server = $this->getServer($serverId); + Configure::write('CurrentUserId', $userId); if (!empty($this->args[2])) { $technique = $this->args[2]; } else { @@ -146,7 +148,7 @@ class ServerShell extends AppShell try { $result = $this->Server->pull($user, $technique, $server, $jobId, $force); if (is_array($result)) { - $message = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4]); + $message = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled, %s analyst data pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4], $result[5]); $this->Job->saveStatus($jobId, true, $message); } else { $message = __('ERROR: %s', $result); @@ -169,6 +171,7 @@ class ServerShell extends AppShell $user = $this->getUser($userId); $serverId = $this->args[1]; $server = $this->getServer($serverId); + Configure::write('CurrentUserId', $userId); $technique = empty($this->args[2]) ? 'full' : $this->args[2]; if (!empty($this->args[3])) { $jobId = $this->args[3]; @@ -203,6 +206,7 @@ class ServerShell extends AppShell $userId = $this->args[0]; $user = $this->getUser($userId); + Configure::write('CurrentUserId', $userId); $technique = isset($this->args[1]) ? $this->args[1] : 'full'; $servers = $this->Server->find('list', array( diff --git a/app/Controller/AnalystDataBlocklistsController.php b/app/Controller/AnalystDataBlocklistsController.php new file mode 100644 index 000000000..ed1489cee --- /dev/null +++ b/app/Controller/AnalystDataBlocklistsController.php @@ -0,0 +1,103 @@ + 60, + 'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 entries <- no we won't, this is the max a user van view/page. + 'order' => array( + 'AnalystDataBlocklist.created' => 'DESC' + ), + ); + + public function index() + { + $passedArgsArray = array(); + $passedArgs = $this->passedArgs; + $params = array(); + $validParams = array('analyst_data_uuid', 'comment', 'analyst_data_info', 'analyst_data_orgc'); + foreach ($validParams as $validParam) { + if (!empty($this->params['named'][$validParam])) { + $params[$validParam] = $this->params['named'][$validParam]; + } + } + if (!empty($this->params['named']['searchall'])) { + $params['AND']['OR'] = array( + 'analyst_data_uuid' => $this->params['named']['searchall'], + 'comment' => $this->params['named']['searchall'], + 'analyst_data_info' => $this->params['named']['searchall'], + 'analyst_data_orgc' => $this->params['named']['searchall'] + ); + } + $this->set('passedArgs', json_encode($passedArgs)); + $this->set('passedArgsArray', $passedArgsArray); + return $this->BlockList->index($this->_isRest(), $params); + } + + public function add() + { + $this->set('action', 'add'); + return $this->BlockList->add($this->_isRest()); + } + + public function edit($id) + { + $this->set('action', 'edit'); + return $this->BlockList->edit($this->_isRest(), $id); + } + + public function delete($id) + { + if (Validation::uuid($id)) { + $entry = $this->AnalystDataBlocklist->find('first', array( + 'conditions' => array('analyst_data_uuid' => $id) + )); + if (empty($entry)) { + throw new NotFoundException(__('Invalid blocklist entry')); + } + $id = $entry['AnalystDataBlocklist']['id']; + } + return $this->BlockList->delete($this->_isRest(), $id); + } + + public function massDelete() + { + if ($this->request->is('post') || $this->request->is('put')) { + if (!isset($this->request->data['AnalystDataBlocklist'])) { + $this->request->data = array('AnalystDataBlocklist' => $this->request->data); + } + $ids = $this->request->data['AnalystDataBlocklist']['ids']; + $analyst_data_ids = json_decode($ids, true); + if (empty($analyst_data_ids)) { + throw new NotFoundException(__('Invalid Analyst Data IDs.')); + } + $result = $this->AnalystDataBlocklist->deleteAll(array('AnalystDataBlocklist.id' => $analyst_data_ids)); + if ($result) { + if ($this->_isRest()) { + return $this->RestResponse->saveSuccessResponse('AnalystDataBlocklist', 'Deleted', $ids, $this->response->type()); + } else { + $this->Flash->success('Blocklist entry removed'); + $this->redirect(array('controller' => 'AnalystDataBlocklist', 'action' => 'index')); + } + } else { + $error = __('Failed to delete Analyst Data from AnalystDataBlocklist. Error: ') . PHP_EOL . h($result); + if ($this->_isRest()) { + return $this->RestResponse->saveFailResponse('AnalystDataBlocklist', 'Deleted', false, $error, $this->response->type()); + } else { + $this->Flash->error($error); + $this->redirect(array('controller' => 'AnalystDataBlocklists', 'action' => 'index')); + } + } + } else { + $ids = json_decode($this->request->query('ids'), true); + if (empty($ids)) { + throw new NotFoundException(__('Invalid analyst data IDs.')); + + } + $this->set('analyst_data_ids', $ids); + } + } +} diff --git a/app/Controller/AnalystDataController.php b/app/Controller/AnalystDataController.php new file mode 100644 index 000000000..ef7b305fb --- /dev/null +++ b/app/Controller/AnalystDataController.php @@ -0,0 +1,327 @@ + 60, + 'order' => [] + ]; + + public $uses = [ + 'Opinion', + 'Note', + 'Relationship' + ]; + + private $__valid_types = [ + 'Opinion', + 'Note', + 'Relationship' + ]; + + // public $modelSelection = 'Note'; + + private function _setViewElements() + { + $dropdownData = []; + $this->loadModel('Event'); + $dropdownData['distributionLevels'] = $this->Event->distributionLevels; + $this->set('initialDistribution', Configure::read('MISP.default_event_distribution')); + $dropdownData['sgs'] = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1); + $dropdownData['valid_targets'] = array_combine($this->AnalystData->valid_targets, $this->AnalystData->valid_targets); + $this->set(compact('dropdownData')); + $this->set('modelSelection', $this->modelSelection); + $this->set('distributionLevels', $this->Event->distributionLevels); + App::uses('LanguageRFC5646Tool', 'Tools'); + $this->set('languageRFC5646', ['' => __('- No language -'), LanguageRFC5646Tool::getLanguages()]); + } + + public function add($type = 'Note', $object_uuid = null, $object_type = null) + { + $this->__typeSelector($type); + if (!empty($object_uuid)) { + $this->request->data[$this->modelSelection]['object_uuid'] = $object_uuid; + } + if (!empty($object_type)) { + $this->request->data[$this->modelSelection]['object_type'] = $object_type; + } + + if (empty($this->request->data[$this->modelSelection]['object_type']) && !empty($this->request->data[$this->modelSelection]['object_uuid'])) { + $this->request->data[$this->modelSelection]['object_type'] = $this->AnalystData->deduceType($object_uuid); + } + $params = []; + $this->CRUD->add($params); + if ($this->restResponsePayload) { + return $this->restResponsePayload; + } + $this->_setViewElements(); + if ($type == 'Relationship') { + $this->set('existingRelations', $this->AnalystData->getExistingRelationships()); + } + $this->set('menuData', array('menuList' => 'analyst_data', 'menuItem' => 'add_' . strtolower($type))); + $this->render('add'); + } + + public function edit($type = 'Note', $id) + { + if ($type === 'all' && Validation::uuid($id)) { + $this->loadModel('AnalystData'); + $type = $this->AnalystData->deduceType($id); + } + $this->__typeSelector($type); + if (!is_numeric($id) && Validation::uuid($id)) { + $id = $this->AnalystData->getIDFromUUID($type, $id); + } + + $this->set('id', $id); + $conditions = $this->AnalystData->buildConditions($this->Auth->user()); + $params = [ + 'conditions' => $conditions, + 'afterFind' => function(array $analystData): array { + $canEdit = $this->ACL->canEditAnalystData($this->Auth->user(), $analystData, $this->modelSelection); + if (!$canEdit) { + throw new MethodNotAllowedException(__('You are not authorised to do that.')); + } + return $analystData; + }, + 'beforeSave' => function(array $analystData): array { + $analystData[$this->modelSelection]['modified'] = date ('Y-m-d H:i:s'); + return $analystData; + } + ]; + $this->CRUD->edit($id, $params); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + $this->_setViewElements(); + if ($type == 'Relationship') { + $this->set('existingRelations', $this->AnalystData->getExistingRelationships()); + } + $this->set('menuData', array('menuList' => 'analyst_data', 'menuItem' => 'edit')); + $this->render('add'); + } + + public function delete($type = 'Note', $id, $hard=true) + { + if ($type === 'all' && Validation::uuid($id)) { + $this->loadModel('AnalystData'); + $type = $this->AnalystData->deduceType($id); + } + $this->__typeSelector($type); + if (!is_numeric($id) && Validation::uuid($id)) { + $id = $this->AnalystData->getIDFromUUID($type, $id); + } + + $params = [ + 'afterFind' => function(array $analystData) { + $canEdit = $this->ACL->canEditAnalystData($this->Auth->user(), $analystData, $this->modelSelection); + if (!$canEdit) { + throw new MethodNotAllowedException(__('You are not authorised to do that.')); + } + return $analystData; + }, + 'afterDelete' => function($deletedAnalystData) use ($hard) { + if (empty($hard)) { + return; + } + $type = $this->AnalystData->deduceAnalystDataType($deletedAnalystData); + $info = '- Unsupported analyst type -'; + if ($type === 'Note') { + $info = $deletedAnalystData[$type]['note']; + } else if ($type === 'Opinion') { + $info = sprintf('%s/100 :: %s', $deletedAnalystData[$type]['opinion'], $deletedAnalystData[$type]['comment']); + } else if ($type === 'Relationship') { + $info = sprintf('-- %s --> %s :: %s', $deletedAnalystData[$type]['relationship_type'] ?? '[undefined]', $deletedAnalystData[$type]['related_object_type'], $deletedAnalystData[$type]['related_object_uuid']); + } + $this->AnalystDataBlocklist = ClassRegistry::init('AnalystDataBlocklist'); + $this->AnalystDataBlocklist->create(); + if (!empty($deletedAnalystData[$type]['orgc_uuid'])) { + if (!empty($deletedAnalystData[$type]['Orgc'])) { + $orgc = $deletedAnalystData[$type]; + } else { + $orgc = $this->Orgc->find('first', array( + 'conditions' => ['Orgc.uuid' => $deletedAnalystData[$type]['orgc_uuid']], + 'recursive' => -1, + 'fields' => ['Orgc.name'], + )); + } + } else { + $orgc = ['Orgc' => ['name' => 'MISP']]; + } + $this->AnalystDataBlocklist->save(['analyst_data_uuid' => $deletedAnalystData[$type]['uuid'], 'analyst_data_info' => $info, 'analyst_data_orgc' => $orgc['Orgc']['name']]); + } + ]; + $this->CRUD->delete($id, $params); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + + } + + public function view($type = 'Note', $id) + { + if ($type === 'all' && Validation::uuid($id)) { + $this->loadModel('AnalystData'); + $type = $this->AnalystData->getAnalystDataTypeFromUUID($id); + } + $this->__typeSelector($type); + if (!is_numeric($id) && Validation::uuid($id)) { + $id = $this->AnalystData->getIDFromUUID($type, $id); + } + + if (!$this->IndexFilter->isRest()) { + $this->AnalystData->fetchRecursive = true; + } + $conditions = $this->AnalystData->buildConditions($this->Auth->user()); + $this->CRUD->view($id, [ + 'conditions' => $conditions, + 'contain' => ['Org', 'Orgc'], + 'afterFind' => function(array $analystData) { + if (!$this->request->is('ajax')) { + unset($analystData[$this->modelSelection]['_canEdit']); + } + return $analystData; + } + ]); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + $this->set('id', $id); + $this->loadModel('Event'); + $this->_setViewElements(); + $this->set('distributionLevels', $this->Event->distributionLevels); + $this->set('shortDist', $this->Event->shortDist); + $this->set('menuData', array('menuList' => 'analyst_data', 'menuItem' => 'view')); + $this->render('view'); + } + + public function index($type = 'Note') + { + $this->__typeSelector($type); + + $conditions = $this->AnalystData->buildConditions($this->Auth->user()); + $params = [ + 'filters' => ['uuid', 'target_object'], + 'quickFilters' => ['name'], + 'conditions' => $conditions, + 'afterFind' => function(array $data) { + foreach ($data as $i => $analystData) { + if (!$this->request->is('ajax')) { + unset($analystData[$this->modelSelection]['_canEdit']); + } + } + return $data; + } + ]; + $this->CRUD->index($params); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + $this->_setViewElements(); + $this->set('menuData', array('menuList' => 'analyst_data', 'menuItem' => 'index')); + } + + public function getRelatedElement($type, $uuid) + { + $this->__typeSelector('Relationship'); + $data = $this->AnalystData->getRelatedElement($this->Auth->user(), $type, $uuid); + return $this->RestResponse->viewData($data, 'json'); + } + + public function getChildren($type = 'Note', $uuid, $depth=2) + { + $this->__typeSelector($type); + $data = $this->AnalystData->getChildren($this->Auth->user(), $uuid, $depth); + return $this->RestResponse->viewData($data, 'json'); + } + + public function filterAnalystDataForPush() + { + if (!$this->request->is('post')) { + throw new MethodNotAllowedException(__('This function is only accessible via POST requests.')); + } + + $this->loadModel('AnalystData'); + + $allIncomingAnalystData = $this->request->data; + $allData = $this->AnalystData->filterAnalystDataForPush($allIncomingAnalystData); + + return $this->RestResponse->viewData($allData, $this->response->type()); + } + + public function indexMinimal() + { + $this->loadModel('AnalystData'); + $filters = []; + if ($this->request->is('post')) { + $filters = $this->request->data; + } + $options = []; + if (!empty($filters['orgc_name'])) { + $orgcNames = $filters['orgc_name']; + if (!is_array($orgcNames)) { + $orgcName = [$orgcNames]; + } + $filterName = 'orgc_uuid'; + foreach ($orgcNames as $orgcName) { + if ($orgcName[0] === '!') { + $orgc = $this->AnalystData->Orgc->fetchOrg(substr($orgcName, 1)); + if ($orgc === false) { + continue; + } + $options[]['AND'][] = ["{$filterName} !=" => $orgc['uuid']]; + } else { + $orgc = $this->AnalystData->Orgc->fetchOrg($orgcName); + if ($orgc === false) { + continue; + } + $options['OR'][] = [$filterName => $orgc['uuid']]; + } + } + } + $allData = $this->AnalystData->indexMinimal($this->Auth->user(), $options); + + return $this->RestResponse->viewData($allData, $this->response->type()); + } + + public function pushAnalystData() + { + if (!$this->Auth->user()['Role']['perm_sync'] || !$this->Auth->user()['Role']['perm_analyst_data']) { + throw new MethodNotAllowedException(__('You do not have the permission to do that.')); + } + if (!$this->_isRest()) { + throw new MethodNotAllowedException(__('This action is only accessible via a REST request.')); + } + if ($this->request->is('post')) { + $this->loadModel('AnalystData'); + $analystData = $this->request->data; + $saveResult = $this->AnalystData->captureAnalystData($this->Auth->user(), $analystData); + $messageInfo = __('%s imported, %s ignored, %s failed. %s', $saveResult['imported'], $saveResult['ignored'], $saveResult['failed'], !empty($saveResult['errors']) ? implode(', ', $saveResult['errors']) : ''); + if ($saveResult['success']) { + $message = __('Analyst Data imported. ') . $messageInfo; + return $this->RestResponse->saveSuccessResponse('AnalystData', 'pushAnalystData', false, $this->response->type(), $message); + } else { + $message = __('Could not import analyst data. ') . $messageInfo; + return $this->RestResponse->saveFailResponse('AnalystData', 'pushAnalystData', false, $message); + } + } + } + + private function __typeSelector($type) { + foreach ($this->__valid_types as $vt) { + if ($type === $vt) { + $this->modelSelection = $vt; + $this->loadModel($vt); + $this->AnalystData = $this->{$vt}; + $this->modelClass = $vt; + $this->{$vt}->current_user = $this->Auth->user(); + return $vt; + } + } + throw new MethodNotAllowedException(__('Invalid type.')); + } +} diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index 8997cf30c..c7828fdce 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -33,7 +33,7 @@ class AppController extends Controller public $helpers = array('OrgImg', 'FontAwesome', 'UserName'); - private $__queryVersion = '158'; + private $__queryVersion = '159'; public $pyMispVersion = '2.4.185'; public $phpmin = '7.2'; public $phprec = '7.4'; @@ -306,6 +306,7 @@ class AppController extends Controller $this->set('isAclTagger', $role['perm_tagger']); $this->set('isAclGalaxyEditor', !empty($role['perm_galaxy_editor'])); $this->set('isAclSighting', $role['perm_sighting'] ?? false); + $this->set('isAclAnalystDataCreator', $role['perm_analyst_data'] ?? false); $this->set('aclComponent', $this->ACL); $this->userRole = $role; diff --git a/app/Controller/CollectionElementsController.php b/app/Controller/CollectionElementsController.php new file mode 100644 index 000000000..2087c40d3 --- /dev/null +++ b/app/Controller/CollectionElementsController.php @@ -0,0 +1,146 @@ + 60, + 'order' => [] + ]; + + public $uses = [ + ]; + + public function add($collection_id) + { + $this->CollectionElement->Collection->current_user = $this->Auth->user(); + if (!$this->CollectionElement->Collection->mayModify($this->Auth->user('id'), intval($collection_id))) { + throw new MethodNotAllowedException(__('Invalid Collection or insuficient privileges')); + } + $this->CRUD->add([ + 'beforeSave' => function (array $collectionElement) use ($collection_id) { + $collectionElement['CollectionElement']['collection_id'] = intval($collection_id); + return $collectionElement; + } + ]); + if ($this->restResponsePayload) { + return $this->restResponsePayload; + } + $dropdownData = [ + 'types' => array_combine($this->CollectionElement->valid_types, $this->CollectionElement->valid_types) + ]; + $this->set(compact('dropdownData')); + $this->set('menuData', array('menuList' => 'collections', 'menuItem' => 'add_element')); + } + + public function delete($element_id) + { + $collectionElement = $this->CollectionElement->find('first', [ + 'recursive' => -1, + 'conditions' => [ + 'CollectionElement.id' => $element_id + ] + ]); + $collection_id = $collectionElement['CollectionElement']['collection_id']; + if (!$this->CollectionElement->Collection->mayModify($this->Auth->user('id'), $collection_id)) { + throw new MethodNotAllowedException(__('Invalid Collection or insuficient privileges')); + } + $this->CRUD->delete($element_id); + if ($this->restResponsePayload) { + return $this->restResponsePayload; + } + } + + public function index($collection_id) + { + $this->set('menuData', array('menuList' => 'collections', 'menuItem' => 'index')); + if (!$this->CollectionElement->Collection->mayView($this->Auth->user('id'), intval($collection_id))) { + throw new NotFoundException(__('Invalid collection or no access.')); + } + $params = [ + 'filters' => ['uuid', 'type', 'name'], + 'quickFilters' => ['name'], + 'conditions' => ['collection_id' => $collection_id] + ]; + $this->loadModel('Event'); + $this->set('distributionLevels', $this->Event->distributionLevels); + $this->CRUD->index($params); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + } + + public function addElementToCollection($element_type, $element_uuid) + { + if ($this->request->is('get')) { + $validCollections = $this->CollectionElement->Collection->find('list', [ + 'recursive' => -1, + 'fields' => ['Collection.id', 'Collection.name'], + 'conditions' => ['Collection.orgc_id' => $this->Auth->user('org_id')] + ]); + if (empty($validCollections)) { + if ($this->request->is('ajax')) { + return $this->redirect(['controller' => 'collections', 'action' => 'add']); + } + throw new NotFoundException(__('You don\'t have any collections yet. Make sure you create one first before you can start adding elements.')); + } + $dropdownData = [ + 'collections' => $validCollections + ]; + $this->set(compact('dropdownData')); + } else if ($this->request->is('post')) { + if (!isset($this->request->data['CollectionElement'])) { + $this->request->data = ['CollectionElement' => $this->request->data]; + } + if (!isset($this->request->data['CollectionElement']['collection_id'])) { + throw new NotFoundException(__('No collection_id specified.')); + } + $collection_id = intval($this->request->data['CollectionElement']['collection_id']); + if (!$this->CollectionElement->Collection->mayModify($this->Auth->user('id'), $collection_id)) { + throw new NotFoundException(__('Invalid collection or not authorized.')); + } + $description = empty($this->request->data['CollectionElement']['description']) ? '' : $this->request->data['CollectionElement']['description']; + $dataToSave = [ + 'CollectionElement' => [ + 'element_uuid' => $element_uuid, + 'element_type' => $element_type, + 'description' => $description, + 'collection_id' => $collection_id + ] + ]; + $this->CollectionElement->create(); + $error = ''; + try { + $result = $this->CollectionElement->save($dataToSave); + } catch (PDOException $e) { + if ($e->errorInfo[0] == 23000) { + $error = __(' Element already in Collection.'); + } + } + + if ($result) { + $message = __('Element added to the Collection.'); + if ($this->IndexFilter->isRest()) { + return $this->RestResponse->saveSuccessResponse('CollectionElements', 'addElementToCollection', false, $this->response->type(), $message); + } else { + $this->Flash->success($message); + $this->redirect(Router::url($this->referer(), true)); + } + } else { + $message = __('Element could not be added to the Collection.%s', $error); + if ($this->IndexFilter->isRest()) { + return $this->RestResponse->saveFailResponse('CollectionElements', 'addElementToCollection', false, $message, $this->response->type()); + } else { + $this->Flash->error($message); + $this->redirect(Router::url($this->referer(), true)); + } + } + } + } +} diff --git a/app/Controller/CollectionsController.php b/app/Controller/CollectionsController.php new file mode 100644 index 000000000..a547c5c62 --- /dev/null +++ b/app/Controller/CollectionsController.php @@ -0,0 +1,171 @@ + 60, + 'order' => [] + ]; + + public $uses = [ + ]; + + private $valid_types = [ + 'campaign', + 'intrusion_set', + 'named_threat', + 'other', + 'research' + ]; + + public function add() + { + $this->Collection->current_user = $this->Auth->user(); + $params = []; + if ($this->request->is('post')) { + $data = $this->request->data; + $params = [ + 'afterSave' => function (array $collection) use ($data) { + $this->Collection->CollectionElement->captureElements($collection); + return $collection; + } + ]; + } + $this->CRUD->add($params); + if ($this->restResponsePayload) { + return $this->restResponsePayload; + } + $this->set('menuData', array('menuList' => 'collections', 'menuItem' => 'add')); + $this->loadModel('Event'); + $dropdownData = [ + 'types' => array_combine($this->valid_types, $this->valid_types), + 'distributionLevels' => $this->Event->distributionLevels, + 'sgs' => $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1) + ]; + $this->set('initialDistribution', Configure::read('MISP.default_event_distribution')); + $this->set(compact('dropdownData')); + $this->render('add'); + } + + public function edit($id) + { + $this->Collection->current_user = $this->Auth->user(); + if (!$this->Collection->mayModify($this->Auth->user('id'), $id)) { + throw new MethodNotAllowedException(__('Invalid Collection or insuficient privileges')); + } + $params = []; + if ($this->request->is('post') || $this->request->is('put')) { + $oldCollection = $this->Collection->find('first', [ + 'recursive' => -1, + 'conditions' => ['Collection.id' => intval($id)] + ]); + if (empty($oldCollection)) { + throw new NotFoundException(__('Invalid collection.')); + } + if (empty($this->request->data['Collection'])) { + $this->request->data = ['Collection' => $this->request->data]; + } + $data = $this->request->data; + if ( + isset($data['Collection']['modified']) && + $data['Collection']['modified'] <= $oldCollection['Collection']['modified'] + ) { + throw new ForbiddenException(__('Collection received older or same as local version.')); + } + $params = [ + 'afterSave' => function (array &$collection) use ($data) { + $collection = $this->Collection->CollectionElement->captureElements($collection); + return $collection; + } + ]; + } + $this->set('id', $id); + $this->CRUD->edit($id, $params); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + $this->set('menuData', array('menuList' => 'collections', 'menuItem' => 'edit')); + $this->loadModel('Event'); + $dropdownData = [ + 'types' => $this->valid_types, + 'distributionLevels' => $this->Event->distributionLevels, + 'sgs' => $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1) + ]; + $this->set(compact('dropdownData')); + $this->render('add'); + } + + public function delete($id) + { + if (!$this->Collection->mayModify($this->Auth->user('id'), $id)) { + throw new MethodNotAllowedException(__('Invalid Collection or insuficient privileges')); + } + $this->CRUD->delete($id); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + } + + public function view($id) + { + $this->set('mayModify', $this->Collection->mayModify($this->Auth->user('id'), $id)); + if (!$this->Collection->mayView($this->Auth->user('id'), $id)) { + throw new MethodNotAllowedException(__('Invalid Collection or insuficient privileges')); + } + $this->set('menuData', array('menuList' => 'collections', 'menuItem' => 'view')); + $params = [ + 'contain' => [ + 'Orgc', + 'Org', + 'User', + 'CollectionElement' + ], + 'afterFind' => function (array $collection){ + return $this->Collection->rearrangeCollection($collection); + } + ]; + $this->CRUD->view($id, $params); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + $this->set('id', $id); + $this->loadModel('Event'); + $this->set('distributionLevels', $this->Event->distributionLevels); + $this->render('view'); + } + + public function index($filter = null) + { + $this->set('menuData', array('menuList' => 'collections', 'menuItem' => 'index')); + $params = [ + 'filters' => ['Collection.uuid', 'Collection.type', 'Collection.name'], + 'quickFilters' => ['Collection.name'], + 'contain' => ['Orgc'], + 'afterFind' => function($collections) { + foreach ($collections as $k => $collection) { + $collections[$k]['Collection']['element_count'] = $this->Collection->CollectionElement->find('count', [ + 'recursive' => -1, + 'conditions' => ['CollectionElement.collection_id' => $collection['Collection']['id']] + ]); + } + return $collections; + } + ]; + if ($filter === 'my_collections') { + $params['conditions']['Collection.user_id'] = $this->Auth->user('id'); + } + if ($filter === 'org_collections') { + $params['conditions']['Collection.orgc_id'] = $this->Auth->user('org_id'); + } + $this->loadModel('Event'); + $this->set('distributionLevels', $this->Event->distributionLevels); + $this->CRUD->index($params); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + } +} diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php index e86e3e642..675cc42a5 100644 --- a/app/Controller/Component/ACLComponent.php +++ b/app/Controller/Component/ACLComponent.php @@ -19,6 +19,25 @@ class ACLComponent extends Component 'queryACL' => array(), 'restSearch' => array('*'), ), + 'analystData' => [ + 'add' => ['AND' => ['perm_add', 'perm_analyst_data']], + 'delete' => ['AND' => ['perm_add', 'perm_analyst_data']], + 'edit' => ['AND' => ['perm_add', 'perm_analyst_data']], + 'filterAnalystDataForPush' => ['perm_sync'], + 'getChildren' => ['*'], + 'getRelatedElement' => ['*'], + 'index' => ['*'], + 'indexMinimal' => ['*'], + 'pushAnalystData' => ['perm_sync'], + 'view' => ['*'], + ], + 'analystDataBlocklists' => array( + 'add' => array(), + 'delete' => array(), + 'edit' => array(), + 'index' => array(), + 'massDelete' => array(), + ), 'api' => [ 'rest' => ['perm_auth'], 'viewDeprecatedFunctionUse' => [], @@ -89,6 +108,19 @@ class ACLComponent extends Component 'pull_sgs' => [], 'view' => [] ], + 'collections' => [ + 'add' => ['perm_modify'], + 'delete' => ['perm_modify'], + 'edit' => ['perm_modify'], + 'index' => ['*'], + 'view' => ['*'] + ], + 'collectionElements' => [ + 'add' => ['perm_modify'], + 'addElementToCollection' => ['perm_modify'], + 'delete' => ['perm_modify'], + 'index' => ['*'] + ], 'correlationExclusions' => [ 'add' => [], 'edit' => [], @@ -1079,6 +1111,27 @@ class ACLComponent extends Component return $cluster['GalaxyCluster']['orgc_id'] == $user['org_id']; } + /** + * Checks if user can modify given analyst data + * + * @param array $user + * @param array $analystData + * @return bool + */ + public function canEditAnalystData(array $user, array $analystData, $modelType): bool + { + if (!isset($analystData[$modelType])) { + throw new InvalidArgumentException('Passed object does not contain a(n) ' . $modelType); + } + if ($user['Role']['perm_site_admin']) { + return true; + } + if ($analystData[$modelType]['orgc_uuid'] == $user['Organisation']['uuid']) { + return true; + } + return false; + } + /** * Checks if user can publish given galaxy cluster * diff --git a/app/Controller/Component/CRUDComponent.php b/app/Controller/Component/CRUDComponent.php index 81ad2bfb1..04ee82c55 100644 --- a/app/Controller/Component/CRUDComponent.php +++ b/app/Controller/Component/CRUDComponent.php @@ -25,6 +25,7 @@ class CRUDComponent extends Component } $options['filters'][] = 'quickFilter'; } + $this->Controller->{$this->Controller->modelClass}->includeAnalystData = true; $params = $this->Controller->IndexFilter->harvestParameters(empty($options['filters']) ? [] : $options['filters']); $query = []; $query = $this->setFilters($params, $query); @@ -39,6 +40,7 @@ class CRUDComponent extends Component if (!empty($this->Controller->paginate['fields'])) { $query['fields'] = $this->Controller->paginate['fields']; } + $query['includeAnalystData'] = true; $data = $this->Controller->{$this->Controller->modelClass}->find('all', $query); if (isset($options['afterFind'])) { if (is_callable($options['afterFind'])) { @@ -49,6 +51,7 @@ class CRUDComponent extends Component } $this->Controller->restResponsePayload = $this->Controller->RestResponse->viewData($data, 'json'); } else { + $query['includeAnalystData'] = true; $this->Controller->paginate = $query; $data = $this->Controller->paginate(); if (isset($options['afterFind'])) { @@ -93,7 +96,7 @@ class CRUDComponent extends Component $savedData = $model->save($data); if ($savedData) { if (isset($params['afterSave'])) { - $params['afterSave']($data); + $params['afterSave']($savedData); } $data = $model->find('first', [ 'recursive' => -1, @@ -200,7 +203,7 @@ class CRUDComponent extends Component if (isset($params['beforeSave'])) { $data = $params['beforeSave']($data); } - if ($model->save($data)) { + if ($data = $model->save($data)) { if (isset($params['afterSave'])) { $params['afterSave']($data); } @@ -231,6 +234,8 @@ class CRUDComponent extends Component if (empty($id)) { throw new NotFoundException(__('Invalid %s.', $modelName)); } + $this->Controller->{$modelName}->includeAnalystData = true; + $this->Controller->{$modelName}->includeAnalystDataRecursive = true; $query = [ 'recursive' => -1, 'conditions' => [$modelName . '.id' => $id], @@ -297,6 +302,9 @@ class CRUDComponent extends Component $result = $this->Controller->{$modelName}->delete($id); } if ($result) { + if (isset($params['afterDelete']) && is_callable($params['afterDelete'])) { + $params['afterDelete']($data); + } $message = __('%s deleted.', $modelName); if ($this->Controller->IndexFilter->isRest()) { $this->Controller->restResponsePayload = $this->Controller->RestResponse->saveSuccessResponse($modelName, 'delete', $id, 'json', $message); @@ -336,7 +344,9 @@ class CRUDComponent extends Component if ($filter === 'quickFilter') { continue; } - if (strlen(trim($filterValue, '%')) === strlen($filterValue)) { + if (is_array($filterValue)) { + $query['conditions']['AND'][] = [$filter => $filterValue]; + } else if (strlen(trim($filterValue, '%')) === strlen($filterValue)) { $query['conditions']['AND'][] = [$filter => $filterValue]; } else { $query['conditions']['AND'][] = [$filter . ' LIKE' => $filterValue]; diff --git a/app/Controller/Component/RestResponseComponent.php b/app/Controller/Component/RestResponseComponent.php index 96f4d36ef..0d9863d70 100644 --- a/app/Controller/Component/RestResponseComponent.php +++ b/app/Controller/Component/RestResponseComponent.php @@ -213,6 +213,7 @@ class RestResponseComponent extends Component 'perm_tag_editor', 'default_role', 'perm_sighting', + 'perm_analyst_data', 'permission' ) ), @@ -234,6 +235,7 @@ class RestResponseComponent extends Component 'perm_tag_editor', 'default_role', 'perm_sighting', + 'perm_analyst_data', 'permission' ) ) @@ -1557,6 +1559,11 @@ class RestResponseComponent extends Component 'type' => 'integer', 'values' => array(1 => 'True', 0 => 'False') ), + 'perm_analyst_data' => array( + 'input' => 'radio', + 'type' => 'integer', + 'values' => array(1 => 'True', 0 => 'False') + ), 'permission' => array( 'input' => 'select', 'type' => 'string', diff --git a/app/Controller/Component/RestSearchComponent.php b/app/Controller/Component/RestSearchComponent.php index c9a4aa451..ae8a4a34e 100644 --- a/app/Controller/Component/RestSearchComponent.php +++ b/app/Controller/Component/RestSearchComponent.php @@ -124,6 +124,7 @@ class RestSearchComponent extends Component 'extended', 'extensionList', 'excludeGalaxy', + 'includeAnalystData', 'includeRelatedTags', 'includeDecayScore', 'includeScoresOnEvent', diff --git a/app/Controller/EventReportsController.php b/app/Controller/EventReportsController.php index e73b1fd50..04199f2b5 100644 --- a/app/Controller/EventReportsController.php +++ b/app/Controller/EventReportsController.php @@ -61,6 +61,8 @@ class EventReportsController extends AppController public function view($reportId, $ajax=false) { + $this->EventReport->includeAnalystData = true; + $this->EventReport->includeAnalystDataRecursive = true; $report = $this->EventReport->simpleFetchById($this->Auth->user(), $reportId); if ($this->_isRest()) { return $this->RestResponse->viewData($report, $this->response->type()); @@ -175,6 +177,7 @@ class EventReportsController extends AppController $filters = $this->IndexFilter->harvestParameters(['event_id', 'value', 'context', 'index_for_event', 'extended_event']); $filters['embedded_view'] = $this->request->is('ajax'); $compiledConditions = $this->__generateIndexConditions($filters); + $this->EventReport->includeAnalystData = true; if ($this->_isRest()) { $reports = $this->EventReport->find('all', [ 'recursive' => -1, @@ -515,6 +518,7 @@ class EventReportsController extends AppController { $distributionLevels = $this->EventReport->Event->Attribute->distributionLevels; $this->set('distributionLevels', $distributionLevels); + $this->set('shortDist', $this->EventReport->Event->Attribute->shortDist); $this->set('initialDistribution', $this->EventReport->Event->Attribute->defaultDistribution()); } diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 7c87891fb..c65a98721 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -755,7 +755,8 @@ class EventsController extends AppController if ($nothing) { $this->paginate['conditions']['AND'][] = ['Event.id' => -1]; // do not fetch any event } - + $this->Event->includeAnalystData = true; + $this->paginate['includeAnalystData'] = true; $events = $this->paginate(); if (count($events) === 1 && isset($this->passedArgs['searchall'])) { @@ -811,6 +812,7 @@ class EventsController extends AppController $rules = [ 'contain' => ['EventTag'], 'fields' => array_keys($fieldNames), + 'includeAnalystData' => isset($passedArgs['includeAnalystData']) ? $passedArgs['includeAnalystData'] : true, ]; } if (isset($passedArgs['sort']) && isset($fieldNames[$passedArgs['sort']])) { @@ -1227,6 +1229,9 @@ class EventsController extends AppController } } + $this->Event->Attribute->includeAnalystData = true; + $this->Event->Attribute->includeAnalystDataRecursive = true; + if (isset($filters['focus'])) { $this->set('focus', $filters['focus']); } @@ -1691,7 +1696,7 @@ class EventsController extends AppController } $namedParams = $this->request->params['named']; - + $conditions['includeAnalystData'] = true; if ($this->_isRest()) { $conditions['includeAttachments'] = isset($namedParams['includeAttachments']) ? $namedParams['includeAttachments'] : true; } else { @@ -1786,7 +1791,6 @@ class EventsController extends AppController } else { $user = $this->Auth->user(); } - $results = $this->Event->fetchEvent($user, $conditions); if (empty($results)) { throw new NotFoundException(__('Invalid event')); @@ -2692,7 +2696,7 @@ class EventsController extends AppController $this->request->data = $this->request->data['Event']; } $eventToSave = $event; - $capturedObjects = ['Attribute', 'Object', 'Tag', 'Galaxy', 'EventReport']; + $capturedObjects = ['Attribute', 'Object', 'Tag', 'Galaxy', 'EventReport', 'Note', 'Opinion', 'Relationship',]; foreach ($capturedObjects as $objectType) { if (!empty($this->request->data[$objectType])) { if (!empty($regenerateUUIDs)) { @@ -4373,12 +4377,12 @@ class EventsController extends AppController $id = $event['Event']['id']; $exports = array( 'json' => array( - 'url' => $this->baseurl . '/events/restSearch/json/eventid:' . $id . '.json', + 'url' => $this->baseurl . '/events/restSearch/json/includeAnalystData:1/eventid:' . $id . '.json', 'text' => __('MISP JSON (metadata + all attributes)'), 'requiresPublished' => false, 'checkbox' => true, 'checkbox_text' => __('Encode Attachments'), - 'checkbox_set' => $this->baseurl . '/events/restSearch/json/withAttachments:1/eventid:' . $id . '.json', + 'checkbox_set' => $this->baseurl . '/events/restSearch/json/withAttachments:1/includeAnalystData:1/eventid:' . $id . '.json', 'checkbox_default' => true, ), 'xml' => array( diff --git a/app/Controller/GalaxyClustersController.php b/app/Controller/GalaxyClustersController.php index eac9803e5..bc18807e0 100644 --- a/app/Controller/GalaxyClustersController.php +++ b/app/Controller/GalaxyClustersController.php @@ -173,6 +173,8 @@ class GalaxyClustersController extends AppController */ public function view($id) { + $this->GalaxyCluster->includeAnalystData = true; + $this->GalaxyCluster->includeAnalystDataRecursive = true; $cluster = $this->GalaxyCluster->fetchIfAuthorized($this->Auth->user(), $id, 'view', $throwErrors=true, $full=true); $tag = $this->GalaxyCluster->Tag->find('first', array( 'conditions' => array( @@ -208,6 +210,7 @@ class GalaxyClustersController extends AppController $this->loadModel('Attribute'); $distributionLevels = $this->Attribute->distributionLevels; $this->set('distributionLevels', $distributionLevels); + $this->set('shortDist', $this->Attribute->shortDist); if (!$cluster['GalaxyCluster']['default'] && !$cluster['GalaxyCluster']['published'] && $cluster['GalaxyCluster']['orgc_id'] == $this->Auth->user()['org_id']) { $this->Flash->warning(__('This cluster is not published. Users will not be able to use it')); } diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index d540e005a..1355d39c7 100644 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -529,7 +529,7 @@ class ServersController extends AppController if (!$fail) { // say what fields are to be updated - $fieldList = array('id', 'url', 'push', 'pull', 'push_sightings', 'push_galaxy_clusters', 'pull_galaxy_clusters', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'remove_missing_tags', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy'); + $fieldList = array('id', 'url', 'push', 'pull', 'push_sightings', 'push_galaxy_clusters', 'pull_galaxy_clusters', 'push_analyst_data', 'pull_analyst_data', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'remove_missing_tags', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy'); $this->request->data['Server']['id'] = $id; if (isset($this->request->data['Server']['authkey']) && "" != $this->request->data['Server']['authkey']) { $fieldList[] = 'authkey'; @@ -776,7 +776,7 @@ class ServersController extends AppController if (!Configure::read('MISP.background_jobs')) { $result = $this->Server->pull($this->Auth->user(), $technique, $s); if (is_array($result)) { - $success = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4]); + $success = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled, %s analyst data pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4], $result[5]); } else { $error = $result; } @@ -784,6 +784,7 @@ class ServersController extends AppController $this->set('fails', $result[1]); $this->set('pulledProposals', $result[2]); $this->set('pulledSightings', $result[3]); + $this->set('pulledAnalystData', $result[5]); } else { $this->loadModel('Job'); $jobId = $this->Job->createJob( @@ -1889,6 +1890,7 @@ class ServersController extends AppController 'perm_sync' => (bool) $user['Role']['perm_sync'], 'perm_sighting' => (bool) $user['Role']['perm_sighting'], 'perm_galaxy_editor' => (bool) $user['Role']['perm_galaxy_editor'], + 'perm_analyst_data' => (bool) $user['Role']['perm_analyst_data'], 'uuid' => $user['Role']['perm_sync'] ? Configure::read('MISP.uuid') : '-', 'request_encoding' => $this->CompressedRequestHandler->supportedEncodings(), 'filter_sightings' => true, // check if Sightings::filterSightingUuidsForPush method is supported diff --git a/app/Lib/Dashboard/MispAdminSyncTestWidget.php b/app/Lib/Dashboard/MispAdminSyncTestWidget.php index 17493237e..e2c9e4347 100644 --- a/app/Lib/Dashboard/MispAdminSyncTestWidget.php +++ b/app/Lib/Dashboard/MispAdminSyncTestWidget.php @@ -37,6 +37,10 @@ class MispAdminSyncTestWidget $colour = 'orange'; $message .= ' ' . __('No sighting access.'); } + if (empty($result['info']['perm_analyst_data'])) { + $colour = 'orange'; + $message .= ' ' . __('No analyst data sync access.'); + } } else { $colour = 'red'; $message = $syncTestErrorCodes[$result['status']]; diff --git a/app/Lib/Tools/JSONConverterTool.php b/app/Lib/Tools/JSONConverterTool.php index 04666aee1..eabf5c476 100644 --- a/app/Lib/Tools/JSONConverterTool.php +++ b/app/Lib/Tools/JSONConverterTool.php @@ -21,7 +21,7 @@ class JSONConverterTool public static function convertObject($object, $isSiteAdmin = false, $raw = false) { - $toRearrange = array('SharingGroup', 'Attribute', 'ShadowAttribute', 'Event', 'CryptographicKey'); + $toRearrange = array('SharingGroup', 'Attribute', 'ShadowAttribute', 'Event', 'CryptographicKey', 'Note', 'Opinion', 'Relationship'); foreach ($toRearrange as $element) { if (isset($object[$element])) { $object['Object'][$element] = $object[$element]; @@ -40,7 +40,7 @@ class JSONConverterTool public static function convert($event, $isSiteAdmin=false, $raw = false) { - $toRearrange = array('Org', 'Orgc', 'SharingGroup', 'Attribute', 'ShadowAttribute', 'RelatedAttribute', 'RelatedEvent', 'Galaxy', 'Object', 'EventReport', 'CryptographicKey'); + $toRearrange = array('Org', 'Orgc', 'SharingGroup', 'Attribute', 'ShadowAttribute', 'RelatedAttribute', 'RelatedEvent', 'Galaxy', 'Object', 'EventReport', 'CryptographicKey', 'Note', 'Opinion', 'Relationship'); foreach ($toRearrange as $object) { if (isset($event[$object])) { $event['Event'][$object] = $event[$object]; @@ -69,6 +69,16 @@ class JSONConverterTool } } + if (isset($event['Event']['Note'])) { + $event['Event']['Note'] = self::__cleanAnalystData($event['Event']['Note']); + } + if (isset($event['Event']['Opinion'])) { + $event['Event']['Opinion'] = self::__cleanAnalystData($event['Event']['Opinion']); + } + if (isset($event['Event']['Relationship'])) { + $event['Event']['Relationship'] = self::__cleanAnalystData($event['Event']['Relationship']); + } + // cleanup the array from things we do not want to expose $tempSightings = array(); if (!empty($event['Sighting'])) { @@ -209,6 +219,17 @@ class JSONConverterTool return $objects; } + private function __cleanAnalystData($data) + { + foreach ($data as $k => $entry) { + if (empty($entry['SharingGroup'])) { + unset($data[$k]['SharingGroup']); + } + } + $data = array_values($data); + return $data; + } + public static function arrayPrinter($array, $root = true) { if (is_array($array)) { diff --git a/app/Lib/Tools/LanguageRFC5646Tool.php b/app/Lib/Tools/LanguageRFC5646Tool.php new file mode 100644 index 000000000..63c482446 --- /dev/null +++ b/app/Lib/Tools/LanguageRFC5646Tool.php @@ -0,0 +1,245 @@ + 'Afrikaans', + 'af-ZA' => 'Afrikaans (South Africa)', + 'ar' => 'Arabic', + 'ar-AE' => 'Arabic (U.A.E.)', + 'ar-BH' => 'Arabic (Bahrain)', + 'ar-DZ' => 'Arabic (Algeria)', + 'ar-EG' => 'Arabic (Egypt)', + 'ar-IQ' => 'Arabic (Iraq)', + 'ar-JO' => 'Arabic (Jordan)', + 'ar-KW' => 'Arabic (Kuwait)', + 'ar-LB' => 'Arabic (Lebanon)', + 'ar-LY' => 'Arabic (Libya)', + 'ar-MA' => 'Arabic (Morocco)', + 'ar-OM' => 'Arabic (Oman)', + 'ar-QA' => 'Arabic (Qatar)', + 'ar-SA' => 'Arabic (Saudi Arabia)', + 'ar-SY' => 'Arabic (Syria)', + 'ar-TN' => 'Arabic (Tunisia)', + 'ar-YE' => 'Arabic (Yemen)', + 'az' => 'Azeri (Latin)', + 'az-AZ' => 'Azeri (Latin) (Azerbaijan)', + 'az-Cyrl-AZ' => 'Azeri (Cyrillic) (Azerbaijan)', + 'be' => 'Belarusian', + 'be-BY' => 'Belarusian (Belarus)', + 'bg' => 'Bulgarian', + 'bg-BG' => 'Bulgarian (Bulgaria)', + 'bs-BA' => 'Bosnian (Bosnia and Herzegovina)', + 'ca' => 'Catalan', + 'ca-ES' => 'Catalan (Spain)', + 'cs' => 'Czech', + 'cs-CZ' => 'Czech (Czech Republic)', + 'cy' => 'Welsh', + 'cy-GB' => 'Welsh (United Kingdom)', + 'da' => 'Danish', + 'da-DK' => 'Danish (Denmark)', + 'de' => 'German', + 'de-AT' => 'German (Austria)', + 'de-CH' => 'German (Switzerland)', + 'de-DE' => 'German (Germany)', + 'de-LI' => 'German (Liechtenstein)', + 'de-LU' => 'German (Luxembourg)', + 'dv' => 'Divehi', + 'dv-MV' => 'Divehi (Maldives)', + 'el' => 'Greek', + 'el-GR' => 'Greek (Greece)', + 'en' => 'English', + 'en-AU' => 'English (Australia)', + 'en-BZ' => 'English (Belize)', + 'en-CA' => 'English (Canada)', + 'en-CB' => 'English (Caribbean)', + 'en-GB' => 'English (United Kingdom)', + 'en-IE' => 'English (Ireland)', + 'en-JM' => 'English (Jamaica)', + 'en-NZ' => 'English (New Zealand)', + 'en-PH' => 'English (Republic of the Philippines)', + 'en-TT' => 'English (Trinidad and Tobago)', + 'en-US' => 'English (United States)', + 'en-ZA' => 'English (South Africa)', + 'en-ZW' => 'English (Zimbabwe)', + 'eo' => 'Esperanto', + 'es' => 'Spanish', + 'es-AR' => 'Spanish (Argentina)', + 'es-BO' => 'Spanish (Bolivia)', + 'es-CL' => 'Spanish (Chile)', + 'es-CO' => 'Spanish (Colombia)', + 'es-CR' => 'Spanish (Costa Rica)', + 'es-DO' => 'Spanish (Dominican Republic)', + 'es-EC' => 'Spanish (Ecuador)', + 'es-ES' => 'Spanish (Spain)', + 'es-GT' => 'Spanish (Guatemala)', + 'es-HN' => 'Spanish (Honduras)', + 'es-MX' => 'Spanish (Mexico)', + 'es-NI' => 'Spanish (Nicaragua)', + 'es-PA' => 'Spanish (Panama)', + 'es-PE' => 'Spanish (Peru)', + 'es-PR' => 'Spanish (Puerto Rico)', + 'es-PY' => 'Spanish (Paraguay)', + 'es-SV' => 'Spanish (El Salvador)', + 'es-UY' => 'Spanish (Uruguay)', + 'es-VE' => 'Spanish (Venezuela)', + 'et' => 'Estonian', + 'et-EE' => 'Estonian (Estonia)', + 'eu' => 'Basque', + 'eu-ES' => 'Basque (Spain)', + 'fa' => 'Farsi', + 'fa-IR' => 'Farsi (Iran)', + 'fi' => 'Finnish', + 'fi-FI' => 'Finnish (Finland)', + 'fo' => 'Faroese', + 'fo-FO' => 'Faroese (Faroe Islands)', + 'fr' => 'French', + 'fr-BE' => 'French (Belgium)', + 'fr-CA' => 'French (Canada)', + 'fr-CH' => 'French (Switzerland)', + 'fr-FR' => 'French (France)', + 'fr-LU' => 'French (Luxembourg)', + 'fr-MC' => 'French (Principality of Monaco)', + 'gl' => 'Galician', + 'gl-ES' => 'Galician (Spain)', + 'gu' => 'Gujarati', + 'gu-IN' => 'Gujarati (India)', + 'he' => 'Hebrew', + 'he-IL' => 'Hebrew (Israel)', + 'hi' => 'Hindi', + 'hi-IN' => 'Hindi (India)', + 'hr' => 'Croatian', + 'hr-BA' => 'Croatian (Bosnia and Herzegovina)', + 'hr-HR' => 'Croatian (Croatia)', + 'hu' => 'Hungarian', + 'hu-HU' => 'Hungarian (Hungary)', + 'hy' => 'Armenian', + 'hy-AM' => 'Armenian (Armenia)', + 'id' => 'Indonesian', + 'id-ID' => 'Indonesian (Indonesia)', + 'is' => 'Icelandic', + 'is-IS' => 'Icelandic (Iceland)', + 'it' => 'Italian', + 'it-CH' => 'Italian (Switzerland)', + 'it-IT' => 'Italian (Italy)', + 'ja' => 'Japanese', + 'ja-JP' => 'Japanese (Japan)', + 'ka' => 'Georgian', + 'ka-GE' => 'Georgian (Georgia)', + 'kk' => 'Kazakh', + 'kk-KZ' => 'Kazakh (Kazakhstan)', + 'kn' => 'Kannada', + 'kn-IN' => 'Kannada (India)', + 'ko' => 'Korean', + 'ko-KR' => 'Korean (Korea)', + 'kok' => 'Konkani', + 'kok-IN' => 'Konkani (India)', + 'ky' => 'Kyrgyz', + 'ky-KG' => 'Kyrgyz (Kyrgyzstan)', + 'lt' => 'Lithuanian', + 'lt-LT' => 'Lithuanian (Lithuania)', + 'lv' => 'Latvian', + 'lv-LV' => 'Latvian (Latvia)', + 'mi' => 'Maori', + 'mi-NZ' => 'Maori (New Zealand)', + 'mk' => 'FYRO Macedonian', + 'mk-MK' => 'FYRO Macedonian (Former Yugoslav Republic of Macedonia)', + 'mn' => 'Mongolian', + 'mn-MN' => 'Mongolian (Mongolia)', + 'mr' => 'Marathi', + 'mr-IN' => 'Marathi (India)', + 'ms' => 'Malay', + 'ms-BN' => 'Malay (Brunei Darussalam)', + 'ms-MY' => 'Malay (Malaysia)', + 'mt' => 'Maltese', + 'mt-MT' => 'Maltese (Malta)', + 'nb' => 'Norwegian (Bokm?l)', + 'nb-NO' => 'Norwegian (Bokm?l) (Norway)', + 'nl' => 'Dutch', + 'nl-BE' => 'Dutch (Belgium)', + 'nl-NL' => 'Dutch (Netherlands)', + 'nn-NO' => 'Norwegian (Nynorsk) (Norway)', + 'ns' => 'Northern Sotho', + 'ns-ZA' => 'Northern Sotho (South Africa)', + 'pa' => 'Punjabi', + 'pa-IN' => 'Punjabi (India)', + 'pl' => 'Polish', + 'pl-PL' => 'Polish (Poland)', + 'ps' => 'Pashto', + 'ps-AR' => 'Pashto (Afghanistan)', + 'pt' => 'Portuguese', + 'pt-BR' => 'Portuguese (Brazil)', + 'pt-PT' => 'Portuguese (Portugal)', + 'qu' => 'Quechua', + 'qu-BO' => 'Quechua (Bolivia)', + 'qu-EC' => 'Quechua (Ecuador)', + 'qu-PE' => 'Quechua (Peru)', + 'ro' => 'Romanian', + 'ro-RO' => 'Romanian (Romania)', + 'ru' => 'Russian', + 'ru-RU' => 'Russian (Russia)', + 'sa' => 'Sanskrit', + 'sa-IN' => 'Sanskrit (India)', + 'se' => 'Sami', + 'se-FI' => 'Sami (Finland)', + 'se-NO' => 'Sami (Norway)', + 'se-SE' => 'Sami (Sweden)', + 'sk' => 'Slovak', + 'sk-SK' => 'Slovak (Slovakia)', + 'sl' => 'Slovenian', + 'sl-SI' => 'Slovenian (Slovenia)', + 'sq' => 'Albanian', + 'sq-AL' => 'Albanian (Albania)', + 'sr-BA' => 'Serbian (Latin) (Bosnia and Herzegovina)', + 'sr-Cyrl-BA' => 'Serbian (Cyrillic) (Bosnia and Herzegovina)', + 'sr-SP' => 'Serbian (Latin) (Serbia and Montenegro)', + 'sr-Cyrl-SP' => 'Serbian (Cyrillic) (Serbia and Montenegro)', + 'sv' => 'Swedish', + 'sv-FI' => 'Swedish (Finland)', + 'sv-SE' => 'Swedish (Sweden)', + 'sw' => 'Swahili', + 'sw-KE' => 'Swahili (Kenya)', + 'syr' => 'Syriac', + 'syr-SY' => 'Syriac (Syria)', + 'ta' => 'Tamil', + 'ta-IN' => 'Tamil (India)', + 'te' => 'Telugu', + 'te-IN' => 'Telugu (India)', + 'th' => 'Thai', + 'th-TH' => 'Thai (Thailand)', + 'tl' => 'Tagalog', + 'tl-PH' => 'Tagalog (Philippines)', + 'tn' => 'Tswana', + 'tn-ZA' => 'Tswana (South Africa)', + 'tr' => 'Turkish', + 'tr-TR' => 'Turkish (Turkey)', + 'tt' => 'Tatar', + 'tt-RU' => 'Tatar (Russia)', + 'ts' => 'Tsonga', + 'uk' => 'Ukrainian', + 'uk-UA' => 'Ukrainian (Ukraine)', + 'ur' => 'Urdu', + 'ur-PK' => 'Urdu (Islamic Republic of Pakistan)', + 'uz' => 'Uzbek (Latin)', + 'uz-UZ' => 'Uzbek (Latin) (Uzbekistan)', + 'uz-Cyrl-UZ' => 'Uzbek (Cyrillic) (Uzbekistan)', + 'vi' => 'Vietnamese', + 'vi-VN' => 'Vietnamese (Viet Nam)', + 'xh' => 'Xhosa', + 'xh-ZA' => 'Xhosa (South Africa)', + 'zh' => 'Chinese', + 'zh-CN' => 'Chinese (S)', + 'zh-HK' => 'Chinese (Hong Kong)', + 'zh-MO' => 'Chinese (Macau)', + 'zh-SG' => 'Chinese (Singapore)', + 'zh-TW' => 'Chinese (T)', + 'zu' => 'Zulu', + 'zu-ZA' => 'Zulu (South Africa)' + ]; + + public static function getLanguages() + { + return self::RFC5646_LANGUAGE; + } +} diff --git a/app/Lib/Tools/ServerSyncTool.php b/app/Lib/Tools/ServerSyncTool.php index b22ad08b4..ce8088f24 100644 --- a/app/Lib/Tools/ServerSyncTool.php +++ b/app/Lib/Tools/ServerSyncTool.php @@ -14,6 +14,7 @@ class ServerSyncTool FEATURE_EDIT_OF_GALAXY_CLUSTER = 'edit_of_galaxy_cluster', PERM_SYNC = 'perm_sync', PERM_GALAXY_EDITOR = 'perm_galaxy_editor', + PERM_ANALYST_DATA = 'perm_analyst_data', FEATURE_SIGHTING_REST_SEARCH = 'sighting_rest'; /** @var array */ @@ -215,6 +216,71 @@ class ServerSyncTool return $this->post('/galaxies/pushCluster', [$cluster], $logMessage); } + /** + * @param array $rules + * @return HttpSocketResponseExtended + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + */ + public function filterAnalystDataForPush(array $candidates) + { + if (!$this->isSupported(self::PERM_ANALYST_DATA)) { + return []; + } + + return $this->post('/analyst_data/filterAnalystDataForPush', $candidates); + } + + /** + * @param array $rules + * @return HttpSocketResponseExtended + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + */ + public function fetchIndexMinimal(array $rules) + { + if (!$this->isSupported(self::PERM_ANALYST_DATA)) { + return []; + } + + return $this->post('/analyst_data/indexMinimal', $rules); + } + + /** + * @throws HttpSocketJsonException + * @throws HttpSocketHttpException + */ + public function fetchAnalystData($type, array $uuids) + { + if (!$this->isSupported(self::PERM_ANALYST_DATA)) { + return []; + } + + $params = [ + 'uuid' => $uuids, + ]; + + $url = '/analyst_data/index/' . $type; + $url .= $this->createParams($params); + $url .= '.json'; + return $this->get($url); + + // $response = $this->post('/analyst_data/restSearch' , $params); + // return $response->json(); + } + + /** + * @param array $analystData + * @return HttpSocketResponseExtended + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + */ + public function pushAnalystData($type, array $analystData) + { + $logMessage = "Pushing Analyst Data #{$analystData[$type]['uuid']} to Server #{$this->serverId()}"; + return $this->post('/analyst_data/pushAnalystData', $analystData, $logMessage); + } + /** * @param array $params * @return HttpSocketResponseExtended @@ -414,6 +480,8 @@ class ServerSyncTool return isset($info['perm_sync']) && $info['perm_sync']; case self::PERM_GALAXY_EDITOR: return isset($info['perm_galaxy_editor']) && $info['perm_galaxy_editor']; + case self::PERM_ANALYST_DATA: + return isset($info['perm_analyst_data']) && $info['perm_analyst_data']; case self::FEATURE_SIGHTING_REST_SEARCH: $version = explode('.', $info['version']); return $version[0] == 2 && $version[1] == 4 && $version[2] > 164; diff --git a/app/Model/AnalystData.php b/app/Model/AnalystData.php new file mode 100644 index 000000000..f7023afe4 --- /dev/null +++ b/app/Model/AnalystData.php @@ -0,0 +1,1028 @@ + [ + 'className' => 'SharingGroup', + 'foreignKey' => 'sharing_group_id' + ], + ]; + + public function __construct($id = false, $table = null, $ds = null) + { + parent::__construct($id, $table, $ds); + $this->bindModel([ + 'belongsTo' => [ + 'Org' => [ + 'className' => 'Organisation', + 'foreignKey' => false, + 'conditions' => [ + sprintf('%s.org_uuid = Org.uuid', $this->alias) + ], + ], + 'Orgc' => [ + 'className' => 'Organisation', + 'foreignKey' => false, + 'conditions' => [ + sprintf('%s.orgc_uuid = Orgc.uuid', $this->alias) + ], + ], + 'SharingGroup' => [ + 'className' => 'SharingGroup', + 'foreignKey' => false, + 'conditions' => [ + sprintf('%s.sharing_group_id = SharingGroup.id', $this->alias) + ], + ], + ] + ]); + $this->Org = ClassRegistry::init('Organisation'); + $this->Orgc = ClassRegistry::init('Organisation'); + } + + public function afterFind($results, $primary = false) + { + parent::afterFind($results, $primary); + + $this->setUser(); + + foreach ($results as $i => $v) { + $results[$i][$this->alias]['note_type'] = $this->current_type_id; + $results[$i][$this->alias]['note_type_name'] = $this->current_type; + + $results[$i] = $this->rearrangeOrganisation($results[$i], $this->current_user); + $results[$i] = $this->rearrangeSharingGroup($results[$i], $this->current_user); + + $results[$i][$this->alias]['_canEdit'] = $this->canEditAnalystData($this->current_user, $v, $this->alias); + + if (!empty($this->fetchRecursive) && !empty($results[$i][$this->alias]['uuid'])) { + $this->Note = ClassRegistry::init('Note'); + $this->Opinion = ClassRegistry::init('Opinion'); + $this->Note->fetchRecursive = false; + $this->Opinion->fetchRecursive = false; + $results[$i][$this->alias] = $this->fetchChildNotesAndOpinions($this->current_user, $results[$i][$this->alias]); + $this->Note->fetchRecursive = true; + $this->Opinion->fetchRecursive = true; + } + } + return $results; + } + + public function beforeValidate($options = array()) + { + parent::beforeValidate(); + if (empty($this->id) && empty($this->data[$this->current_type]['uuid'])) { + $this->data[$this->current_type]['uuid'] = CakeText::uuid(); + } + if (empty($this->id)) { + if (empty($this->data[$this->current_type]['orgc_uuid']) || empty($this->current_user['Role']['perm_sync'])) { + $this->data[$this->current_type]['orgc_uuid'] = $this->current_user['Organisation']['uuid']; + } + $this->data[$this->current_type]['org_uuid'] = $this->current_user['Organisation']['uuid']; + if (empty($this->data[$this->current_type]['authors'])) { + $this->data[$this->current_type]['authors'] = $this->current_user['email']; + } + } + return true; + } + + /** + * Checks if user can modify given analyst data + * + * @param array $user + * @param array $analystData + * @return bool + */ + public function canEditAnalystData(array $user, array $analystData, $modelType): bool + { + if (!isset($analystData[$modelType])) { + return false; // This can happen when using find('count') + } + if ($user['Role']['perm_site_admin']) { + return true; + } + if (isset($analystData[$modelType]['orgc_uuid']) && $analystData[$modelType]['orgc_uuid'] == $user['Organisation']['uuid']) { + return true; + } + return false; + } + + public function buildConditions(array $user): array + { + $conditions = []; + if (!$user['Role']['perm_site_admin']) { + $sgids = $this->SharingGroup->authorizedIds($user); + $alias = $this->alias; + $prefix = $alias != 'AnalystData' ? "{$alias}." : ''; + $conditions['AND']['OR'] = [ + "{$prefix}org_uuid" => $user['Organisation']['uuid'], + [ + 'AND' => [ + "{$prefix}distribution >" => 0, + "{$prefix}distribution <" => 4 + ], + ], + [ + 'AND' => [ + "{$prefix}sharing_group_id" => $sgids, + "{$prefix}distribution" => 4 + ] + ] + ]; + } + return $conditions; + } + + protected function setUser() + { + if (empty($this->current_user)) { + $user_id = Configure::read('CurrentUserId'); + $this->User = ClassRegistry::init('User'); + if ($user_id) { + $this->current_user = $this->User->getAuthUser($user_id); + } + } + } + + private function rearrangeOrganisation(array $analystData): array + { + if (!empty($analystData[$this->alias]['orgc_uuid'])) { + if (!isset($analystData['Orgc'])) { + $analystData[$this->alias]['Orgc'] = $this->Orgc->find('first', ['conditions' => ['uuid' => $analystData[$this->alias]['orgc_uuid']]])['Organisation']; + } else { + $analystData[$this->alias]['Orgc'] = $analystData['Orgc']; + } + unset($analystData['Orgc']); + } + if (!empty($analystData[$this->alias]['org_uuid'])) { + if (!isset($analystData['Org'])) { + $analystData[$this->alias]['Org'] = $this->Org->find('first', ['conditions' => ['uuid' => $analystData[$this->alias]['org_uuid']]])['Organisation']; + } else { + $analystData[$this->alias]['Org'] = $analystData['Org']; + } + unset($analystData['Org']); + } + return $analystData; + } + + private function rearrangeSharingGroup(array $analystData, array $user): array + { + if (isset($analystData[$this->alias]['distribution'])) { + if ($analystData[$this->alias]['distribution'] == 4) { + if (!isset($analystData['SharingGroup'])) { + $this->SharingGroup = ClassRegistry::init('SharingGroup'); + $sg = $this->SharingGroup->fetchSG($analystData[$this->alias]['sharing_group_id'], $user, true); + $analystData[$this->alias]['SharingGroup'] = $sg['SharingGroup']; + } else { + $analystData[$this->alias]['SharingGroup'] = $analystData['SharingGroup']; + } + } else { + unset($analystData['SharingGroup']); + } + } + return $analystData; + } + + public function deduceType(string $uuid) + { + foreach ($this->valid_targets as $valid_target) { + $this->{$valid_target} = ClassRegistry::init($valid_target); + $result = $this->$valid_target->find('first', [ + 'conditions' => [$valid_target.'.uuid' => $uuid], + 'recursive' => -1 + ]); + if (!empty($result)) { + return $valid_target; + } + } + throw new NotFoundException(__('Invalid UUID')); + } + + public function getAnalystDataTypeFromUUID($uuid) + { + foreach (self::ANALYST_DATA_TYPES as $type) { + $this->{$type} = ClassRegistry::init($type); + $result = $this->{$type}->find('first', [ + 'conditions' => [$type.'.uuid' => $uuid], + 'recursive' => -1 + ]); + if (!empty($result)) { + return $type; + } + } + throw new NotFoundException(__('Invalid UUID')); + } + + public function deduceAnalystDataType(array $analystData) + { + if (!empty($analystData['note_type_name']) && in_array($analystData['note_type_name'], self::ANALYST_DATA_TYPES)) { + return $analystData['note_type_name']; + } + foreach (self::ANALYST_DATA_TYPES as $type) { + if (isset($analystData[$type])) { + return $type; + } + } + throw new NotFoundException(__('Invalid or could not deduce analyst data type')); + } + + public function getIDFromUUID($type, $id): int + { + $tmpForID = $this->find('first', [ + 'conditions' => [ + 'uuid' => $id, + ], + 'fields' => ['id', 'uuid',], + ]); + $id = -1; + if (!empty($tmpForID)) { + $id = $tmpForID[$type]['id']; + } + return $id; + } + + public function fetchSimple(array $user, $id): array + { + $conditions = [ + 'AND' => [ + $this->buildConditions($user) + ], + ]; + if (Validation::uuid($id)) { + $conditions[$this->alias . '.uuid'] = $id; + } else { + $conditions[$this->alias . '.id'] = $id; + } + return $this->find('first', [ + 'conditions' => $conditions, + 'contain' => ['Org', 'Orgc'], + ]); + } + + public function fetchChildNotesAndOpinions(array $user, array $analystData, $depth = 2): array + { + if ($depth == 0 || !empty($this->fetchedUUIDFromRecursion[$analystData['uuid']])) { + $hasMoreNotesOrOpinions = $this->hasMoreNotesOrOpinions($analystData, $user); + $analystData['_max_depth_reached'] = $hasMoreNotesOrOpinions; + return $analystData; + } + $this->fetchedUUIDFromRecursion[$analystData['uuid']] = true; + + $paramsNote = [ + 'recursive' => -1, + 'contain' => ['Org', 'Orgc'], + 'conditions' => [ + 'AND' => [ + $this->Note->buildConditions($user) + ], + 'object_type' => $analystData['note_type_name'], + 'object_uuid' => $analystData['uuid'], + ] + ]; + $paramsOpinion = [ + 'recursive' => -1, + 'contain' => ['Org', 'Orgc'], + 'conditions' => [ + 'AND' => [ + $this->Opinion->buildConditions($user) + ], + 'object_type' => $analystData['note_type_name'], + 'object_uuid' => $analystData['uuid'], + ] + ]; + + // recursively fetch and include nested notes and opinions + $childNotes = array_map(function ($item) use ($user, $depth) { + $expandedNotes = $this->fetchChildNotesAndOpinions($user, $item['Note'], $depth-1); + return $expandedNotes; + }, $this->Note->find('all', $paramsNote)); + $childOpinions = array_map(function ($item) use ($user, $depth) { + $expandedNotes = $this->fetchChildNotesAndOpinions($user, $item['Opinion'], $depth-1); + return $expandedNotes; + }, $this->Opinion->find('all', $paramsOpinion)); + + if (!empty($childNotes)) { + foreach ($childNotes as $childNote) { + $this->fetchedUUIDFromRecursion[$childNote['uuid']] = true; + } + $analystData['Note'] = $childNotes; + } + if (!empty($childOpinions)) { + foreach ($childNotes as $childNote) { + $this->fetchedUUIDFromRecursion[$childNote['uuid']] = true; + } + $analystData['Opinion'] = $childOpinions; + } + return $analystData; + } + + protected function hasMoreNotesOrOpinions($analystData, array $user): bool + { + $hasMoreNotes = $this->Note->find('first', [ + 'recursive' => -1, + 'conditions' => [ + 'AND' => [ + $this->Note->buildConditions($user) + ], + 'object_type' => $analystData['note_type_name'], + 'object_uuid' => $analystData['uuid'], + ] + ]); + if (!empty($hasMoreNotes)) { + return true; + } + $hasMoreOpinions = $this->Note->find('first', [ + 'recursive' => -1, + 'conditions' => [ + 'AND' => [ + $this->Opinion->buildConditions($user) + ], + 'object_type' => $analystData['note_type_name'], + 'object_uuid' => $analystData['uuid'], + ] + ]); + if (!empty($hasMoreOpinions)) { + return true; + } + return false; + } + + public function getExistingRelationships() + { + $existingRelationships = $this->find('column', [ + 'recursive' => -1, + 'fields' => ['relationship_type'], + 'unique' => true, + ]); + $this->ObjectRelationship = ClassRegistry::init('ObjectRelationship'); + $objectRelationships = $this->ObjectRelationship->find('column', [ + 'recursive' => -1, + 'fields' => ['name'], + 'unique' => true, + ]); + return array_unique(array_merge($existingRelationships, $objectRelationships)); + } + + public function getChildren($user, $uuid, $depth=2): array + { + $analystData = $this->fetchSimple($user, $uuid); + if (empty($analystData)) { + return []; + } + $analystData = $analystData[$this->alias]; + $this->Note = ClassRegistry::init('Note'); + $this->Opinion = ClassRegistry::init('Opinion'); + $analystData = $this->fetchChildNotesAndOpinions($user, $analystData, $depth); + return $analystData; + } + + /** + * Gets a cluster then save it. + * + * @param array $user + * @param array $analystData Analyst data to be saved + * @param bool $fromPull If the current capture is performed from a PULL sync + * @param int $orgId The organisation id that should own the analyst data + * @param array $server The server for which to capture is ongoing + * @return array Result of the capture including successes, fails and errors + */ + public function captureAnalystData(array $user, array $analystData, $fromPull=false, $orgUUId=false, $server=false): array + { + $this->Note = ClassRegistry::init('Note'); + $this->Opinion = ClassRegistry::init('Opinion'); + $this->Relationship = ClassRegistry::init('Relationship'); + $results = ['success' => false, 'imported' => 0, 'ignored' => 0, 'failed' => 0, 'errors' => []]; + $type = $this->deduceAnalystDataType($analystData); + if (!isset($analystData[$type])) { + $analystData = [$type => $analystData]; + } + $analystModel = ClassRegistry::init($type); + + if ($fromPull && !empty($orgUUId)) { + $analystData[$type]['org_uuid'] = $orgUUId; + } else { + $analystData[$type]['org_uuid'] = $user['Organisation']['uuid']; + } + + $this->AnalystDataBlocklist = ClassRegistry::init('AnalystDataBlocklist'); + if ($this->AnalystDataBlocklist->checkIfBlocked($analystData[$type]['uuid'])) { + $results['errors'][] = __('Blocked by blocklist'); + $results['ignored']++; + return $results; + } + + if (!isset($analystData[$type]['orgc_uuid']) && !isset($analystData[$type]['Orgc'])) { + $analystData[$type]['orgc_uuid'] = $analystData[$type]['org_uuid']; + } else { + if (!isset($analystData[$type]['Orgc'])) { + if (isset($analystData[$type]['orgc_uuid']) && $analystData[$type]['orgc_uuid'] != $user['Organisation']['uuid'] && !$user['Role']['perm_sync'] && !$user['Role']['perm_site_admin']) { + $analystData[$type]['orgc_uuid'] = $analystData[$type]['org_uuid']; // Only sync user can create analyst data on behalf of other users + } + } else { + if ($analystData[$type]['Orgc']['uuid'] != $user['Organisation']['uuid'] && !$user['Role']['perm_sync'] && !$user['Role']['perm_site_admin']) { + $analystData[$type]['orgc_uuid'] = $analystData[$type]['org_uuid']; // Only sync user can create analyst data on behalf of other users + } + } + if (isset($analystData[$type]['orgc_uuid']) && $analystData[$type]['orgc_uuid'] != $user['Organisation']['uuid'] && !$user['Role']['perm_sync'] && !$user['Role']['perm_site_admin']) { + $analystData[$type]['orgc_uuid'] = $analystData[$type]['org_uuid']; // Only sync user can create analyst data on behalf of other users + } + } + + if (!Configure::check('MISP.enableOrgBlocklisting') || Configure::read('MISP.enableOrgBlocklisting') !== false) { + $analystModel->OrgBlocklist = ClassRegistry::init('OrgBlocklist'); + $orgcUUID = $analystData[$type]['Orgc']['uuid']; + if ($analystData[$type]['orgc_uuid'] != 0 && $analystModel->OrgBlocklist->hasAny(array('OrgBlocklist.org_uuid' => $orgcUUID))) { + $results['errors'][] = __('Organisation blocklisted (%s)', $orgcUUID); + $results['ignored']++; + return $results; + } + } + + $analystData = $analystModel->captureOrganisationAndSG($analystData, $type, $user); + if (!isset($analystData[$type]['distribution'])) { + $analystData[$type]['distribution'] = Configure::read('MISP.default_event_distribution'); // use default event distribution + } + if ($analystData[$type]['distribution'] != 4) { + $analystData[$type]['sharing_group_id'] = null; + } + + // Start saving from the leaf since to make sure child elements get saved even if the parent should not be saved (or updated due to locked or timestamp) + foreach (self::ANALYST_DATA_TYPES as $childType) { + if (!empty($analystData[$type][$childType])) { + foreach ($analystData[$type][$childType] as $childAnalystData) { + $captureResult = $this->{$childType}->captureAnalystData($user, $childAnalystData, $fromPull, $orgUUId, $server); + $results['imported'] += $captureResult['imported']; + $results['ignored'] += $captureResult['ignored']; + $results['failed'] += $captureResult['failed']; + $results['errors'] = array_merge($results['errors'], $captureResult['errors']); + } + } + } + + $existingAnalystData = $analystModel->find('first', [ + 'conditions' => ["{$type}.uuid" => $analystData[$type]['uuid'],], + ]); + if (empty($existingAnalystData)) { + unset($analystData[$type]['id']); + $analystModel->create(); + $saveSuccess = $analystModel->save($analystData); + $saveSuccess = true; + } else { + if (!$existingAnalystData[$type]['locked'] && empty($server['Server']['internal'])) { + $results['errors'][] = __('Blocked an edit to an analyst data that was created locally. This can happen if a synchronised analyst data that was created on this instance was modified by an administrator on the remote side.'); + $results['failed']++; + return $results; + } + if ($analystData[$type]['modified'] > $existingAnalystData[$type]['modified']) { + $analystData[$type]['id'] = $existingAnalystData[$type]['id']; + $saveSuccess = $analystModel->save($analystData); + } else { + $results['errors'][] = __('Remote version is not newer than local one for analyst data (%s)', $analystData[$type]['uuid']); + $results['ignored']++; + return $results; + } + } + if ($saveSuccess) { + $results['imported']++; + } else { + $results['failed']++; + foreach ($analystModel->validationErrors as $validationError) { + $results['errors'][] = $validationError[0]; + } + } + $results['success'] = $results['imported'] > 0; + return $results; + } + + public function captureOrganisationAndSG($element, $model, $user) + { + $this->Event = ClassRegistry::init('Event'); + if (isset($element[$model]['distribution']) && $element[$model]['distribution'] == 4) { + $element[$model] = $this->Event->captureSGForElement($element[$model], $user); + } + // first we want to see how the creator organisation is encoded + // The options here are either by passing an organisation object along or simply passing a string along + if (isset($element[$model]['Orgc'])) { + $element[$model]['orgc_uuid'] = $this->Orgc->captureOrg($element[$model]['Orgc'], $user, false, true); + unset($element[$model]['Orgc']); + } else { + // Can't capture the Orgc, default to the current user + $element[$model]['orgc_uuid'] = $user['Organisation']['uuid']; + } + return $element; + } + + /** + * Push Analyst Data to remote server. Collect elligible data locally and propose the list to the remote. + * Remote will then return the list of UUIDs it's willing to get. Then, upload these entries. + * + * @param array $user + * @param ServerSyncTool $serverSync + * @return array + * @throws Exception + */ + public function push(array $user, ServerSyncTool $serverSync): array + { + $server = $serverSync->server(); + + if (!$server['Server']['push_analyst_data']) { + return []; + } + $this->Server = ClassRegistry::init('Server'); + $this->AnalystData = ClassRegistry::init('AnalystData'); + + $this->log("Starting Analyst Data sync with server #{$server['Server']['id']}", LOG_INFO); + + $analystData = $this->collectDataForPush($serverSync->server()); + $keyedAnalystData = []; + foreach ($analystData as $type => $entries) { + foreach ($entries as $entry) { + $entry = $entry[$type]; + $keyedAnalystData[$type][$entry['uuid']] = $entry['modified']; + } + } + if (empty($analystData)) { + return []; + } + + try { + $conditions = []; + foreach ($keyedAnalystData as $type => $entry) { + $conditions[$type] = array_keys($entry); + } + $analystDataToPush = $this->identifyUUIDsForPush($serverSync, $analystData, $conditions); + } catch (Exception $e) { + $this->logException("Could not get eligible Analyst Data IDs from server #{$server['Server']['id']} for push.", $e); + return []; + } + $successes = []; + foreach ($analystDataToPush as $type => $entries) { + foreach ($entries as $entry) { + $result = $this->uploadEntryToServer($type, $entry, $server, $serverSync, $user); + if ($result === 'Success') { + $successes[] = __('AnalystData %s', $entry[$type]['uuid']); + } + } + } + return $successes; + } + + /** + * Collect elligible data to be pushed on a server + * + * @param array $user + * @return array + */ + public function collectDataForPush(array $server): array + { + $sgIDs = $this->collectValidSharingGroupIDs($server); + $options = [ + 'recursive' => -1, + 'conditions' => [ + 'OR' => [ + [ + 'AND' => [ + ['distribution >' => 0], + ['distribution <' => 4], + ] + ], + [ + 'AND' => [ + 'distribution' => 4, + 'sharing_group_id' => $sgIDs, + ] + ], + ] + ], + ]; + $dataForPush = $this->getAllAnalystData('all', $options); + $this->Event = ClassRegistry::init('Event'); + foreach ($dataForPush as $type => $entries) { + foreach ($entries as $i => $analystData) { + if (!$this->Event->checkDistributionForPush($analystData, $server, $type)) { + unset($dataForPush[$type][$i]); + } + if (!$this->isPushableForServerSyncRules($analystData[$type], $server)) { + unset($dataForPush[$type][$i]); + } + } + } + return $dataForPush; + } + + private function collectValidSharingGroupIDs(array $server): array + { + $this->SharingGroup = ClassRegistry::init('SharingGroup'); + $sgs = $this->SharingGroup->find('all', [ + 'recursive' => -1, + 'contain' => ['Organisation', 'SharingGroupOrg' => ['Organisation'], 'SharingGroupServer'] + ]); + $sgIDs = []; + foreach ($sgs as $sg) { + if ($this->SharingGroup->checkIfServerInSG($sg, $server)) { + $sgIDs[] = $sg['SharingGroup']['id']; + } + } + if (empty($sgIDs)) { + $sgIDs = [-1]; + } + return $sgIDs; + } + + private function isPushableForServerSyncRules(array $analystData, array $server): bool + { + $push_rules = json_decode($server['Server']['push_rules'], true); + if (!empty($push_rules['orgs']['OR'])) { + if (!in_array($analystData['Orgc']['id'], $push_rules['orgs']['OR'])) { + return false; + } + } + if (!empty($push_rules['orgs']['NOT'])) { + if (in_array($analystData['Orgc']['id'], $push_rules['orgs']['NOT'])) { + return false; + } + } + return true; + } + + + /** + * Get an array of analyst data that the remote is willing to get and returns analyst data that should be pushed. + * @param ServerSyncTool $serverSync + * @param array $localAnalystData + * @param array $conditions + * @return array + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + * @throws JsonException + */ + public function identifyUUIDsForPush(ServerSyncTool $serverSync, array $localAnalystData=[], array $conditions=[]): array + { + $this->log("Fetching eligible analyst data from server #{$serverSync->serverId()} for push: " . JsonTool::encode($conditions), LOG_INFO); + $candidates = []; + foreach ($localAnalystData as $type => $entries) { + foreach ($entries as $entry) { + $entry = $entry[$type]; + $candidates[$type][$entry['uuid']] = $entry['modified']; + } + } + $remoteDataArray = $this->proposeDataToRemote($serverSync, $candidates); + foreach ($localAnalystData as $type => $entries) { + foreach ($entries as $i => $entry) { + $entry = $entry[$type]; + if (!isset($remoteDataArray[$type][$entry['uuid']])) { + unset($localAnalystData[$type][$i]); + } + } + } + return $localAnalystData; + } + + public function proposeDataToRemote(ServerSyncTool $serverSync, array $candidates): array + { + $acceptedDataForPush = $this->Server->filterAnalystDataForPush($serverSync, $candidates); + return $acceptedDataForPush; + } + + public function filterAnalystDataForPush($allIncomingAnalystData): array + { + $validModels = [ + 'Note' => ClassRegistry::init('Note'), + 'Opinion' => ClassRegistry::init('Opinion'), + 'Relationship' => ClassRegistry::init('Relationship'), + ]; + + $allData = ['Note' => [], 'Opinion' => [], 'Relationship' => []]; + foreach ($allIncomingAnalystData as $model => $entries) { + $incomingAnalystData = $entries; + $incomingUuids = array_keys($entries); + $options = [ + 'conditions' => ["{$model}.uuid" => $incomingUuids], + 'recursive' => -1, + 'fields' => ['uuid', 'modified', 'locked'] + ]; + $analystData = $validModels[$model]->find('all', $options); + foreach ($analystData as $entry) { + if (empty($incomingAnalystData[$entry[$model]['uuid']])) { + continue; + } + if (!$this->isCandidateValidForPush($incomingAnalystData[$entry[$model]['uuid']], $entry[$model])) { + unset($incomingAnalystData[$entry[$model]['uuid']]); + } + } + $allData[$model] = $incomingAnalystData; + } + return $allData; + } + + private function isCandidateValidForPush($candidateModified, array $existingEntry): bool + { + if ($existingEntry['locked'] == 0) { + return false; + } + if (strtotime($existingEntry['modified']) >= strtotime($candidateModified)) { + return false; + } + return true; + } + + public function indexMinimal(array $user, $filters = []): array + { + $options = [ + 'recursive' => -1, + 'conditions' => [ + 'AND' => [ + $this->buildConditions($user), + 'AND' => [$filters], + ], + ], + 'fields' => ['uuid', 'modified', 'locked'] + ]; + $tmp = $this->getAllAnalystData('all', $options); + $allData = []; + foreach ($tmp as $type => $entries) { + foreach ($entries as $i => $entry) { + $entry = $entry[$type]; + $allData[$type][$entry['uuid']] = $entry['modified']; + } + } + return $allData; + } + + /** + * getAllAnalystData Collect all analyst data regardless if they are notes, opinions or relationships + * + * @param array $user + * @return array + */ + public function getAllAnalystData($findType='all', array $findOptions=[]): array + { + $allData = []; + $validModels = [ + 'Note' => ClassRegistry::init('Note'), + 'Opinion' => ClassRegistry::init('Opinion'), + 'Relationship' => ClassRegistry::init('Relationship'), + ]; + foreach ($validModels as $model) { + $result = $model->find($findType, $findOptions); + $allData[$model->alias] = $result; + } + return $allData; + } + + public function uploadEntryToServer($type, array $analystData, array $server, ServerSyncTool $serverSync, array $user) + { + $analystDataID = $analystData[$type]['id']; + $analystData = $this->prepareForPushToServer($type, $analystData, $server); + if (is_numeric($analystData)) { + return $analystData; + } + + try { + if (!$serverSync->isSupported(ServerSyncTool::PERM_SYNC) || !$serverSync->isSupported(ServerSyncTool::PERM_ANALYST_DATA)) { + return __('The remote user does not have the permission to manipulate analyst data, the upload of the analyst data has been blocked.'); + } + $serverSync->pushAnalystData($type, $analystData)->json(); + } catch (Exception $e) { + $title = __('Uploading AnalystData (%s::%s) to Server (%s)', $type, $analystDataID, $server['Server']['id']); + $this->loadLog()->createLogEntry($user, 'push', 'AnalystData', $analystDataID, $title, $e->getMessage()); + + $this->logException("Could not push analyst data to remote server {$serverSync->serverId()}", $e); + return $e->getMessage(); + } + + return 'Success'; + } + + private function prepareForPushToServer($type, array $analystData, array $server) + { + if ($analystData[$type]['distribution'] == 4) { + if (!empty($analystData[$type]['SharingGroup']['SharingGroupServer'])) { + $found = false; + foreach ($analystData[$type]['SharingGroup']['SharingGroupServer'] as $sgs) { + if ($sgs['server_id'] == $server['Server']['id']) { + $found = true; + } + } + if (!$found) { + return 403; + } + } elseif (empty($analystData[$type]['SharingGroup']['roaming'])) { + return 403; + } + } + $this->Event = ClassRegistry::init('Event'); + if ($this->Event->checkDistributionForPush($analystData, $server, $type)) { + return $this->updateAnalystDataForSync($type, $analystData, $server); + } + return 403; + } + + private function updateAnalystDataForSync($type, array $analystData, array $server): array + { + $this->Event = ClassRegistry::init('Event'); + // cleanup the array from things we do not want to expose + foreach (['id'] as $field) { + unset($analystData[$type][$field]); + } + // Add the local server to the list of instances in the SG + if (isset($analystData[$type]['SharingGroup']) && isset($analystData[$type]['SharingGroup']['SharingGroupServer'])) { + foreach ($analystData[$type]['SharingGroup']['SharingGroupServer'] as &$s) { + if ($s['server_id'] == 0) { + $s['Server'] = array( + 'id' => 0, + 'url' => $this->Event->__getAnnounceBaseurl(), + 'name' => $this->Event->__getAnnounceBaseurl() + ); + } + } + } + + $analystData[$type]['locked'] = true; + // Downgrade the event from connected communities to community only + if (!$server['Server']['internal'] && $analystData[$type]['distribution'] == 2) { + $analystData[$type]['distribution'] = 1; + } + return $analystData; + } + + /** + * Collect all UUIDs with their modified time on the remote side, then filter the list based on what we have locally. + * Afterward, iteratively pull what should be pulled. + * + * @param array $user + * @param ServerSyncTool $serverSync + */ + public function pull(array $user, ServerSyncTool $serverSync) + { + $this->Server = ClassRegistry::init('Server'); + $this->AnalystData = ClassRegistry::init('AnalystData'); + try { + $filterRules = $this->buildPullFilterRules($serverSync->server()); + $remoteData = $serverSync->fetchIndexMinimal($filterRules)->json(); + } catch (Exception $e) { + $this->logException("Could not fetch analyst data IDs from server {$serverSync->server()['Server']['name']}", $e); + return 0; + } + + $allRemoteUUIDs = []; + if (empty($remoteData)) { + return 0; + } + foreach (self::ANALYST_DATA_TYPES as $type) { + $allRemoteUUIDs = array_merge($allRemoteUUIDs, array_keys($remoteData[$type])); + } + + $localAnalystData = $this->getAllAnalystData('list', [ + 'conditions' => ['uuid' => $allRemoteUUIDs], + 'fields' => ['uuid', 'modified'], + ]); + + $remoteUUIDsToFetch = []; + foreach ($remoteData as $type => $remoteAnalystData) { + foreach ($remoteAnalystData as $remoteUUID => $remoteModified) { + if (!isset($localAnalystData[$type][$remoteUUID])) { + $remoteUUIDsToFetch[$type][$remoteUUID] = $remoteModified; + } elseif (strtotime($localAnalystData[$type][$remoteUUID]) < strtotime($remoteModified)) { + $remoteUUIDsToFetch[$type][$remoteUUID] = $remoteModified; + } + } + } + unset($remoteData, $allRemoteUUIDs, $localAnalystData); + + if (empty($remoteUUIDsToFetch)) { + return 0; + } + + if ($serverSync->isSupported(ServerSyncTool::PERM_ANALYST_DATA)) { + return $this->pullInChunks($user, $remoteUUIDsToFetch, $serverSync); + } + } + + public function pullInChunks(array $user, array $analystDataUuids, ServerSyncTool $serverSync) + { + $uuids = array_keys($analystDataUuids); + $saved = 0; + $serverOrgUUID = $this->Org->find('first', [ + 'recursive' => -1, + 'conditions' => ['id' => $serverSync->server()['Server']['org_id']], + 'fields' => ['id', 'uuid'] + ])['Organisation']['uuid']; + + foreach ($analystDataUuids as $type => $entries) { + $uuids = array_keys($entries); + if (empty($uuids)) { + continue; + } + + foreach (array_chunk($uuids, 100) as $uuidChunk) { + try { + $chunkedAnalystData = $serverSync->fetchAnalystData($type, $uuidChunk)->json(); + } catch (Exception $e) { + $this->logException("Failed downloading the chunked analyst data from {$serverSync->server()['Server']['name']}.", $e); + continue; + } + + foreach ($chunkedAnalystData as $analystData) { + $analystData = $this->updatePulledBeforeInsert($analystData, $type, $serverSync->server(), $user, $serverSync->pullRules()); + $savedResult = $this->captureAnalystData($user, $analystData, true, $serverOrgUUID, $serverSync->server()); + if ($savedResult['success']) { + $saved += $savedResult['imported']; + } + } + } + } + + return $saved; + } + + private function updatePulledBeforeInsert(array $analystData, $type, array $server, array $user, array $pullRules): array + { + $analystData[$type]['locked'] = true; + + if (empty(Configure::read('MISP.host_org_id')) || !$server['Server']['internal'] || Configure::read('MISP.host_org_id') != $server['Server']['org_id']) { + switch ($analystData[$type]['distribution']) { + case 1: + // if community only, downgrade to org only after pull + $analystData[$type]['distribution'] = '0'; + break; + case 2: + // if connected communities downgrade to community only + $analystData[$type]['distribution'] = '1'; + break; + } + } + return $analystData; + } + + private function buildPullFilterRules(array $server): array + { + $filterRules = ['orgc_name' => []]; + $pullRules = $this->jsonDecode($server['Server']['pull_rules']); + if (!empty($pullRules['orgs']['OR'])) { + $filterRules['orgc_name'] = $pullRules['orgs']['OR']; + } + if (!empty($pullRules['orgs']['NOT'])) { + $filterRules['orgc_name'] = array_merge($filterRules['orgc_name'], array_map(function($orgName) { + return '!' . $orgName; + }, $pullRules['orgs']['NOT'])); + } + return $filterRules; + } +} diff --git a/app/Model/AnalystDataBlocklist.php b/app/Model/AnalystDataBlocklist.php new file mode 100644 index 000000000..3ff893308 --- /dev/null +++ b/app/Model/AnalystDataBlocklist.php @@ -0,0 +1,58 @@ + array( // TODO Audit, logable + 'userModel' => 'User', + 'userKey' => 'user_id', + 'change' => 'full' + ), + 'Containable', + ); + + public $blocklistFields = array('analyst_data_uuid', 'comment', 'analyst_data_info', 'analyst_data_orgc'); + public $blocklistTarget = 'analyst_data'; + + public $validate = array( + 'analyst_data_uuid' => array( + 'unique' => array( + 'rule' => 'isUnique', + 'message' => 'Analyst Data already blocklisted.' + ), + 'uuid' => array( + 'rule' => array('uuid'), + 'message' => 'Please provide a valid UUID' + ), + ) + ); + + public function beforeValidate($options = array()) + { + parent::beforeValidate(); + if (empty($this->data['AnalystDataBlocklist']['id'])) { + $this->data['AnalystDataBlocklist']['date_created'] = date('Y-m-d H:i:s'); + } + if (empty($this->data['AnalystDataBlocklist']['comment'])) { + $this->data['AnalystDataBlocklist']['comment'] = ''; + } + return true; + } + + /** + * @param string $analystDataUUID + * @return bool + */ + public function checkIfBlocked($analystDataUUID) + { + return $this->hasAny([ + 'analyst_data_uuid' => $analystDataUUID, + ]); + } +} diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 7f7bea0a4..3077fa34e 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -48,6 +48,9 @@ class AppModel extends Model /** @var Workflow|null */ private $Workflow; + public $includeAnalystData; + public $includeAnalystDataRecursive; + // deprecated, use $db_changes // major -> minor -> hotfix -> requires_logout const OLD_DB_CHANGES = array( @@ -87,7 +90,7 @@ class AppModel extends Model 99 => false, 100 => false, 101 => false, 102 => false, 103 => false, 104 => false, 105 => false, 106 => false, 107 => false, 108 => false, 109 => false, 110 => false, 111 => false, 112 => false, 113 => true, 114 => false, 115 => false, 116 => false, - 117 => false, 118 => false, 119 => false, 120 => false + 117 => false, 118 => false, 119 => false, 120 => false, 121 => false, 122 => false, ); const ADVANCED_UPDATES_DESCRIPTION = array( @@ -2013,6 +2016,145 @@ class AppModel extends Model case 119: $sqlArray[] = "ALTER TABLE `access_logs` MODIFY `action` varchar(191) NOT NULL"; break; + case 121: + $sqlArray[] = "CREATE TABLE `notes` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + `object_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + `object_type` varchar(80) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + `authors` text, + `org_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + `orgc_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + `created` datetime DEFAULT CURRENT_TIMESTAMP, + `modified` datetime ON UPDATE CURRENT_TIMESTAMP, + `distribution` tinyint(4) NOT NULL, + `sharing_group_id` int(10) unsigned, + `locked` tinyint(1) NOT NULL DEFAULT 0, + `note` mediumtext, + `language` varchar(16) DEFAULT 'en', + PRIMARY KEY (`id`), + UNIQUE KEY `uuid` (`uuid`), + KEY `object_uuid` (`object_uuid`), + KEY `object_type` (`object_type`), + KEY `org_uuid` (`org_uuid`), + KEY `orgc_uuid` (`orgc_uuid`), + KEY `distribution` (`distribution`), + KEY `sharing_group_id` (`sharing_group_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"; + + $sqlArray[] = "CREATE TABLE `opinions` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + `object_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + `object_type` varchar(80) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + `authors` text, + `org_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + `orgc_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + `created` datetime DEFAULT CURRENT_TIMESTAMP, + `modified` datetime ON UPDATE CURRENT_TIMESTAMP, + `distribution` tinyint(4) NOT NULL, + `sharing_group_id` int(10) unsigned, + `locked` tinyint(1) NOT NULL DEFAULT 0, + `opinion` int(10) unsigned, + `comment` text, + PRIMARY KEY (`id`), + UNIQUE KEY `uuid` (`uuid`), + KEY `object_uuid` (`object_uuid`), + KEY `object_type` (`object_type`), + KEY `org_uuid` (`org_uuid`), + KEY `orgc_uuid` (`orgc_uuid`), + KEY `distribution` (`distribution`), + KEY `sharing_group_id` (`sharing_group_id`), + KEY `opinion` (`opinion`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"; + + $sqlArray[] = "CREATE TABLE `relationships` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `uuid` varchar(40) CHARACTER SET ascii NOT NULL, + `object_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + `object_type` varchar(80) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + `authors` text, + `org_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + `orgc_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + `created` datetime DEFAULT CURRENT_TIMESTAMP, + `modified` datetime ON UPDATE CURRENT_TIMESTAMP, + `distribution` tinyint(4) NOT NULL, + `sharing_group_id` int(10) unsigned, + `locked` tinyint(1) NOT NULL DEFAULT 0, + `relationship_type` varchar(255) CHARACTER SET utf8 COLLATE utf8_unicode_ci, + `related_object_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + `related_object_type` varchar(80) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `uuid` (`uuid`), + KEY `object_uuid` (`object_uuid`), + KEY `object_type` (`object_type`), + KEY `org_uuid` (`org_uuid`), + KEY `orgc_uuid` (`orgc_uuid`), + KEY `distribution` (`distribution`), + KEY `sharing_group_id` (`sharing_group_id`), + KEY `relationship_type` (`relationship_type`), + KEY `related_object_uuid` (`related_object_uuid`), + KEY `related_object_type` (`related_object_type`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"; + + $sqlArray[] = "CREATE TABLE IF NOT EXISTS `analyst_data_blocklists` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `analyst_data_uuid` varchar(40) COLLATE utf8_bin NOT NULL, + `created` datetime NOT NULL, + `analyst_data_info` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL, + `comment` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci, + `analyst_data_orgc` VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + PRIMARY KEY (`id`), + KEY `analyst_data_uuid` (`analyst_data_uuid`), + KEY `analyst_data_orgc` (`analyst_data_orgc`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;"; + + $sqlArray[] = "ALTER TABLE `roles` ADD `perm_analyst_data` tinyint(1) NOT NULL DEFAULT 0;"; + $sqlArray[] = "UPDATE `roles` SET `perm_analyst_data`=1 WHERE `perm_add` = 1;"; + + $sqlArray[] = "ALTER TABLE `servers` ADD `push_analyst_data` tinyint(1) NOT NULL DEFAULT 0 AFTER `push_galaxy_clusters`;"; + $sqlArray[] = "ALTER TABLE `servers` ADD `pull_analyst_data` tinyint(1) NOT NULL DEFAULT 0 AFTER `push_analyst_data`;"; + break; + case 122: + $sqlArray[] = "CREATE TABLE `collections` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + `org_id` int(10) unsigned NOT NULL, + `orgc_id` int(10) unsigned NOT NULL, + `user_id` int(10) unsigned NOT NULL, + `created` datetime DEFAULT CURRENT_TIMESTAMP, + `modified` datetime ON UPDATE CURRENT_TIMESTAMP, + `distribution` tinyint(4) NOT NULL, + `sharing_group_id` int(10) unsigned, + `name` varchar(191) NOT NULL, + `type` varchar(80) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + `description` mediumtext, + PRIMARY KEY (`id`), + UNIQUE KEY `uuid` (`uuid`), + KEY `name` (`name`), + KEY `type` (`type`), + KEY `org_id` (`org_id`), + KEY `orgc_id` (`orgc_id`), + KEY `user_id` (`user_id`), + KEY `distribution` (`distribution`), + KEY `sharing_group_id` (`sharing_group_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"; + + $sqlArray[] = "CREATE TABLE `collection_elements` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + `element_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + `element_type` varchar(80) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL, + `collection_id` int(10) unsigned NOT NULL, + `description` text, + PRIMARY KEY (`id`), + UNIQUE KEY `uuid` (`uuid`), + KEY `element_uuid` (`element_uuid`), + KEY `element_type` (`element_type`), + KEY `collection_id` (`collection_id`), + UNIQUE KEY `unique_element` (`element_uuid`, `collection_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"; + break; case 'fixNonEmptySharingGroupID': $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; @@ -4051,8 +4193,17 @@ class AppModel extends Model if (!empty($query['order']) && $this->validOrderClause($query['order']) === false) { throw new InvalidArgumentException('Invalid order clause'); } - - return parent::find($type, $query); + $results = parent::find($type, $query); + if (!empty($query['includeAnalystData']) && $this->Behaviors->enabled('AnalystDataParent')) { + if ($type === 'first') { + $results[$this->alias] = array_merge($results[$this->alias], $this->attachAnalystData($results[$this->alias])); + } else if ($type === 'all') { + foreach ($results as $k => $result) { + $results[$k][$this->alias] = array_merge($results[$k][$this->alias], $this->attachAnalystData($results[$k][$this->alias])); + } + } + } + return $results; } private function validOrderClause($order) @@ -4087,7 +4238,6 @@ class AppModel extends Model $newImageDir = APP . 'files/img'; $oldOrgDir = new Folder($oldImageDir . '/orgs'); $oldCustomDir = new Folder($oldImageDir . '/custom'); - $result = false; $result = $oldOrgDir->copy([ 'from' => $oldImageDir . '/orgs', 'to' => $newImageDir . '/orgs', diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 2467c4089..99830885d 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -36,7 +36,8 @@ class Attribute extends AppModel 'Trim', 'Containable', 'Regexp' => array('fields' => array('value')), - 'LightPaginator' + 'LightPaginator', + 'AnalystDataParent', ); public $displayField = 'value'; @@ -2658,6 +2659,7 @@ class Attribute extends AppModel if (!empty($attribute['Sighting'])) { $this->Sighting->captureSightings($attribute['Sighting'], $this->id, $eventId, $user); } + $this->Event->captureAnalystData($user, $attribute); } if (!empty($this->validationErrors)) { $validationErrors = $this->validationErrors; @@ -2798,6 +2800,7 @@ class Attribute extends AppModel if (!empty($attribute['Sighting'])) { $this->Sighting->captureSightings($attribute['Sighting'], $attributeId, $eventId, $user); } + $this->Event->captureAnalystData($user, $attribute); if ($user['Role']['perm_tagger']) { /* We should unwrap the line below and remove the server option in the future once we have tag soft-delete diff --git a/app/Model/Behavior/AnalystDataBehavior.php b/app/Model/Behavior/AnalystDataBehavior.php new file mode 100644 index 000000000..3ad99b86d --- /dev/null +++ b/app/Model/Behavior/AnalystDataBehavior.php @@ -0,0 +1,49 @@ +__current_type = $Model->alias; + } + + // Return the analystData of the current type for a given UUID (this only checks the ACL of the analystData, NOT of the parent.) + public function fetchForUuid(Model $Model, $uuid, $user = null) + { + $conditions = [ + 'object_uuid' => $uuid + ]; + $type = $Model->current_type; + if (empty($user['Role']['perm_site_admin'])) { + $validSharingGroups = $Model->SharingGroup->authorizedIds($user, true); + $conditions['AND'][] = [ + 'OR' => [ + $type . '.orgc_uuid' => $user['Organisation']['uuid'], + $type . '.org_uuid' => $user['Organisation']['uuid'], + $type . '.distribution IN' => [1, 2, 3], + 'AND' => [ + $type . '.distribution' => 4, + $type . '.sharing_group_id IN' => $validSharingGroups + ] + ] + ]; + } + return $Model->find('all', [ + 'recursive' => -1, + 'conditions' => $conditions, + 'contain' => ['Org', 'Orgc', 'SharingGroup'], + ]); + } + + public function checkACL() + { + + } +} diff --git a/app/Model/Behavior/AnalystDataParentBehavior.php b/app/Model/Behavior/AnalystDataParentBehavior.php new file mode 100644 index 000000000..a3d0f46bf --- /dev/null +++ b/app/Model/Behavior/AnalystDataParentBehavior.php @@ -0,0 +1,53 @@ +__currentUser)) { + $user_id = Configure::read('CurrentUserId'); + $this->User = ClassRegistry::init('User'); + if ($user_id) { + $this->__currentUser = $this->User->getAuthUser($user_id); + } + } + $data = []; + foreach ($types as $type) { + $this->{$type} = ClassRegistry::init($type); + $this->{$type}->fetchRecursive = !empty($model->includeAnalystDataRecursive); + $temp = $this->{$type}->fetchForUuid($object['uuid'], $this->__currentUser); + if (!empty($temp)) { + foreach ($temp as $k => $temp_element) { + $data[$type][] = $temp_element[$type]; + } + } + } + return $data; + } + + public function afterFind(Model $model, $results, $primary = false) + { + if (!empty($model->includeAnalystData)) { + foreach ($results as $k => $item) { + if (isset($item[$model->alias])) { + $results[$k] = array_merge($results[$k], $this->attachAnalystData($model, $item[$model->alias])); + } + } + } + return $results; + } + +} diff --git a/app/Model/Collection.php b/app/Model/Collection.php new file mode 100644 index 000000000..25e571282 --- /dev/null +++ b/app/Model/Collection.php @@ -0,0 +1,145 @@ + array( + 'className' => 'Organisation', + 'foreignKey' => 'orgc_id', + 'fields' => [ + 'Orgc.id', + 'Orgc.uuid', + 'Orgc.name' + ] + ), + 'Org' => array( + 'className' => 'Organisation', + 'foreignKey' => 'org_id', + 'fields' => [ + 'Org.id', + 'Org.uuid', + 'Org.name' + ] + ), + 'User' => array( + 'className' => 'User', + 'foreignKey' => 'user_id', + 'fields' => [ + 'User.id', + 'User.email' + ] + ) + ]; + + public $hasMany = [ + 'CollectionElement' => [ + 'dependent' => true + ] + ]; + + public $valid_targets = [ + 'Attribute', + 'Event', + 'GalaxyCluster', + 'Galaxy', + 'Object', + 'Note', + 'Opinion', + 'Relationship', + 'Organisation', + 'SharingGroup' + ]; + + public $current_user = null; + + + public function beforeValidate($options = array()) + { + if (empty($this->data['Collection'])) { + $this->data = ['Collection' => $this->data]; + } + if (empty($this->id) && empty($this->data['Collection']['uuid'])) { + $this->data['Collection']['uuid'] = CakeText::uuid(); + } + if (empty($this->id)) { + $this->data['Collection']['user_id'] = $this->current_user['id']; + if (empty($this->data['Collection']['orgc_id']) || empty($this->current_user['Role']['perm_sync'])) { + $this->data['Collection']['orgc_id'] = $this->current_user['Organisation']['id']; + } + $this->data['Collection']['org_id'] = $this->current_user['Organisation']['id']; + $this->data['Collection']['user_id'] = $this->current_user['id']; + } + return true; + } + + public function mayModify($user_id, $collection_id) + { + $user = $this->User->getAuthUser($user_id); + $collection = $this->find('first', [ + 'recursive' => -1, + 'conditions' => ['Collection.id' => $collection_id] + ]); + if ($user['Role']['perm_site_admin']) { + return true; + } + if (empty($user['Role']['perm_modify'])) { + return false; + } + if (!empty($user['Role']['perm_modify_org'])) { + if ($user['org_id'] == $collection['Collection']['Orgc_id']) { + return true; + } + if ($user['Role']['perm_sync'] && $user['org_id'] == $collection['Collection']['Org_id']) { + return true; + } + } + if (!empty($user['Role']['perm_modify']) && $user['id'] === $collection['Collection']['user_id']) { + } + } + + public function mayView($user_id, $collection_id) + { + $user = $this->User->getAuthUser($user_id); + $collection = $this->find('first', [ + 'recursive' => -1, + 'conditions' => ['Collection.id' => $collection_id] + ]); + if ($user['Role']['perm_site_admin']) { + return true; + } + if ($collection['Collection']['org_id'] == $user('org_id')) { + return true; + } + if (in_array($collection['Collection']['distribution'], [1,2,3])) { + return true; + } + if ($collection['Collection']['distribution'] === 4) { + $SharingGroup = ClassRegistry::init('SharingGroup'); + $sgs = $this->SharingGroup->fetchAllAuthorised($user, 'uuid'); + if (isset($sgs[$collection['Collection']['sharing_group_id']])) { + return true; + } else { + return false; + } + } + return false; + } + + public function rearrangeCollection(array $collection) { + foreach ($collection as $key => $elements) { + if ($key !== 'Collection') { + $collection['Collection'][$key] = $elements; + unset($collection[$key]); + } + } + return $collection; + } +} diff --git a/app/Model/CollectionElement.php b/app/Model/CollectionElement.php new file mode 100644 index 000000000..1667eb86e --- /dev/null +++ b/app/Model/CollectionElement.php @@ -0,0 +1,205 @@ + array( + 'className' => 'Collection', + 'foreignKey' => 'collection_id' + ) + ); + + // Make sure you also update the validation for element_type to include anything you add here. + public $valid_types = [ + 'Event', + 'GalaxyCluster' + ]; + + public $validate = [ + 'collection_id' => [ + 'numeric' => [ + 'rule' => ['numeric'] + ] + ], + 'uuid' => [ + 'uuid' => [ + 'rule' => 'uuid', + 'message' => 'Please provide a valid RFC 4122 UUID' + ] + ], + 'element_uuid' => [ + 'element_uuid' => [ + 'rule' => 'uuid', + 'message' => 'Please provide a valid RFC 4122 UUID' + ] + ], + 'element_type' => [ + 'element_type' => [ + 'rule' => ['inList', ['Event', 'GalaxyCluster']], + 'message' => 'Invalid object type.' + ] + ] + ]; + + + public function beforeValidate($options = array()) + { + // Massage to a common format + if (empty($this->data['CollectionElement'])) { + $this->data = ['CollectionElement' => $this->data]; + } + + // if we're creating a new element, assign a uuid (unless provided) + if (empty($this->id) && empty($this->data['CollectionElement']['uuid'])) { + $this->data['CollectionElement']['uuid'] = CakeText::uuid(); + } + if ( + empty($this->id) && + empty($this->data['CollectionElement']['element_type']) && + !empty($this->data['CollectionElement']['element_uuid']) + ) { + $this->data['CollectionElement']['element_type'] = $this->deduceType($this->data['CollectionElement']['element_uuid']); + } + return true; + } + + public function mayModify(int $user_id, int $collection_id) + { + $user = $this->User->getAuthUser($user_id); + $collection = $this->find('first', [ + 'recursive' => -1, + 'conditions' => ['Collection.id' => $collection_id] + ]); + if ($user['Role']['perm_site_admin']) { + return true; + } + if (empty($user['Role']['perm_modify'])) { + return false; + } + if (!empty($user['Role']['perm_modify_org'])) { + if ($user['org_id'] == $collection['Collection']['Orgc_id']) { + return true; + } + if ($user['Role']['perm_sync'] && $user['org_id'] == $collection['Collection']['Org_id']) { + return true; + } + } + if (!empty($user['Role']['perm_modify']) && $user['id'] === $collection['Collection']['user_id']) { + } + } + + public function mayView(int $user_id, int $collection_id) + { + $user = $this->User->getAuthUser($user_id); + $collection = $this->find('first', [ + 'recursive' => -1, + 'conditions' => ['Collection.id' => $collection_id] + ]); + if ($user['Role']['perm_site_admin']) { + return true; + } + if ($collection['Collection']['org_id'] == $user('org_id')) { + return true; + } + if (in_array($collection['Collection']['distribution'], [1,2,3])) { + return true; + } + if ($collection['Collection']['distribution'] === 4) { + $SharingGroup = ClassRegistry::init('SharingGroup'); + $sgs = $this->SharingGroup->fetchAllAuthorised($user, 'uuid'); + if (isset($sgs[$collection['Collection']['sharing_group_id']])) { + return true; + } else { + return false; + } + } + return false; + } + + public function deduceType(string $uuid) + { + foreach ($this->valid_types as $valid_type) { + $this->{$valid_type} = ClassRegistry::init($valid_type); + $result = $this->$valid_type->find('first', [ + 'conditions' => [$valid_type.'.uuid' => $uuid], + 'recursive' => -1 + ]); + if (!empty($result)) { + return $valid_type; + } + } + throw new NotFoundException(__('Invalid UUID')); + } + + /* + * Pass a Collection as received from another instance to this function to capture the elements + * The received object is authoritative, so all elements that no longer exist in the upstream will be culled. + */ + public function captureElements($data) { + $temp = $this->find('all', [ + 'recursive' => -1, + 'conditions' => ['CollectionElement.collection_id' => $data['Collection']['id']] + ]); + $oldElements = []; + foreach ($temp as $oldElement) { + $oldElements[$oldElement['CollectionElement']['uuid']] = $oldElement['CollectionElement']; + } + if (isset($data['Collection']['CollectionElement'])) { + $elementsToSave = []; + foreach ($data['Collection']['CollectionElement'] as $k => $element) { + if (empty($element['uuid'])) { + $element['uuid'] = CakeText::uuid(); + } + if (isset($oldElements[$element['uuid']])) { + if (isset($element['description'])) { + $oldElements[$element['uuid']]['description'] = $element['description']; + } + $elementsToSave[$k] = $oldElements[$element['uuid']]; + unset($oldElements[$element['uuid']]); + } else { + $elementsToSave[$k] = [ + 'CollectionElement' => [ + 'uuid' => $element['uuid'], + 'element_uuid' => $element['element_uuid'], + 'element_type' => $element['element_type'], + 'description' => $element['description'], + 'collection_id' => $data['Collection']['id'] + ] + ]; + + } + } + foreach ($elementsToSave as $k => $element) { + if (empty($element['CollectionElement']['id'])) { + $this->create(); + } + try{ + $this->save($element); + } catch (PDOException $e) { + // duplicate value? + } + } + foreach ($oldElements as $toDelete) { + $this->delete($toDelete['id']); + } + $temp = $this->find('all', [ + 'conditions' => ['CollectionElement.collection_id' => $data['Collection']['id']], + 'recursive' => -1 + ]); + $data['Collection']['CollectionElement'] = []; + foreach ($temp as $element) { + $data['Collection']['CollectionElement'][] = $element['CollectionElement']; + } + } + + return $data; + } +} diff --git a/app/Model/Event.php b/app/Model/Event.php index 75c52b4fc..834db06b0 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -19,6 +19,9 @@ App::uses('ProcessTool', 'Tools'); * @property Organisation $Org * @property Organisation $Orgc * @property CryptographicKey $CryptographicKey + * @property Note $Note + * @property Opinion $Opinion + * @property Relationship $Relationship */ class Event extends AppModel { @@ -40,7 +43,8 @@ class Event extends AppModel 'change' => 'full'), 'Trim', 'Containable', - 'EventWarning' + 'EventWarning', + 'AnalystDataParent' ); public $displayField = 'id'; @@ -1760,6 +1764,11 @@ class Event extends AppModel if (!isset($options['fetchFullClusterRelationship'])) { $options['fetchFullClusterRelationship'] = false; } + if (!isset($options['includeAnalystData'])) { + $options['includeAnalystData'] = false; + } else { + $options['includeAnalystData'] = !empty($options['includeAnalystData']); + } foreach ($this->possibleOptions as $opt) { if (!isset($options[$opt])) { $options[$opt] = false; @@ -2038,6 +2047,8 @@ class Event extends AppModel if (!empty($options['page'])) { $params['page'] = $options['page']; } + $this->includeAnalystData = $options['includeAnalystData']; + $this->includeAnalystDataRecursive = $options['includeAnalystData']; if (!empty($options['order'])) { $params['order'] = $this->findOrder( $options['order'], @@ -2198,6 +2209,13 @@ class Event extends AppModel if (!empty($options['includeGranularCorrelations'])) { $event['Attribute'] = $this->Attribute->Correlation->attachCorrelationExclusion($event['Attribute']); } + if (!empty($options['includeAnalystData'])) { + foreach ($event['Attribute'] as $k => $attribute) { + $this->Attribute->includeAnalystDataRecursive = true; + $analyst_data = $this->Attribute->attachAnalystData($attribute); + $event['Attribute'][$k] = array_merge($event['Attribute'][$k], $analyst_data); + } + } // move all object attributes to a temporary container $tempObjectAttributeContainer = array(); @@ -2248,6 +2266,11 @@ class Event extends AppModel if (isset($tempObjectAttributeContainer[$objectValue['id']])) { $objectValue['Attribute'] = $tempObjectAttributeContainer[$objectValue['id']]; } + if (!empty($options['includeAnalystData'])) { + $this->Object->includeAnalystDataRecursive = true; + $analyst_data = $this->Object->attachAnalystData($objectValue); + $objectValue = array_merge($objectValue, $analyst_data); + } } unset($tempObjectAttributeContainer); } @@ -3633,6 +3656,7 @@ class Event extends AppModel $created_id = 0; $event['Event']['locked'] = 1; $event['Event']['published'] = $publish; + $event = $this->updatedLockedFieldForAllAnalystData($event); $result = $this->_add($event, true, $user, '', null, false, null, $created_id, $validationIssues); $results[] = [ 'info' => $event['Event']['info'], @@ -3644,6 +3668,59 @@ class Event extends AppModel return $results; } + private function updatedLockedFieldForAllAnalystData(array $event): array + { + $event = $this->updatedLockedFieldForAnalystData($event, 'Event'); + if (!empty($event['Event']['Attribute'])) { + for ($i=0; $i < count($event['Event']['Attribute']); $i++) { + $event['Event']['Attribute'][$i] = $this->updatedLockedFieldForAnalystData($event['Event']['Attribute'][$i]); + } + } + if (!empty($event['Event']['Object'])) { + for ($i=0; $i < count($event['Event']['Object']); $i++) { + $event['Event']['Object'][$i] = $this->updatedLockedFieldForAnalystData($event['Event']['Object'][$i]); + if (!empty($event['Event']['Object'][$i])) { + for ($j=0; $j < count($event['Event']['Object'][$i]['Attribute']); $j++) { + $event['Event']['Object'][$i]['Attribute'][$j] = $this->updatedLockedFieldForAnalystData($event['Event']['Object'][$i]['Attribute'][$j]); + } + } + } + } + if (!empty($event['Event']['EventReport'])) { + for ($i=0; $i < count($event['Event']['EventReport']); $i++) { + $event['Event']['EventReport'][$i] = $this->updatedLockedFieldForAnalystData($event['Event']['EventReport'][$i]); + } + } + return $event; + } + + private function updatedLockedFieldForAnalystData(array $data, $model=false): array + { + $this->AnalystData = ClassRegistry::init('AnalystData'); + if (!empty($model)) { + $data = $data[$model]; + } + foreach ($this->AnalystData::ANALYST_DATA_TYPES as $type) { + if (!empty($data[$type])) { + for ($i=0; $i < count($data[$type]); $i++) { + $data[$type][$i]['locked'] = true; + foreach ($this->AnalystData::ANALYST_DATA_TYPES as $childType) { + if (!empty($data[$type][$i][$childType])) { + for ($j=0; $j < count($data[$type][$i][$childType]); $j++) { + $data[$type][$i][$childType][$j]['locked'] = true; + $data[$type][$i][$childType][$j] = $this->updatedLockedFieldForAnalystData($data[$type][$i][$childType][$j]); + } + } + } + } + } + } + if (!empty($model)) { + $data = [$model => $data]; + } + return $data; + } + /** * Low level function to add an Event based on an Event $data array. * @@ -3881,6 +3958,8 @@ class Event extends AppModel if (isset($data['Sighting']) && !empty($data['Sighting'])) { $this->Sighting->captureSightings($data['Sighting'], null, $this->id, $user); } + + $this->captureAnalystData($user, $data['Event']); if ($fromXml) { $created_id = $this->id; } @@ -4182,6 +4261,8 @@ class Event extends AppModel if (isset($data['Sighting']) && !empty($data['Sighting'])) { $this->Sighting->captureSightings($data['Sighting'], null, $this->id, $user); } + + $this->captureAnalystData($user, $data['Event']); // if published -> do the actual publishing if ($changed && (!empty($data['Event']['published']) && 1 == $data['Event']['published'])) { // The edited event is from a remote server ? @@ -7948,6 +8029,20 @@ class Event extends AppModel } } + public function captureAnalystData($user, $data) + { + $this->Note = ClassRegistry::init('Note'); + $this->Opinion = ClassRegistry::init('Opinion'); + $this->Relationship = ClassRegistry::init('Relationship'); + foreach ($this->Note::ANALYST_DATA_TYPES as $type) { + if (!empty($data[$type])) { + foreach ($data[$type] as $analystData) { + $this->{$type}->captureAnalystData($user, $analystData); + } + } + } + } + public function getTrendsForTags(array $user, array $eventFilters=[], int $baseDayRange, int $rollingWindows=3, $tagFilterPrefixes=null): array { $fullDayNumber = $baseDayRange + $baseDayRange * $rollingWindows; diff --git a/app/Model/EventReport.php b/app/Model/EventReport.php index cdce53419..cbf552612 100644 --- a/app/Model/EventReport.php +++ b/app/Model/EventReport.php @@ -17,6 +17,7 @@ class EventReport extends AppModel 'change' => 'full' ), 'Regexp' => array('fields' => array('value')), + 'AnalystDataParent', ); public $validate = array( @@ -118,6 +119,8 @@ class EventReport extends AppModel __('Event Report dropped due to validation for Event report %s failed: %s', $this->data['EventReport']['uuid'], $this->data['EventReport']['name']), __('Validation errors: %s.%sFull report: %s', json_encode($errors), PHP_EOL, json_encode($report['EventReport'])) ); + } else { + $this->Event->captureAnalystData($user, $report); } return $errors; } @@ -188,6 +191,7 @@ class EventReport extends AppModel } $errors = $this->saveAndReturnErrors($report, ['fieldList' => self::CAPTURE_FIELDS], $errors); if (empty($errors)) { + $this->Event->captureAnalystData($user, $report['EventReport']); $this->Event->unpublishEvent($eventId); } return $errors; diff --git a/app/Model/GalaxyCluster.php b/app/Model/GalaxyCluster.php index 781d31033..c97e708f1 100644 --- a/app/Model/GalaxyCluster.php +++ b/app/Model/GalaxyCluster.php @@ -22,6 +22,7 @@ class GalaxyCluster extends AppModel 'userKey' => 'user_id', 'change' => 'full'), 'Containable', + 'AnalystDataParent', ); private $__assetCache = array(); @@ -196,7 +197,7 @@ class GalaxyCluster extends AppModel */ public function arrangeData($cluster) { - $models = array('Galaxy', 'SharingGroup', 'GalaxyElement', 'GalaxyClusterRelation', 'Org', 'Orgc', 'TargetingClusterRelation'); + $models = array('Galaxy', 'SharingGroup', 'GalaxyElement', 'GalaxyClusterRelation', 'Org', 'Orgc', 'TargetingClusterRelation', 'Note', 'Opinion', 'Relationship'); foreach ($models as $model) { if (isset($cluster[$model])) { $cluster['GalaxyCluster'][$model] = $cluster[$model]; diff --git a/app/Model/MispObject.php b/app/Model/MispObject.php index 172a5ae21..dfae8b047 100644 --- a/app/Model/MispObject.php +++ b/app/Model/MispObject.php @@ -20,11 +20,12 @@ class MispObject extends AppModel public $actsAs = array( 'AuditLog', - 'Containable', - 'SysLogLogable.SysLogLogable' => array( // TODO Audit, logable - 'userModel' => 'User', - 'userKey' => 'user_id', - 'change' => 'full'), + 'Containable', + 'SysLogLogable.SysLogLogable' => array( // TODO Audit, logable + 'userModel' => 'User', + 'userKey' => 'user_id', + 'change' => 'full'), + 'AnalystDataParent' ); public $belongsTo = array( @@ -571,11 +572,18 @@ class MispObject extends AppModel if (isset($options['fields'])) { $params['fields'] = $options['fields']; } + $contain = []; + if (isset($options['contain'])) { + $contain = $options['contain']; + } + if (empty($contain['Event'])) { + $contain = ['Event' => ['distribution', 'id', 'user_id', 'orgc_id', 'org_id']]; + } $results = $this->find('all', array( 'conditions' => $params['conditions'], 'recursive' => -1, 'fields' => $params['fields'], - 'contain' => array('Event' => array('distribution', 'id', 'user_id', 'orgc_id', 'org_id')), + 'contain' => $contain, 'sort' => false )); return $results; @@ -1131,6 +1139,7 @@ class MispObject extends AppModel $this->Attribute->captureAttribute($attribute, $eventId, $user, $objectId, false, $parentEvent); } } + $this->Event->captureAnalystData($user, $object['Object']); return true; } @@ -1210,6 +1219,7 @@ class MispObject extends AppModel ); return $this->validationErrors; } + $this->Event->captureAnalystData($user, $object); if (!empty($object['Attribute'])) { $attributes = []; foreach ($object['Attribute'] as $attribute) { diff --git a/app/Model/Note.php b/app/Model/Note.php new file mode 100644 index 000000000..9f41df961 --- /dev/null +++ b/app/Model/Note.php @@ -0,0 +1,26 @@ +create(); $this->save($organisation); - return $this->id; + return $returnUUID ? $organisation['uuid'] : $this->id; } else { $changed = false; if (isset($org['uuid']) && empty($existingOrg[$this->alias]['uuid'])) { @@ -262,7 +262,7 @@ class Organisation extends AppModel $this->save($existingOrg); } } - return $existingOrg[$this->alias]['id']; + return $returnUUID ? $existingOrg[$this->alias]['uuid']: $existingOrg[$this->alias]['id']; } /** diff --git a/app/Model/Relationship.php b/app/Model/Relationship.php new file mode 100644 index 000000000..ea73cd777 --- /dev/null +++ b/app/Model/Relationship.php @@ -0,0 +1,146 @@ +__currentUser)) { + $user_id = Configure::read('CurrentUserId'); + $this->User = ClassRegistry::init('User'); + if ($user_id) { + $this->__currentUser = $this->User->getAuthUser($user_id); + } + } + foreach ($results as $i => $v) { + if (!empty($v[$this->alias]['related_object_type']) && !empty($v[$this->alias]['related_object_uuid'])) { + $results[$i][$this->alias]['related_object'] = $this->getRelatedElement($this->__currentUser, $v[$this->alias]['related_object_type'], $v[$this->alias]['related_object_uuid']); + } + } + return $results; + } + + public function getRelatedElement(array $user, $type, $uuid): array + { + $data = []; + if ($type == 'Event') { + $this->Event = ClassRegistry::init('Event'); + $params = [ + ]; + $backup = $this->Event->includeAnalystData; + $this->Event->includeAnalystData = false; + $data = $this->Event->fetchSimpleEvent($user, $uuid, $params); + $this->Event->includeAnalystData = $backup; + } else if ($type == 'Attribute') { + $this->Attribute = ClassRegistry::init('Attribute'); + $params = [ + 'conditions' => [ + ['Attribute.uuid' => $uuid], + ], + 'contain' => ['Event' => 'Orgc', 'Object',] + ]; + $backup = $this->Attribute->includeAnalystData; + $this->Attribute->includeAnalystData = false; + $data = $this->Attribute->fetchAttributeSimple($user, $params); + $this->Attribute->includeAnalystData = $backup; + $data = $this->rearrangeData($data, 'Attribute'); + } else if ($type == 'Object') { + $this->Object = ClassRegistry::init('MispObject'); + $params = [ + 'conditions' => [ + ['Object.uuid' => $uuid], + ], + 'contain' => ['Event' => 'Orgc',] + ]; + $backup = $this->Object->includeAnalystData; + $this->Object->includeAnalystData = false; + $data = $this->Object->fetchObjectSimple($user, $params); + $this->Object->includeAnalystData = $backup; + if (!empty($data)) { + $data = $data[0]; + } + $data = $this->rearrangeData($data, 'Object'); + } else if ($type == 'Note') { + $this->Note = ClassRegistry::init('Note'); + $params = [ + + ]; + $backup = $this->Note->includeAnalystData; + $this->Note->includeAnalystData = false; + $data = $this->Note->fetchNote(); + $this->Note->includeAnalystData = $backup; + } else if ($type == 'Opinion') { + $this->Opinion = ClassRegistry::init('Opinion'); + $params = [ + + ]; + $backup = $this->Opinion->includeAnalystData; + $this->Opinion->includeAnalystData = false; + $data = $this->Opinion->fetchOpinion(); + $this->Opinion->includeAnalystData = $backup; + } else if ($type == 'Relationship') { + $this->Relationship = ClassRegistry::init('Relationship'); + $params = [ + + ]; + $backup = $this->Relationship->includeAnalystData; + $this->Relationship->includeAnalystData = false; + $data = $this->Relationship->fetchRelationship(); + $this->Relationship->includeAnalystData = $backup; + } + return $data; + } + + private function rearrangeData(array $data, $objectType): array + { + $models = ['Event', 'Attribute', 'Object', 'Organisation', ]; + if (!empty($data) && !empty($data[$objectType])) { + foreach ($models as $model) { + if ($model == $objectType) { + continue; + } + if (isset($data[$model])) { + $data[$objectType][$model] = $data[$model]; + unset($data[$model]); + } + } + $data[$objectType]['Organisation'] = $data[$objectType]['Event']['Orgc']; + $data[$objectType]['orgc_uuid'] = $data[$objectType]['Event']['Orgc']['uuid']; + unset($data[$objectType]['Event']['Orgc']); + } + return $data; + } +} diff --git a/app/Model/Role.php b/app/Model/Role.php index b4274e33b..94754bbd1 100644 --- a/app/Model/Role.php +++ b/app/Model/Role.php @@ -331,6 +331,12 @@ class Role extends AppModel 'readonlyenabled' => true, 'title' => __('Allow the viewing of feed correlations. Enabling this can come at a performance cost.'), ), + 'perm_analyst_data' => array( + 'id' => 'RolePermAnalystData', + 'text' => 'Analyst Data Creator', + 'readonlyenabled' => false, + 'title' => __('Create or modify Analyst Data such as Analyst Notes or Opinions.'), + ), ); } } diff --git a/app/Model/Server.php b/app/Model/Server.php index 14a3f4ab6..9d3e21858 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -683,21 +683,24 @@ class Server extends AppModel $job->saveProgress($jobId, 'Pulling sightings.', 75); } $pulledSightings = $eventModel->Sighting->pullSightings($user, $serverSync); + $this->AnalystData = ClassRegistry::init('AnalystData'); + $pulledAnalystData = $this->AnalystData->pull($user, $serverSync); } if ($jobId) { $job->saveStatus($jobId, true, 'Pull completed.'); } $change = sprintf( - '%s events, %s proposals, %s sightings and %s galaxy clusters pulled or updated. %s events failed or didn\'t need an update.', + '%s events, %s proposals, %s sightings, %s galaxy clusters and %s analyst data pulled or updated. %s events failed or didn\'t need an update.', count($successes), $pulledProposals, $pulledSightings, $pulledClusters, + $pulledAnalystData, count($fails) ); $this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['Server']['id'], 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $email, $change); - return [$successes, $fails, $pulledProposals, $pulledSightings, $pulledClusters]; + return [$successes, $fails, $pulledProposals, $pulledSightings, $pulledClusters, $pulledAnalystData]; } public function filterRuleToParameter($filter_rules) @@ -757,6 +760,41 @@ class Server extends AppModel return $clusterArray; } + /** + * fetchUUIDsFromServer Fetch remote analyst datas' UUIDs and timestamp + * + * @param ServerSyncTool $serverSync + * @param array $conditions + * @return array The list of analyst data + * @throws JsonException|HttpSocketHttpException|HttpSocketJsonException + */ + public function fetchUUIDsFromServer(ServerSyncTool $serverSync, array $conditions = []) + { + $filterRules = $conditions; + $dataArray = $serverSync->fetchIndexMinimal($filterRules)->json(); + if (isset($dataArray['response'])) { + $dataArray = $dataArray['response']; + } + return $dataArray; + } + + /** + * filterAnalystDataForPush Send a candidate data to be pushed and returns the list of accepted entries + * + * @param ServerSyncTool $serverSync + * @param array $conditions + * @return array The list of analyst data + * @throws JsonException|HttpSocketHttpException|HttpSocketJsonException + */ + public function filterAnalystDataForPush(ServerSyncTool $serverSync, array $candidates = []) + { + $dataArray = $serverSync->filterAnalystDataForPush($candidates)->json(); + if (isset($dataArray['response'])) { + $dataArray = $dataArray['response']; + } + return $dataArray; + } + /** * Get a list of cluster IDs that are present on the remote server and returns clusters that should be pulled * @@ -1239,6 +1277,20 @@ class Server extends AppModel } else { $successes = array_merge($successes, $sightingSuccesses); } + + if ($push['canPush'] || $push['canEditAnalystData']) { + $this->AnalystData = ClassRegistry::init('AnalystData'); + $analystDataSuccesses = $this->AnalystData->push($user, $serverSync); + } else { + $analystDataSuccesses = array(); + } + + if (!isset($successes)) { + $successes = $analystDataSuccesses; + } else { + $successes = array_merge($successes, $analystDataSuccesses); + } + if (!isset($fails)) { $fails = array(); } @@ -2543,33 +2595,33 @@ class Server extends AppModel public function getFileRules() { - return array( - 'orgs' => array( + return [ + 'orgs' => [ 'name' => __('Organisation logos'), 'description' => __('The logo used by an organisation on the event index, event view, discussions, proposals, etc. Make sure that the filename is in the org.png format, where org is the case-sensitive organisation name.'), - 'expected' => array(), + 'expected' => [], 'valid_format' => __('48x48 pixel .png files or .svg file'), - 'path' => APP . 'webroot' . DS . 'img' . DS . 'orgs', + 'path' => APP . 'files' . DS . 'img' . DS . 'orgs', 'regex' => '.*\.(png|svg)$', 'regex_error' => __('Filename must be in the following format: *.png or *.svg'), - 'files' => array(), - ), - 'img' => array( + 'files' => [], + ], + 'img' => [ 'name' => __('Additional image files'), 'description' => __('Image files uploaded into this directory can be used for various purposes, such as for the login page logos'), - 'expected' => array( + 'expected' => [ 'MISP.footer_logo' => Configure::read('MISP.footer_logo'), 'MISP.home_logo' => Configure::read('MISP.home_logo'), 'MISP.welcome_logo' => Configure::read('MISP.welcome_logo'), 'MISP.welcome_logo2' => Configure::read('MISP.welcome_logo2'), - ), + ], 'valid_format' => __('PNG or SVG file'), - 'path' => APP . 'webroot' . DS . 'img' . DS . 'custom', + 'path' => APP . 'files' . DS . 'img' . DS . 'custom', 'regex' => '.*\.(png|svg)$', 'regex_error' => __('Filename must be in the following format: *.png or *.svg'), 'files' => array(), - ), - ); + ], + ]; } public function grabFiles() @@ -2752,6 +2804,7 @@ class Server extends AppModel $canPush = isset($remoteVersion['perm_sync']) ? $remoteVersion['perm_sync'] : false; $canSight = isset($remoteVersion['perm_sighting']) ? $remoteVersion['perm_sighting'] : false; $canEditGalaxyCluster = isset($remoteVersion['perm_galaxy_editor']) ? $remoteVersion['perm_galaxy_editor'] : false; + $canEditAnalystData = isset($remoteVersion['perm_analyst_data']) ? $remoteVersion['perm_analyst_data'] : false; $remoteVersionString = $remoteVersion['version']; $remoteVersion = explode('.', $remoteVersion['version']); if (!isset($remoteVersion[0])) { @@ -2801,6 +2854,7 @@ class Server extends AppModel 'response' => $response, 'canPush' => $canPush, 'canSight' => $canSight, + 'canEditAnalystData' => $canEditAnalystData, 'canEditGalaxyCluster' => $canEditGalaxyCluster, 'version' => $remoteVersion, 'protectedMode' => $protectedMode, diff --git a/app/Model/WorkflowModules/action/Module_attribute_edition_operation.php b/app/Model/WorkflowModules/action/Module_attribute_edition_operation.php index aaec37941..d30e54098 100644 --- a/app/Model/WorkflowModules/action/Module_attribute_edition_operation.php +++ b/app/Model/WorkflowModules/action/Module_attribute_edition_operation.php @@ -39,22 +39,21 @@ class Module_attribute_edition_operation extends WorkflowBaseActionModule protected function __saveAttributes(array $attributes, array $rData, array $params, array $user): array { $success = false; - $attributes = []; $newAttributes = []; foreach ($attributes as $k => $attribute) { $newAttribute = $this->_editAttribute($attribute, $rData, $params); - $newAttributes[] = $newAttribute; unset($newAttribute['timestamp']); + $newAttributes[] = $newAttribute; $result = $this->Attribute->editAttribute($newAttribute, $rData, $user, $newAttribute['object_id']); if (is_array($result)) { $attributes[] = $result; } } - $this->Attribute->editAttributeBulk($attributes, $rData, $user); - foreach ($attributes as $k => $attribute) { + $this->Attribute->editAttributeBulk($newAttributes, $rData, $user); + foreach ($newAttributes as $k => $attribute) { $saveSuccess = empty($this->Attribute->validationErrors[$k]); if ($saveSuccess) { - $rData = $this->_overrideAttribute($attribute, $newAttribute, $rData); + $rData = $this->_overrideAttribute($attribute, $attribute, $rData); } $success = $success || !empty($saveSuccess); } diff --git a/app/View/AnalystData/add.ctp b/app/View/AnalystData/add.ctp new file mode 100644 index 000000000..de7ae5437 --- /dev/null +++ b/app/View/AnalystData/add.ctp @@ -0,0 +1,228 @@ +request->params['action'] === 'edit' ? true : false; +$fields = [ + [ + 'field' => 'object_type', + 'class' => 'span2', + 'disabled' => !empty($this->data[$modelSelection]['object_type']), + 'default' => empty($this->data[$modelSelection]['object_type']) ? null : $this->data[$modelSelection]['object_type'], + 'options' => $dropdownData['valid_targets'], + 'type' => 'dropdown', + 'stayInLine' => 1 + ], + [ + 'field' => 'object_uuid', + 'class' => 'span4', + 'disabled' => !empty($this->data[$modelSelection]['object_uuid']), + 'default' => empty($this->data[$modelSelection]['object_uuid']) ? null : $this->data[$modelSelection]['object_uuid'] + ], + [ + 'field' => 'distribution', + 'class' => 'input', + 'options' => $dropdownData['distributionLevels'], + 'default' => isset($attribute['Attribute']['distribution']) ? $attribute['Attribute']['distribution'] : $initialDistribution, + 'stayInLine' => 1, + 'type' => 'dropdown' + ], + [ + 'field' => 'sharing_group_id', + 'class' => 'input', + 'options' => $dropdownData['sgs'], + 'label' => __("Sharing Group"), + 'type' => 'dropdown' + ], + [ + 'field' => 'authors', + 'class' => 'span3', + 'stayInLine' => $modelSelection === 'Note', + 'default' => $me['email'], + ], +]; + +if ($modelSelection === 'Note') { + $fields = array_merge($fields, + [ + [ + 'field' => 'language', + 'class' => 'span3', + 'options' => $languageRFC5646, + 'type' => 'dropdown', + ], + [ + 'field' => 'note', + 'type' => 'textarea', + 'class' => 'input span6', + ] + ] + ); +} else if ($modelSelection === 'Opinion') { + $fields = array_merge($fields, + [ + [ + 'field' => 'opinion', + 'class' => '', + 'type' => 'opinion', + ], + [ + 'field' => 'comment', + 'type' => 'textarea', + 'class' => 'input span6', + ] + ] + ); +} else if ($modelSelection === 'Relationship') { + $fields = array_merge($fields, + [ + [ + 'field' => 'relationship_type', + 'class' => 'span4', + 'options' => $existingRelations, + 'type' => 'text', + 'picker' => array( + 'text' => __('Pick Relationship'), + 'function' => 'pickerRelationshipTypes', + ) + ], + [ + 'field' => 'related_object_type', + 'class' => 'span2', + 'options' => $dropdownData['valid_targets'], + 'type' => 'dropdown', + 'stayInLine' => 1, + ], + [ + 'field' => 'related_object_uuid', + 'class' => 'span4', + ], + sprintf('