diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 7cce89e66..46ad250e8 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -4115,29 +4115,31 @@ class EventsController extends AppController public function filterEventIdsForPush() { - if ($this->request->is('post')) { - $incomingIDs = array(); - $incomingEvents = array(); - foreach ($this->request->data as $event) { - $incomingIDs[] = $event['Event']['uuid']; - $incomingEvents[$event['Event']['uuid']] = $event['Event']['timestamp']; - } - $events = $this->Event->find('all', array( - 'conditions' => array('Event.uuid' => $incomingIDs), - 'recursive' => -1, - 'fields' => array('Event.uuid', 'Event.timestamp', 'Event.locked'), - )); - foreach ($events as $event) { - if ($event['Event']['timestamp'] >= $incomingEvents[$event['Event']['uuid']]) { - unset($incomingEvents[$event['Event']['uuid']]); - continue; - } - if ($event['Event']['locked'] == 0) { - unset($incomingEvents[$event['Event']['uuid']]); - } - } - return $this->RestResponse->viewData(array_keys($incomingEvents), $this->response->type()); + if (!$this->request->is('post')) { + throw new MethodNotAllowedException(__('This endpoint requires a POST request.')); } + + $incomingUuids = []; + $incomingEvents = []; + foreach ($this->request->data as $event) { + $incomingUuids[] = $event['Event']['uuid']; + $incomingEvents[$event['Event']['uuid']] = $event['Event']['timestamp']; + } + $events = $this->Event->find('all', [ + 'conditions' => ['Event.uuid' => $incomingUuids], + 'recursive' => -1, + 'fields' => ['Event.uuid', 'Event.timestamp', 'Event.locked'], + ]); + foreach ($events as $event) { + if ($event['Event']['timestamp'] >= $incomingEvents[$event['Event']['uuid']]) { + unset($incomingEvents[$event['Event']['uuid']]); + continue; + } + if ($event['Event']['locked'] == 0) { + unset($incomingEvents[$event['Event']['uuid']]); + } + } + return $this->RestResponse->viewData(array_keys($incomingEvents), $this->response->type()); } public function checkuuid($uuid) diff --git a/app/Lib/Tools/ServerSyncTool.php b/app/Lib/Tools/ServerSyncTool.php index cff26a9f4..9cdbaaca5 100644 --- a/app/Lib/Tools/ServerSyncTool.php +++ b/app/Lib/Tools/ServerSyncTool.php @@ -1,5 +1,6 @@ get($url); } + /** + * @param array $events + * @return HttpSocketResponseExtended + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + */ + public function filterEventIdsForPush(array $events) + { + return $this->post('/events/filterEventIdsForPush', $events); + } + /** * @param array $event * @return HttpSocketResponseExtended @@ -164,6 +179,39 @@ class ServerSyncTool return $this->post('/attributes/restSearch.json', $rules); } + /** + * @param array $rules + * @return HttpSocketResponseExtended + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + */ + public function galaxyClusterSearch(array $rules) + { + return $this->post('/galaxy_clusters/restSearch', $rules); + } + + /** + * @param int|string $galaxyClusterId Galaxy Cluster ID or UUID + * @return HttpSocketResponseExtended + * @throws HttpSocketHttpException + */ + public function fetchGalaxyCluster($galaxyClusterId) + { + return $this->get('/galaxy_clusters/view/' . $galaxyClusterId); + } + + /** + * @param array $cluster + * @return HttpSocketResponseExtended + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + */ + public function pushGalaxyCluster(array $cluster) + { + $logMessage = "Pushing Galaxy Cluster #{$cluster['GalaxyCluster']['id']} to Server #{$this->serverId()}"; + return $this->post('/galaxies/pushCluster', [$cluster], $logMessage); + } + /** * @param array $params * @return HttpSocketResponseExtended @@ -332,6 +380,12 @@ class ServerSyncTool case self::FEATURE_PROTECTED_EVENT: $version = explode('.', $info['version']); return $version[0] == 2 && $version[1] == 4 && $version[2] > 155; + case self::FEATURE_EDIT_OF_GALAXY_CLUSTER: + return isset($info['perm_galaxy_editor']); + case self::PERM_SYNC: + return isset($info['perm_sync']) && $info['perm_sync']; + case self::PERM_GALAXY_EDITOR: + return isset($info['perm_galaxy_editor']) && $info['perm_galaxy_editor']; default: throw new InvalidArgumentException("Invalid flag `$flag` provided"); } @@ -373,7 +427,7 @@ class ServerSyncTool private function post($url, $data, $logMessage = null) { $protectedMode = !empty($data['Event']['protected']); - $data = json_encode($data, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); + $data = JsonTool::encode($data); if ($logMessage && !empty(Configure::read('Security.sync_audit'))) { $pushLogEntry = sprintf( diff --git a/app/Model/Event.php b/app/Model/Event.php index f468dd0c7..2305d6c94 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -17,6 +17,7 @@ App::uses('ProcessTool', 'Tools'); * @property ThreatLevel $ThreatLevel * @property Sighting $Sighting * @property Organisation $Org + * @property Organisation $Orgc * @property CryptographicKey $CryptographicKey */ class Event extends AppModel @@ -301,6 +302,9 @@ class Event extends AppModel private $assetCache = []; + /** @var array|null */ + private $eventBlockRule; + public function beforeDelete($cascade = true) { // blocklist the event UUID if the feature is enabled @@ -836,12 +840,13 @@ class Event extends AppModel /** * @param array $event * @param array $server + * @param ServerSyncTool $serverSync * @return false|string * @throws HttpSocketJsonException * @throws JsonException * @throws Exception */ - public function uploadEventToServer(array $event, array $server) + public function uploadEventToServer(array $event, array $server, ServerSyncTool $serverSync) { $this->Server = ClassRegistry::init('Server'); @@ -852,8 +857,6 @@ class Event extends AppModel return 'The distribution level of this event blocks it from being pushed.'; } - $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); - $push = $this->Server->checkVersionCompatibility($server, false, $serverSync); if (empty($push['canPush'])) { return 'The remote user is not a sync user - the upload of the event has been blocked.'; @@ -3437,17 +3440,18 @@ class Event extends AppModel return $attributes; } - public function checkEventBlockRules($event) + /** + * @param array $event + * @return bool + */ + private function checkEventBlockRules(array $event) { - if (!isset($this->AdminSetting)) { + if (!isset($this->eventBlockRule)) { $this->AdminSetting = ClassRegistry::init('AdminSetting'); + $setting = $this->AdminSetting->getSetting('eventBlockRule'); + $this->eventBlockRule = $setting ? json_decode($setting, true) : false; } - $setting = $this->AdminSetting->getSetting('eventBlockRule'); - if (empty($setting)) { - return true; - } - $rules = json_decode($setting, true); - if (empty($rules)) { + if (empty($this->eventBlockRule)) { return true; } if (!empty($rules['tags'])) { @@ -3548,7 +3552,9 @@ class Event extends AppModel public function _add(array &$data, $fromXml, array $user, $org_id = 0, $passAlong = null, $fromPull = false, $jobId = null, &$created_id = 0, &$validationErrors = array()) { if (Configure::read('MISP.enableEventBlocklisting') !== false && isset($data['Event']['uuid'])) { - $this->EventBlocklist = ClassRegistry::init('EventBlocklist'); + if (!isset($this->EventBlocklist)) { + $this->EventBlocklist = ClassRegistry::init('EventBlocklist'); + } if ($this->EventBlocklist->isBlocked($data['Event']['uuid'])) { return 'Blocked by blocklist'; } @@ -4219,8 +4225,15 @@ class Event extends AppModel $failedServers = []; foreach ($servers as $server) { + $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); try { - $this->pushSightingsToServer($server, $event, $sightingsUuidsToPush); + try { + if ($serverSync->eventExists($event) === false) { + continue; // skip if event not exists on remote server + } + } catch (Exception $e) {} + + $this->pushSightingsToServer($serverSync, $event, $sightingsUuidsToPush); } catch (Exception $e) { $this->logException("Uploading sightings to server {$server['Server']['id']} failed.", $e); $failedServers[] = $server['Server']['url']; @@ -4233,25 +4246,16 @@ class Event extends AppModel } /** - * @param array $server + * @param ServerSyncTool $serverSync * @param array $event * @param array $sightingsUuidsToPush * @throws HttpSocketJsonException - * @throws JsonException * @throws Exception */ - private function pushSightingsToServer(array $server, array $event, array $sightingsUuidsToPush = []) + private function pushSightingsToServer(ServerSyncTool $serverSync, array $event, array $sightingsUuidsToPush = []) { - App::uses('ServerSyncTool', 'Tools'); - $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); - try { - if ($serverSync->eventExists($event) === false) { - return; // skip if event not exists on remote server - } - } catch (Exception $e) {} - $fakeSyncUser = [ - 'org_id' => $server['Server']['remote_org_id'], + 'org_id' => $serverSync->server()['Server']['remote_org_id'], 'Role' => [ 'perm_site_admin' => 0, ], @@ -4320,8 +4324,6 @@ class Event extends AppModel } $uploaded = true; $failedServers = []; - App::uses('SyncTool', 'Tools'); - $syncTool = new SyncTool(); foreach ($servers as $server) { if ( @@ -4331,7 +4333,7 @@ class Event extends AppModel } // Skip servers where the event has come from. if ($passAlong != $server['Server']['id']) { - $HttpSocket = $syncTool->setupHttpSocket($server); + $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); $params = [ 'eventid' => $id, 'includeAttachments' => true, @@ -4359,17 +4361,17 @@ class Event extends AppModel 'perm_site_admin' => 0 ) ); - $this->Server->syncGalaxyClusters($HttpSocket, $server, $fakeSyncUser, $technique=$event['Event']['id'], $event=$event); - $thisUploaded = $this->uploadEventToServer($event, $server); + $this->Server->syncGalaxyClusters($serverSync, $server, $fakeSyncUser, $technique=$event['Event']['id'], $event=$event); + $thisUploaded = $this->uploadEventToServer($event, $server, $serverSync); if ($thisUploaded === 'Success') { try { - $this->pushSightingsToServer($server, $event); // push sighting by method that check for duplicates + $this->pushSightingsToServer($serverSync, $event); // push sighting by method that check for duplicates } catch (Exception $e) { $this->logException("Uploading sightings to server {$server['Server']['id']} failed.", $e); } } if (isset($this->data['ShadowAttribute'])) { - $this->Server->syncProposals($HttpSocket, $server, null, $id, $this); + $this->Server->syncProposals(null, $server, null, $id, $this); } if (!$thisUploaded) { $uploaded = !$uploaded ? $uploaded : $thisUploaded; @@ -4757,30 +4759,6 @@ class Event extends AppModel return $xmlArray; } - public function removeOlder(array &$events, $scope = 'events') - { - $field = $scope === 'sightings' ? 'sighting_timestamp' : 'timestamp'; - $localEvents = $this->find('all', [ - 'recursive' => -1, - 'fields' => ['Event.uuid', 'Event.' . $field, 'Event.locked'], - ]); - $localEvents = array_column(array_column($localEvents, 'Event'), null, 'uuid'); - foreach ($events as $k => $event) { - // remove all events for the sighting sync if the remote is not aware of the new field yet - if (!isset($event[$field])) { - unset($events[$k]); - } else { - $uuid = $event['uuid']; - if (isset($localEvents[$uuid]) - && ($localEvents[$uuid][$field] >= $event[$field] - || ($scope === 'events' && !$localEvents[$uuid]['locked']))) - { - unset($events[$k]); - } - } - } - } - public function sharingGroupRequired($field) { if ($this->data[$this->alias]['distribution'] == 4) { diff --git a/app/Model/GalaxyCluster.php b/app/Model/GalaxyCluster.php index cee5857c4..73d7d5a32 100644 --- a/app/Model/GalaxyCluster.php +++ b/app/Model/GalaxyCluster.php @@ -628,13 +628,11 @@ class GalaxyCluster extends AppModel $elevatedUser = array( 'Role' => array( 'perm_site_admin' => 1, - 'perm_sync' => 1 + 'perm_sync' => 1, + 'perm_audit' => 0, ), 'org_id' => $clusterOrgcId['GalaxyCluster']['orgc_id'] ); - $elevatedUser['Role']['perm_site_admin'] = 1; - $elevatedUser['Role']['perm_sync'] = 1; - $elevatedUser['Role']['perm_audit'] = 0; $cluster = $this->fetchGalaxyClusters($elevatedUser, array('minimal' => true, 'conditions' => array('id' => $clusterId)), $full=false); if (empty($cluster)) { return true; @@ -655,14 +653,10 @@ class GalaxyCluster extends AppModel return true; } $uploaded = false; - $failedServers = array(); - App::uses('SyncTool', 'Tools'); - foreach ($servers as &$server) { + foreach ($servers as $server) { if ((!isset($server['Server']['internal']) || !$server['Server']['internal']) && $cluster['GalaxyCluster']['distribution'] < 2) { continue; } - $syncTool = new SyncTool(); - $HttpSocket = $syncTool->setupHttpSocket($server); $fakeSyncUser = array( 'id' => 0, 'email' => 'fakeSyncUser@user.test', @@ -678,14 +672,13 @@ class GalaxyCluster extends AppModel ); $cluster = $this->fetchGalaxyClusters($fakeSyncUser, array('conditions' => array('GalaxyCluster.id' => $clusterId)), $full=true); if (empty($cluster)) { - return true; + continue; } $cluster = $cluster[0]; - $result = $this->uploadClusterToServer($cluster, $server, $HttpSocket, $fakeSyncUser); - if ($result == 'Success') { + $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); + $result = $this->uploadClusterToServer($cluster, $server, $serverSync, $fakeSyncUser); + if ($result === 'Success') { $uploaded = true; - } else { - $failedServers[] = $server; } } return $uploaded; @@ -1672,79 +1665,29 @@ class GalaxyCluster extends AppModel /** * @return string|bool The result of the upload. True if success, a string otherwise + * @throws Exception */ - public function uploadClusterToServer($cluster, $server, $HttpSocket, $user) - { - $this->Server = ClassRegistry::init('Server'); - $this->Log = ClassRegistry::init('Log'); - $push = $this->Server->checkVersionCompatibility($server, false); - if (empty($push['canPush']) && empty($push['canPushGalaxyCluster'])) { - return __('The remote user does not have the permission to manipulate galaxies - the upload of the galaxy clusters has been blocked.'); - } - $updated = null; - $newLocation = $newTextBody = ''; - $result = $this->__executeRestfulGalaxyClusterToServer($cluster, $server, null, $newLocation, $newTextBody, $HttpSocket, $user); - if ($result !== true) { - return $result; - } - if (strlen($newLocation)) { // HTTP/1.1 302 Found and Location: http:// - $result = $this->__executeRestfulGalaxyClusterToServer($cluster, $server, $newLocation, $newLocation, $newTextBody, $HttpSocket, $user); - if ($result !== true) { - return $result; - } - } - $uploadFailed = false; - try { - $json = json_decode($newTextBody, true); - } catch (Exception $e) { - $uploadFailed = true; - } - if (!is_array($json) || $uploadFailed) { - $this->Log->createLogEntry($user, 'push', 'GalaxyCluster', $cluster['GalaxyCluster']['id'], 'push', $newTextBody); - } - return 'Success'; - } - - private function __executeRestfulGalaxyClusterToServer($cluster, $server, $resourceId, &$newLocation, &$newTextBody, $HttpSocket, $user) - { - $result = $this->restfulGalaxyClusterToServer($cluster, $server, $resourceId, $newLocation, $newTextBody, $HttpSocket); - if (is_numeric($result)) { - $error = $this->__resolveErrorCode($result, $cluster, $server, $user); - if ($error) { - return $error . ' Error code: ' . $result; - } - } - return true; - } - - /** - * @return string|bool|int The result of the upload. - */ - public function restfulGalaxyClusterToServer($cluster, $server, $urlPath, &$newLocation, &$newTextBody, $HttpSocket = null) + public function uploadClusterToServer(array $cluster, array $server, ServerSyncTool $serverSync, array $user) { $cluster = $this->__prepareForPushToServer($cluster, $server); if (is_numeric($cluster)) { return $cluster; } - $url = $server['Server']['url']; - $HttpSocket = $this->setupHttpSocket($server, $HttpSocket); - $request = $this->setupSyncRequest($server); - $scope = 'galaxies/pushCluster'; - $uri = $url . '/' . $scope; - $clusters = array($cluster); - $data = json_encode($clusters); - if (!empty(Configure::read('Security.sync_audit'))) { - $pushLogEntry = sprintf( - "==============================================================\n\n[%s] Pushing Galaxy Cluster #%d to Server #%d:\n\n%s\n\n", - date("Y-m-d H:i:s"), - $cluster['GalaxyCluster']['id'], - $server['Server']['id'], - $data - ); - file_put_contents(APP . 'files/scripts/tmp/debug_server_' . $server['Server']['id'] . '.log', $pushLogEntry, FILE_APPEND); + + try { + if (!$serverSync->isSupported(ServerSyncTool::PERM_SYNC) || $serverSync->isSupported(ServerSyncTool::PERM_GALAXY_EDITOR)) { + return __('The remote user does not have the permission to manipulate galaxies - the upload of the galaxy clusters has been blocked.'); + } + $serverSync->pushGalaxyCluster($cluster)->json(); + } catch (Exception $e) { + $title = __('Uploading GalaxyCluster (%s) to Server (%s)', $cluster['GalaxyCluster']['id'], $server['Server']['id']); + $this->loadLog()->createLogEntry($user, 'push', 'GalaxyCluster', $cluster['GalaxyCluster']['id'], $title, $e->getMessage()); + + $this->logException("Could not push galaxy cluster to remote server {$serverSync->serverId()}", $e); + return $e->getMessage(); } - $response = $HttpSocket->post($uri, $data, $request); - return $this->__handleRestfulGalaxyClusterToServerResponse($response, $newLocation, $newTextBody); + + return 'Success'; } /** @@ -1752,7 +1695,7 @@ class GalaxyCluster extends AppModel * * @param array $cluster * @param array $server - * @return array The cluster ready to be pushed + * @return array|int The cluster ready to be pushed */ private function __prepareForPushToServer(array $cluster, array $server) { @@ -1773,11 +1716,9 @@ class GalaxyCluster extends AppModel } $this->Event = ClassRegistry::init('Event'); if ($this->Event->checkDistributionForPush($cluster, $server, 'GalaxyCluster')) { - $cluster = $this->__updateClusterForSync($cluster, $server); - } else { - return 403; + return $this->__updateClusterForSync($cluster, $server); } - return $cluster; + return 403; } /** @@ -1886,90 +1827,39 @@ class GalaxyCluster extends AppModel return $relation; } - /** - * @return string|bool|int The result of the upload. - */ - private function __handleRestfulGalaxyClusterToServerResponse($response, &$newLocation, &$newTextBody) - { - switch ($response->code) { - case '200': // 200 (OK) + entity-action-result - if ($response->isOk()) { - $newTextBody = $response->body(); - return true; - } else { - try { - $jsonArray = json_decode($response->body, true); - } catch (Exception $e) { - return true; - } - return $jsonArray['name']; - } - // no break - case '302': // Found - $newLocation = $response->headers['Location']; - $newTextBody = $response->body(); - return true; - case '404': // Not Found - $newLocation = $response->headers['Location']; - $newTextBody = $response->body(); - return 404; - case '405': - return 405; - case '403': // Not authorised - return 403; - } - } - - private function __resolveErrorCode($code, &$cluster, &$server, $user) - { - $this->Log = ClassRegistry::init('Log'); - $error = false; - switch ($code) { - case 403: - return __('The distribution level of the cluster blocks it from being pushed.'); - case 405: - $error = __('The sync user on the remote instance does not have the required privileges to handle this cluster.'); - break; - } - if ($error) { - $newTextBody = 'Uploading GalaxyCluster (' . $cluster['GalaxyCluster']['id'] . ') to Server (' . $server['Server']['id'] . ')'; - $newTextBody = __('Uploading GalaxyCluster (%s) to Server (%s)', $cluster['GalaxyCluster']['id'], $server['Server']['id']); - $this->Log->createLogEntry($user, 'push', 'GalaxyCluster', $cluster['GalaxyCluster']['id'], 'push', $newTextBody); - } - return $error; - } - /** * pullGalaxyClusters * - * @param array $user - * @param array $server - * @param string|int $technique The technique startegy used for pulling + * @param array $user + * @param ServerSyncTool $serverSync + * @param string|int $technique The technique startegy used for pulling * allowed: * - int event containing the clusters to pulled * - string pull everything * - string pull updates of cluster present locally * - string pull clusters based on tags present locally * @return int The number of pulled clusters + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException */ - public function pullGalaxyClusters(array $user, array $server, $technique = 'full') + public function pullGalaxyClusters(array $user, ServerSyncTool $serverSync, $technique = 'full') { - $this->Server = ClassRegistry::init('Server'); - $compatible = $this->Server->checkVersionCompatibility($server, $user)['supportEditOfGalaxyCluster']; + $compatible = $serverSync->isSupported(ServerSyncTool::FEATURE_EDIT_OF_GALAXY_CLUSTER); if (!$compatible) { return 0; } - $clusterIds = $this->getClusterIdListBasedOnPullTechnique($user, $technique, $server); - $successes = array(); - $fails = array(); + $clusterIds = $this->getClusterIdListBasedOnPullTechnique($user, $technique, $serverSync); + $successes = 0; // now process the $clusterIds to pull each of the events sequentially if (!empty($clusterIds)) { // download each cluster - foreach ($clusterIds as $k => $clusterId) { - $this->__pullGalaxyCluster($clusterId, $successes, $fails, $server, $user); + foreach ($clusterIds as $clusterId) { + if ($this->__pullGalaxyCluster($clusterId, $serverSync, $user)) { + $successes++; + } } } - return count($successes); + return $successes; } /** @@ -1977,16 +1867,16 @@ class GalaxyCluster extends AppModel * * @param array $user * @param string|int $technique - * @param array $server + * @param ServerSyncTool $serverSync * @return array cluster ID list to be pulled */ - private function getClusterIdListBasedOnPullTechnique(array $user, $technique, array $server) + private function getClusterIdListBasedOnPullTechnique(array $user, $technique, ServerSyncTool $serverSync) { $this->Server = ClassRegistry::init('Server'); try { if ("update" === $technique) { $localClustersToUpdate = $this->getElligibleLocalClustersToUpdate($user); - $clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = true, $elligibleClusters = $localClustersToUpdate); + $clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($serverSync, $onlyUpdateLocalCluster = true, $elligibleClusters = $localClustersToUpdate); } elseif ("pull_relevant_clusters" === $technique) { // Fetch all local custom cluster tags then fetch their corresponding clusters on the remote end $tagNames = $this->Tag->find('column', array( @@ -2005,55 +1895,39 @@ class GalaxyCluster extends AppModel } $localClustersToUpdate = $this->getElligibleLocalClustersToUpdate($user); $conditions = array('uuid' => array_keys($clusterUUIDs)); - $clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = false, $elligibleClusters = $localClustersToUpdate, $conditions = $conditions); + $clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($serverSync, $onlyUpdateLocalCluster = false, $elligibleClusters = $localClustersToUpdate, $conditions = $conditions); } elseif (is_numeric($technique)) { $conditions = array('eventid' => $technique); - $clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = false, $elligibleClusters = array(), $conditions = $conditions); + $clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($serverSync, $onlyUpdateLocalCluster = false, $elligibleClusters = array(), $conditions = $conditions); } else { - $clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = false); + $clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($serverSync, $onlyUpdateLocalCluster = false); } } catch (HttpSocketHttpException $e) { if ($e->getCode() === 403) { return array('error' => array(1, null)); } else { - $this->logException("Could not get eligible cluster IDs from server {$server['Server']['id']} for pull.", $e); + $this->logException("Could not get eligible cluster IDs from server {$serverSync->serverId()} for pull.", $e); return array('error' => array(2, $e->getMessage())); } } catch (Exception $e) { - $this->logException("Could not get eligible cluster IDs from server {$server['Server']['id']} for pull.", $e); + $this->logException("Could not get eligible cluster IDs from server {$serverSync->serverId()} for pull.", $e); return array('error' => array(2, $e->getMessage())); } return $clusterIds; } - private function __pullGalaxyCluster($clusterId, &$successes, &$fails, $server, $user) + private function __pullGalaxyCluster($clusterId, ServerSyncTool $serverSync, array $user) { - $cluster = $this->downloadGalaxyClusterFromServer($clusterId, $server); - if (!empty($cluster)) { - $cluster = $this->updatePulledClusterBeforeInsert($cluster, $server, $user); - $result = $this->captureCluster($user, $cluster, $fromPull=true, $orgId=$server['Server']['org_id']); - if ($result['success']) { - $successes[] = $clusterId; - } else { - $fails[$clusterId] = __('Failed because of errors: ') . json_encode($result['errors']); - } - } else { - $fails[$clusterId] = __('failed downloading the galaxy cluster'); + try { + $cluster = $serverSync->fetchGalaxyCluster($clusterId)->json(); + } catch (Exception $e) { + $this->logException("Could not fetch galaxy cluster $clusterId from server {$serverSync->serverId()}", $e); + return false; } - return true; - } - public function downloadGalaxyClusterFromServer($clusterId, $server, $HttpSocket=null) - { - $url = $server['Server']['url']; - $HttpSocket = $this->setupHttpSocket($server, $HttpSocket); - $request = $this->setupSyncRequest($server); - $uri = $url . '/galaxy_clusters/view/' . $clusterId; - $response = $HttpSocket->get($uri, $data = '', $request); - if ($response->isOk()) { - return json_decode($response->body, true); - } - return null; + $cluster = $this->updatePulledClusterBeforeInsert($cluster, $serverSync->server(), $user); + $result = $this->captureCluster($user, $cluster, $fromPull=true, $orgId=$serverSync->server()['Server']['org_id']); + return $result['success']; } private function updatePulledClusterBeforeInsert($cluster, $server, $user) diff --git a/app/Model/Server.php b/app/Model/Server.php index 7618b479c..a2b635260 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -222,25 +222,14 @@ class Server extends AppModel throw new InvalidArgumentException("Invalid pull technique `$technique`."); } - private function __checkIfEventIsBlockedBeforePull($event) - { - if (Configure::read('MISP.enableEventBlocklisting') !== false) { - if (!isset($this->EventBlocklist)) { - $this->EventBlocklist = ClassRegistry::init('EventBlocklist'); - } - if ($this->EventBlocklist->isBlocked($event['Event']['uuid'])) { - return true; - } - } - return false; - } - /** * @param array $event * @param array $server * @param array $user + * @param array $pullRules + * @param bool $pullRulesEmptiedEvent */ - private function __updatePulledEventBeforeInsert(array &$event, array $server, array $user, array $pullRules, bool &$pullRulesEmptiedEvent=false) + private function __updatePulledEventBeforeInsert(array &$event, array $server, array $user, array $pullRules, bool &$pullRulesEmptiedEvent = false) { // we have an Event array // The event came from a pull, so it should be locked. @@ -269,14 +258,12 @@ class Server extends AppModel } } + $typeFilteringEnabled = !empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && + !empty($pullRules['type_attributes']['NOT']); if (isset($event['Event']['Attribute'])) { $originalCount = count($event['Event']['Attribute']); foreach ($event['Event']['Attribute'] as $key => $attribute) { - if ( - !empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && - !empty($pullRules['type_attributes']['NOT']) && - in_array($attribute['type'], $pullRules['type_attributes']['NOT']) - ) { + if ($typeFilteringEnabled && in_array($attribute['type'], $pullRules['type_attributes']['NOT'])) { unset($event['Event']['Attribute'][$key]); continue; } @@ -297,7 +284,7 @@ class Server extends AppModel } } } - if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalCount > 0 && count($event['Event']['Attribute']) == 0) { + if ($typeFilteringEnabled && $originalCount > 0 && empty($event['Event']['Attribute'])) { $pullRulesEmptiedEvent = true; } } @@ -323,11 +310,7 @@ class Server extends AppModel if (isset($object['Attribute'])) { $originalAttributeCount = count($object['Attribute']); foreach ($object['Attribute'] as $j => $a) { - if ( - !empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && - !empty($pullRules['type_attributes']['NOT']) && - in_array($a['type'], $pullRules['type_attributes']['NOT']) - ) { + if ($typeFilteringEnabled && in_array($a['type'], $pullRules['type_attributes']['NOT'])) { unset($event['Event']['Object'][$i]['Attribute'][$j]); continue; } @@ -348,13 +331,13 @@ class Server extends AppModel } } } - if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalAttributeCount > 0 && empty($event['Event']['Object'][$i]['Attribute'])) { + if ($typeFilteringEnabled && $originalAttributeCount > 0 && empty($event['Event']['Object'][$i]['Attribute'])) { unset($event['Event']['Object'][$i]); // Object is empty, get rid of it $pullRulesEmptiedEvent = true; } } } - if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalObjectCount > 0 && count($event['Event']['Object']) == 0) { + if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalObjectCount > 0 && empty($event['Event']['Object'])) { $pullRulesEmptiedEvent = true; } } @@ -466,7 +449,18 @@ class Server extends AppModel } } - private function __pullEvent($eventId, &$successes, &$fails, Event $eventModel, ServerSyncTool $serverSync, $user, $jobId, $force = false) + /** + * @param int|string $eventId Event ID or UUID + * @param array $successes + * @param array $fails + * @param Event $eventModel + * @param ServerSyncTool $serverSync + * @param array $user + * @param int $jobId + * @param bool $force + * @return bool + */ + private function __pullEvent($eventId, array &$successes, array &$fails, Event $eventModel, ServerSyncTool $serverSync, $user, $jobId, $force = false) { $params = [ 'deleted' => [0, 1], @@ -489,23 +483,16 @@ class Server extends AppModel return false; } - if (!empty($event)) { - if ($this->__checkIfEventIsBlockedBeforePull($event)) { - return false; + $pullRulesEmptiedEvent = false; + $this->__updatePulledEventBeforeInsert($event, $serverSync->server(), $user, $serverSync->pullRules(), $pullRulesEmptiedEvent); + + if (!$this->__checkIfEventSaveAble($event)) { + if (!$pullRulesEmptiedEvent) { // The event is empty because of the filtering rule. This is not considered a failure + $fails[$eventId] = __('Empty event detected.'); } - $pullRulesEmptiedEvent = false; - $this->__updatePulledEventBeforeInsert($event, $serverSync->server(), $user, $serverSync->pullRules(), $pullRulesEmptiedEvent); - if (!$this->__checkIfEventSaveAble($event)) { - if (!$pullRulesEmptiedEvent) { // The event is empty because of the filtering rule. This is not considered a failure - $fails[$eventId] = __('Empty event detected.'); - } - } else { - $this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $headers, $body); - } - } else { - // error - $fails[$eventId] = __('failed downloading the event'); + return false; } + $this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $headers, $body); return true; } @@ -551,7 +538,7 @@ class Server extends AppModel if ($jobId) { $job->saveProgress($jobId, $technique === 'pull_relevant_clusters' ? __('Pulling relevant galaxy clusters.') : __('Pulling galaxy clusters.')); } - $pulledClusters = $this->GalaxyCluster->pullGalaxyClusters($user, $server, $technique); + $pulledClusters = $this->GalaxyCluster->pullGalaxyClusters($user, $serverSync, $technique); if ($technique === 'pull_relevant_clusters') { if ($jobId) { $job->saveStatus($jobId, true, 'Pulling complete.'); @@ -579,8 +566,8 @@ class Server extends AppModel /** @var Event $eventModel */ $eventModel = ClassRegistry::init('Event'); - $successes = array(); - $fails = array(); + $successes = []; + $fails = []; // now process the $eventIds to pull each of the events sequentially if (!empty($eventIds)) { // download each event @@ -622,7 +609,7 @@ class Server extends AppModel count($fails) ); $this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['Server']['id'], 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $email, $change); - return array($successes, $fails, $pulledProposals, $pulledSightings, $pulledClusters); + return [$successes, $fails, $pulledProposals, $pulledSightings, $pulledClusters]; } public function filterRuleToParameter($filter_rules) @@ -662,30 +649,20 @@ class Server extends AppModel /** * fetchCustomClusterIdsFromServer Fetch custom-published remote clusters' UUIDs and versions * - * @param array $server - * @param HttpSocketExtended|null $HttpSocket + * @param ServerSyncTool $serverSync * @param array $conditions * @return array The list of clusters * @throws JsonException|HttpSocketHttpException|HttpSocketJsonException */ - private function fetchCustomClusterIdsFromServer(array $server, HttpSocketExtended $HttpSocket=null, array $conditions=array()) + private function fetchCustomClusterIdsFromServer(ServerSyncTool $serverSync, array $conditions = []) { - $url = $server['Server']['url']; - $HttpSocket = $this->setupHttpSocket($server, $HttpSocket); - $request = $this->setupSyncRequest($server); - $uri = $url . '/galaxy_clusters/restSearch'; $filterRules = [ 'published' => 1, 'minimal' => 1, 'custom' => 1, ]; $filterRules = array_merge($filterRules, $conditions); - $response = $HttpSocket->post($uri, json_encode($filterRules), $request); - if (!$response->isOk()) { - throw new HttpSocketHttpException($response); - } - - $clusterArray = $response->json(); + $clusterArray = $serverSync->galaxyClusterSearch($filterRules)->json(); if (isset($clusterArray['response'])) { $clusterArray = $clusterArray['response']; } @@ -705,9 +682,9 @@ class Server extends AppModel * @throws HttpSocketJsonException * @throws JsonException */ - public function getElligibleClusterIdsFromServerForPull(array $server, $HttpSocket=null, $onlyUpdateLocalCluster=true, array $elligibleClusters=array(), array $conditions=array()) + public function getElligibleClusterIdsFromServerForPull(ServerSyncTool $serverSyncTool, $onlyUpdateLocalCluster=true, array $elligibleClusters=array(), array $conditions=array()) { - $clusterArray = $this->fetchCustomClusterIdsFromServer($server, $HttpSocket, $conditions=$conditions); + $clusterArray = $this->fetchCustomClusterIdsFromServer($serverSyncTool, $conditions=$conditions); if (!empty($clusterArray)) { foreach ($clusterArray as $cluster) { if (isset($elligibleClusters[$cluster['GalaxyCluster']['uuid']])) { @@ -730,8 +707,7 @@ class Server extends AppModel /** * Get an array of cluster_ids that are present on the remote server and returns clusters that should be pushed. - * @param array $server - * @param HttpSocket|null $HttpSocket + * @param ServerSyncTool $serverSync * @param array $localClusters * @param array $conditions * @return array @@ -739,9 +715,9 @@ class Server extends AppModel * @throws HttpSocketJsonException * @throws JsonException */ - public function getElligibleClusterIdsFromServerForPush(array $server, $HttpSocket=null, $localClusters=array(), $conditions=array()) + private function getElligibleClusterIdsFromServerForPush(ServerSyncTool $serverSync, array $localClusters=array(), array $conditions=array()) { - $clusterArray = $this->fetchCustomClusterIdsFromServer($server, $HttpSocket, $conditions=$conditions); + $clusterArray = $this->fetchCustomClusterIdsFromServer($serverSync, $conditions=$conditions); $keyedClusterArray = Hash::combine($clusterArray, '{n}.GalaxyCluster.uuid', '{n}.GalaxyCluster.version'); if (!empty($localClusters)) { foreach ($localClusters as $k => $localCluster) { @@ -784,13 +760,35 @@ class Server extends AppModel return $eventIndex; } + /** + * @param array $events + * @return void + */ + private function removeOlderEvents(array &$events) + { + $conditions = (count($events) > 10000) ? [] : ['Event.uuid' => array_column($events, 'uuid')]; + $this->Event = ClassRegistry::init('Event'); + $localEvents = $this->Event->find('all', [ + 'recursive' => -1, + 'conditions' => $conditions, + 'fields' => ['Event.uuid', 'Event.timestamp', 'Event.locked'], + ]); + $localEvents = array_column(array_column($localEvents, 'Event'), null, 'uuid'); + foreach ($events as $k => $event) { + $uuid = $event['uuid']; + if (isset($localEvents[$uuid]) && ($localEvents[$uuid]['timestamp'] >= $event['timestamp'] || !$localEvents[$uuid]['locked'])) { + unset($events[$k]); + } + } + } + /** * Get an array of event UUIDs that are present on the remote server. * * @param ServerSyncTool $serverSync * @param bool $all - * @param bool $ignoreFilterRules - * @param bool $force + * @param bool $ignoreFilterRules Ignore defined server pull rules + * @param bool $force If true, returns all events regardless their update timestamp * @return array Array of event UUIDs. * @throws HttpSocketHttpException * @throws HttpSocketJsonException @@ -820,8 +818,7 @@ class Server extends AppModel } } if (!$force) { - $this->Event = ClassRegistry::init('Event'); - $this->Event->removeOlder($eventArray); + $this->removeOlderEvents($eventArray); } return array_column($eventArray, 'uuid'); } @@ -940,7 +937,7 @@ class Server extends AppModel // sync custom galaxy clusters if user is capable if ($push['canEditGalaxyCluster'] && $server['Server']['push_galaxy_clusters'] && "full" == $technique) { - $clustersSuccesses = $this->syncGalaxyClusters($HttpSocket, $this->data, $user, $technique='full'); + $clustersSuccesses = $this->syncGalaxyClusters($serverSync, $this->data, $user, $technique='full'); } else { $clustersSuccesses = array(); } @@ -989,7 +986,7 @@ class Server extends AppModel 'fields' => array('Event.id', 'Event.timestamp', 'Event.sighting_timestamp', 'Event.uuid', 'Event.orgc_id'), // array of field names ); $eventIds = $this->Event->find('all', $findParams); - $eventUUIDsFiltered = $this->getEventIdsForPush($server, $HttpSocket, $eventIds); + $eventUUIDsFiltered = $this->getEventIdsForPush($server, $serverSync, $eventIds); if (!empty($eventUUIDsFiltered)) { $eventCount = count($eventUUIDsFiltered); // now process the $eventIds to push each of the events sequentially @@ -1018,11 +1015,11 @@ class Server extends AppModel $event = $event[0]; $event['Event']['locked'] = 1; if ($push['canEditGalaxyCluster'] && $server['Server']['push_galaxy_clusters'] && "full" != $technique) { - $clustersSuccesses = $this->syncGalaxyClusters($HttpSocket, $this->data, $user, $technique=$event['Event']['id'], $event=$event); + $clustersSuccesses = $this->syncGalaxyClusters($serverSync, $this->data, $user, $technique=$event['Event']['id'], $event=$event); } else { $clustersSuccesses = array(); } - $result = $this->Event->uploadEventToServer($event, $server, $HttpSocket); + $result = $this->Event->uploadEventToServer($event, $server, $serverSync); if ('Success' === $result) { $successes[] = $event['Event']['id']; } else { @@ -1049,7 +1046,7 @@ class Server extends AppModel if ($push['canPush'] || $push['canSight']) { $this->Sighting = ClassRegistry::init('Sighting'); - $sightingSuccesses =$this->Sighting->pushSightings($user, $serverSync); + $sightingSuccesses = $this->Sighting->pushSightings($user, $serverSync); } else { $sightingSuccesses = array(); } @@ -1083,23 +1080,35 @@ class Server extends AppModel return true; } - public function getEventIdsForPush(array $server, HttpSocket $HttpSocket, array $eventIds) + /** + * @param array $server + * @param ServerSyncTool $serverSync + * @param array $events + * @return array|false + */ + private function getEventIdsForPush(array $server, ServerSyncTool $serverSync, array $events) { - foreach ($eventIds as $k => $event) { - if (empty($this->eventFilterPushableServers($event, array($server)))) { - unset($eventIds[$k]); + $request = []; + foreach ($events as $event) { + if (empty($this->eventFilterPushableServers($event, [$server]))) { continue; } - unset($eventIds[$k]['Event']['id']); + $request[] = ['Event' => [ + 'uuid' => $event['Event']['uuid'], + 'timestamp' => $event['Event']['timestamp'], + ]]; } - $request = $this->setupSyncRequest($server); - $data = json_encode($eventIds); - $uri = $server['Server']['url'] . '/events/filterEventIdsForPush'; - $response = $HttpSocket->post($uri, $data, $request); - if ($response->code == '200') { - return $this->jsonDecode($response->body()); + + if (empty($request)) { + return []; + } + + try { + return $serverSync->filterEventIdsForPush($request)->json(); + } catch (Exception $e) { + $this->logException("Could not filter events for push when pushing to server {$serverSync->serverId()}", $e); + return false; } - return false; } /** @@ -1112,7 +1121,7 @@ class Server extends AppModel * @param array|bool $event * @return array List of successfully pushed clusters */ - public function syncGalaxyClusters($HttpSocket, array $server, array $user, $technique='full', $event=false) + public function syncGalaxyClusters(ServerSyncTool $serverSync, array $server, array $user, $technique='full', $event=false) { $successes = array(); if (!$server['Server']['push_galaxy_clusters']) { @@ -1120,7 +1129,6 @@ class Server extends AppModel } $this->GalaxyCluster = ClassRegistry::init('GalaxyCluster'); $this->Event = ClassRegistry::init('Event'); - $HttpSocket = $this->setupHttpSocket($server, $HttpSocket); $clusters = array(); if ($technique == 'full') { $clusters = $this->GalaxyCluster->getElligibleClustersToPush($user, $conditions=array(), $full=true); @@ -1135,13 +1143,13 @@ class Server extends AppModel } $localClusterUUIDs = Hash::extract($clusters, '{n}.GalaxyCluster.uuid'); try { - $clustersToPush = $this->getElligibleClusterIdsFromServerForPush($server, $HttpSocket = $HttpSocket, $localClusters = $clusters, $conditions = array('uuid' => $localClusterUUIDs)); + $clustersToPush = $this->getElligibleClusterIdsFromServerForPush($serverSync, $localClusters = $clusters, $conditions = array('uuid' => $localClusterUUIDs)); } catch (Exception $e) { $this->logException("Could not get eligible cluster IDs from server #{$server['Server']['id']} for push.", $e); return []; } foreach ($clustersToPush as $cluster) { - $result = $this->GalaxyCluster->uploadClusterToServer($cluster, $server, $HttpSocket, $user); + $result = $this->GalaxyCluster->uploadClusterToServer($cluster, $server, $serverSync, $user); if ($result === 'Success') { $successes[] = __('GalaxyCluster %s', $cluster['GalaxyCluster']['uuid']); } @@ -1153,10 +1161,10 @@ class Server extends AppModel { $saModel = ClassRegistry::init('ShadowAttribute'); $HttpSocket = $this->setupHttpSocket($server, $HttpSocket); - $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); if ($sa_id == null) { if ($event_id == null) { // event_id is null when we are doing a push + $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); try { $ids = $this->getEventIdsFromServer($serverSync, true, true); } catch (Exception $e) { @@ -2503,7 +2511,6 @@ class Server extends AppModel $canPush = isset($remoteVersion['perm_sync']) ? $remoteVersion['perm_sync'] : false; $canSight = isset($remoteVersion['perm_sighting']) ? $remoteVersion['perm_sighting'] : false; - $supportEditOfGalaxyCluster = isset($remoteVersion['perm_galaxy_editor']); $canEditGalaxyCluster = isset($remoteVersion['perm_galaxy_editor']) ? $remoteVersion['perm_galaxy_editor'] : false; $remoteVersionString = $remoteVersion['version']; $remoteVersion = explode('.', $remoteVersion['version']); @@ -2555,7 +2562,6 @@ class Server extends AppModel 'canPush' => $canPush, 'canSight' => $canSight, 'canEditGalaxyCluster' => $canEditGalaxyCluster, - 'supportEditOfGalaxyCluster' => $supportEditOfGalaxyCluster, 'version' => $remoteVersion, 'protectedMode' => $protectedMode, ]; @@ -3770,8 +3776,7 @@ class Server extends AppModel public function extensionDiagnostics() { try { - $file = new File(APP . DS . 'composer.json'); - $composer = $this->jsonDecode($file->read()); + $composer = FileAccessTool::readJsonFromFile(APP . DS . 'composer.json'); $extensions = []; foreach ($composer['require'] as $require => $foo) { if (substr($require, 0, 4) === 'ext-') {