server = $server; $this->request = $request; $syncTool = new SyncTool(); $this->socket = $syncTool->setupHttpSocket($server); } /** * Check if event exists on remote server by event UUID. * @param array $event * @return bool * @throws Exception */ public function eventExists(array $event) { $url = $this->server['Server']['url'] . '/events/view/' . $event['Event']['uuid']; $start = microtime(true); $exists = $this->socket->head($url, [], $this->request); $this->log($start, 'HEAD', $url, $exists); if ($exists->code == '404') { return false; } if ($exists->code == '200') { return true; } throw new HttpSocketHttpException($exists, $url); } /** * @param array $params * @param string|null $etag * @return HttpSocketResponseExtended * @throws HttpSocketHttpException * @throws HttpSocketJsonException */ public function eventIndex($params = [], $etag = null) { return $this->post('/events/index', $params, null, $etag); } /** * @param int|string $eventId Event ID or UUID * @param array $params * @return HttpSocketResponseExtended * @throws HttpSocketHttpException */ public function fetchEvent($eventId, array $params = []) { $url = "/events/view/$eventId"; $url .= $this->createParams($params); return $this->get($url); } /** * @param array $events * @return HttpSocketResponseExtended * @throws HttpSocketHttpException * @throws HttpSocketJsonException */ public function filterEventIdsForPush(array $events) { return $this->post('/events/filterEventIdsForPush', $events); } /** * @param array $event * @return HttpSocketResponseExtended * @throws HttpSocketHttpException * @throws HttpSocketJsonException */ public function pushEvent(array $event) { try { // Check if event exists on remote server to use proper endpoint $exists = $this->eventExists($event); } catch (Exception $e) { // In case of failure consider that event doesn't exists $exists = false; } try { return $exists ? $this->updateEvent($event) : $this->createEvent($event); } catch (HttpSocketHttpException $e) { if ($e->getCode() === 404) { // Maybe the check if event exists was not correct, try to create a new event if ($exists) { return $this->createEvent($event); // There is bug in MISP API, that returns response code 404 with Location if event already exists } else if ($e->getResponse()->getHeader('Location')) { $urlPath = $e->getResponse()->getHeader('Location'); $pieces = explode('/', $urlPath); $lastPart = end($pieces); return $this->updateEvent($event, $lastPart); } } throw $e; } } /** * @param array $event * @return HttpSocketResponseExtended * @throws HttpSocketHttpException * @throws HttpSocketJsonException */ public function createEvent(array $event) { $logMessage = "Pushing Event #{$event['Event']['id']} to Server #{$this->serverId()}"; return $this->post("/events/add/metadata:1", $event, $logMessage); } /** * @param array $event * @param int|string|null Event ID or UUID that should be updated. If not provieded, UUID from $event will be used * @return HttpSocketResponseExtended * @throws HttpSocketHttpException * @throws HttpSocketJsonException */ public function updateEvent(array $event, $eventId = null) { if ($eventId === null) { $eventId = $event['Event']['uuid']; } $logMessage = "Pushing Event #{$event['Event']['id']} to Server #{$this->serverId()}"; return $this->post("/events/edit/$eventId/metadata:1", $event, $logMessage); } /** * @param array $rules * @return HttpSocketResponseExtended * @throws HttpSocketHttpException * @throws HttpSocketJsonException */ public function attributeSearch(array $rules) { return $this->post('/attributes/restSearch.json', $rules); } /** * @param array $rules * @return HttpSocketResponseExtended * @throws HttpSocketHttpException * @throws HttpSocketJsonException */ public function galaxyClusterSearch(array $rules) { return $this->post('/galaxy_clusters/restSearch', $rules); } /** * @param int|string $galaxyClusterId Galaxy Cluster ID or UUID * @return HttpSocketResponseExtended * @throws HttpSocketHttpException */ public function fetchGalaxyCluster($galaxyClusterId) { return $this->get('/galaxy_clusters/view/' . $galaxyClusterId); } /** * @param array $cluster * @return HttpSocketResponseExtended * @throws HttpSocketHttpException * @throws HttpSocketJsonException */ public function pushGalaxyCluster(array $cluster) { $logMessage = "Pushing Galaxy Cluster #{$cluster['GalaxyCluster']['id']} to Server #{$this->serverId()}"; return $this->post('/galaxies/pushCluster', [$cluster], $logMessage); } /** * @param array $params * @return HttpSocketResponseExtended * @throws HttpSocketHttpException */ public function fetchProposals(array $params = []) { $url = '/shadow_attributes/index'; $url .= $this->createParams($params); $url .= '.json'; return $this->get($url); } /** * @param array $eventUuids * @return array * @throws HttpSocketHttpException * @throws HttpSocketJsonException * @throws JsonException */ public function fetchSightingsForEvents(array $eventUuids) { return $this->post('/sightings/restSearch/event', [ 'returnFormat' => 'json', 'last' => 0, // fetch all 'includeUuid' => true, 'uuid' => $eventUuids, ])->json()['response']; } /** * @param array $event * @param array $sightingUuids * @return array Sighting UUIDs that exists on remote side * @throws HttpSocketJsonException * @throws HttpSocketHttpException */ public function filterSightingUuidsForPush(array $event, array $sightingUuids) { if (!$this->isSupported(self::FEATURE_FILTER_SIGHTINGS)) { return []; } $response = $this->post('/sightings/filterSightingUuidsForPush/' . $event['Event']['uuid'], $sightingUuids); return $response->json(); } /** * @param array $sightings * @param string $eventUuid * @throws HttpSocketHttpException * @throws HttpSocketJsonException */ public function uploadSightings(array $sightings, $eventUuid) { foreach ($sightings as &$sighting) { if (!isset($sighting['org_id'])) { $sighting['org_id'] = '0'; } } $logMessage = "Pushing Sightings for Event #{$eventUuid} to Server #{$this->serverId()}"; $this->post('/sightings/bulkSaveSightings/' . $eventUuid, $sightings, $logMessage); } /** * @return HttpSocketResponseExtended * @throws HttpSocketHttpException */ public function getAvailableSyncFilteringRules() { return $this->get('/servers/getAvailableSyncFilteringRules'); } /** * @return array * @throws HttpSocketJsonException * @throws HttpSocketHttpException * @throws Exception */ public function info() { if ($this->info) { return $this->info; } $response = $this->get('/servers/getVersion'); $info = $response->json(); if (!isset($info['version'])) { throw new Exception("Invalid response when fetching server version: `version` field missing."); } $this->info = $info; return $info; } /** * @return HttpSocketResponseExtended * @throws HttpSocketHttpException */ public function userInfo() { return $this->get('/users/view/me.json'); } /** * @return HttpSocketResponseExtended * @throws HttpSocketHttpException * @throws HttpSocketJsonException */ public function resetAuthKey() { return $this->post('/users/resetauthkey/me', []); } /** * @param string $testString * @return HttpSocketResponseExtended * @throws Exception */ public function postTest($testString) { return $this->post('/servers/postTest', ['testString' => $testString]); } /** * @return array */ public function server() { return $this->server; } /** * @return int */ public function serverId() { return $this->server['Server']['id']; } /** * @return array */ public function pullRules() { return $this->decodeRule('pull_rules'); } /** * @return array */ public function pushRules() { return $this->decodeRule('push_rules'); } /** * @param string $flag * @return bool * @throws HttpSocketJsonException * @throws HttpSocketHttpException * @throws InvalidArgumentException */ public function isSupported($flag) { $info = $this->info(); switch ($flag) { case self::FEATURE_BR: return isset($info['request_encoding']) && in_array('br', $info['request_encoding'], true); case self::FEATURE_GZIP: return isset($info['request_encoding']) && in_array('gzip', $info['request_encoding'], true); case self::FEATURE_FILTER_SIGHTINGS: return isset($info['filter_sightings']) && $info['filter_sightings']; case self::FEATURE_ORG_RULE: $version = explode('.', $info['version']); return $version[0] == 2 && $version[1] == 4 && $version[2] > 123; case self::FEATURE_PROPOSALS: $version = explode('.', $info['version']); return $version[0] == 2 && $version[1] == 4 && $version[2] >= 111; case self::FEATURE_POST_TEST: $version = explode('.', $info['version']); return $version[0] == 2 && $version[1] == 4 && $version[2] > 68; case self::FEATURE_PROTECTED_EVENT: $version = explode('.', $info['version']); return $version[0] == 2 && $version[1] == 4 && $version[2] > 155; case self::FEATURE_EDIT_OF_GALAXY_CLUSTER: return isset($info['perm_galaxy_editor']); case self::PERM_SYNC: return isset($info['perm_sync']) && $info['perm_sync']; case self::PERM_GALAXY_EDITOR: return isset($info['perm_galaxy_editor']) && $info['perm_galaxy_editor']; case self::FEATURE_SIGHTING_REST_SEARCH: $version = explode('.', $info['version']); return $version[0] == 2 && $version[1] == 4 && $version[2] > 164; default: throw new InvalidArgumentException("Invalid flag `$flag` provided"); } } /** * @return array|null */ public function connectionMetaData() { return $this->socket->getMetaData(); } /** * @params string $url Relative URL * @return HttpSocketResponseExtended * @throws HttpSocketHttpException */ private function get($url) { $url = $this->server['Server']['url'] . $url; $start = microtime(true); $response = $this->socket->get($url, [], $this->request); $this->log($start, 'GET', $url, $response); if (!$response->isOk()) { throw new HttpSocketHttpException($response, $url); } return $response; } /** * @param string $url Relative URL * @param mixed $data * @param string|null $logMessage * @param string|null $etag * @return HttpSocketResponseExtended * @throws HttpSocketHttpException * @throws HttpSocketJsonException * @throws JsonException */ private function post($url, $data, $logMessage = null, $etag = null) { $protectedMode = !empty($data['Event']['protected']); $data = JsonTool::encode($data); if ($logMessage && !empty(Configure::read('Security.sync_audit'))) { $pushLogEntry = sprintf( "==============================================================\n\n[%s] %s:\n\n%s\n\n", date("Y-m-d H:i:s"), $logMessage, $data ); file_put_contents(APP . 'files/scripts/tmp/debug_server_' . $this->serverId() . '.log', $pushLogEntry, FILE_APPEND | LOCK_EX); } $request = $this->request; if ($protectedMode) { $request['header']['x-pgp-signature'] = $this->signEvent($data); } if ($etag) { // Remove compression marks that adds Apache for compressed content $etagWithoutQuotes = trim($etag, '"'); $dashPos = strrpos($etagWithoutQuotes, '-'); if ($dashPos && in_array(substr($etagWithoutQuotes, $dashPos + 1), ['br', 'gzip'], true)) { $etag = '"' . substr($etagWithoutQuotes, 0, $dashPos) . '"'; } $request['header']['If-None-Match'] = $etag; } if (strlen($data) > 1024) { // do not compress small body if ($this->isSupported(self::FEATURE_BR) && function_exists('brotli_compress')) { $request['header']['Content-Encoding'] = 'br'; $data = brotli_compress($data, 1, BROTLI_TEXT); } else if ($this->isSupported(self::FEATURE_GZIP) && function_exists('gzencode')) { $request['header']['Content-Encoding'] = 'gzip'; $data = gzencode($data, 1); } } $url = $this->server['Server']['url'] . $url; $start = microtime(true); $response = $this->socket->post($url, $data, $request); $this->log($start, 'POST', $url, $response); if ($etag && $response->isNotModified()) { return $response; // if etag was provided and response code is 304, it is valid response } if (!$response->isOk()) { throw new HttpSocketHttpException($response, $url); } return $response; } /** * @param string $data Data to sign * @return string base64 encoded signature * @throws Exception */ private function signEvent($data) { if (!$this->isSupported(self::FEATURE_PROTECTED_EVENT)) { throw new Exception(__('Remote instance is not protected event aware yet (< 2.4.156), aborting.')); } if (!$this->cryptographicKey) { $this->cryptographicKey = ClassRegistry::init('CryptographicKey'); } $signature = $this->cryptographicKey->signWithInstanceKey($data); if (empty($signature)) { throw new Exception(__("Invalid signing key. This should never happen.")); } return base64_encode($signature); } /** * @param string $key * @return array */ private function decodeRule($key) { $rules = $this->server['Server'][$key]; return json_decode($rules, true); } /** * @param array $params * @return string */ private function createParams(array $params) { $url = ''; foreach ($params as $key => $value) { if (is_array($value)) { foreach ($value as $v) { $url .= "/{$key}[]:$v"; } } else { $url .= "/$key:$value"; } } return $url; } /** * @param float $start Microtime when request was send * @param string $method HTTP method * @param string $url * @param HttpSocketResponse $response */ private function log($start, $method, $url, HttpSocketResponse $response) { $duration = round(microtime(true) - $start, 3); $responseSize = strlen($response->body); $ce = $response->getHeader('Content-Encoding'); $logEntry = '[' . date('Y-m-d H:i:s', intval($start)) . "] \"$method $url\" {$response->code} $responseSize $duration $ce\n"; file_put_contents(APP . 'tmp/logs/server-sync.log', $logEntry, FILE_APPEND | LOCK_EX); } }