From e107d6e8bb120639eec5daaebc606e912038f4b8 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Thu, 11 Jan 2024 16:18:48 +0100 Subject: [PATCH] new: [wip] migrate feeds controller main api endpoints --- src/Controller/FeedsController.php | 1066 ++++++++ src/Lib/Tools/AttributeValidationTool.php | 757 ++++++ src/Lib/Tools/ComplexTypeTool.php | 671 +++++ src/Lib/Tools/RandomTool.php | 64 + src/Model/Entity/Attribute.php | 83 + src/Model/Entity/Event.php | 116 + src/Model/Entity/Feed.php | 39 + src/Model/Table/FeedsTable.php | 2236 +++++++++++++++++ tests/Fixture/FeedsFixture.php | 39 + tests/TestCase/Api/Feeds/AddFeedApiTest.php | 47 + tests/TestCase/Api/Feeds/EditFeedApiTest.php | 72 + .../TestCase/Api/Feeds/IndexFeedsApiTest.php | 38 + 12 files changed, 5228 insertions(+) create mode 100644 src/Controller/FeedsController.php create mode 100644 src/Lib/Tools/AttributeValidationTool.php create mode 100644 src/Lib/Tools/ComplexTypeTool.php create mode 100644 src/Lib/Tools/RandomTool.php create mode 100644 src/Model/Entity/Attribute.php create mode 100644 src/Model/Entity/Feed.php create mode 100644 src/Model/Table/FeedsTable.php create mode 100644 tests/Fixture/FeedsFixture.php create mode 100644 tests/TestCase/Api/Feeds/AddFeedApiTest.php create mode 100644 tests/TestCase/Api/Feeds/EditFeedApiTest.php create mode 100644 tests/TestCase/Api/Feeds/IndexFeedsApiTest.php diff --git a/src/Controller/FeedsController.php b/src/Controller/FeedsController.php new file mode 100644 index 000000000..2a871e504 --- /dev/null +++ b/src/Controller/FeedsController.php @@ -0,0 +1,1066 @@ + 60, + 'order' => array( + 'Feed.default' => 'DESC', + 'Feed.id' => 'ASC' + ), + ); + + public $uses = array('Feed'); + + public function beforeFilter(EventInterface $event) + { + parent::beforeFilter($event); + $this->Security->setConfig('unlockedActions', ['previewIndex', 'feedCoverage']); + } + + public function loadDefaultFeeds() + { + if ($this->request->is('post')) { + $this->Feeds->load_default_feeds(); + $message = __('Default feed metadata loaded.'); + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveSuccessResponse('Feed', 'loadDefaultFeeds', false, $this->response->getType(), $message); + } else { + $this->Flash->success($message); + $this->redirect(array('controller' => 'Feeds', 'action' => 'index')); + } + } + } + + public function index() + { + $conditions = []; + $scope = isset($this->request->getQueryParams()['scope']) ? $this->request->getQueryParams()['scope'] : 'all'; + if ($scope !== 'all') { + if ($scope == 'enabled') { + $conditions[] = array( + 'OR' => array( + 'Feed.enabled' => 1, + 'Feed.caching_enabled' => 1 + ) + ); + } else { + $conditions[] = array( + 'Feed.default' => $scope == 'custom' ? 0 : 1 + ); + } + } + + $this->CRUD->index([ + 'filters' => [ + 'Feed.name', + 'url', + 'provider', + 'source_format', + 'enabled', + 'caching_enabled', + 'default' + ], + 'quickFilters' => [ + 'Feed.name', + 'url', + 'provider', + 'source_format' + ], + 'contain' => [ + 'Tags', + 'SharingGroups', + 'Orgc' => [ + 'fields' => [ + 'Orgc.id', + 'Orgc.uuid', + 'Orgc.name', + 'Orgc.local' + ] + ] + ], + 'conditions' => $conditions, + 'afterFind' => function (Feed $feed) { + if ($this->isSiteAdmin()) { + $feed = $this->Feeds->attachFeedCacheTimestamps($feed); + } + + if ($this->ParamHandler->isRest()) { + unset($feed['SharingGroup']); + if (empty($feed['Tag']['id'])) { + unset($feed['Tag']); + } + } + + return $feed; + } + ]); + + if ($this->ParamHandler->isRest()) { + return $this->restResponsePayload; + } + + $this->set('title_for_layout', __('Feeds')); + $this->set('menuData', [ + 'menuList' => 'feeds', + 'menuItem' => 'index' + ]); + $distributionLevels = Distribution::ALL; + $distributionLevels[5] = __('Inherit from feed'); + $this->set('distributionLevels', $distributionLevels); + $this->set('scope', $scope); + } + + public function view($feedId) + { + $this->CRUD->view($feedId, [ + 'contain' => ['Tag'], + 'afterFind' => function (array $feed) { + if (!$this->isSiteAdmin()) { + unset($feed['Feed']['headers']); + } + + $feed['Feed']['cached_elements'] = $this->Feeds->getCachedElements($feed['Feed']['id']); + $feed['Feed']['coverage_by_other_feeds'] = $this->Feeds->getFeedCoverage($feed['Feed']['id'], 'feed', 'all') . '%'; + + if ($this->ParamHandler->isRest()) { + if (empty($feed['Tag']['id'])) { + unset($feed['Tag']); + } + } + + return $feed; + } + ]); + if ($this->ParamHandler->isRest()) { + return $this->restResponsePayload; + } + + $otherFeeds = $this->Feeds->getAllCachingEnabledFeeds($feedId, true); + $this->set('other_feeds', $otherFeeds); + $this->set('feedId', $feedId); + } + + public function feedCoverage($feedId) + { + $feed = $this->Feeds->find('all', array( + 'conditions' => array('Feed.id' => $feedId), + 'recursive' => -1, + 'contain' => array('Tag') + ))->first(); + $result = $this->Feeds->getFeedCoverage($feed['Feed']['id'], 'feed', $this->request->getData()); + return $this->RestResponse->viewData($result, $this->response->getType()); + } + + public function importFeeds() + { + if ($this->request->is('post')) { + $data = $this->request->getData(); + if (isset($data['Feed']['json'])) { + $data = $data['Feed']['json']; + } + $results = $this->Feeds->importFeeds($data, $this->Auth->user()); + if ($results['successes'] > 0) { + $flashType = 'success'; + $message = $results['successes'] . ' new feeds added.'; + } else { + $flashType = 'info'; + $message = 'No new feeds to add.'; + } + if ($results['fails']) { + $message .= ' ' . $results['fails'] . ' feeds could not be added (possibly because they already exist)'; + } + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveSuccessResponse('Feed', 'importFeeds', false, $this->response->getType(), $message); + } else { + $this->Flash->{$flashType}($message); + $this->redirect(array('controller' => 'Feeds', 'action' => 'index', 'all')); + } + } + } + + public function add() + { + $params = [ + 'beforeSave' => function (Feed $feed) { + if ($this->ParamHandler->isRest()) { + if (empty($feed['source_format'])) { + $feed['source_format'] = 'freetext'; + } + if (!isset($feed['fixed_event'])) { + $feed['fixed_event'] = '1'; + } + } + + if (!isset($feed['distribution'])) { + $feed['distribution'] = '0'; + } + if ($feed['distribution'] != 4) { + $feed['sharing_group_id'] = '0'; + } + $feed['default'] = '0'; + if (!isset($feed['source_format'])) { + $feed['source_format'] = 'freetext'; + } + if (!empty($feed['source_format']) && ($feed['source_format'] == 'misp')) { + if (!empty($feed['orgc_id'])) { + $feed['orgc_id'] = '0'; + } + } + if ($feed['source_format'] == 'freetext') { + if ($feed['fixed_event'] == 1) { + if (!empty($feed['target_event']) && is_numeric($feed['target_event'])) { + $feed['event_id'] = $feed['target_event']; + } + } + } + if (!isset($feed['settings'])) { + $feed['settings'] = array(); + } else { + if (!empty($feed['settings']['common']['excluderegex']) && !$this->__checkRegex($feed['settings']['common']['excluderegex'])) { + $regexErrorMessage = __('Invalid exclude regex. Make sure it\'s a delimited PCRE regex pattern.'); + if (!$this->ParamHandler->isRest()) { + $this->Flash->error($regexErrorMessage); + } else { + return $this->RestResponse->saveFailResponse( + 'Feeds', + 'add', + false, + $regexErrorMessage, + $this->response->getType() + ); + } + } + } + if (isset($feed['settings']['delimiter']) && empty($feed['settings']['delimiter'])) { + $feed['settings']['delimiter'] = ','; + } + if (empty($feed['target_event'])) { + $feed['target_event'] = 0; + } + if (empty($feed['lookup_visible'])) { + $feed['lookup_visible'] = 0; + } + if (empty($feed['input_source'])) { + $feed['input_source'] = 'network'; + } else { + $feed['input_source'] = strtolower($feed['input_source']); + } + if (!in_array($feed['input_source'], array('network', 'local'))) { + $feed['input_source'] = 'network'; + } + if (!isset($feed['delete_local_file'])) { + $feed['delete_local_file'] = 0; + } + $feed['event_id'] = !empty($feed['fixed_event']) ? $feed['target_event'] : 0; + + return $feed; + } + ]; + + $this->CRUD->add($params); + if ($this->ParamHandler->isRest()) { + return $this->restResponsePayload; + } + + $EventsTable = $this->fetchTable('Events'); + + $sharingGroups = $EventsTable->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1); + $distributionLevels = $EventsTable->distributionLevels; + $distributionLevels[5] = __('Inherit from feed'); + if (empty($sharingGroups)) { + unset($distributionLevels[4]); + } + $inputSources = array('network' => 'Network'); + if (empty(Configure::read('Security.disable_local_feed_access'))) { + $inputSources['local'] = 'Local'; + } + $tags = $EventsTable->EventTag->Tag->find('list', array('fields' => array('Tag.name'), 'order' => array('lower(Tag.name) asc'))); + $tags[0] = 'None'; + + $ServersTable = $this->fetchTable('Servers'); + $allTypes = $ServersTable->getAllTypes(); + + $dropdownData = [ + 'orgs' => $EventsTable->Orgc->find('list', array( + 'fields' => array('id', 'name'), + 'order' => 'LOWER(name)' + )), + 'tags' => $tags, + 'feedTypes' => $this->Feeds->getFeedTypesOptions(), + 'sharingGroups' => $sharingGroups, + 'distributionLevels' => $distributionLevels, + 'inputSources' => $inputSources + ]; + $this->set('allAttributeTypes', $allTypes['attribute']); + $this->set('allObjectTypes', $allTypes['object']); + $this->set('supportedUrlparams', Feed::SUPPORTED_URL_PARAM_FILTERS); + $this->set(compact('dropdownData')); + $this->set('defaultPullRules', json_encode(Feed::DEFAULT_FEED_PULL_RULES)); + $this->set('menuData', array('menuList' => 'feeds', 'menuItem' => 'add')); + $this->set('pull_scope', 'feed'); + } + + private function __checkRegex($pattern) + { + if (@preg_match($pattern, '') === false) { + return false; + } + return true; + } + + public function edit($feedId) + { + $this->CRUD->edit($feedId, [ + 'fields' => [ + 'name', + 'provider', + 'enabled', + 'caching_enabled', + 'pull_rules', + 'rules', + 'url', + 'distribution', + 'sharing_group_id', + 'tag_id', + 'event_id', + 'publish', + 'delta_merge', + 'source_format', + 'override_ids', + 'settings', + 'input_source', + 'delete_local_file', + 'lookup_visible', + 'headers', + 'orgc_id', + 'fixed_event' + ], + 'beforeSave' => function (Feed $feed) use ($feedId) { + if (isset($feed['pull_rules'])) { + $feed['rules'] = $feed['pull_rules']; + } + if (isset($feed['distribution']) && $feed['distribution'] != 4) { + $feed['sharing_group_id'] = '0'; + } + $feed['id'] = $feedId; + if (!empty($feed['source_format']) && ($feed['source_format'] == 'misp')) { + if (!empty($feed['orgc_id'])) { + $feed['orgc_id'] = '0'; + } + } + if (!empty($feed['source_format']) && ($feed['source_format'] == 'freetext' || $feed['source_format'] == 'csv')) { + if ($feed['fixed_event'] == 1) { + if (isset($feed['target_event']) && is_numeric($feed['target_event'])) { + $feed['event_id'] = $feed['target_event']; + } else if (!empty($feed['event_id'])) { + $feed['event_id'] = $feed['event_id']; + } else { + $feed['event_id'] = '0'; + } + } + } + if (!isset($feed['settings'])) { + if (!empty($feed['settings'])) { + $feed['settings'] = $feed['settings']; + } else { + $feed['settings'] = array(); + } + } else { + if (!empty($feed['settings']['common']['excluderegex']) && !$this->__checkRegex($feed['settings']['common']['excluderegex'])) { + $regexErrorMessage = __('Invalid exclude regex. Make sure it\'s a delimited PCRE regex pattern.'); + if (!$this->ParamHandler->isRest()) { + $this->Flash->error($regexErrorMessage); + return true; + } else { + return $this->RestResponse->saveFailResponse( + 'Feeds', + 'edit', + false, + $regexErrorMessage, + $this->response->getType() + ); + } + } + } + if (isset($feed['settings']['delimiter']) && empty($feed['settings']['delimiter'])) { + $feed['settings']['delimiter'] = ','; + } + + return $feed; + }, + 'afterSave' => function (Feed $feed) { + $feedCache = APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . intval($feed['id']) . '.cache'; + if (file_exists($feedCache)) { + unlink($feedCache); + } + return $feed; + } + ]); + if ($this->ParamHandler->isRest()) { + return $this->restResponsePayload; + } + + $EventsTable = $this->fetchTable('Events'); + $sharingGroups = $EventsTable->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1); + $distributionLevels = Distribution::ALL; + $distributionLevels[5] = __('Inherit from feed'); + if (empty($sharingGroups)) { + unset($distributionLevels[4]); + } + $inputSources = array('network' => 'Network'); + if (empty(Configure::read('Security.disable_local_feed_access'))) { + $inputSources['local'] = 'Local'; + } + $tags = $this->Event->EventTag->Tag->find('list', array('fields' => array('Tag.name'), 'order' => array('lower(Tag.name) asc'))); + $tags[0] = 'None'; + + $ServersTable = $this->fetchTable('Servers'); + $allTypes = $ServersTable->getAllTypes(); + $this->set('allAttributeTypes', $allTypes['attribute']); + $this->set('allObjectTypes', $allTypes['object']); + $this->set('supportedUrlparams', Feed::SUPPORTED_URL_PARAM_FILTERS); + + $dropdownData = [ + 'orgs' => $this->Event->Orgc->find('list', array( + 'fields' => array('id', 'name'), + 'order' => 'LOWER(name)' + )), + 'tags' => $tags, + 'feedTypes' => $this->Feeds->getFeedTypesOptions(), + 'sharingGroups' => $sharingGroups, + 'distributionLevels' => $distributionLevels, + 'inputSources' => $inputSources + ]; + $this->set(compact('dropdownData')); + $this->set('menuData', [ + 'menuList' => 'feeds', + 'menuItem' => 'edit', + ]); + + $this->set('feedId', $feedId); + $this->set('pull_scope', 'feed'); + $this->render('add'); + } + + public function delete($feedId) + { + $this->CRUD->delete($feedId); + if ($this->ParamHandler->isRest()) { + return $this->restResponsePayload; + } + } + + public function fetchFromFeed($feedId) + { + $this->Feeds->id = $feedId; + if (!$this->Feeds->exists()) { + throw new NotFoundException(__('Invalid feed.')); + } + $this->Feeds->read(); + if (!$this->Feeds->data['Feed']['enabled']) { + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->viewData( + array('result' => __('Feed is currently not enabled. Make sure you enable it.')), + $this->response->getType() + ); + } else { + $this->Flash->error(__('Feed is currently not enabled. Make sure you enable it.')); + $this->redirect(array('action' => 'index')); + } + } + if (Configure::read('MISP.background_jobs')) { + + /** @var JobsTable $JobsTable */ + $JobsTable = $this->fetchTable('Jobs'); + $jobId = $JobsTable->createJob( + $this->Auth->user(), + Job::WORKER_DEFAULT, + 'fetch_feeds', + 'Feed: ' . $feedId, + __('Starting fetch from Feed.') + ); + + $this->Feeds->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_SERVER, + [ + 'fetchFeed', + $this->Auth->user('id'), + $feedId, + $jobId + ], + true, + $jobId + ); + + $message = __('Pull queued for background execution.'); + } else { + $result = $this->Feeds->downloadFromFeedInitiator($feedId, $this->Auth->user()); + if (!$result) { + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->viewData(array('result' => __('Fetching the feed has failed.')), $this->response->getType()); + } else { + $this->Flash->error(__('Fetching the feed has failed.')); + $this->redirect(array('action' => 'index')); + } + } + $message = __('Fetching the feed has successfully completed.'); + if ($this->Feeds->data['Feed']['source_format'] == 'misp') { + if (isset($result['add'])) { + $message .= ' Downloaded ' . count($result['add']) . ' new event(s).'; + } + if (isset($result['edit'])) { + $message .= ' Updated ' . count($result['edit']) . ' event(s).'; + } + } + } + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->viewData(array('result' => $message), $this->response->getType()); + } else { + $this->Flash->success($message); + $this->redirect(array('action' => 'index')); + } + } + + public function fetchFromAllFeeds() + { + $feeds = $this->Feeds->find('all', array( + 'recursive' => -1, + 'fields' => array('id') + )); + foreach ($feeds as $feed) { + $feedId = $feed['Feed']['id']; + $this->Feeds->id = $feedId; + $this->Feeds->read(); + if (!$this->Feeds->data['Feed']['enabled']) { + continue; + } + if (Configure::read('MISP.background_jobs')) { + + /** @var Job $job */ + $JobsTable = $this->fetchTable('Jobs'); + + $jobId = $JobsTable->createJob( + $this->Auth->user(), + Job::WORKER_DEFAULT, + 'fetch_feed', + 'Feed: ' . $feedId, + __('Starting fetch from Feed.') + ); + + $this->Feeds->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_SERVER, + [ + 'fetchFeed', + $this->Auth->user('id'), + $feedId, + $jobId + ], + true, + $jobId + ); + + $message = 'Pull queued for background execution.'; + } else { + $result = $this->Feeds->downloadFromFeedInitiator($feedId, $this->Auth->user()); + if (!$result) { + continue; + } + $message = __('Fetching the feed has successfully completed.'); + if ($this->Feeds->data['Feed']['source_format'] == 'misp') { + if (isset($result['add'])) { + $message['result'] .= ' Downloaded ' . count($result['add']) . ' new event(s).'; + } + if (isset($result['edit'])) { + $message['result'] .= ' Updated ' . count($result['edit']) . ' event(s).'; + } + } + } + } + if (!isset($message)) { + $message = __('No feed enabled.'); + } + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->viewData(array('result' => $message), $this->response->getType()); + } else { + $this->Flash->success($message); + $this->redirect(array('action' => 'index')); + } + } + + public function getEvent($feedId, $eventUuid, $all = false) + { + $this->Feeds->id = $feedId; + if (!$this->Feeds->exists()) { + throw new NotFoundException(__('Invalid feed.')); + } + $this->Feeds->read(); + if (!$this->Feeds->data['Feed']['enabled']) { + $this->Flash->error(__('Feed is currently not enabled. Make sure you enable it.')); + $this->redirect(array('action' => 'previewIndex', $feedId)); + } + try { + $result = $this->Feeds->downloadAndSaveEventFromFeed($this->Feeds->data, $eventUuid, $this->Auth->user()); + } catch (Exception $e) { + $this->Flash->error(__('Download failed.') . ' ' . $e->getMessage()); + $this->redirect(array('action' => 'previewIndex', $feedId)); + } + + if (isset($result['action'])) { + if ($result['result']) { + if ($result['action'] == 'add') { + $this->Flash->success(__('Event added.')); + } else { + if ($result['result'] === 'No change') { + $this->Flash->info(__('Event already up to date.')); + } else { + $this->Flash->success(__('Event updated.')); + } + } + } else { + $this->Flash->error(__('Could not %s event.', $result['action'])); + } + } else { + $this->Flash->error(__('Download failed.')); + } + $this->redirect(array('action' => 'previewIndex', $feedId)); + } + + public function previewIndex($feedId) + { + $feed = $this->Feeds->find('all', [ + 'conditions' => ['id' => $feedId], + 'recursive' => -1, + ])->first(); + if (empty($feed)) { + throw new NotFoundException(__('Invalid feed.')); + } + $params = array(); + if ($this->request->is('post')) { + $params = $this->request->getData()['Feed']; + } + if ($feed['Feed']['source_format'] === 'misp') { + return $this->__previewIndex($feed, $params); + } elseif (in_array($feed['Feed']['source_format'], ['freetext', 'csv'], true)) { + return $this->__previewFreetext($feed); + } else { + throw new Exception("Invalid feed format `{$feed['Feed']['source_format']}`."); + } + } + + private function __previewIndex(array $feed, $filterParams = array()) + { + $urlparams = ''; + $customPagination = new CustomPaginationTool(); + $passedArgs = array(); + $syncTool = new SyncTool(); + $HttpSocket = $syncTool->setupHttpSocketFeed(); + try { + $events = $this->Feeds->getManifest($feed, $HttpSocket); + } catch (Exception $e) { + $this->Flash->error("Could not fetch manifest for feed: {$e->getMessage()}"); + $this->redirect(array('controller' => 'feeds', 'action' => 'index')); + } + + if (!empty($this->params['named']['searchall'])) { + $searchAll = trim(mb_strtolower($this->params['named']['searchall'])); + foreach ($events as $uuid => $event) { + if ($uuid === $searchAll) { + continue; + } + if (strpos(mb_strtolower($event['info']), $searchAll) !== false) { + continue; + } + if (strpos(mb_strtolower($event['Orgc']['name']), $searchAll) !== false) { + continue; + } + if (!empty($event['Tag'])) { + foreach ($event['Tag'] as $tag) { + if (strpos(mb_strtolower($tag['name']), $searchAll) !== false) { + continue 2; + } + } + } + unset($events[$uuid]); + } + } + foreach ($filterParams as $k => $filter) { + if (!empty($filter)) { + $filterParams[$k] = json_decode($filter); + } + } + if (!empty($filterParams['eventid'])) { + foreach ($events as $k => $event) { + if (!in_array($k, $filterParams['eventid'])) { + unset($events[$k]); + continue; + } + } + } + $params = $customPagination->createPaginationRules($events, $this->request->getQueryParams(), $this->alias); + $this->params->params['paging'] = array('Feeds' => $params); + $events = $customPagination->sortArray($events, $params, true); + $customPagination->truncateByPagination($events, $params); + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->viewData($events, $this->response->getType()); + } + $this->set('events', $events); + $EventsTable = $this->fetchTable('Events'); + $this->set('threatLevels', $EventsTable->ThreatLevel->listThreatLevels()); + $this->set('eventDescriptions', Distribution::DESCRIPTIONS); + $this->set('analysisLevels', Event::ANALYSIS_LEVELS); + $this->set('distributionLevels', Distribution::ALL); + $shortDist = array(0 => 'Organisation', 1 => 'Community', 2 => 'Connected', 3 => 'All', 4 => ' sharing Group'); + $this->set('shortDist', $shortDist); + $this->set('id', $feed['Feed']['id']); + $this->set('feed', $feed); + $this->set('urlparams', $urlparams); + $this->set('passedArgs', json_encode($passedArgs)); + $this->set('passedArgsArray', $passedArgs); + } + + private function __previewFreetext(array $feed) + { + if (isset($this->request->getQueryParams()['page'])) { + $currentPage = $this->request->getQueryParams()['page']; + } elseif (isset($this->request->getQueryParams()['page'])) { + $currentPage = $this->request->getQueryParams()['page']; + } else { + $currentPage = 1; + } + if (!in_array($feed['Feed']['source_format'], array('freetext', 'csv'))) { + throw new MethodNotAllowedException(__('Invalid feed type.')); + } + $syncTool = new SyncTool(); + $HttpSocket = $syncTool->setupHttpSocketFeed(); + // params is passed as reference here, the pagination happens in the method, which isn't ideal but considering the performance gains here it's worth it + try { + $resultArray = $this->Feeds->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format']); + } catch (Exception $e) { + $this->Flash->error("Could not fetch feed: {$e->getMessage()}"); + $this->redirect(array('controller' => 'feeds', 'action' => 'index')); + } + + $customPagination = new CustomPaginationTool(); + $params = $customPagination->createPaginationRules($resultArray, array('page' => $currentPage, 'limit' => 60), 'Feed', $sort = false); + if (!empty($currentPage) && $currentPage !== 'all') { + $start = ($currentPage - 1) * 60; + if ($start > count($resultArray)) { + return false; + } + $resultArray = array_slice($resultArray, $start, 60); + } + + $this->params->params['paging'] = array('Feeds' => $params); + $resultArray = $this->Feeds->getFreetextFeedCorrelations($resultArray, $feed['Feed']['id']); + // remove all duplicates + $correlatingEvents = array(); + foreach ($resultArray as $k => $v) { + if (!empty($resultArray[$k]['correlations'])) { + foreach ($resultArray[$k]['correlations'] as $correlatingEvent) { + if (!in_array($correlatingEvent, $correlatingEvents)) { + $correlatingEvents[] = $correlatingEvent; + } + } + } + } + $resultArray = array_values($resultArray); + $AttributesTable = $this->fetchTable('Attributes'); + $correlatingEventInfos = $AttributesTable->Event->find('list', array( + 'fields' => array('Event.id', 'Event.info'), + 'conditions' => array('Event.id' => $correlatingEvents) + )); + $this->set('correlatingEventInfos', $correlatingEventInfos); + $this->set('distributionLevels', Distribution::ALL); + $this->set('feed', $feed); + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->viewData($resultArray, $this->response->getType()); + } + $this->set('attributes', $resultArray); + $this->render('freetext_index'); + } + + public function previewEvent($feedId, $eventUuid, $all = false) + { + $feed = $this->Feeds->find('all', [ + 'conditions' => ['id' => $feedId], + 'recursive' => -1, + ])->first(); + if (empty($feed)) { + throw new NotFoundException(__('Invalid feed.')); + } + try { + $event = $this->Feeds->downloadEventFromFeed($feed, $eventUuid); + } catch (Exception $e) { + throw new Exception(__('Could not download the selected Event'), 0, $e); + } + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->viewData($event, $this->response->getType()); + } + if (is_array($event)) { + if (isset($event['Event']['Attribute'])) { + $WarninglistsTable = $this->fetchTable('Warninglists'); + $WarninglistsTable->attachWarninglistToAttributes($event['Event']['Attribute']); + } + + $EventsTable = $this->fetchTable('Events'); + $params = $EventsTable->rearrangeEventForView($event, $this->request->getQueryParams(), $all); + $this->params->params['paging'] = array('Feed' => $params); + $this->set('event', $event); + $this->set('feed', $feed); + $dataForView = array( + 'Attribute' => array('attrDescriptions' => 'fieldDescriptions', 'distributionDescriptions' => 'distributionDescriptions', 'distributionLevels' => 'distributionLevels'), + 'Event' => array('eventDescriptions' => 'fieldDescriptions', 'analysisLevels' => 'analysisLevels') + ); + foreach ($dataForView as $m => $variables) { + if ($m === 'Event') { + $currentModel = $EventsTable; + } elseif ($m === 'Attribute') { + $currentModel = $this->fetchTable('Attributes'); + } + foreach ($variables as $alias => $variable) { + $this->set($alias, $currentModel->{$variable}); + } + } + $this->set('threatLevels', $EventsTable->ThreatLevel->find('list')); + } else { + if ($event === 'blocked') { + throw new MethodNotAllowedException(__('This event is blocked by the Feed filters.')); + } else { + throw new NotFoundException(__('Could not download the selected Event')); + } + } + } + + public function enable($id) + { + $result = $this->__toggleEnable($id, true); + $this->set('name', $result['message']); + $this->set('message', $result['message']); + $this->set('url', $this->request->getAttribute('here')); + if ($result) { + $this->viewBuilder()->setOption('serialize', array('name', 'message', 'url')); + } else { + $this->set('errors', $result); + } + } + + public function disable($id) + { + $result = $this->__toggleEnable($id, false); + $this->set('name', $result['message']); + $this->set('message', $result['message']); + $this->set('url', $this->request->getAttribute('here')); + if ($result['result']) { + $this->viewBuilder()->setOption('serialize', array('name', 'message', 'url')); + } else { + $this->set('errors', $result); + $this->viewBuilder()->setOption('serialize', array('name', 'message', 'url', 'errors')); + } + } + + private function __toggleEnable($id, $enable = true) + { + if (!is_numeric($id)) { + throw new MethodNotAllowedException(__('Invalid Feed.')); + } + $feed = $this->Feeds->get($id); + if (!$feed) { + throw new MethodNotAllowedException(__('Invalid Feed.')); + } + $feed['enabled'] = $enable; + $result = array('result' => $this->Feeds->save($feed)); + $fail = false; + if (!$result['result']) { + $fail = true; + $result['result'] = $this->Feeds->validationErrors; + } + $action = $enable ? 'enable' : 'disable'; + $result['message'] = $fail ? 'Could not ' . $action . ' feed.' : 'Feed ' . $action . 'd.'; + return $result; + } + + public function fetchSelectedFromFreetextIndex($id) + { + if (!$this->request->is('Post')) { + throw new MethodNotAllowedException(__('Only POST requests are allowed.')); + } + $this->Feeds->id = $id; + if (!$this->Feeds->exists()) { + throw new NotFoundException(__('Feed not found.')); + } + $feed = $this->Feeds->read(); + $data = json_decode($this->request->getData()['Feed']['data'], true); + try { + $this->Feeds->saveFreetextFeedData($feed, $data, $this->Auth->user()); + $this->Flash->success(__('Data pulled.')); + } catch (Exception $e) { + $this->Flash->error(__('Could not pull the selected data. Reason: %s', $e->getMessage())); + } + $this->redirect(array('controller' => 'feeds', 'action' => 'index')); + } + + public function cacheFeeds($scope = 'freetext') + { + if (Configure::read('MISP.background_jobs')) { + + /** @var JobsTable $JobsTable */ + $JobsTable = $this->fetchTable('Jobs'); + $jobId = $JobsTable->createJob( + $this->Auth->user(), + Job::WORKER_DEFAULT, + 'cache_feeds', + $scope, + __('Starting feed caching.') + ); + + $this->Feeds->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_SERVER, + [ + 'cacheFeed', + $this->Auth->user('id'), + $scope, + $jobId + ], + true, + $jobId + ); + + $message = 'Feed caching job initiated.'; + } else { + $result = $this->Feeds->cacheFeedInitiator($this->Auth->user(), false, $scope); + if ($result['fails'] > 0) { + $this->Flash->error(__('Caching the feeds has failed.')); + $this->redirect(array('action' => 'index')); + } + $message = __('Caching the feeds has successfully completed.'); + } + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveSuccessResponse('Feed', 'cacheFeed', false, $this->response->getType(), $message); + } else { + $this->Flash->info($message); + $this->redirect(array('controller' => 'feeds', 'action' => 'index')); + } + } + + public function compareFeeds($id = false) + { + $feeds = $this->Feeds->compareFeeds($id); + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->viewData($feeds, $this->response->getType()); + } else { + $this->set('feeds', $feeds); + } + } + + public function toggleSelected($enable = false, $cache = false, $feedList = false) + { + $field = $cache ? 'caching_enabled' : 'enabled'; + if (!empty($enable)) { + $enable = 1; + } else { + $enable = 0; + } + try { + $feedIds = json_decode($feedList, true); + } catch (Exception $e) { + $this->Flash->error(__('Invalid feed list received.')); + $this->redirect(array('controller' => 'feeds', 'action' => 'index')); + } + if ($this->request->is('post')) { + $feeds = $this->Feeds->find('all', array( + 'conditions' => array('Feed.id' => $feedIds), + 'recursive' => -1 + )); + $count = 0; + foreach ($feeds as $feed) { + if ($feed['Feed'][$field] != $enable) { + $feed['Feed'][$field] = $enable; + $this->Feeds->save($feed); + $count++; + } + } + if ($count > 0) { + $this->Flash->success($count . ' feeds ' . array('disabled', 'enabled')[$enable] . '.'); + $this->redirect(array('controller' => 'feeds', 'action' => 'index')); + } else { + $this->Flash->info('All selected feeds are already ' . array('disabled', 'enabled')[$enable] . ', nothing to update.'); + $this->redirect(array('controller' => 'feeds', 'action' => 'index')); + } + } else { + $this->set('feedList', $feedList); + $this->set('enable', $enable); + $this->render('ajax/feedToggleConfirmation'); + } + } + + public function searchCaches() + { + if (isset($this->request->getQueryParams()['pages'])) { + $currentPage = $this->request->getQueryParams()['pages']; + } else { + $currentPage = 1; + } + $urlparams = ''; + $customPagination = new CustomPaginationTool(); + $passedArgs = array(); + $value = false; + $data = $this->request->getData(); + if ($this->request->is('post')) { + if (isset($data['Feed'])) { + $data = $data['Feed']; + } + if (isset($data['value'])) { + $data = $data['value']; + } + $value = $data; + } + if (!empty($this->params['named']['value'])) { + $value = $this->params['named']['value']; + } + $hits = $this->Feeds->searchCaches($value); + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->viewData($hits, $this->response->getType()); + } else { + $this->set('hits', $hits); + } + $params = $customPagination->createPaginationRules($hits, $this->request->getQueryParams(), $this->alias); + $this->params->params['paging'] = array('Feed' => $params); + $hits = $customPagination->sortArray($hits, $params, true); + if (is_array($hits)) { + $customPagination->truncateByPagination($hits, $params); + } + $this->set('urlparams', $urlparams); + $this->set('passedArgs', json_encode($passedArgs)); + $this->set('passedArgsArray', $passedArgs); + } +} diff --git a/src/Lib/Tools/AttributeValidationTool.php b/src/Lib/Tools/AttributeValidationTool.php new file mode 100644 index 000000000..19266b8b5 --- /dev/null +++ b/src/Lib/Tools/AttributeValidationTool.php @@ -0,0 +1,757 @@ + 64, + 'md5' => 32, + 'imphash' => 32, + 'telfhash' => 70, + 'sha1' => 40, + 'git-commit-id' => 40, + 'x509-fingerprint-md5' => 32, + 'x509-fingerprint-sha1' => 40, + 'x509-fingerprint-sha256' => 64, + 'ja3-fingerprint-md5' => 32, + 'jarm-fingerprint' => 62, + 'hassh-md5' => 32, + 'hasshserver-md5' => 32, + 'pehash' => 40, + 'sha224' => 56, + 'sha256' => 64, + 'sha384' => 96, + 'sha512' => 128, + 'sha512/224' => 56, + 'sha512/256' => 64, + 'sha3-224' => 56, + 'sha3-256' => 64, + 'sha3-384' => 96, + 'sha3-512' => 128, + ]; + + /** + * Do some last second modifications before the validation + * @param string $type + * @param mixed $value + * @return string + */ + public static function modifyBeforeValidation($type, $value) + { + $value = self::handle4ByteUnicode($value); + switch ($type) { + case 'ip-src': + case 'ip-dst': + return self::normalizeIp($value); + case 'md5': + case 'sha1': + case 'sha224': + case 'sha256': + case 'sha384': + case 'sha512': + case 'sha512/224': + case 'sha512/256': + case 'sha3-224': + case 'sha3-256': + case 'sha3-384': + case 'sha3-512': + case 'ja3-fingerprint-md5': + case 'jarm-fingerprint': + case 'hassh-md5': + case 'hasshserver-md5': + case 'hostname': + case 'pehash': + case 'authentihash': + case 'vhash': + case 'imphash': + case 'telfhash': + case 'tlsh': + case 'anonymised': + case 'cdhash': + case 'email': + case 'email-src': + case 'email-dst': + case 'target-email': + case 'whois-registrant-email': + return strtolower($value); + case 'domain': + $value = strtolower($value); + $value = trim($value, '.'); + // Domain is not valid, try to convert to punycode + if (!self::isDomainValid($value) && function_exists('idn_to_ascii')) { + $punyCode = idn_to_ascii($value); + if ($punyCode !== false) { + $value = $punyCode; + } + } + return $value; + case 'domain|ip': + $value = strtolower($value); + $parts = explode('|', $value); + if (!isset($parts[1])) { + return $value; // not a composite + } + $parts[0] = trim($parts[0], '.'); + // Domain is not valid, try to convert to punycode + if (!self::isDomainValid($parts[0]) && function_exists('idn_to_ascii')) { + $punyCode = idn_to_ascii($parts[0]); + if ($punyCode !== false) { + $parts[0] = $punyCode; + } + } + $parts[1] = self::normalizeIp($parts[1]); + return "$parts[0]|$parts[1]"; + case 'filename|md5': + case 'filename|sha1': + case 'filename|imphash': + case 'filename|sha224': + case 'filename|sha256': + case 'filename|sha384': + case 'filename|sha512': + case 'filename|sha512/224': + case 'filename|sha512/256': + case 'filename|sha3-224': + case 'filename|sha3-256': + case 'filename|sha3-384': + case 'filename|sha3-512': + case 'filename|authentihash': + case 'filename|vhash': + case 'filename|pehash': + case 'filename|tlsh': + // Convert hash to lowercase + $pos = strpos($value, '|'); + return substr($value, 0, $pos) . strtolower(substr($value, $pos)); + case 'http-method': + case 'hex': + return strtoupper($value); + case 'vulnerability': + case 'weakness': + $value = str_replace('–', '-', $value); + return strtoupper($value); + case 'cc-number': + case 'bin': + return preg_replace('/[^0-9]+/', '', $value); + case 'iban': + case 'bic': + $value = strtoupper($value); + return preg_replace('/[^0-9A-Z]+/', '', $value); + case 'prtn': + case 'whois-registrant-phone': + case 'phone-number': + if (substr($value, 0, 2) == '00') { + $value = '+' . substr($value, 2); + } + $value = preg_replace('/\(0\)/', '', $value); + return preg_replace('/[^\+0-9]+/', '', $value); + case 'x509-fingerprint-md5': + case 'x509-fingerprint-sha256': + case 'x509-fingerprint-sha1': + $value = str_replace(':', '', $value); + return strtolower($value); + case 'ip-dst|port': + case 'ip-src|port': + if (substr_count($value, ':') >= 2) { // (ipv6|port) - tokenize ip and port + if (strpos($value, '|')) { // 2001:db8::1|80 + $parts = explode('|', $value); + } elseif (strpos($value, '[') === 0 && strpos($value, ']') !== false) { // [2001:db8::1]:80 + $ipv6 = substr($value, 1, strpos($value, ']') - 1); + $port = explode(':', substr($value, strpos($value, ']')))[1]; + $parts = array($ipv6, $port); + } elseif (strpos($value, '.')) { // 2001:db8::1.80 + $parts = explode('.', $value); + } elseif (strpos($value, ' port ')) { // 2001:db8::1 port 80 + $parts = explode(' port ', $value); + } elseif (strpos($value, 'p')) { // 2001:db8::1p80 + $parts = explode('p', $value); + } elseif (strpos($value, '#')) { // 2001:db8::1#80 + $parts = explode('#', $value); + } else { // 2001:db8::1:80 this one is ambiguous + $temp = explode(':', $value); + $parts = array(implode(':', array_slice($temp, 0, count($temp) - 1)), end($temp)); + } + } elseif (strpos($value, ':')) { // (ipv4:port) + $parts = explode(':', $value); + } elseif (strpos($value, '|')) { // (ipv4|port) + $parts = explode('|', $value); + } else { + return $value; + } + return self::normalizeIp($parts[0]) . '|' . $parts[1]; + case 'mac-address': + case 'mac-eui-64': + $value = str_replace(array('.', ':', '-', ' '), '', strtolower($value)); + return wordwrap($value, 2, ':', true); + case 'hostname|port': + $value = strtolower($value); + return str_replace(':', '|', $value); + case 'boolean': + $value = trim(strtolower($value)); + if ('true' === $value) { + $value = 1; + } else if ('false' === $value) { + $value = 0; + } + return $value ? '1' : '0'; + case 'datetime': + try { + return (new FrozenTime($value, 'GMT'))->format('Y-m-d\TH:i:s.uO'); // ISO8601 formatting with microseconds + } catch (Exception $e) { + return $value; // silently skip. Rejection will be done in validation() + } + case 'AS': + if (strtoupper(substr($value, 0, 2)) === 'AS') { + $value = substr($value, 2); // remove 'AS' + } + if (strpos($value, '.') !== false) { // maybe value is in asdot notation + $parts = explode('.', $value, 2); + if (self::isPositiveInteger($parts[0]) && self::isPositiveInteger($parts[1])) { + return $parts[0] * 65536 + $parts[1]; + } + } + return $value; + } + return $value; + } + + /** + * Validate if value is valid for given attribute type. + * At this point, we can be sure, that composite type is really composite. + * @param string $type + * @param string $value + * @return bool|string + */ + public static function validate($type, $value) + { + switch ($type) { + case 'md5': + case 'imphash': + case 'sha1': + case 'sha224': + case 'sha256': + case 'sha384': + case 'sha512': + case 'sha512/224': + case 'sha512/256': + case 'sha3-224': + case 'sha3-256': + case 'sha3-384': + case 'sha3-512': + case 'authentihash': + case 'ja3-fingerprint-md5': + case 'jarm-fingerprint': + case 'hassh-md5': + case 'hasshserver-md5': + case 'x509-fingerprint-md5': + case 'x509-fingerprint-sha256': + case 'x509-fingerprint-sha1': + case 'git-commit-id': + if (self::isHashValid($type, $value)) { + return true; + } + $length = self::HASH_HEX_LENGTH[$type]; + return __('Checksum has an invalid length or format (expected: %s hexadecimal characters). Please double check the value or select type "other".', $length); + case 'tlsh': + if (self::isTlshValid($value)) { + return true; + } + return __('Checksum has an invalid length or format (expected: at least 35 hexadecimal characters, optionally starting with t1 instead of hexadecimal characters). Please double check the value or select type "other".'); + case 'telfhash': + if (self::isTelfhashValid($value)) { + return true; + } + return __('Checksum has an invalid length or format (expected: %s or %s hexadecimal characters). Please double check the value or select type "other".', 70, 72); + case 'pehash': + if (self::isHashValid('pehash', $value)) { + return true; + } + return __('The input doesn\'t match the expected sha1 format (expected: 40 hexadecimal characters). Keep in mind that MISP currently only supports SHA1 for PEhashes, if you would like to get the support extended to other hash types, make sure to create a github ticket about it at https://github.com/MISP/MISP!'); + case 'ssdeep': + if (self::isSsdeep($value)) { + return true; + } + return __('Invalid SSDeep hash. The format has to be blocksize:hash:hash'); + case 'impfuzzy': + if (substr_count($value, ':') === 2) { + $parts = explode(':', $value); + if (self::isPositiveInteger($parts[0])) { + return true; + } + } + return __('Invalid impfuzzy format. The format has to be imports:hash:hash'); + case 'cdhash': + if (preg_match("#^[0-9a-f]{40,}$#", $value)) { + return true; + } + return __('The input doesn\'t match the expected format (expected: 40 or more hexadecimal characters)'); + case 'http-method': + if (preg_match("#(OPTIONS|GET|HEAD|POST|PUT|DELETE|TRACE|CONNECT|PROPFIND|PROPPATCH|MKCOL|COPY|MOVE|LOCK|UNLOCK|VERSION-CONTROL|REPORT|CHECKOUT|CHECKIN|UNCHECKOUT|MKWORKSPACE|UPDATE|LABEL|MERGE|BASELINE-CONTROL|MKACTIVITY|ORDERPATCH|ACL|PATCH|SEARCH)#", $value)) { + return true; + } + return __('Unknown HTTP method.'); + case 'filename|pehash': + // no newline + if (preg_match("#^.+\|[0-9a-f]{40}$#", $value)) { + return true; + } + return __('The input doesn\'t match the expected filename|sha1 format (expected: filename|40 hexadecimal characters). Keep in mind that MISP currently only supports SHA1 for PEhashes, if you would like to get the support extended to other hash types, make sure to create a github ticket about it at https://github.com/MISP/MISP!'); + case 'filename|md5': + case 'filename|sha1': + case 'filename|imphash': + case 'filename|sha224': + case 'filename|sha256': + case 'filename|sha384': + case 'filename|sha512': + case 'filename|sha512/224': + case 'filename|sha512/256': + case 'filename|sha3-224': + case 'filename|sha3-256': + case 'filename|sha3-384': + case 'filename|sha3-512': + case 'filename|authentihash': + $hashType = substr($type, 9); // strip `filename|` + $length = self::HASH_HEX_LENGTH[$hashType]; + if (preg_match("#^.+\|[0-9a-f]{" . $length . "}$#", $value)) { + return true; + } + return __('Checksum has an invalid length or format (expected: filename|%s hexadecimal characters). Please double check the value or select type "other".', $length); + case 'filename|ssdeep': + $composite = explode('|', $value); + if (strpos($composite[0], "\n") !== false) { + return __('Filename must not contain new line character.'); + } + if (self::isSsdeep($composite[1])) { + return true; + } + return __('Invalid ssdeep hash (expected: blocksize:hash:hash).'); + case 'filename|tlsh': + $composite = explode('|', $value); + if (strpos($composite[0], "\n") !== false) { + return __('Filename must not contain new line character.'); + } + if (self::isTlshValid($composite[1])) { + return true; + } + return __('TLSH hash has an invalid length or format (expected: filename|at least 35 hexadecimal characters, optionally starting with t1 instead of hexadecimal characters). Please double check the value or select type "other".'); + case 'filename|vhash': + if (preg_match('#^.+\|.+$#', $value)) { + return true; + } + return __('Checksum has an invalid length or format (expected: filename|string characters). Please double check the value or select type "other".'); + case 'ip-src': + case 'ip-dst': + if (strpos($value, '/') !== false) { + $parts = explode("/", $value); + if (count($parts) !== 2 || !self::isPositiveInteger($parts[1])) { + return __('Invalid CIDR notation value found.'); + } + + if (filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) { + if ($parts[1] > 32) { + return __('Invalid CIDR notation value found, for IPv4 must be lower or equal 32.'); + } + } else if (filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) { + if ($parts[1] > 128) { + return __('Invalid CIDR notation value found, for IPv6 must be lower or equal 128.'); + } + } else { + return __('IP address has an invalid format.'); + } + } else if (!filter_var($value, FILTER_VALIDATE_IP)) { + return __('IP address has an invalid format.'); + } + return true; + case 'port': + if (!self::isPortValid($value)) { + return __('Port numbers have to be integers between 1 and 65535.'); + } + return true; + case 'ip-dst|port': + case 'ip-src|port': + $parts = explode('|', $value); + if (!filter_var($parts[0], FILTER_VALIDATE_IP)) { + return __('IP address has an invalid format.'); + } + if (!self::isPortValid($parts[1])) { + return __('Port numbers have to be integers between 1 and 65535.'); + } + return true; + case 'mac-address': + return preg_match('/^([a-fA-F0-9]{2}[:]?){6}$/', $value) === 1; + case 'mac-eui-64': + return preg_match('/^([a-fA-F0-9]{2}[:]?){8}$/', $value) === 1; + case 'hostname': + case 'domain': + if (self::isDomainValid($value)) { + return true; + } + return __('%s has an invalid format. Please double check the value or select type "other".', ucfirst($type)); + case 'hostname|port': + $parts = explode('|', $value); + if (!self::isDomainValid($parts[0])) { + return __('Hostname has an invalid format.'); + } + if (!self::isPortValid($parts[1])) { + return __('Port numbers have to be integers between 1 and 65535.'); + } + return true; + case 'domain|ip': + $parts = explode('|', $value); + if (!self::isDomainValid($parts[0])) { + return __('Domain has an invalid format.'); + } + if (!filter_var($parts[1], FILTER_VALIDATE_IP)) { + return __('IP address has an invalid format.'); + } + return true; + case 'email': + case 'email-src': + case 'eppn': + case 'email-dst': + case 'target-email': + case 'whois-registrant-email': + case 'dns-soa-email': + case 'jabber-id': + // we don't use the native function to prevent issues with partial email addresses + if (preg_match("#^.[^\s]*\@.*\..*$#i", $value)) { + return true; + } + return __('Email address has an invalid format. Please double check the value or select type "other".'); + case 'vulnerability': + if (preg_match("#^CVE-[0-9]{4}-[0-9]{4,}$#", $value)) { + return true; + } + return __('Invalid format. Expected: CVE-xxxx-xxxx...'); + case 'weakness': + if (preg_match("#^CWE-[0-9]+$#", $value)) { + return true; + } + return __('Invalid format. Expected: CWE-x...'); + case 'windows-service-name': + case 'windows-service-displayname': + if (strlen($value) > 256 || preg_match('#[\\\/]#', $value)) { + return __('Invalid format. Only values shorter than 256 characters that don\'t include any forward or backward slashes are allowed.'); + } + return true; + case 'mutex': + case 'process-state': + case 'snort': + case 'bro': + case 'zeek': + case 'community-id': + case 'anonymised': + case 'pattern-in-file': + case 'pattern-in-traffic': + case 'pattern-in-memory': + case 'filename-pattern': + case 'pgp-public-key': + case 'pgp-private-key': + case 'yara': + case 'stix2-pattern': + case 'sigma': + case 'gene': + case 'kusto-query': + case 'mime-type': + case 'identity-card-number': + case 'cookie': + case 'attachment': + case 'malware-sample': + case 'comment': + case 'text': + case 'other': + case 'cpe': + case 'email-attachment': + case 'email-body': + case 'email-header': + case 'first-name': + case 'middle-name': + case 'last-name': + case 'full-name': + return true; + case 'link': + // Moved to a native function whilst still enforcing the scheme as a requirement + return (bool)filter_var($value); + case 'hex': + return ctype_xdigit($value); + case 'target-user': + case 'campaign-name': + case 'campaign-id': + case 'threat-actor': + case 'target-machine': + case 'target-org': + case 'target-location': + case 'target-external': + case 'email-subject': + case 'malware-type': + // TODO: review url/uri validation + case 'url': + case 'uri': + case 'user-agent': + case 'regkey': + case 'regkey|value': + case 'filename': + case 'pdb': + case 'windows-scheduled-task': + case 'whois-registrant-name': + case 'whois-registrant-org': + case 'whois-registrar': + case 'whois-creation-date': + case 'date-of-birth': + case 'place-of-birth': + case 'gender': + case 'passport-number': + case 'passport-country': + case 'passport-expiration': + case 'redress-number': + case 'nationality': + case 'visa-number': + case 'issue-date-of-the-visa': + case 'primary-residence': + case 'country-of-residence': + case 'special-service-request': + case 'frequent-flyer-number': + case 'travel-details': + case 'payment-details': + case 'place-port-of-original-embarkation': + case 'place-port-of-clearance': + case 'place-port-of-onward-foreign-destination': + case 'passenger-name-record-locator-number': + case 'email-dst-display-name': + case 'email-src-display-name': + case 'email-reply-to': + case 'email-x-mailer': + case 'email-mime-boundary': + case 'email-thread-index': + case 'email-message-id': + case 'github-username': + case 'github-repository': + case 'github-organisation': + case 'twitter-id': + case 'dkim': + case 'dkim-signature': + case 'favicon-mmh3': + case 'chrome-extension-id': + case 'mobile-application-id': + case 'azure-application-id': + case 'named pipe': + if (strpos($value, "\n") !== false) { + return __('Value must not contain new line character.'); + } + return true; + case 'ssh-fingerprint': + if (self::isSshFingerprint($value)) { + return true; + } + return __('SSH fingerprint must be in MD5 or SHA256 format.'); + case 'datetime': + if (strtotime($value) !== false) { + return true; + } + return __('Datetime has to be in the ISO 8601 format.'); + case 'size-in-bytes': + case 'counter': + if (self::isPositiveInteger($value)) { + return true; + } + return __('The value has to be a whole number greater or equal 0.'); + /* case 'targeted-threat-index': + if (!is_numeric($value) || $value < 0 || $value > 10) { + return __('The value has to be a number between 0 and 10.'); + } + return true;*/ + case 'iban': + case 'bic': + case 'btc': + case 'dash': + case 'xmr': + return preg_match('/^[a-zA-Z0-9]+$/', $value) === 1; + case 'vhash': + return preg_match('/^.+$/', $value) === 1; + case 'bin': + case 'cc-number': + case 'bank-account-nr': + case 'aba-rtn': + case 'prtn': + case 'phone-number': + case 'whois-registrant-phone': + case 'float': + return is_numeric($value); + case 'cortex': + return JsonTool::isValid($value); + case 'boolean': + return $value == 1 || $value == 0; + case 'AS': + if (self::isPositiveInteger($value) && $value <= 4294967295) { + return true; + } + return __('AS number have to be integer between 1 and 4294967295'); + } + throw new InvalidArgumentException("Unknown type $type."); + } + + /** + * This method will generate all valid types for given value. + * @param array $types Typos to check + * @param array $compositeTypes Composite types + * @param string $value Values to check + * @return array + */ + public static function validTypesForValue(array $types, array $compositeTypes, $value) + { + $possibleTypes = []; + foreach ($types as $type) { + if (in_array($type, $compositeTypes, true) && substr_count($value, '|') !== 1) { + continue; // value is not in composite format + } + $modifiedValue = AttributeValidationTool::modifyBeforeValidation($type, $value); + if (AttributeValidationTool::validate($type, $modifiedValue) === true) { + $possibleTypes[] = $type; + } + } + return $possibleTypes; + } + + /** + * @param string $value + * @return bool + */ + private static function isDomainValid($value) + { + return preg_match("#^[A-Z0-9.\-_]+\.[A-Z0-9\-]{2,}$#i", $value) === 1; + } + + /** + * @param string $value + * @return bool + */ + private static function isPortValid($value) + { + return self::isPositiveInteger($value) && $value >= 1 && $value <= 65535; + } + + /** + * @param string $value + * @return bool + */ + private static function isTlshValid($value) + { + if ($value[0] === 't') { + $value = substr($value, 1); + } + return strlen($value) > 35 && ctype_xdigit($value); + } + + /** + * @param string $value + * @return bool + */ + private static function isTelfhashValid($value) + { + return strlen($value) == 70 || strlen($value) == 72; + } + + + /** + * @param string $type + * @param string $value + * @return bool + */ + private static function isHashValid($type, $value) + { + return strlen($value) === self::HASH_HEX_LENGTH[$type] && ctype_xdigit($value); + } + + /** + * Returns true if input value is positive integer or zero. + * @param int|string $value + * @return bool + */ + private static function isPositiveInteger($value) + { + return (is_int($value) && $value >= 0) || ctype_digit($value); + } + + /** + * @param string $value + * @return bool + */ + private static function isSsdeep($value) + { + return preg_match('#^([0-9]+):([0-9a-zA-Z/+]*):([0-9a-zA-Z/+]*)$#', $value); + } + + /** + * @param string $value + * @return bool + */ + private static function isSshFingerprint($value) + { + if (substr($value, 0, 7) === 'SHA256:') { + $value = substr($value, 7); + $decoded = base64_decode($value, true); + return $decoded && strlen($decoded) === 32; + } else if (substr($value, 0, 4) === 'MD5:') { + $value = substr($value, 4); + } + + $value = str_replace(':', '', $value); + return self::isHashValid('md5', $value); + } + + /** + * @param string $value + * @return string + */ + private static function normalizeIp($value) + { + // If IP is a CIDR + if (strpos($value, '/')) { + list($ip, $range) = explode('/', $value, 2); + + // Compress IPv6 + if (strpos($ip, ':') && $converted = inet_pton($ip)) { + $ip = inet_ntop($converted); + } + + // If IP is in CIDR format, but the network is 32 for IPv4 or 128 for IPv6, normalize to non CIDR type + if (($range === '32' && strpos($value, '.')) || ($range === '128' && strpos($value, ':'))) { + return $ip; + } + + return "$ip/$range"; + } + + // Compress IPv6 + if (strpos($value, ':') && $converted = inet_pton($value)) { + return inet_ntop($converted); + } + + return $value; + } + + /** + * Temporary solution for utf8 columns until we migrate to utf8mb4. + * via https://stackoverflow.com/questions/16496554/can-php-detect-4-byte-encoded-utf8-chars + * @param string $input + * @return string + */ + private static function handle4ByteUnicode($input) + { + return preg_replace( + '%(?: + \xF0[\x90-\xBF][\x80-\xBF]{2} + | [\xF1-\xF3][\x80-\xBF]{3} + | \xF4[\x80-\x8F][\x80-\xBF]{2} + )%xs', + '?', + $input + ); + } +} diff --git a/src/Lib/Tools/ComplexTypeTool.php b/src/Lib/Tools/ComplexTypeTool.php new file mode 100644 index 000000000..7916ed882 --- /dev/null +++ b/src/Lib/Tools/ComplexTypeTool.php @@ -0,0 +1,671 @@ + '/^(hxxp|hxtp|htxp|meow|h\[tt\]p)/i', + 'to' => 'http', + 'types' => array('link', 'url') + ), + array( + 'from' => '/(\[\.\]|\[dot\]|\(dot\))/', + 'to' => '.', + 'types' => array('link', 'url', 'ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname') + ), + array( + 'from' => '/\[hxxp:\/\/\]/', + 'to' => 'http://', + 'types' => array('link', 'url') + ), + array( + 'from' => '/[\@]|\[at\]/', + 'to' => '@', + 'types' => array('email', 'email-src', 'email-dst') + ), + array( + 'from' => '/\[:\]/', + 'to' => ':', + 'types' => array('url', 'link') + ) + ); + + const HEX_HASH_TYPES = [ + 32 => ['single' => ['md5', 'imphash', 'x509-fingerprint-md5', 'ja3-fingerprint-md5'], 'composite' => ['filename|md5', 'filename|imphash']], + 40 => ['single' => ['sha1', 'pehash', 'x509-fingerprint-sha1', 'cdhash'], 'composite' => ['filename|sha1', 'filename|pehash']], + 56 => ['single' => ['sha224', 'sha512/224'], 'composite' => ['filename|sha224', 'filename|sha512/224']], + 64 => ['single' => ['sha256', 'authentihash', 'sha512/256', 'x509-fingerprint-sha256'], 'composite' => ['filename|sha256', 'filename|authentihash', 'filename|sha512/256']], + 96 => ['single' => ['sha384'], 'composite' => ['filename|sha384']], + 128 => ['single' => ['sha512'], 'composite' => ['filename|sha512']], + ]; + + private $__tlds; + + /** + * Hardcoded list if properly warninglist is not available + * @var string[] + */ + private $securityVendorDomains = ['virustotal.com', 'hybrid-analysis.com']; + + public static function refangValue($value, $type) + { + foreach (self::REFANG_REGEX_TABLE as $regex) { + if (in_array($type, $regex['types'], true)) { + $value = preg_replace($regex['from'], $regex['to'], $value); + } + } + return $value; + } + + public function setTLDs($tlds = array()) + { + $this->__tlds = []; + foreach ($tlds as $tld) { + $this->__tlds[$tld] = true; + } + } + + public function setSecurityVendorDomains(array $securityVendorDomains) + { + if (empty($securityVendorDomains)) { + return; // if provided warninglist is empty, keep hardcoded domains + } + $this->securityVendorDomains = $securityVendorDomains; + } + + public function checkComplexRouter($input, $type, $settings = array()) + { + switch ($type) { + case 'File': + return $this->checkComplexFile($input); + case 'CnC': + return $this->checkComplexCnC($input); + case 'freetext': + case 'FreeText': + return $this->checkFreeText($input, $settings); + case 'csv': + return $this->checkCSV($input, $settings); + default: + return false; + } + } + + // checks if the passed input matches a valid file description attribute's pattern (filename, md5, sha1, sha256, filename|md5, filename|sha1, filename|sha256) + public function checkComplexFile($input) + { + $original = $input; + $type = ''; + $composite = false; + if (strpos($input, '|')) { + $composite = true; + $result = explode('|', $input); + if (count($result) != 2 || !preg_match("#^.+#", $result[0])) { + $type = 'other'; + } else { + $type = 'filename|'; + } + $input = $result[1]; + } + if (strlen($input) == 32 && preg_match("#[0-9a-f]{32}$#", $input)) { + $type .= 'md5'; + } + if (strlen($input) == 40 && preg_match("#[0-9a-f]{40}$#", $input)) { + $type .= 'sha1'; + } + if (strlen($input) == 64 && preg_match("#[0-9a-f]{64}$#", $input)) { + $type .= 'sha256'; + } + if ($type == '' && !$composite && preg_match("#^.+#", $input)) { + $type = 'filename'; + } + if ($type == '') { + $type = 'other'; + } + return array('type' => $type, 'value' => $original); + } + + public function checkComplexCnC($input) + { + $toReturn = array(); + // check if it's an IP address + if (filter_var($input, FILTER_VALIDATE_IP)) { + return array('type' => 'ip-dst', 'value' => $input); + } + if (preg_match("#^[A-Z0-9.-]+\.[A-Z]{2,4}$#i", $input)) { + $result = explode('.', $input); + if (count($result) > 2) { + $toReturn['multi'][] = array('type' => 'hostname', 'value' => $input); + $pos = strpos($input, '.'); + $toReturn['multi'][] = array('type' => 'domain', 'value' => substr($input, (1 + $pos))); + return $toReturn; + } + return array('type' => 'domain', 'value' => $input); + } + + if (!preg_match("#\n#", $input)) { + return array('type' => 'url', 'value' => $input); + } + return array('type' => 'other', 'value' => $input); + } + + /** + * Parse a CSV file with the given settings + * All lines starting with # are stripped + * The settings can contain the following: + * delimiter: Expects a delimiter string (default is a simple comma). + * For example, to split the following line: "value1##comma##value2" simply pass $settings['delimiter'] = "##comma##"; + * values: Expects an array (or a comma separated string) with numeric values denoting the columns containing indicators. If this is not set then every value will be checked. (column numbers start at 1) + * @param string $input + * @param array $settings + * @return array + * @throws Exception + */ + public function checkCSV($input, $settings = array()) + { + if (empty($input)) { + return []; + } + + $delimiter = !empty($settings['delimiter']) ? $settings['delimiter'] : ","; + if ($delimiter === '\t') { + $delimiter = "\t"; + } + $values = !empty($settings['value']) ? $settings['value'] : array(); + if (!is_array($values)) { + $values = explode(',', $values); + } + foreach ($values as $key => $value) { + $values[$key] = intval($value); + } + + // Write to tmp file to save memory + $tmpFile = new TmpFileTool(); + $tmpFile->write($input); + unset($input); + + $iocArray = []; + foreach ($tmpFile->intoParsedCsv($delimiter) as $row) { + if (!empty($row[0][0]) && $row[0][0] === '#') { // Comment + continue; + } + foreach ($row as $elementPos => $element) { + if (empty($element)) { + continue; + } + if (empty($values) || in_array(($elementPos + 1), $values)) { + $element = trim($element, " \t\n\r\0\x0B\"\'"); + if (empty($element)) { + continue; + } + if (!empty($settings['excluderegex']) && preg_match($settings['excluderegex'], $element)) { + continue; + } + $resolvedResult = $this->__resolveType($element); + // Do not extract datetime from CSV + if ($resolvedResult) { + $iocArray[] = $resolvedResult; + } + } + } + } + + return $iocArray; + } + + /** + * @param string $input + * @param array $settings + * @return array + */ + public function checkFreeText($input, array $settings = []) + { + if (empty($input)) { + return []; + } + + if ($input[0] === '{') { + // If input looks like JSON, try to parse it as JSON + try { + return $this->parseJson($input, $settings); + } catch (Exception $e) { + } + } + + $iocArray = $this->parseFreetext($input); + + $resultArray = []; + foreach ($iocArray as $ioc) { + $ioc = trim($ioc, '\'".,() ' . "\t\n\r\0\x0B"); // custom + default PHP trim + if (empty($ioc)) { + continue; + } + if (!empty($settings['excluderegex']) && preg_match($settings['excluderegex'], $ioc)) { + continue; + } + $typeArray = $this->__resolveType($ioc); + if ($typeArray === false) { + continue; + } + // Remove duplicates + if (isset($resultArray[$typeArray['value']])) { + continue; + } + $typeArray['original_value'] = $ioc; + $resultArray[$typeArray['value']] = $typeArray; + } + return array_values($resultArray); + } + + /** + * @param string $input + * @throws JsonException + */ + private function parseJson($input, array $settings) + { + $parsed = JsonTool::decode($input); + + $values = []; + array_walk_recursive($parsed, function ($value) use (&$values) { + if (is_bool($value) || is_int($value) || empty($value)) { + return; // skip boolean, integer or empty values + } + + $values[] = $value; + foreach ($this->parseFreetext($value) as $v) { + if ($v !== $value) { + $values[] = $v; + } + } + }); + unset($parsed); + + $resultArray = []; + foreach ($values as $ioc) { + $ioc = trim($ioc, '\'".,() ' . "\t\n\r\0\x0B"); // custom + default PHP trim + if (empty($ioc)) { + continue; + } + if (!empty($settings['excluderegex']) && preg_match($settings['excluderegex'], $ioc)) { + continue; + } + $typeArray = $this->__resolveType($ioc); + if ($typeArray === false) { + continue; + } + // Remove duplicates + if (isset($resultArray[$typeArray['value']])) { + continue; + } + $typeArray['original_value'] = $ioc; + $resultArray[$typeArray['value']] = $typeArray; + } + return array_values($resultArray); + } + + /** + * @param string $input + * @return array|string[] + */ + private function parseFreetext($input) + { + $input = str_replace("\xc2\xa0", ' ', $input); // non breaking space to normal space + $input = preg_replace('/\p{C}+/u', ' ', $input); + $iocArray = preg_split("/\r\n|\n|\r|\s|\s+|,|\<|\>|;/", $input); + + preg_match_all('/\"([^\"]+)\"/', $input, $matches); + foreach ($matches[1] as $match) { + $iocArray[] = $match; + } + return $iocArray; + } + + /** + * @param string $raw_input Trimmed value + * @return array|false + */ + private function __resolveType($raw_input) + { + // Check if value is clean IP without doing expensive operations. + if (filter_var($raw_input, FILTER_VALIDATE_IP)) { + return [ + 'types' => ['ip-dst', 'ip-src', 'ip-src/ip-dst'], + 'default_type' => 'ip-dst', + 'value' => $raw_input, + ]; + } + + $input = ['raw' => $raw_input]; + + // Check hashes before refang and port extracting, it is not necessary for hashes. This speedups parsing + // freetexts or CSVs with a lot of hashes. + if ($result = $this->__checkForHashes($input)) { + return $result; + } + + $input = $this->__refangInput($input); + + // Check email before port extracting, it is not necessary for email. This speedups parsing + // freetexts or CSVs with a lot of emails. + if ($result = $this->__checkForEmail($input)) { + return $result; + } + + $input = $this->__extractPort($input); + if ($result = $this->__checkForIP($input)) { + return $result; + } + if ($result = $this->__checkForDomainOrFilename($input)) { + return $result; + } + if ($result = $this->__checkForSimpleRegex($input)) { + return $result; + } + if ($result = $this->__checkForAS($input)) { + return $result; + } + if ($result = $this->__checkForBTC($input)) { + return $result; + } + return false; + } + + private function __checkForBTC($input) + { + if (preg_match("#^([13][a-km-zA-HJ-NP-Z1-9]{25,34})|(bc|tb)1([023456789acdefghjklmnpqrstuvwxyz]{11,71})$#i", $input['raw'])) { + return [ + 'types' => ['btc'], + 'default_type' => 'btc', + 'value' => $input['raw'], + ]; + } + return false; + } + + private function __checkForEmail($input) + { + // quick filter for an @ to see if we should validate a potential e-mail address + if (strpos($input['refanged'], '@') !== false) { + if (filter_var($input['refanged'], FILTER_VALIDATE_EMAIL)) { + return [ + 'types' => array('email', 'email-src', 'email-dst', 'target-email', 'whois-registrant-email'), + 'default_type' => 'email-src', + 'value' => $input['refanged'], + ]; + } + } + return false; + } + + private function __checkForAS($input) + { + if (preg_match('#^as[0-9]+$#i', $input['raw'])) { + $input['raw'] = strtoupper($input['raw']); + return array('types' => array('AS'), 'default_type' => 'AS', 'value' => $input['raw']); + } + return false; + } + + private function __checkForHashes($input) + { + // handle prepared composite values with the filename|hash format + if (strpos($input['raw'], '|')) { + $compositeParts = explode('|', $input['raw']); + if (count($compositeParts) === 2) { + if ($this->__resolveFilename($compositeParts[0])) { + $hash = $this->__resolveHash($compositeParts[1]); + if ($hash) { + return array('types' => $hash['composite'], 'default_type' => $hash['composite'][0], 'value' => $input['raw']); + } + if ($this->__resolveSsdeep($compositeParts[1])) { + return array('types' => array('filename|ssdeep'), 'default_type' => 'filename|ssdeep', 'value' => $input['raw']); + } + } + } + } + + // check for hashes + $hash = $this->__resolveHash($input['raw']); + if ($hash) { + $types = $hash['single']; + if ($this->__checkForBTC($input)) { + $types[] = 'btc'; + } + return array('types' => $types, 'default_type' => $types[0], 'value' => $input['raw']); + } + // ssdeep has a different pattern + if ($this->__resolveSsdeep($input['raw'])) { + return array('types' => array('ssdeep'), 'default_type' => 'ssdeep', 'value' => $input['raw']); + } + return false; + } + + private function __extractPort($input) + { + // note down and remove the port if it's a url / domain name / hostname / ip + // input2 from here on is the variable containing the original input with the port removed. It is only used by url / domain name / hostname / ip + if (preg_match('/(:[0-9]{2,5})$/', $input['refanged'], $port)) { + $input['comment'] = 'On port ' . substr($port[0], 1); + $input['refanged_no_port'] = str_replace($port[0], '', $input['refanged']); + $input['port'] = substr($port[0], 1); + } else { + $input['comment'] = false; + $input['refanged_no_port'] = $input['refanged']; + } + return $input; + } + + private function __refangInput($input) + { + $refanged = $input['raw']; + foreach (self::REFANG_REGEX_TABLE as $regex) { + $refanged = preg_replace($regex['from'], $regex['to'], $refanged); + } + $refanged = rtrim($refanged, "."); + $input['refanged'] = preg_replace_callback( + '/\[.\]/', + function ($matches) { + return trim($matches[0], '[]'); + }, + $refanged + ); + return $input; + } + + private function __checkForSimpleRegex($input) + { + // CVE numbers + if (preg_match("#^cve-[0-9]{4}-[0-9]{4,9}$#i", $input['raw'])) { + return [ + 'types' => ['vulnerability'], + 'default_type' => 'vulnerability', + 'value' => strtoupper($input['raw']), // 'CVE' must be uppercase + ]; + } + + // Phone numbers - for automatic recognition, needs to start with + or include dashes + if ($input['raw'][0] === '+' || strpos($input['raw'], '-')) { + if (!preg_match('#^[0-9]{4}-[0-9]{2}-[0-9]{2}$#i', $input['raw']) && preg_match("#^(\+)?([0-9]{1,3}(\(0\))?)?[0-9\/\-]{5,}[0-9]$#i", $input['raw'])) { + return array('types' => array('phone-number', 'prtn', 'whois-registrant-phone'), 'default_type' => 'phone-number', 'value' => $input['raw']); + } + } + return false; + } + + private function __checkForIP(array $input) + { + if (filter_var($input['refanged_no_port'], FILTER_VALIDATE_IP)) { + if (isset($input['port'])) { + return array('types' => array('ip-dst|port', 'ip-src|port', 'ip-src|port/ip-dst|port'), 'default_type' => 'ip-dst|port', 'comment' => $input['comment'], 'value' => $input['refanged_no_port'] . '|' . $input['port']); + } else { + return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']); + } + } + // IPv6 address that is considered as IP address with port + if (filter_var($input['refanged'], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) { + return [ + 'types' => ['ip-dst', 'ip-src', 'ip-src/ip-dst'], + 'default_type' => 'ip-dst', + 'comment' => '', + 'value' => $input['refanged'], + ]; + } + // IPv6 with port in `[1fff:0:a88:85a3::ac1f]:8001` format + if ( + isset($input['port']) && + !empty($input['refanged_no_port']) && + $input['refanged_no_port'][0] === '[' && + filter_var(substr($input['refanged_no_port'], 1, -1), FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) + ) { + $value = substr($input['refanged_no_port'], 1, -1); // remove brackets + return [ + 'types' => ['ip-dst|port', 'ip-src|port', 'ip-src|port/ip-dst|port'], + 'default_type' => 'ip-dst|port', + 'comment' => $input['comment'], + 'value' => "$value|{$input['port']}", + ]; + } + // it could still be a CIDR block + if (strpos($input['refanged_no_port'], '/')) { + $temp = explode('/', $input['refanged_no_port']); + if (count($temp) === 2 && filter_var($temp[0], FILTER_VALIDATE_IP) && is_numeric($temp[1])) { + return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']); + } + } + return false; + } + + private function __checkForDomainOrFilename(array $input) + { + if (strpos($input['refanged_no_port'], '.') !== false) { + $temp = explode('.', $input['refanged_no_port']); + $domainDetection = true; + if (preg_match('/^([-\pL\pN]+\.)+[a-z0-9-]+$/iu', $input['refanged_no_port'])) { + if (!$this->isTld(end($temp))) { + $domainDetection = false; + } + } else { + $domainDetection = false; + } + if ($domainDetection) { + if (count($temp) > 2) { + return array('types' => array('hostname', 'domain', 'url', 'filename'), 'default_type' => 'hostname', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']); + } else { + return array('types' => array('domain', 'filename'), 'default_type' => 'domain', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']); + } + } else { + // check if it is a URL + // Adding http:// infront of the input in case it was left off. github.com/MISP/MISP should still be counted as a valid link + if (count($temp) > 1 && (filter_var($input['refanged_no_port'], FILTER_VALIDATE_URL) || filter_var('http://' . $input['refanged_no_port'], FILTER_VALIDATE_URL))) { + // Even though some domains are valid, we want to exclude them as they are known security vendors / etc + if ($this->isLink($input['refanged_no_port'])) { + return array('types' => array('link'), 'default_type' => 'link', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']); + } + if (strpos($input['refanged_no_port'], '/')) { + return array('types' => array('url'), 'default_type' => 'url', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']); + } + } + if ($this->__resolveFilename($input['raw'])) { + return array('types' => array('filename'), 'default_type' => 'filename', 'value' => $input['raw']); + } + } + } + if (strpos($input['raw'], '\\') !== false) { + $temp = explode('\\', $input['raw']); + if (strpos(end($temp), '.') || preg_match('/^.:/i', $temp[0])) { + if ($this->__resolveFilename(end($temp))) { + return array('types' => array('filename'), 'default_type' => 'filename', 'value' => $input['raw']); + } + } else if (!empty($temp[0])) { + return array('types' => array('regkey'), 'default_type' => 'regkey', 'value' => $input['raw']); + } + } + return false; + } + + private function __resolveFilename($param) + { + if ((preg_match('/^.:/', $param) || strpos($param, '.') != 0)) { + $parts = explode('.', $param); + if (!is_numeric(end($parts)) && ctype_alnum(end($parts))) { + return true; + } + } + return false; + } + + /** + * @param string $value + * @return bool + */ + private function __resolveSsdeep($value) + { + return preg_match('#^[0-9]+:[0-9a-zA-Z/+]+:[0-9a-zA-Z/+]+$#', $value) && !preg_match('#^[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$#', $value); + } + + /** + * @param string $value + * @return bool|string[][] + */ + private function __resolveHash($value) + { + $strlen = strlen($value); + if (isset(self::HEX_HASH_TYPES[$strlen]) && ctype_xdigit($value)) { + return self::HEX_HASH_TYPES[$strlen]; + } + return false; + } + + /** + * @param string $tld + * @return bool + */ + private function isTld($tld) + { + if ($this->__tlds === null) { + $this->setTLDs($this->__generateTLDList()); + } + return isset($this->__tlds[strtolower($tld)]); + } + + /** + * Check if URL should be considered as link attribute type + * @param string $value + * @return bool + */ + private function isLink($value) + { + if (!preg_match('/^https:\/\/([^\/]*)/i', $value, $matches)) { + return false; + } + + $domainToCheck = ''; + $domainParts = array_reverse(explode('.', strtolower($matches[1]))); + foreach ($domainParts as $domainPart) { + $domainToCheck = $domainPart . $domainToCheck; + if (in_array($domainToCheck, $this->securityVendorDomains, true)) { + return true; + } + $domainToCheck = '.' . $domainToCheck; + } + return false; + } + + private function __generateTLDList() + { + $tlds = array('biz', 'cat', 'com', 'edu', 'gov', 'int', 'mil', 'net', 'org', 'pro', 'tel', 'aero', 'arpa', 'asia', 'coop', 'info', 'jobs', 'mobi', 'name', 'museum', 'travel', 'onion'); + $char1 = $char2 = 'a'; + for ($i = 0; $i < 26; $i++) { + for ($j = 0; $j < 26; $j++) { + $tlds[] = $char1 . $char2; + $char2++; + } + $char1++; + $char2 = 'a'; + } + return $tlds; + } +} diff --git a/src/Lib/Tools/RandomTool.php b/src/Lib/Tools/RandomTool.php new file mode 100644 index 000000000..5be885b15 --- /dev/null +++ b/src/Lib/Tools/RandomTool.php @@ -0,0 +1,64 @@ + 'aaaaa...' + throw new LogicException('random_str - Argument 3 - expected a string that contains at least 2 distinct characters'); + } + // Now that we have good data, this is the meat of our function: + $random_str = ''; + for ($i = 0; $i < $length; ++$i) { + $r = $crypto_secure ? random_int(0, $charset_max) : mt_rand(0, $charset_max); + $random_str .= $charset[$r]; + } + return $random_str; + } +} diff --git a/src/Model/Entity/Attribute.php b/src/Model/Entity/Attribute.php new file mode 100644 index 000000000..45dbff939 --- /dev/null +++ b/src/Model/Entity/Attribute.php @@ -0,0 +1,83 @@ + ['attachment', 'pattern-in-file', 'filename-pattern', 'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|pehash', 'malware-sample', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'x509-fingerprint-md5'], + 'network' => ['ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port', 'mac-address', 'mac-eui-64', 'hostname', 'hostname|port', 'domain', 'domain|ip', 'email-dst', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'bro', 'zeek', 'pattern-in-traffic', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'jarm-fingerprint', 'favicon-mmh3', 'hassh-md5', 'hasshserver-md5', 'community-id'], + 'financial' => ['btc', 'xmr', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'phone-number'] + ]; +} diff --git a/src/Model/Entity/Event.php b/src/Model/Entity/Event.php index 053e89e16..c3380189f 100644 --- a/src/Model/Entity/Event.php +++ b/src/Model/Entity/Event.php @@ -6,4 +6,120 @@ use App\Model\Entity\AppModel; class Event extends AppModel { + public const NO_PUSH_DISTRIBUTION = 'distribution', + NO_PUSH_SERVER_RULES = 'push_rules'; + + public $displayField = 'id'; + + public $fieldDescriptions = [ + 'threat_level_id' => ['desc' => 'Risk levels: *low* means mass-malware, *medium* means APT malware, *high* means sophisticated APT malware or 0-day attack', 'formdesc' => 'Risk levels: low: mass-malware medium: APT malware high: sophisticated APT malware or 0-day attack'], + 'classification' => ['desc' => 'Set the Traffic Light Protocol classification.
  1. TLP:AMBER- Share only within the organization on a need-to-know basis
  2. TLP:GREEN:NeedToKnow- Share within your constituency on the need-to-know basis.
  3. TLP:GREEN- Share within your constituency.
'], + 'submittedioc' => ['desc' => '', 'formdesc' => ''], + 'analysis' => ['desc' => 'Analysis Levels: *Initial* means the event has just been created, *Ongoing* means that the event is being populated, *Complete* means that the event\'s creation is complete', 'formdesc' => 'Analysis levels: Initial: event has been started Ongoing: event population is in progress Complete: event creation has finished'], + 'distribution' => ['desc' => 'Describes who will have access to the event.'] + ]; + + public $analysisDescriptions = [ + 0 => ['desc' => '*Initial* means the event has just been created', 'formdesc' => 'Event has just been created and is in an initial state'], + 1 => ['desc' => '*Ongoing* means that the event is being populated', 'formdesc' => 'The analysis is still ongoing'], + 2 => ['desc' => '*Complete* means that the event\'s creation is complete', 'formdesc' => 'The event creator considers the analysis complete'] + ]; + + public $distributionDescriptions = [ + Distribution::ORGANISATION_ONLY => [ + 'desc' => 'This field determines the current distribution of the event', + 'formdesc' => "This setting will only allow members of your organisation on this server to see it.", + ], + Distribution::COMMUNITY_ONLY => [ + 'desc' => 'This field determines the current distribution of the event', + 'formdesc' => "Organisations that are part of this MISP community will be able to see the event.", + ], + Distribution::CONNECTED_COMMUNITIES => [ + 'desc' => 'This field determines the current distribution of the event', + 'formdesc' => "Organisations that are either part of this MISP community or part of a directly connected MISP community will be able to see the event.", + ], + Distribution::ALL => [ + 'desc' => 'This field determines the current distribution of the event', + 'formdesc' => "This will share the event with all MISP communities, allowing the event to be freely propagated from one server to the next.", + ], + Distribution::SHARING_GROUP => [ + 'desc' => 'This field determines the current distribution of the event', + 'formdesc' => "This distribution of this event will be handled by the selected sharing group.", + ], + ]; + + public const ANALYSIS_LEVELS = [ + 0 => 'Initial', 1 => 'Ongoing', 2 => 'Completed' + ]; + + public $shortDist = [0 => 'Organisation', 1 => 'Community', 2 => 'Connected', 3 => 'All', 4 => ' sharing Group']; + + public $validFormats = [ + 'attack' => ['html', 'AttackExport', 'html'], + 'attack-sightings' => ['json', 'AttackSightingsExport', 'json'], + 'cache' => ['txt', 'CacheExport', 'cache'], + 'context' => ['html', 'ContextExport', 'html'], + 'context-markdown' => ['txt', 'ContextMarkdownExport', 'md'], + 'count' => ['txt', 'CountExport', 'txt'], + 'csv' => ['csv', 'CsvExport', 'csv'], + 'hashes' => ['txt', 'HashesExport', 'txt'], + 'hosts' => ['txt', 'HostsExport', 'txt'], + 'json' => ['json', 'JsonExport', 'json'], + 'netfilter' => ['txt', 'NetfilterExport', 'sh'], + 'opendata' => ['txt', 'OpendataExport', 'txt'], + 'openioc' => ['xml', 'OpeniocExport', 'ioc'], + 'rpz' => ['txt', 'RPZExport', 'rpz'], + 'snort' => ['txt', 'NidsSnortExport', 'rules'], + 'stix' => ['xml', 'Stix1Export', 'xml'], + 'stix-json' => ['json', 'Stix1Export', 'json'], + 'stix2' => ['json', 'Stix2Export', 'json'], + 'suricata' => ['txt', 'NidsSuricataExport', 'rules'], + 'text' => ['text', 'TextExport', 'txt'], + 'xml' => ['xml', 'XmlExport', 'xml'], + 'yara' => ['txt', 'YaraExport', 'yara'], + 'yara-json' => ['json', 'YaraExport', 'json'] + ]; + + public $possibleOptions = [ + 'eventid', + 'idList', + 'tags', + 'from', + 'to', + 'last', + 'to_ids', + 'includeAllTags', // include also non exportable tags, default `false` + 'includeAttachments', + 'event_uuid', + 'distribution', + 'sharing_group_id', + 'disableSiteAdmin', + 'metadata', + 'enforceWarninglist', // return just attributes that contains no warnings + 'sgReferenceOnly', // do not fetch additional information about sharing groups + 'flatten', + 'blockedAttributeTags', + 'eventsExtendingUuid', + 'extended', + 'extensionList', + 'excludeGalaxy', + // 'includeCustomGalaxyCluster', // not used + 'includeRelatedTags', + 'excludeLocalTags', + 'includeDecayScore', + 'includeScoresOnEvent', + 'includeSightingdb', + 'includeFeedCorrelations', + 'includeServerCorrelations', + 'includeWarninglistHits', + 'includeGranularCorrelations', + 'noEventReports', // do not include event report in event data + 'noShadowAttributes', // do not fetch proposals, + 'limit', + 'page', + 'order', + 'protected', + 'published', + 'orgc_id', + ]; } diff --git a/src/Model/Entity/Feed.php b/src/Model/Entity/Feed.php new file mode 100644 index 000000000..c8efc3139 --- /dev/null +++ b/src/Model/Entity/Feed.php @@ -0,0 +1,39 @@ + [ + "OR" => [], + "NOT" => [], + ], + 'orgs' => [ + "OR" => [], + "NOT" => [], + ], + 'url_params' => '' + ]; + + public const SUPPORTED_URL_PARAM_FILTERS = [ + 'timestamp', + 'publish_timestamp', + ]; + + public const CACHE_DIR = APP . 'tmp' . DS . 'cache' . DS . 'feeds' . DS; + + public const FEED_TYPES = array( + 'misp' => array( + 'name' => 'MISP Feed' + ), + 'freetext' => array( + 'name' => 'Freetext Parsed Feed' + ), + 'csv' => array( + 'name' => 'Simple CSV Parsed Feed' + ) + ); +} diff --git a/src/Model/Table/FeedsTable.php b/src/Model/Table/FeedsTable.php new file mode 100644 index 000000000..40bd3a766 --- /dev/null +++ b/src/Model/Table/FeedsTable.php @@ -0,0 +1,2236 @@ +belongsTo( + 'SharingGroups', + [ + 'foreignKey' => 'sharing_group_id', + 'className' => 'SharingGroups', + 'propertyName' => 'Organisation' + ] + ); + $this->belongsTo( + 'Tags', + [ + 'foreignKey' => 'sharing_group_id', + 'className' => 'SharingGroups', + 'propertyName' => 'Organisation' + ] + ); + $this->belongsTo( + 'Orgc', + [ + 'foreignKey' => 'orgc_id', + 'className' => 'Organisations', + 'propertyName' => 'Orgc' + ] + ); + + $this->addBehavior( + 'JsonFields', + [ + 'fields' => [ + 'settings' => ['default' => []], + 'rules' => ['default' => []] + ], + ] + ); + + $this->setDisplayField('name'); + } + + public function validationDefault(Validator $validator): Validator + { + $validator + ->requirePresence('url', 'name') + ->notEmptyString('provider') + ->add('url', 'custom', [ + 'rule' => function ($value, $context) { + if ($this->isFeedLocal($context['data'])) { + $path = mb_ereg_replace("/\:\/\//", '', $value); + if ($value['Feed']['source_format'] == 'misp') { + if (!is_dir($path)) { + return 'For MISP type local feeds, please specify the containing directory.'; + } + } else { + if (!file_exists($path)) { + return 'Invalid path or file not found. Make sure that the path points to an existing file that is readable and watch out for typos.'; + } + } + } else { + if (!filter_var($context['data']['url'], FILTER_VALIDATE_URL)) { + return false; + } + } + return true; + }, + 'message' => 'Invalid URL/File Path.' + ]) + ->add('input_source', 'custom', [ + 'rule' => function ($value, $context) { + if (!empty($value['Feed']['input_source'])) { + $localAllowed = empty(Configure::read('Security.disable_local_feed_access')); + $validOptions = array('network'); + if ($localAllowed) { + $validOptions[] = 'local'; + } + if (!in_array($value['Feed']['input_source'], $validOptions)) { + return __( + 'Invalid input source. The only valid options are %s. %s', + implode(', ', $validOptions), + (!$localAllowed && $value['Feed']['input_source'] === 'local') ? + __('Security.disable_local_feed_access is currently enabled, local feeds are thereby not allowed.') : + '' + ); + } + } + return true; + }, + 'message' => 'Invalid input source' + ]) + ->add('event_id', 'valid', [ + 'rule' => 'numeric', + 'message' => 'Please enter a numeric event ID or leave this field blank.' + ]); + + return $validator; + } + + public function validateInputSource($fields) + { + if (!empty($this->data['Feed']['input_source'])) { + $localAllowed = empty(Configure::read('Security.disable_local_feed_access')); + $validOptions = array('network'); + if ($localAllowed) { + $validOptions[] = 'local'; + } + if (!in_array($this->data['Feed']['input_source'], $validOptions)) { + return __( + 'Invalid input source. The only valid options are %s. %s', + implode(', ', $validOptions), + (!$localAllowed && $this->data['Feed']['input_source'] === 'local') ? + __('Security.disable_local_feed_access is currently enabled, local feeds are thereby not allowed.') : + '' + ); + } + } + return true; + } + + public function urlOrExistingFilepath($value, $context) + { + if ($this->isFeedLocal($value)) { + $path = mb_ereg_replace("/\:\/\//", '', $value['Feed']['url']); + if ($value['Feed']['source_format'] == 'misp') { + if (!is_dir($path)) { + return 'For MISP type local feeds, please specify the containing directory.'; + } + } else { + if (!file_exists($path)) { + return 'Invalid path or file not found. Make sure that the path points to an existing file that is readable and watch out for typos.'; + } + } + } else { + if (!filter_var($value['Feed']['url'], FILTER_VALIDATE_URL)) { + return false; + } + } + return true; + } + + public function getFeedTypesOptions() + { + $result = array(); + foreach (Feed::FEED_TYPES as $key => $value) { + $result[$key] = $value['name']; + } + return $result; + } + + /** + * Gets the event UUIDs from the feed by ID + * Returns an array with the UUIDs of events that are new or that need updating. + * + * @param array $feed + * @param HttpClient|null $HttpSocket + * @return array + * @throws Exception + */ + public function getNewEventUuids($feed, HttpClient $HttpSocket = null) + { + $manifest = $this->isFeedLocal($feed) ? $this->downloadManifest($feed) : $this->getRemoteManifest($feed, $HttpSocket); + $EventsTable = $this->fetchTable('Events'); + $events = $EventsTable->find('all', array( + 'conditions' => array( + 'Event.uuid' => array_keys($manifest), + ), + 'recursive' => -1, + 'fields' => array('Event.uuid', 'Event.timestamp') + )); + $result = array('add' => array(), 'edit' => array()); + foreach ($events as $event) { + $eventUuid = $event['Event']['uuid']; + if ($event['Event']['timestamp'] < $manifest[$eventUuid]['timestamp']) { + $result['edit'][] = $eventUuid; + } else { + $this->__cleanupFile($feed, '/' . $eventUuid . '.json'); + } + unset($manifest[$eventUuid]); + } + // Rest events in manifest does't exists, they will be added + $result['add'] = array_keys($manifest); + return $result; + } + + /** + * @param array $feed + * @param HttpClient|null $HttpSocket Null can be for local feed + * @return Generator + * @throws Exception + */ + public function getCache(array $feed, HttpClient $HttpSocket = null) + { + $uri = $feed['Feed']['url'] . '/hashes.csv'; + $data = $this->feedGetUri($feed, $uri, $HttpSocket); + + if (empty($data)) { + throw new Exception("File '$uri' with hashes for cache filling is empty."); + } + + // CSV file can be pretty big to do operations in memory, so we save content to temp and iterate line by line. + $tmpFile = new TmpFileTool(); + $tmpFile->write(trim($data)); + unset($data); + + return $tmpFile->intoParsedCsv(); + } + + /** + * @param array $feed + * @param HttpClient|null $HttpSocket Null can be for local feed + * @return array + * @throws Exception + */ + private function downloadManifest($feed, HttpClient $HttpSocket = null) + { + $manifestUrl = $feed['Feed']['url'] . '/manifest.json'; + $data = $this->feedGetUri($feed, $manifestUrl, $HttpSocket); + + try { + return JsonTool::decodeArray($data); + } catch (Exception $e) { + throw new Exception("Could not parse '$manifestUrl' manifest JSON", 0, $e); + } + } + + /** + * @param int $feedId + */ + private function cleanFileCache($feedId) + { + $cacheFiles = [ + "misp_feed_{$feedId}_manifest.cache.gz", + "misp_feed_{$feedId}_manifest.cache", + "misp_feed_{$feedId}_manifest.etag", + "misp_feed_$feedId.cache.gz", + "misp_feed_$feedId.cache", // old file name + "misp_feed_$feedId.etag", + ]; + foreach ($cacheFiles as $fileName) { + FileAccessTool::deleteFileIfExists(Feed::CACHE_DIR . $fileName); + } + } + + /** + * Get remote manifest for feed with etag checking. + * @param array $feed + * @param HttpClient $HttpSocket + * @return array + * @throws HttpException + * @throws JsonException + */ + private function getRemoteManifest(array $feed, HttpClient $HttpSocket) + { + $feedCache = Feed::CACHE_DIR . 'misp_feed_' . (int)$feed['Feed']['id'] . '_manifest.cache.gz'; + $feedCacheEtag = Feed::CACHE_DIR . 'misp_feed_' . (int)$feed['Feed']['id'] . '_manifest.etag'; + + $etag = null; + if (file_exists($feedCache) && file_exists($feedCacheEtag)) { + $etag = file_get_contents($feedCacheEtag); + } + + $manifestUrl = $feed['Feed']['url'] . '/manifest.json'; + + try { + $response = $this->feedGetUriRemote($feed, $manifestUrl, $HttpSocket, $etag); + } catch (HttpException $e) { + if ($e->getCode() === 304) { // not modified + try { + return JsonTool::decodeArray(FileAccessTool::readCompressedFile($feedCache)); + } catch (Exception $e) { + return $this->feedGetUriRemote($feed, $manifestUrl, $HttpSocket)->json(); // cache file is not readable, fetch without etag + } + } else { + throw $e; + } + } + + if ($response->getHeader('etag')) { + try { + FileAccessTool::writeCompressedFile($feedCache, $response->body); + FileAccessTool::writeToFile($feedCacheEtag, $response->getHeader('etag')); + } catch (Exception $e) { + FileAccessTool::deleteFileIfExists($feedCacheEtag); + $this->logException("Could not save file `$feedCache` to cache.", $e, LOG_NOTICE); + } + } else { + FileAccessTool::deleteFileIfExists($feedCacheEtag); + } + + return $response->json(); + } + + /** + * @param array $feed + * @param HttpClient|null $HttpSocket Null can be for local feed + * @return array + * @throws Exception + */ + public function getManifest(array $feed, HttpClient $HttpSocket = null) + { + $events = $this->isFeedLocal($feed) ? $this->downloadManifest($feed) : $this->getRemoteManifest($feed, $HttpSocket); + $events = $this->__filterEventsIndex($events, $feed); + return $events; + } + + /** + * Load remote file with cache support and etag checking. + * @param array $feed + * @param HttpClient $HttpSocket + * @return string + * @throws HttpException + */ + private function getFreetextFeedRemote(array $feed, HttpClient $HttpSocket) + { + $feedCache = Feed::CACHE_DIR . 'misp_feed_' . (int)$feed['Feed']['id'] . '.cache.gz'; + $feedCacheEtag = Feed::CACHE_DIR . 'misp_feed_' . (int)$feed['Feed']['id'] . '.etag'; + + $etag = null; + if (file_exists($feedCache)) { + if (time() - filemtime($feedCache) < 600) { + try { + return FileAccessTool::readCompressedFile($feedCache); + } catch (Exception $e) { + // ignore + } + } else if (file_exists($feedCacheEtag)) { + $etag = file_get_contents($feedCacheEtag); + } + } + + try { + $response = $this->feedGetUriRemote($feed, $feed['Feed']['url'], $HttpSocket, $etag); + } catch (HttpException $e) { + if ($e->getCode() === 304) { // not modified + try { + return FileAccessTool::readCompressedFile($feedCache); + } catch (Exception $e) { + return $this->feedGetUriRemote($feed, $feed['Feed']['url'], $HttpSocket); // cache file is not readable, fetch without etag + } + } else { + throw $e; + } + } + + try { + FileAccessTool::writeCompressedFile($feedCache, $response->body); + if ($response->getHeader('etag')) { + FileAccessTool::writeToFile($feedCacheEtag, $response->getHeader('etag')); + } + } catch (Exception $e) { + FileAccessTool::deleteFileIfExists($feedCacheEtag); + $this->logException("Could not save file `$feedCache` to cache.", $e, LOG_NOTICE); + } + + return $response->body; + } + + /** + * @param array $feed + * @param HttpClient|null $HttpSocket Null can be for local feed + * @param string $type + * @return array|bool + * @throws Exception + */ + public function getFreetextFeed($feed, HttpClient $HttpSocket = null, $type = 'freetext') + { + if ($this->isFeedLocal($feed)) { + $feedUrl = $feed['Feed']['url']; + $data = $this->feedGetUri($feed, $feedUrl, $HttpSocket); + } else { + $data = $this->getFreetextFeedRemote($feed, $HttpSocket); + } + + $complexTypeTool = new ComplexTypeTool(); + $WarninglistsTable = $this->fetchTable('Warninglists'); + $complexTypeTool->setTLDs($WarninglistsTable->fetchTLDLists()); + $complexTypeTool->setSecurityVendorDomains($WarninglistsTable->fetchSecurityVendorDomains()); + $settings = array(); + if (!empty($feed['Feed']['settings']) && !is_array($feed['Feed']['settings'])) { + $feed['Feed']['settings'] = json_decode($feed['Feed']['settings'], true); + } + if (isset($feed['Feed']['settings'][$type])) { + $settings = $feed['Feed']['settings'][$type]; + } + if (isset($feed['Feed']['settings']['common'])) { + $settings = array_merge($settings, $feed['Feed']['settings']['common']); + } + $resultArray = $complexTypeTool->checkComplexRouter($data, $type, $settings); + $AttributesTable = $this->fetchTable('Attributes'); + $typeDefinitions = $AttributesTable->typeDefinitions; + foreach ($resultArray as &$value) { + $definition = $typeDefinitions[$value['default_type']]; + $value['category'] = $definition['default_category']; + $value['to_ids'] = $definition['to_ids']; + } + return $resultArray; + } + + public function getFreetextFeedCorrelations($data, $feedId) + { + $values = array(); + foreach ($data as $key => $value) { + $values[] = $value['value']; + } + $AttributesTable = $this->fetchTable('Attributes'); + $redis = RedisTool::init(); + if ($redis !== false) { + $feeds = $this->find('all', array( + 'recursive' => -1, + 'conditions' => array('Feed.id !=' => $feedId), + 'fields' => array('id', 'name', 'url', 'provider', 'source_format') + )); + foreach ($feeds as $k => $v) { + if (!$redis->exists('misp:feed_cache:' . $v['Feed']['id'])) { + unset($feeds[$k]); + } + } + } else { + return array(); + } + // Adding a 3rd parameter to a list find seems to allow grouping several results into a key. If we ran a normal list with value => event_id we'd only get exactly one entry for each value + // The cost of this method is orders of magnitude lower than getting all id - event_id - value triplets and then doing a double loop comparison + $correlations = $AttributesTable->find('list', array('conditions' => array('Attribute.value1' => $values, 'Attribute.deleted' => 0), 'fields' => array('Attribute.event_id', 'Attribute.event_id', 'Attribute.value1'))); + $correlations2 = $AttributesTable->find('list', array('conditions' => array('Attribute.value2' => $values, 'Attribute.deleted' => 0), 'fields' => array('Attribute.event_id', 'Attribute.event_id', 'Attribute.value2'))); + $correlations = array_merge_recursive($correlations, $correlations2); + foreach ($data as $key => $value) { + if (isset($correlations[$value['value']])) { + $data[$key]['correlations'] = array_values($correlations[$value['value']]); + } + if ($redis) { + foreach ($feeds as $k => $v) { + if ($redis->sismember('misp:feed_cache:' . $v['Feed']['id'], md5($value['value']))) { + $data[$key]['feed_correlations'][] = array($v); + } + } + } + } + return $data; + } + + /** + * Attach correlations from cached servers or feeds. + * + * @param array $attributes + * @param array $user + * @param array $event + * @param bool $overrideLimit Override hardcoded limit for 10 000 correlations. + * @param string $scope `Feed` or `Server` + * @return array + */ + public function attachFeedCorrelations(array $attributes, array $user, array &$event, $overrideLimit = false, $scope = 'Feed') + { + if (!isset($user['Role']['perm_view_feed_correlations']) || $user['Role']['perm_view_feed_correlations'] != true) { + return $attributes; + } + if (empty($attributes)) { + return $attributes; + } + + try { + $redis = RedisTool::init(); + } catch (Exception $e) { + return $attributes; + } + + $cachePrefix = 'misp:' . strtolower($scope) . '_cache:'; + + // Skip if redis cache for $scope is empty. + if ($redis->sCard($cachePrefix . 'combined') === 0) { + return $attributes; + } + + $AttributesTable = $this->fetchTable('Attributes'); + $compositeTypes = $AttributesTable->getCompositeTypes(); + + $pipe = $redis->pipeline(); + $hashTable = []; + $redisResultToAttributePosition = []; + + foreach ($attributes as $k => $attribute) { + if (in_array($attribute['type'], Attribute::NON_CORRELATING_TYPES, true)) { + continue; // attribute type is not correlateable + } + if (!empty($attribute['disable_correlation'])) { + continue; // attribute correlation is disabled + } + + if (in_array($attribute['type'], $compositeTypes, true)) { + list($value1, $value2) = explode('|', $attribute['value']); + $parts = [$value1]; + + if (!in_array($attribute['type'], Attribute::PRIMARY_ONLY_CORRELATING_TYPES, true)) { + $parts[] = $value2; + } + } else { + $parts = [$attribute['value']]; + + // Some feeds contains URL without protocol, so if attribute is URL and value contains protocol, + // we will check also value without protocol. + if ($attribute['type'] === 'url' || $attribute['type'] === 'uri') { + $protocolPos = strpos($attribute['value'], '://'); + if ($protocolPos !== false) { + $parts[] = substr($attribute['value'], $protocolPos + 3); + } + } + } + + foreach ($parts as $part) { + $md5 = md5($part); + $hashTable[] = $md5; + $redis->sismember($cachePrefix . 'combined', $md5); + $redisResultToAttributePosition[] = $k; + } + } + + if (empty($redisResultToAttributePosition)) { + // No attribute that can be correlated + $pipe->discard(); + return $attributes; + } + + $results = $pipe->exec(); + + $hitIds = []; + foreach ($results as $k => $result) { + if ($result) { + $hitIds[] = $k; + } + } + + if (empty($hitIds)) { + return $attributes; // nothing matches, skip + } + + $hitCount = count($hitIds); + if (!$overrideLimit && $hitCount > 10000) { + $event['FeedCount'] = $hitCount; + foreach ($hitIds as $k) { + $attributes[$redisResultToAttributePosition[$k]]['FeedHit'] = true; + } + return $attributes; + } + + $sources = $this->getCachedFeedsOrServers($user, $scope); + if ($scope == 'Server' && !$user['Role']['perm_site_admin'] && $user['org_id'] != Configure::read('MISP.host_org_id')) { + // Filter fields that shouldn't be visible to everyone + $allowedFieldsForAllUsers = array_flip(['id', 'name',]); + $sources = array_map(function ($source) use ($scope, $allowedFieldsForAllUsers) { + return [$scope => array_intersect_key($source[$scope], $allowedFieldsForAllUsers)]; + }, $sources); + } + foreach ($sources as $source) { + $sourceId = $source[$scope]['id']; + + $pipe = $redis->pipeline(); + foreach ($hitIds as $k) { + $redis->sismember($cachePrefix . $sourceId, $hashTable[$k]); + } + $sourceHits = $pipe->exec(); + $sourceHasHit = false; + foreach ($sourceHits as $k => $hit) { + if ($hit) { + if (!isset($event[$scope][$sourceId])) { + $event[$scope][$sourceId] = $source[$scope]; + } + + $attributePosition = $redisResultToAttributePosition[$hitIds[$k]]; + $alreadyAttached = isset($attributes[$attributePosition][$scope]) && + in_array($sourceId, array_column($attributes[$attributePosition][$scope], 'id')); + if (!$alreadyAttached) { + $attributes[$attributePosition][$scope][] = $source[$scope]; + } + $sourceHasHit = true; + } + } + // Append also exact MISP feed or server event UUID + // TODO: This can be optimised in future to do that in one pass + if ($sourceHasHit && ($scope === 'Server' || $source[$scope]['source_format'] === 'misp')) { + if ( + $scope === 'Server' && + !$user['Role']['perm_site_admin'] && $user['org_id'] != Configure::read('MISP.host_org_id') + ) { + continue; // Non-privileged users cannot see the hits for server + } + $pipe = $redis->pipeline(); + $eventUuidHitPosition = []; + foreach ($hitIds as $sourceHitPos => $k) { + if ($sourceHits[$sourceHitPos]) { + $redis->smembers($cachePrefix . 'event_uuid_lookup:' . $hashTable[$k]); + $eventUuidHitPosition[] = $redisResultToAttributePosition[$k]; + } + } + $mispFeedHits = $pipe->exec(); + foreach ($mispFeedHits as $sourceHitPos => $feedUuidMatches) { + if (empty($feedUuidMatches)) { + continue; + } + foreach ($feedUuidMatches as $url) { + list($feedId, $eventUuid) = explode('/', $url); + if ($feedId != $sourceId) { + continue; // just process current source, skip others + } + + if (empty($event[$scope][$feedId]['event_uuids']) || !in_array($eventUuid, $event[$scope][$feedId]['event_uuids'])) { + $event[$scope][$feedId]['event_uuids'][] = $eventUuid; + } + $attributePosition = $eventUuidHitPosition[$sourceHitPos]; + foreach ($attributes[$attributePosition][$scope] as $tempKey => $tempFeed) { + if ($tempFeed['id'] == $feedId) { + if (empty($attributes[$attributePosition][$scope][$tempKey]['event_uuids']) || !in_array($eventUuid, $attributes[$attributePosition][$scope][$tempKey]['event_uuids'])) { + $attributes[$attributePosition][$scope][$tempKey]['event_uuids'][] = $eventUuid; + } + break; + } + } + } + } + } + } + + if (isset($event[$scope])) { + $event[$scope] = array_values($event[$scope]); + } + + return $attributes; + } + + /** + * Return just feeds or servers that has some data in Redis cache. + * @param array $user + * @param string $scope 'Feed' or 'Server' + * @return array + */ + private function getCachedFeedsOrServers(array $user, $scope) + { + if ($scope === 'Feed') { + $params = array( + 'recursive' => -1, + 'fields' => array('id', 'name', 'url', 'provider', 'source_format') + ); + if (!$user['Role']['perm_site_admin']) { + $params['conditions'] = array('Feed.lookup_visible' => 1); + } + $sources = $this->find('all', $params); + } else { + $params = array( + 'recursive' => -1, + 'fields' => array('id', 'name', 'url') + ); + if (!$user['Role']['perm_site_admin']) { + $params['conditions'] = array('Server.caching_enabled' => 1); + } + $ServersTable = $this->fetchTable('Servers'); + $sources = $ServersTable->find('all', $params); + } + + try { + $redis = RedisTool::init(); + $pipe = $redis->pipeline(); + $cachePrefix = 'misp:' . strtolower($scope) . '_cache:'; + foreach ($sources as $source) { + $pipe->exists($cachePrefix . $source[$scope]['id']); + } + $results = $pipe->exec(); + foreach ($sources as $k => $source) { + if (!$results[$k]) { + unset($sources[$k]); + } + } + } catch (Exception $e) { + } + + return $sources; + } + + /** + * @param array $actions + * @param array $feed + * @param HttpClient|null $HttpSocket + * @param array $user + * @param int|false $jobId + * @return array + * @throws Exception + */ + private function downloadFromFeed(array $actions, array $feed, HttpClient $HttpSocket = null, array $user, $jobId = false) + { + $total = count($actions['add']) + count($actions['edit']); + $currentItem = 0; + $results = array(); + $filterRules = $this->__prepareFilterRules($feed); + + foreach ($actions['add'] as $uuid) { + try { + $result = $this->__addEventFromFeed($HttpSocket, $feed, $uuid, $user, $filterRules); + if ($result === true) { + $results['add']['success'] = $uuid; + } else if ($result !== 'blocked') { + $results['add']['fail'] = ['uuid' => $uuid, 'reason' => $result]; + $this->log("Could not add event '$uuid' from feed {$feed['Feed']['id']}: $result", LOG_WARNING); + } + } catch (Exception $e) { + $this->logException("Could not add event '$uuid' from feed {$feed['Feed']['id']}.", $e); + $results['add']['fail'] = array('uuid' => $uuid, 'reason' => $e->getMessage()); + } + + $this->__cleanupFile($feed, '/' . $uuid . '.json'); + $this->jobProgress($jobId, null, 100 * (($currentItem + 1) / $total)); + $currentItem++; + } + + foreach ($actions['edit'] as $uuid) { + try { + $result = $this->__updateEventFromFeed($HttpSocket, $feed, $uuid, $user, $filterRules); + if ($result === true) { + $results['add']['success'] = $uuid; + } else if ($result !== 'blocked') { + $results['add']['fail'] = ['uuid' => $uuid, 'reason' => $result]; + $this->log("Could not edit event '$uuid' from feed {$feed['Feed']['id']}: " . json_encode($result), LOG_WARNING); + } + } catch (Exception $e) { + $this->logException("Could not edit event '$uuid' from feed {$feed['Feed']['id']}.", $e); + $results['edit']['fail'] = array('uuid' => $uuid, 'reason' => $e->getMessage()); + } + + $this->__cleanupFile($feed, '/' . $uuid . '.json'); + if ($currentItem % 10 === 0) { + $this->jobProgress($jobId, null, 100 * (($currentItem + 1) / $total)); + } + $currentItem++; + } + + return $results; + } + + private function __createFeedRequest($headers = false) + { + $version = $this->checkMISPVersion(); + $version = implode('.', $version); + + $result = array( + 'header' => array( + 'Accept' => array('application/json', 'text/plain', 'text/*'), + 'MISP-version' => $version, + 'MISP-uuid' => Configure::read('MISP.uuid'), + ) + ); + + $commit = $this->checkMIPSCommit(); + if ($commit) { + $result['header']['commit'] = $commit; + } + if (!empty($headers)) { + $lines = explode("\n", $headers); + foreach ($lines as $line) { + if (!empty($line)) { + $kv = explode(':', $line); + if (!empty($kv[0]) && !empty($kv[1])) { + if (!in_array($kv[0], array('commit', 'MISP-version', 'MISP-uuid'))) { + $result['header'][trim($kv[0])] = trim($kv[1]); + } + } + } + } + } + return $result; + } + + private function __checkIfEventBlockedByFilter($event, $filterRules) + { + $fields = array('tags' => 'Tag', 'orgs' => 'Orgc'); + $prefixes = array('OR', 'NOT'); + foreach ($fields as $field => $fieldModel) { + foreach ($prefixes as $prefix) { + if (!empty($filterRules[$field][$prefix])) { + $found = false; + if (isset($event['Event'][$fieldModel]) && !empty($event['Event'][$fieldModel])) { + if (!isset($event['Event'][$fieldModel][0])) { + $event['Event'][$fieldModel] = array(0 => $event['Event'][$fieldModel]); + } + foreach ($event['Event'][$fieldModel] as $object) { + foreach ($filterRules[$field][$prefix] as $temp) { + if (stripos($object['name'], $temp) !== false) { + $found = true; + break 2; + } + } + } + } + if ($prefix === 'OR' && !$found) { + return false; + } + if ($prefix !== 'OR' && $found) { + return false; + } + } + } + } + $url_params = !empty($filterRules['url_params']) ? $filterRules['url_params'] : []; + if (!$this->passesURLParamFilters($url_params, $event['Event'])) { + return false; + } + return true; + } + + private function __filterEventsIndex(array $events, array $feed) + { + $filterRules = $this->__prepareFilterRules($feed); + if (!$filterRules) { + $filterRules = array(); + } + foreach ($events as $k => $event) { + if (isset($event['orgc']) && !isset($event['Orgc'])) { // fix key case + $event['Orgc'] = $event['orgc']; + unset($event['orgc']); + $events[$k] = $event; + } + + if (isset($filterRules['orgs']['OR']) && !empty($filterRules['orgs']['OR']) && !in_array($event['Orgc']['name'], $filterRules['orgs']['OR'])) { + unset($events[$k]); + continue; + } + if (isset($filterRules['orgs']['NO']) && !empty($filterRules['orgs']['NOT']) && in_array($event['Orgc']['name'], $filterRules['orgs']['OR'])) { + unset($events[$k]); + continue; + } + if (isset($filterRules['tags']['OR']) && !empty($filterRules['tags']['OR'])) { + if (!isset($event['Tag']) || empty($event['Tag'])) { + unset($events[$k]); + } + $found = false; + foreach ($event['Tag'] as $tag) { + foreach ($filterRules['tags']['OR'] as $filterTag) { + if (strpos(strtolower($tag['name']), strtolower($filterTag))) { + $found = true; + } + } + } + if (!$found) { + unset($events[$k]); + continue; + } + } + if (isset($filterRules['tags']['NOT']) && !empty($filterRules['tags']['NOT'])) { + if (isset($event['Tag']) && !empty($event['Tag'])) { + $found = false; + foreach ($event['Tag'] as $tag) { + foreach ($filterRules['tags']['NOT'] as $filterTag) { + if (strpos(strtolower($tag['name']), strtolower($filterTag))) { + $found = true; + } + } + } + if ($found) { + unset($events[$k]); + } + } + } + $url_params = !empty($filterRules['url_params']) ? $filterRules['url_params'] : []; + if (!$this->passesURLParamFilters($url_params, $event)) { + unset($events[$k]); + } + } + return $events; + } + + private function passesURLParamFilters($url_params, $event): bool + { + $AttributesTable = $this->fetchTable('Attributes'); + if (!empty($url_params['timestamp'])) { + $timestamps = $AttributesTable->setTimestampConditions($url_params['timestamp'], [], '', true); + if (is_array($timestamps)) { + if ($event['timestamp'] < $timestamps[0] || $event['timestamp'] > $timestamps[1]) { + return false; + } + } else { + if ($event['timestamp'] < $timestamps) { + return false; + } + } + } + if (!empty($url_params['publish_timestamp'])) { + $timestamps = $AttributesTable->setTimestampConditions($url_params['publish_timestamp'], [], '', true); + if (is_array($timestamps)) { + if ($event['timestamp'] < $timestamps[0] || $event['timestamp'] > $timestamps[1]) { + return false; + } + } else { + if ($event['timestamp'] < $timestamps) { + return false; + } + } + } + return true; + } + + /** + * @param array $feed + * @param string $uuid + * @param array $user + * @return array|bool + * @throws Exception + */ + public function downloadAndSaveEventFromFeed(array $feed, $uuid, array $user) + { + $event = $this->downloadEventFromFeed($feed, $uuid); + if (!is_array($event) || isset($event['code'])) { + return false; + } + return $this->__saveEvent($event, $user); + } + + /** + * @param array $feed + * @param string $uuid + * @return bool|string|array + * @throws Exception + */ + public function downloadEventFromFeed(array $feed, $uuid) + { + $filerRules = $this->__prepareFilterRules($feed); + $HttpSocket = $this->isFeedLocal($feed) ? null : $this->__setupHttpClient(); + $event = $this->downloadAndParseEventFromFeed($feed, $uuid, $HttpSocket); + return $this->__prepareEvent($event, $feed, $filerRules); + } + + /** + * @param array $event + * @param array $user + * @return array + */ + private function __saveEvent(array $event, array $user) + { + $EventsTable = $this->fetchTable('Events'); + $existingEvent = $EventsTable->find('first', array( + 'conditions' => array('Event.uuid' => $event['Event']['uuid']), + 'recursive' => -1, + 'fields' => array('Event.uuid', 'Event.id', 'Event.timestamp') + )); + $result = array(); + if (!empty($existingEvent)) { + $result['action'] = 'edit'; + if ($existingEvent['Event']['timestamp'] < $event['Event']['timestamp']) { + $result['result'] = $EventsTable->_edit($event, $user); + } else { + $result['result'] = 'No change'; + } + } else { + $result['action'] = 'add'; + $result['result'] = $EventsTable->_add($event, true, $user); + } + return $result; + } + + /** + * @param array $event + * @param array $feed + * @param array $filterRules + * @return array|string + */ + private function __prepareEvent($event, array $feed, $filterRules) + { + if (isset($event['response'])) { + $event = $event['response']; + } + if (isset($event[0])) { + $event = $event[0]; + } + if (!isset($event['Event']['uuid'])) { + throw new InvalidArgumentException("Event UUID field missing."); + } + if (isset($event['Event']['orgc']) && !isset($event['Event']['Orgc'])) { // fix key case + $event['Event']['Orgc'] = $event['Event']['orgc']; + unset($event['Event']['orgc']); + } + if (5 == $feed['Feed']['distribution'] && isset($event['Event']['distribution'])) { + // We inherit the distribution from the feed and should not rewrite the distributions. + // MISP magically parses the Sharing Group info and creates the SG if it didn't exist. + } else { + // rewrite the distributions to the one configured by the Feed settings + // overwrite Event distribution + if (5 == $feed['Feed']['distribution']) { + // We said to inherit the distribution from the feed, but the feed does not contain distribution + // rewrite the event to My org only distribution, and set all attributes to inherit the event distribution + $event['Event']['distribution'] = 0; + $event['Event']['sharing_group_id'] = 0; + } else { + $event['Event']['distribution'] = $feed['Feed']['distribution']; + $event['Event']['sharing_group_id'] = $feed['Feed']['sharing_group_id']; + if ($feed['Feed']['sharing_group_id']) { + $sg = $this->SharingGroup->find('first', array( + 'recursive' => -1, + 'conditions' => array('SharingGroup.id' => $feed['Feed']['sharing_group_id']) + )); + if (!empty($sg)) { + $event['Event']['SharingGroup'] = $sg['SharingGroup']; + } else { + // We have an SG ID for the feed, but the SG is gone. Make the event private as a fall-back. + $event['Event']['distribution'] = 0; + $event['Event']['sharing_group_id'] = 0; + } + } + } + // overwrite Attributes and Objects distribution to Inherit + if (!empty($event['Event']['Attribute'])) { + foreach ($event['Event']['Attribute'] as $key => $attribute) { + $event['Event']['Attribute'][$key]['distribution'] = 5; + } + } + if (!empty($event['Event']['Object'])) { + foreach ($event['Event']['Object'] as $key => $object) { + $event['Event']['Object'][$key]['distribution'] = 5; + if (!empty($event['Event']['Object'][$key]['Attribute'])) { + foreach ($event['Event']['Object'][$key]['Attribute'] as $a_key => $attribute) { + $event['Event']['Object'][$key]['Attribute'][$a_key]['distribution'] = 5; + } + } + } + } + } + if ($feed['Feed']['tag_id']) { + if (empty($feed['Tag']['name'])) { + $feed_tag = $this->Tag->find('first', [ + 'conditions' => [ + 'Tag.id' => $feed['Feed']['tag_id'] + ], + 'recursive' => -1, + 'fields' => ['Tag.name', 'Tag.colour', 'Tag.id'] + ]); + $feed['Tag'] = $feed_tag['Tag']; + } + if (!isset($event['Event']['Tag'])) { + $event['Event']['Tag'] = array(); + } + + $feedTag = $this->Tag->find('first', array('conditions' => array('Tag.id' => $feed['Feed']['tag_id']), 'recursive' => -1, 'fields' => array('Tag.name', 'Tag.colour', 'Tag.exportable'))); + if (!empty($feedTag)) { + $found = false; + foreach ($event['Event']['Tag'] as $tag) { + if (strtolower($tag['name']) === strtolower($feedTag['Tag']['name'])) { + $found = true; + break; + } + } + if (!$found) { + $event['Event']['Tag'][] = $feedTag['Tag']; + } + } + } + if (!$this->__checkIfEventBlockedByFilter($event, $filterRules)) { + return 'blocked'; + } + if (!empty($feed['Feed']['settings'])) { + if (!empty($feed['Feed']['settings']['disable_correlation'])) { + $event['Event']['disable_correlation'] = (bool) $feed['Feed']['settings']['disable_correlation']; + } + } + return $event; + } + + /** + * @param array $feed + * @return bool|mixed + * @throws Exception + */ + private function __prepareFilterRules($feed) + { + $filterRules = false; + if (isset($feed['Feed']['rules']) && !empty($feed['Feed']['rules'])) { + $filterRules = json_decode($feed['Feed']['rules'], true); + if ($filterRules === null) { + throw new Exception('Could not parse feed filter rules JSON: ' . json_last_error_msg(), json_last_error()); + } + $filterRules['url_params'] = !empty($filterRules['url_params']) ? $this->jsonDecode($filterRules['url_params']) : []; + } + return $filterRules; + } + + private function __setupHttpSocket() + { + $syncTool = new SyncTool(); + return $syncTool->setupHttpSocketFeed(); + } + + /** + * @param HttpClient|null $HttpSocket + * @param array $feed + * @param string $uuid + * @param array $user + * @param array|bool $filterRules + * @return array|bool|string + * @throws Exception + */ + private function __addEventFromFeed(HttpClient $HttpSocket = null, $feed, $uuid, $user, $filterRules) + { + $event = $this->downloadAndParseEventFromFeed($feed, $uuid, $HttpSocket); + $event = $this->__prepareEvent($event, $feed, $filterRules); + if (is_array($event)) { + return $this->Event->_add($event, true, $user); + } else { + return $event; + } + } + + /** + * @param HttpClient|null $HttpSocket Null can be for local feed + * @param array $feed + * @param string $uuid + * @param $user + * @param array|bool $filterRules + * @return mixed + * @throws Exception + */ + private function __updateEventFromFeed(HttpClient $HttpSocket = null, $feed, $uuid, $user, $filterRules) + { + $event = $this->downloadAndParseEventFromFeed($feed, $uuid, $HttpSocket); + $event = $this->__prepareEvent($event, $feed, $filterRules); + return $this->Event->_edit($event, $user, $uuid, $jobId = null); + } + + public function addDefaultFeeds($newFeeds) + { + foreach ($newFeeds as $newFeed) { + $existingFeed = $this->find('list', array('conditions' => array('Feed.url' => $newFeed['url']))); + $success = true; + if (empty($existingFeed)) { + $this->create(); + $feed = $this->newEntity( + array( + 'name' => $newFeed['name'], + 'provider' => $newFeed['provider'], + 'url' => $newFeed['url'], + 'enabled' => $newFeed['enabled'], + 'caching_enabled' => !empty($newFeed['caching_enabled']) ? $newFeed['caching_enabled'] : 0, + 'distribution' => 3, + 'sharing_group_id' => 0, + 'tag_id' => 0, + 'default' => true, + ) + ); + $result = $this->save($feed) && $success; + } + } + return $success; + } + + /** + * @param int $feedId + * @param array $user + * @param int|false $jobId + * @return array|bool + * @throws Exception + */ + public function downloadFromFeedInitiator($feedId, $user, $jobId = false) + { + $feed = $this->find('all', array( + 'conditions' => ['Feed.id' => $feedId], + 'recursive' => -1, + ))->first(); + if (empty($feed)) { + throw new Exception("Feed with ID $feedId not found."); + } + + if (!empty($feed['Feed']['settings'])) { + $feed['Feed']['settings'] = json_decode($feed['Feed']['settings'], true); + } + + $HttpSocket = $this->isFeedLocal($feed) ? null : $this->__setupHttpClient(); + if ($feed['Feed']['source_format'] === 'misp') { + $this->jobProgress($jobId, 'Fetching event manifest.'); + try { + $actions = $this->getNewEventUuids($feed, $HttpSocket); + } catch (Exception $e) { + $this->logException("Could not get new event uuids for feed $feedId.", $e); + $this->jobProgress($jobId, 'Could not fetch event manifest. See error log for more details.'); + return false; + } + + if (empty($actions['add']) && empty($actions['edit'])) { + return true; + } + + $total = count($actions['add']) + count($actions['edit']); + $this->jobProgress($jobId, __("Fetching %s events.", $total)); + $result = $this->downloadFromFeed($actions, $feed, $HttpSocket, $user, $jobId); + $this->__cleanupFile($feed, '/manifest.json'); + } else { + $this->jobProgress($jobId, 'Fetching data.'); + try { + $temp = $this->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format']); + } catch (Exception $e) { + $this->logException("Could not get freetext feed $feedId", $e); + $this->jobProgress($jobId, 'Could not fetch freetext feed. See error log for more details.'); + return false; + } + + if (empty($temp)) { + return true; + } + + $data = array(); + foreach ($temp as $value) { + $data[] = array( + 'category' => $value['category'], + 'type' => $value['default_type'], + 'value' => $value['value'], + 'to_ids' => $value['to_ids'] + ); + } + unset($temp); + + $this->jobProgress($jobId, 'Saving data.', 50); + + try { + $result = $this->saveFreetextFeedData($feed, $data, $user, $jobId); + } catch (Exception $e) { + $this->logException("Could not save freetext feed data for feed $feedId.", $e); + return false; + } + + $this->__cleanupFile($feed, ''); + } + return $result; + } + + private function __cleanupFile($feed, $file) + { + if ($this->isFeedLocal($feed)) { + if (isset($feed['Feed']['delete_local_file']) && $feed['Feed']['delete_local_file']) { + FileAccessTool::deleteFileIfExists($feed['Feed']['url'] . $file); + } + } + return true; + } + + /** + * @param array $feed + * @param array $data + * @param array $user + * @param int|bool $jobId + * @return bool + * @throws Exception + */ + public function saveFreetextFeedData(array $feed, array $data, array $user, $jobId = false) + { + $EventsTable = $this->fetchTable('Events'); + + if ($feed['Feed']['fixed_event'] && $feed['Feed']['event_id']) { + $event = $this->Event->find('first', array('conditions' => array('Event.id' => $feed['Feed']['event_id']), 'recursive' => -1)); + if (empty($event)) { + throw new Exception("The target event is no longer valid. Make sure that the target event {$feed['Feed']['event_id']} exists."); + } + } else { + $this->Event->create(); + $orgc_id = $user['org_id']; + if (!empty($feed['Feed']['orgc_id'])) { + $orgc_id = $feed['Feed']['orgc_id']; + } + $disableCorrelation = false; + if (!empty($feed['Feed']['settings'])) { + $disableCorrelation = (bool) $feed['Feed']['settings']['disable_correlation'] ?? false; + } + $event = array( + 'info' => $feed['Feed']['name'] . ' feed', + 'analysis' => 2, + 'threat_level_id' => 4, + 'orgc_id' => $orgc_id, + 'org_id' => $user['org_id'], + 'date' => date('Y-m-d'), + 'distribution' => $feed['Feed']['distribution'], + 'sharing_group_id' => $feed['Feed']['sharing_group_id'], + 'user_id' => $user['id'], + 'disable_correlation' => $disableCorrelation, + ); + $result = $this->Event->save($event); + if (!$result) { + throw new Exception('Something went wrong while creating a new event.'); + } + $event = $this->Event->find('first', array('conditions' => array('Event.id' => $this->Event->id), 'recursive' => -1)); + if (empty($event)) { + throw new Exception("The newly created event is no longer valid. Make sure that the target event {$this->Event->id} exists."); + } + if ($feed['Feed']['fixed_event']) { + $feed['Feed']['event_id'] = $event['Event']['id']; + if (!empty($feed['Feed']['settings'])) { + $feed['Feed']['settings'] = json_encode($feed['Feed']['settings']); + } + $feedEntity = $this->newEntity($feed['Feed']); + $this->save($feedEntity); + } + } + if ($feed['Feed']['fixed_event']) { + $existsAttributesValueToId = $this->Event->Attribute->find('list', array( + 'conditions' => array( + 'Attribute.deleted' => 0, + 'Attribute.event_id' => $event['Event']['id'] + ), + 'recursive' => -1, + 'fields' => array('value', 'id') + )); + + // Create event diff. After this cycle, `$data` will contains just attributes that do not exists in current + // event and in `$existsAttributesValueToId` will contains just attributes that do not exists in current feed. + foreach ($data as $k => $dataPoint) { + if (isset($existsAttributesValueToId[$dataPoint['value']])) { + unset($data[$k]); + unset($existsAttributesValueToId[$dataPoint['value']]); + continue; + } + + // Because some types can be saved in modified version (for example, IPv6 address is convert to compressed + // format, we should also check if current event contains modified value. + $modifiedValue = AttributeValidationTool::modifyBeforeValidation($dataPoint['type'], $dataPoint['value']); + if (isset($existsAttributesValueToId[$modifiedValue])) { + unset($data[$k]); + unset($existsAttributesValueToId[$modifiedValue]); + } + } + if ($feed['Feed']['delta_merge'] && !empty($existsAttributesValueToId)) { + $attributesToDelete = $this->Event->Attribute->find('all', array( + 'conditions' => array( + 'Attribute.id' => array_values($existsAttributesValueToId) + ), + 'recursive' => -1 + )); + foreach ($attributesToDelete as $k => $attribute) { + $attributesToDelete[$k]['Attribute']['deleted'] = 1; + unset($attributesToDelete[$k]['Attribute']['timestamp']); + } + $this->Event->Attribute->saveMany($attributesToDelete); // We need to trigger callback methods + if (!empty($attributesToDelete)) { + $this->Event->unpublishEvent($feed['Feed']['event_id']); + } + } + } + if (empty($data) && empty($attributesToDelete)) { + return true; + } + + $uniqueValues = array(); + foreach ($data as $key => $value) { + if (isset($uniqueValues[$value['value']])) { + unset($data[$key]); + continue; + } + $data[$key]['event_id'] = $event['Event']['id']; + $data[$key]['distribution'] = 5; + $data[$key]['sharing_group_id'] = 0; + $data[$key]['to_ids'] = $feed['Feed']['override_ids'] ? 0 : $value['to_ids']; + $uniqueValues[$value['value']] = true; + } + $chunks = array_chunk($data, 100); + foreach ($chunks as $k => $chunk) { + $this->Event->Attribute->saveMany($chunk, ['validate' => true, 'parentEvent' => $event]); + $this->jobProgress($jobId, null, 50 + round(($k * 100 + 1) / count($data) * 50)); + } + if (!empty($data) || !empty($attributesToDelete)) { + unset($event['Event']['timestamp']); + unset($event['Event']['attribute_count']); + $this->Event->save($event); + } + if ($feed['Feed']['publish']) { + $this->Event->publishRouter($event['Event']['id'], null, $user); + } + if ($feed['Feed']['tag_id']) { + $this->Event->EventTag->attachTagToEvent($event['Event']['id'], ['id' => $feed['Feed']['tag_id']]); + } + return true; + } + + /** + * @param $user - Not used + * @param int|bool $jobId + * @param string $scope + * @return array + * @throws Exception + */ + public function cacheFeedInitiator($user, $jobId = false, $scope = 'freetext') + { + $params = array( + 'conditions' => array('caching_enabled' => 1), + 'recursive' => -1, + 'fields' => array('source_format', 'input_source', 'url', 'id', 'settings', 'headers') + ); + $redis = RedisTool::init(); + if ($scope !== 'all') { + if (is_numeric($scope)) { + $params['conditions']['id'] = $scope; + } elseif ($scope == 'freetext' || $scope == 'csv') { + $params['conditions']['source_format'] = array('csv', 'freetext'); + } elseif ($scope == 'misp') { + $redis->del($redis->keys('misp:feed_cache:event_uuid_lookup:*')); + $params['conditions']['source_format'] = 'misp'; + } else { + throw new InvalidArgumentException("Invalid value for scope, it must be integer or 'freetext', 'csv', 'misp' or 'all' string."); + } + } else { + $redis->del('misp:feed_cache:combined'); + $redis->del($redis->keys('misp:feed_cache:event_uuid_lookup:*')); + } + $feeds = $this->find('all', $params)->toArray(); + + $results = array('successes' => 0, 'fails' => 0); + foreach ($feeds as $k => $feed) { + if ($this->__cacheFeed($feed, $redis, $jobId)) { + $message = 'Feed ' . $feed['Feed']['id'] . ' cached.'; + $results['successes']++; + } else { + $message = 'Failed to cache feed ' . $feed['Feed']['id'] . '. See logs for more details.'; + $results['fails']++; + } + + $this->jobProgress($jobId, $message, 100 * $k / count($feeds)); + } + return $results; + } + + /** + * @param Feed $feed + * @return Feed + */ + public function attachFeedCacheTimestamps(Feed $feed) + { + try { + $redis = RedisTool::init(); + } catch (Exception $e) { + return $feed; + } + + $pipe = $redis->pipeline(); + $pipe->get('misp:feed_cache_timestamp:' . $feed['id']); + $result = $redis->exec(); + $feed['cache_timestamp'] = $result[0]; + + return $feed; + } + + /** + * @param array $feed + * @param Redis $redis + * @param int|false $jobId + * @return bool + */ + private function __cacheFeed($feed, $redis, $jobId = false) + { + $HttpSocket = $this->isFeedLocal($feed) ? null : $this->__setupHttpClient(); + if ($feed['Feed']['source_format'] === 'misp') { + $result = true; + if (!$this->__cacheMISPFeedCache($feed, $redis, $HttpSocket, $jobId)) { + $result = $this->__cacheMISPFeedTraditional($feed, $redis, $HttpSocket, $jobId); + } + } else { + $result = $this->__cacheFreetextFeed($feed, $redis, $HttpSocket, $jobId); + } + + if ($result) { + $redis->set('misp:feed_cache_timestamp:' . $feed['Feed']['id'], time()); + } + return $result; + } + + /** + * @param array $feed + * @param Redis $redis + * @param HttpClient|null $HttpSocket + * @param int|false $jobId + * @return bool + */ + private function __cacheFreetextFeed(array $feed, $redis, HttpClient $HttpSocket = null, $jobId = false) + { + $feedId = $feed['Feed']['id']; + + $this->jobProgress($jobId, __("Feed %s: Fetching.", $feedId)); + + try { + $values = $this->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format']); + } catch (Exception $e) { + $this->logException("Could not get freetext feed $feedId", $e); + $this->jobProgress($jobId, __('Could not fetch freetext feed %s. See error log for more details.', $feedId)); + return false; + } + + // Convert values to MD5 hashes + $md5Values = array_map('md5', array_column($values, 'value')); + + $redis->del('misp:feed_cache:' . $feedId); + foreach (array_chunk($md5Values, 5000) as $k => $chunk) { + $pipe = $redis->pipeline(); + if (method_exists($redis, 'sAddArray')) { + $redis->sAddArray('misp:feed_cache:' . $feedId, $chunk); + $redis->sAddArray('misp:feed_cache:combined', $chunk); + } else { + foreach ($chunk as $value) { + $redis->sAdd('misp:feed_cache:' . $feedId, $value); + $redis->sAdd('misp:feed_cache:combined', $value); + } + } + $pipe->exec(); + $this->jobProgress($jobId, __('Feed %s: %s/%s values cached.', $feedId, $k * 5000, count($md5Values))); + } + return true; + } + + /** + * @param array $feed + * @param Redis $redis + * @param HttpClient|null $HttpSocket + * @param false $jobId + * @return bool + */ + private function __cacheMISPFeedTraditional($feed, $redis, HttpClient $HttpSocket = null, $jobId = false) + { + $feedId = $feed['Feed']['id']; + try { + $manifest = $this->getManifest($feed, $HttpSocket); + } catch (Exception $e) { + $this->logException("Could not get manifest for feed $feedId.", $e); + return false; + } + + $redis->del('misp:feed_cache:' . $feedId); + + $k = 0; + $AttributesTable = $this->fetchTable('Attributes'); + foreach ($manifest as $uuid => $event) { + try { + $event = $this->downloadAndParseEventFromFeed($feed, $uuid, $HttpSocket); + } catch (Exception $e) { + $this->logException("Could not get and parse event '$uuid' for feed $feedId.", $e); + return false; + } + + if (!empty($event['Event']['Attribute'])) { + $pipe = $redis->pipeline(); + foreach ($event['Event']['Attribute'] as $attribute) { + if (!in_array($attribute['type'], Attribute::NON_CORRELATING_TYPES, true)) { + if (in_array($attribute['type'], $AttributesTable->getCompositeTypes(), true)) { + $value = explode('|', $attribute['value']); + if (in_array($attribute['type'], Attribute::PRIMARY_ONLY_CORRELATING_TYPES, true)) { + unset($value[1]); + } + } else { + $value = [$attribute['value']]; + } + + foreach ($value as $v) { + $md5 = md5($v); + $redis->sAdd('misp:feed_cache:' . $feedId, $md5); + $redis->sAdd('misp:feed_cache:combined', $md5); + $redis->sAdd('misp:feed_cache:event_uuid_lookup:' . $md5, $feedId . '/' . $event['Event']['uuid']); + } + } + } + $pipe->exec(); + } + + $k++; + if ($k % 10 == 0) { + $this->jobProgress($jobId, "Feed $feedId: $k/" . count($manifest) . " events cached."); + } + } + return true; + } + + /** + * @param array $feed + * @param Redis $redis + * @param HttpClient|null $HttpSocket + * @param false $jobId + * @return bool + */ + private function __cacheMISPFeedCache($feed, $redis, HttpClient $HttpSocket = null, $jobId = false) + { + $feedId = $feed['Feed']['id']; + + try { + $cache = $this->getCache($feed, $HttpSocket); + } catch (Exception $e) { + $this->logException("Could not get cache file for $feedId.", $e, LOG_NOTICE); + return false; + } + + $pipe = $redis->pipeline(); + $redis->del('misp:feed_cache:' . $feedId); + foreach ($cache as $v) { + list($hash, $eventUuid) = $v; + $redis->sAdd('misp:feed_cache:' . $feedId, $hash); + $redis->sAdd('misp:feed_cache:combined', $hash); + $redis->sAdd('misp:feed_cache:event_uuid_lookup:' . $hash, "$feedId/$eventUuid"); + } + $pipe->exec(); + $this->jobProgress($jobId, "Feed $feedId: cached via quick cache."); + return true; + } + + public function compareFeeds($id = false) + { + $redis = RedisTool::init(); + if ($redis === false) { + return array(); + } + $fields = array('id', 'input_source', 'source_format', 'url', 'provider', 'name', 'default'); + $feeds = $this->find('all', array( + 'recursive' => -1, + 'fields' => $fields, + 'conditions' => array('Feed.caching_enabled' => 1) + ))->toArray(); + // we'll use this later for the intersect + $fields[] = 'values'; + $fields = array_flip($fields); + // Get all of the feed cache cardinalities for all feeds - if a feed is not cached remove it from the list + foreach ($feeds as $k => $feed) { + if (!$redis->exists('misp:feed_cache:' . $feed['Feed']['id'])) { + unset($feeds[$k]); + continue; + } + $feeds[$k]['Feed']['values'] = $redis->sCard('misp:feed_cache:' . $feed['Feed']['id']); + } + $feeds = array_values($feeds); + $ServersTable = $this->fetchTable('Servers'); + $servers = $ServersTable->find('all', array( + 'recursive' => -1, + 'fields' => array('id', 'url', 'name'), + 'contain' => array('RemoteOrg' => array('fields' => array('RemoteOrg.id', 'RemoteOrg.name'))), + 'conditions' => array('Server.caching_enabled' => 1) + )); + foreach ($servers as $k => $server) { + if (!$redis->exists('misp:server_cache:' . $server['Server']['id'])) { + unset($servers[$k]); + continue; + } + $servers[$k]['Server']['input_source'] = 'network'; + $servers[$k]['Server']['source_format'] = 'misp'; + $servers[$k]['Server']['provider'] = $servers[$k]['RemoteOrg']['name']; + $servers[$k]['Server']['default'] = false; + $servers[$k]['Server']['is_misp_server'] = true; + $servers[$k]['Server']['values'] = $redis->sCard('misp:server_cache:' . $server['Server']['id']); + } + foreach ($feeds as $k => $feed) { + foreach ($feeds as $k2 => $feed2) { + if ($k == $k2) { + continue; + } + $intersect = $redis->sInter('misp:feed_cache:' . $feed['Feed']['id'], 'misp:feed_cache:' . $feed2['Feed']['id']); + $feeds[$k]['Feed']['ComparedFeed'][] = array_merge(array_intersect_key($feed2['Feed'], $fields), array( + 'overlap_count' => count($intersect), + 'overlap_percentage' => round(100 * count($intersect) / $feeds[$k]['Feed']['values']), + )); + } + foreach ($servers as $k2 => $server) { + $intersect = $redis->sInter('misp:feed_cache:' . $feed['Feed']['id'], 'misp:server_cache:' . $server['Server']['id']); + $feeds[$k]['Feed']['ComparedFeed'][] = array_merge(array_intersect_key($server['Server'], $fields), array( + 'overlap_count' => count($intersect), + 'overlap_percentage' => round(100 * count($intersect) / $feeds[$k]['Feed']['values']), + )); + } + } + foreach ($servers as $k => $server) { + foreach ($feeds as $k2 => $feed2) { + $intersect = $redis->sInter('misp:server_cache:' . $server['Server']['id'], 'misp:feed_cache:' . $feed2['Feed']['id']); + $servers[$k]['Server']['ComparedFeed'][] = array_merge(array_intersect_key($feed2['Feed'], $fields), array( + 'overlap_count' => count($intersect), + 'overlap_percentage' => round(100 * count($intersect) / $servers[$k]['Server']['values']), + )); + } + foreach ($servers as $k2 => $server2) { + if ($k == $k2) { + continue; + } + $intersect = $redis->sInter('misp:server_cache:' . $server['Server']['id'], 'misp:server_cache:' . $server2['Server']['id']); + $servers[$k]['Server']['ComparedFeed'][] = array_merge(array_intersect_key($server2['Server'], $fields), array( + 'overlap_count' => count($intersect), + 'overlap_percentage' => round(100 * count($intersect) / $servers[$k]['Server']['values']), + )); + } + } + foreach ($servers as $k => $server) { + $server['Feed'] = $server['Server']; + unset($server['Server']); + $feeds[] = $server; + } + return $feeds; + } + + public function importFeeds($feeds, $user, $default = false) + { + if (is_string($feeds)) { + $feeds = json_decode($feeds, true); + } + if ($feeds && !isset($feeds[0])) { + $feeds = array($feeds); + } + $results = array('successes' => 0, 'fails' => 0); + if (empty($feeds)) { + return $results; + } + $existingFeeds = $this->find('all', array()); + foreach ($feeds as $feed) { + if ($default) { + $feed['Feed']['default'] = 1; + } else { + $feed['Feed']['default'] = 0; + } + if (isset($feed['Feed']['id'])) { + unset($feed['Feed']['id']); + } + $found = false; + foreach ($existingFeeds as $existingFeed) { + if ($existingFeed['Feed']['url'] == $feed['Feed']['url']) { + $found = true; + } + } + if (!$found) { + $feed['Feed']['tag_id'] = 0; + if (isset($feed['Tag'])) { + $tag_id = $this->Tag->captureTag($feed['Tag'], $user); + if ($tag_id) { + $feed['Feed']['tag_id'] = $tag_id; + } + } + $this->create(); + if (!$this->save($feed, true, array('name', 'provider', 'url', 'rules', 'source_format', 'fixed_event', 'delta_merge', 'override_ids', 'publish', 'settings', 'tag_id', 'default', 'lookup_visible', 'headers'))) { + $results['fails']++; + } else { + $results['successes']++; + } + } + } + return $results; + } + + public function load_default_feeds() + { + $user = array('Role' => array('perm_tag_editor' => 1, 'perm_site_admin' => 1)); + $json = file_get_contents(APP . 'files/feed-metadata/defaults.json'); + $this->importFeeds($json, $user, true); + return true; + } + + public function setEnableFeedCachingDefaults() + { + $feeds = $this->find('all', array( + 'conditions' => array( + 'Feed.enabled' => 1 + ), + 'recursive' => -1 + )); + if (empty($feeds)) { + return true; + } + foreach ($feeds as $feed) { + $feed['Feed']['caching_enabled'] = 1; + $this->save($feed); + } + return true; + } + + public function getFeedCoverage($id, $source_scope = 'feed', $dataset = 'all') + { + $redis = RedisTool::init(); + if ($redis === false) { + return 'Could not reach Redis.'; + } + $ServersTable = $this->fetchTable('Servers'); + $feed_conditions = array('Feed.caching_enabled' => 1); + $server_conditions = array('Server.caching_enabled' => 1); + if ($source_scope === 'feed') { + $feed_conditions['NOT'] = array('Feed.id' => $id); + } else { + $server_conditions['NOT'] = array('Server.id' => $id); + } + if ($dataset !== 'all') { + if (empty($dataset['Feed'])) { + $feed_conditions['OR'] = array('Feed.id' => -1); + } else { + $feed_conditions['OR'] = array('Feed.id' => $dataset['Feed']); + } + if (empty($dataset['Server'])) { + $server_conditions['OR'] = array('Server.id' => -1); + } else { + $server_conditions['OR'] = array('Server.id' => $dataset['Server']); + } + } + $other_feeds = $this->find('list', array( + 'recursive' => -1, + 'conditions' => $feed_conditions, + 'fields' => array('Feed.id', 'Feed.id') + )); + $other_servers = $ServersTable->find('list', array( + 'recursive' => -1, + 'conditions' => $server_conditions, + 'fields' => array('Server.id', 'Server.id') + )); + $feed_element_count = $redis->scard('misp:feed_cache:' . $id); + $temp_store = (new RandomTool())->random_str(false, 12); + $params = array('misp:feed_temp:' . $temp_store); + foreach ($other_feeds as $other_feed) { + $params[] = 'misp:feed_cache:' . $other_feed; + } + foreach ($other_servers as $other_server) { + $params[] = 'misp:server_cache:' . $other_server; + } + if (count($params) != 1 && $feed_element_count > 0) { + call_user_func_array(array($redis, 'sunionstore'), $params); + call_user_func_array(array($redis, 'sinterstore'), array('misp:feed_temp:' . $temp_store . '_intersect', 'misp:feed_cache:' . $id, 'misp:feed_temp:' . $temp_store)); + $cardinality_intersect = $redis->scard('misp:feed_temp:' . $temp_store . '_intersect'); + $coverage = round(100 * $cardinality_intersect / $feed_element_count, 2); + $redis->del('misp:feed_temp:' . $temp_store); + $redis->del('misp:feed_temp:' . $temp_store . '_intersect'); + } else { + $coverage = 0; + } + return $coverage; + } + + public function getCachedElements($feedId) + { + $redis = RedisTool::init(); + $cardinality = $redis->sCard('misp:feed_cache:' . $feedId); + return $cardinality; + } + + public function getAllCachingEnabledFeeds($feedId, $intersectingOnly = false) + { + if ($intersectingOnly) { + $redis = RedisTool::init(); + } + $result['Feed'] = $this->find('all', array( + 'conditions' => array( + 'Feed.id !=' => $feedId, + 'caching_enabled' => 1 + ), + 'recursive' => -1, + 'fields' => array('Feed.id', 'Feed.name', 'Feed.url') + )); + $ServersTable = $this->fetchTable('Servers'); + $result['Server'] = $ServersTable->find('all', array( + 'conditions' => array( + 'caching_enabled' => 1 + ), + 'recursive' => -1, + 'fields' => array('Server.id', 'Server.name', 'Server.url') + )); + $scopes = array('Feed', 'Server'); + foreach ($scopes as $scope) { + foreach ($result[$scope] as $k => $v) { + $result[$scope][$k] = $v[$scope]; + } + } + if ($intersectingOnly) { + foreach ($scopes as $scope) { + if (!empty($result[$scope])) { + foreach ($result[$scope] as $k => $feed) { + $intersect = $redis->sInter('misp:feed_cache:' . $feedId, 'misp:' . lcfirst($scope) . '_cache:' . $feed['id']); + if (empty($intersect)) { + unset($result[$scope][$k]); + } else { + $result[$scope][$k]['matching_values'] = count($intersect); + } + } + } + } + } + return $result; + } + + public function searchCaches($value) + { + $hits = array(); + $ServersTable = $this->fetchTable('Servers'); + $redis = RedisTool::init(); + $is_array = true; + if (!is_array($value)) { + $is_array = false; + if (empty($value)) { + // old behaviour allowed for empty values to return all data + $value = [false]; + } else { + $value = [$value]; + } + } + foreach ($value as $v) { + if ($v !== false) { + $v = strtolower(trim($v)); + } + if ($v === false || $redis->sismember('misp:feed_cache:combined', md5($v))) { + $feeds = $this->find('all', array( + 'conditions' => array( + 'caching_enabled' => 1 + ), + 'recursive' => -1, + 'fields' => array('Feed.id', 'Feed.name', 'Feed.url', 'Feed.source_format') + )); + foreach ($feeds as $feed) { + if (($v === false) || $redis->sismember('misp:feed_cache:' . $feed['Feed']['id'], md5($v))) { + if ($feed['Feed']['source_format'] === 'misp') { + $uuid = $redis->smembers('misp:feed_cache:event_uuid_lookup:' . md5($v)); + foreach ($uuid as $k => $url) { + $uuid[$k] = explode('/', $url)[1]; + } + $feed['Feed']['uuid'] = $uuid; + if (!empty($feed['Feed']['uuid'])) { + foreach ($feed['Feed']['uuid'] as $uuid) { + $feed['Feed']['direct_urls'][] = array( + 'url' => sprintf( + '%s/feeds/previewEvent/%s/%s', + Configure::read('MISP.baseurl'), + h($feed['Feed']['id']), + h($uuid) + ), + 'name' => __('Event %s', $uuid) + ); + } + } + $feed['Feed']['type'] = 'MISP Feed'; + } else { + $feed['Feed']['type'] = 'Feed'; + if (!empty($v)) { + $feed['Feed']['direct_urls'][] = array( + 'url' => sprintf( + '%s/feeds/previewIndex/%s', + Configure::read('MISP.baseurl'), + h($feed['Feed']['id']) + ), + 'name' => __('Feed %s', $feed['Feed']['id']) + ); + } + } + if ($is_array) { + $hits[$v][] = $feed; + } else { + $hits[] = $feed; + } + } + } + } + if ($v === false || $redis->sismember('misp:server_cache:combined', md5($v))) { + $servers = $ServersTable->find('all', array( + 'conditions' => array( + 'caching_enabled' => 1 + ), + 'recursive' => -1, + 'fields' => array('Server.id', 'Server.name', 'Server.url') + )); + foreach ($servers as $server) { + if ($v === false || $redis->sismember('misp:server_cache:' . $server['Server']['id'], md5($v))) { + $uuid = $redis->smembers('misp:server_cache:event_uuid_lookup:' . md5($v)); + if (!empty($uuid)) { + foreach ($uuid as $k => $url) { + $uuid[$k] = explode('/', $url)[1]; + $server['Server']['direct_urls'][] = array( + 'url' => sprintf( + '%s/servers/previewEvent/%s/%s', + Configure::read('MISP.baseurl'), + h($server['Server']['id']), + h($uuid[$k]) + ), + 'name' => __('Event %s', h($uuid[$k])) + ); + } + } + $server['Server']['uuid'] = $uuid; + $server['Server']['type'] = 'MISP Server'; + if ($is_array) { + $hits[$v][] = array('Feed' => $server['Server']); + } else { + $hits[] = array('Feed' => $server['Server']); + } + } + } + } + } + return $hits; + } + + /** + * Download and parse event from feed. + * + * @param array $feed + * @param string $eventUuid + * @param HttpClient|null $HttpSocket Null can be for local feed + * @return array + * @throws Exception + */ + private function downloadAndParseEventFromFeed($feed, $eventUuid, HttpClient $HttpSocket = null) + { + if (!Validation::uuid($eventUuid)) { + throw new InvalidArgumentException("Given event UUID '$eventUuid' is invalid."); + } + + $path = $feed['Feed']['url'] . '/' . $eventUuid . '.json'; + $data = $this->feedGetUri($feed, $path, $HttpSocket); + + try { + return JsonTool::decodeArray($data); + } catch (Exception $e) { + throw new Exception("Could not parse event JSON with UUID '$eventUuid' from feed", 0, $e); + } + } + + /** + * @param array $feed + * @param string $uri + * @param HttpClient|null $HttpSocket Null can be for local feed + * @return string + * @throws Exception + */ + private function feedGetUri($feed, $uri, HttpClient $HttpSocket = null) + { + if ($this->isFeedLocal($feed)) { + $uri = mb_ereg_replace("/\:\/\//", '', $uri); + if (file_exists($uri)) { + return FileAccessTool::readFromFile($uri); + } else { + throw new Exception("Local file '$uri' doesn't exists."); + } + } + + return $this->feedGetUriRemote($feed, $uri, $HttpSocket)->body; + } + + /** + * @param array $feed + * @param string $uri + * @param HttpClient $HttpSocket + * @param string|null $etag + * @return false|HttpClientResponse + * @throws HttpException + */ + private function feedGetUriRemote(array $feed, $uri, HttpClient $HttpSocket, $etag = null) + { + $request = $this->__createFeedRequest($feed['Feed']['headers']); + if ($etag) { + $request['header']['If-None-Match'] = $etag; + } + + try { + $response = $this->getFollowRedirect($HttpSocket, $uri, $request); + } catch (Exception $e) { + throw new Exception("Fetching the '$uri' failed with exception: {$e->getMessage()}", 0, $e); + } + + if ($response->code != 200) { // intentionally != + throw new HttpException($response, $uri); + } + + $contentType = $response->getHeader('content-type'); + if ($contentType === 'application/zip') { + $zipFilePath = FileAccessTool::writeToTempFile($response->body); + + try { + $response->body = $this->unzipFirstFile($zipFilePath); + } catch (Exception $e) { + throw new Exception("Fetching the '$uri' failed: {$e->getMessage()}"); + } finally { + FileAccessTool::deleteFile($zipFilePath); + } + } + + return $response; + } + + /** + * It should be possible to use 'redirect' $request attribute, but because HttpClient contains bug that require + * certificate for first domain even when redirect to another domain, we need to use own solution. + * + * @param HttpClient $HttpSocket + * @param string $url + * @param array $request + * @param int $iterations + * @return false|HttpClientResponse + * @throws Exception + */ + private function getFollowRedirect(HttpClient $HttpSocket, $url, $request, $iterations = 5) + { + for ($i = 0; $i < $iterations; $i++) { + $response = $HttpSocket->get($url, array(), $request); + if ($response->isRedirect()) { + $HttpSocket = $this->__setupHttpClient(); // Replace $HttpSocket with fresh instance + $url = trim($response->getHeader('Location')[0], '='); + } else { + return $response; + } + } + + throw new Exception("Maximum number of iteration reached."); + } + + /** + * @param array $feed + * @return bool + */ + private function isFeedLocal($feed) + { + return isset($feed['Feed']['input_source']) && $feed['Feed']['input_source'] === 'local'; + } + + /** + * @param int|null $jobId + * @param string|null $message + * @param int|null $progress + */ + private function jobProgress($jobId = null, $message = null, $progress = null) + { + if ($jobId) { + $JobsTable = $this->fetchTable('Jobs'); + $JobsTable->saveProgress($jobId, $message, $progress); + } + } + + /** + * remove all events tied to a feed. Returns int on success, error message + * as string on failure + */ + public function cleanupFeedEvents($user_id, $id) + { + $feed = $this->find('first', array( + 'conditions' => array('Feed.id' => $id), + 'recursive' => -1 + )); + if (empty($feed)) { + return __('Invalid feed id.'); + } + if (!in_array($feed['Feed']['source_format'], array('csv', 'freetext'))) { + return __('Feed has to be either a CSV or a freetext feed for the purging to work.'); + } + $UsersTable = $this->fetchTable('Users'); + $user = $UsersTable->getAuthUser($user_id); + if (empty($user)) { + return __('Invalid user id.'); + } + $conditions = array('Event.info' => $feed['Feed']['name'] . ' feed'); + $EventsTable = $this->fetchTable('Events'); + $events = $EventsTable->find('list', array( + 'conditions' => $conditions, + 'fields' => array('Event.id', 'Event.id') + ))->toArray(); + $count = count($events); + foreach ($events as $event_id) { + $EventsTable->delete($event_id); + } + $LogsTable = $this->fetchTable('Logs'); + $LogsTable->saveOrFailSilently(array( + 'org' => 'SYSTEM', + 'model' => 'Feed', + 'model_id' => $id, + 'email' => $user['email'], + 'action' => 'purge_events', + 'title' => __('Events related to feed %s purged.', $id), + 'change' => null, + )); + $feed['Feed']['fixed_event'] = 1; + $feed['Feed']['event_id'] = 0; + $feedEntity = $this->newEntity($feed['Feed']); + + $this->save($feedEntity); + return $count; + } + + /** + * @param string $zipFile + * @return string Uncompressed data + * @throws Exception + */ + private function unzipFirstFile($zipFile) + { + if (!class_exists('ZipArchive')) { + throw new Exception('ZIP archive decompressing is not supported. ZIP extension is missing in PHP.'); + } + + $zip = new ZipArchive(); + $result = $zip->open($zipFile); + if ($result !== true) { + $errorCodes = [ + ZipArchive::ER_EXISTS => 'file already exists', + ZipArchive::ER_INCONS => 'zip archive inconsistent', + ZipArchive::ER_INVAL => 'invalid argument', + ZipArchive::ER_MEMORY => 'malloc failure', + ZipArchive::ER_NOENT => 'no such file', + ZipArchive::ER_NOZIP => 'not a zip archive', + ZipArchive::ER_OPEN => 'can\'t open file', + ZipArchive::ER_READ => 'read error', + ZipArchive::ER_SEEK => 'seek error', + ]; + $message = isset($errorCodes[$result]) ? $errorCodes[$result] : 'error ' . $result; + throw new Exception("Remote server returns ZIP file, that cannot be open ($message)"); + } + + if ($zip->numFiles !== 1) { + throw new Exception("Remote server returns ZIP file, that contains multiple files."); + } + + $filename = $zip->getNameIndex(0); + if ($filename === false) { + throw new Exception("Remote server returns ZIP file, but there is a problem with reading filename."); + } + + $zip->close(); + + $destinationFile = FileAccessTool::createTempFile(); + $result = copy("zip://$zipFile#$filename", $destinationFile); + if ($result === false) { + throw new Exception("Remote server returns ZIP file, that contains '$filename' file, but this file cannot be extracted."); + } + + return FileAccessTool::readAndDelete($destinationFile); + } +} diff --git a/tests/Fixture/FeedsFixture.php b/tests/Fixture/FeedsFixture.php new file mode 100644 index 000000000..c0a1f0f38 --- /dev/null +++ b/tests/Fixture/FeedsFixture.php @@ -0,0 +1,39 @@ +records = [ + [ + 'id' => self::FEED_1_ID, + 'name' => self::FEED_1_NAME, + 'provider' => 'test-provider', + 'url' => 'http://localhost/test-feed-1' + ], + [ + 'id' => self::FEED_2_ID, + 'name' => self::FEED_2_NAME, + 'provider' => 'test-provider', + 'url' => 'http://localhost/test-feed-2' + ] + ]; + parent::init(); + } +} diff --git a/tests/TestCase/Api/Feeds/AddFeedApiTest.php b/tests/TestCase/Api/Feeds/AddFeedApiTest.php new file mode 100644 index 000000000..97dae9c98 --- /dev/null +++ b/tests/TestCase/Api/Feeds/AddFeedApiTest.php @@ -0,0 +1,47 @@ +skipOpenApiValidations(); + + $this->setAuthToken(AuthKeysFixture::ADMIN_API_KEY); + + $faker = \Faker\Factory::create(); + + $this->post( + self::ENDPOINT, + [ + "name" => "feed-osint", + "provider" => "CIRCL", + "url" => "https://www.circl.lu/doc/misp/feed-osint", + "rules" => "{\"tags\":{\"OR\":[],\"NOT\":[]},\"orgs\":{\"OR\":[],\"NOT\":[]},\"url_params\":\"\"}", + "source_format" => "1" + ] + ); + + $this->assertResponseOk(); + $this->assertDbRecordExists('Feeds', ['name' => 'feed-osint']); + } +} diff --git a/tests/TestCase/Api/Feeds/EditFeedApiTest.php b/tests/TestCase/Api/Feeds/EditFeedApiTest.php new file mode 100644 index 000000000..5005e0dfa --- /dev/null +++ b/tests/TestCase/Api/Feeds/EditFeedApiTest.php @@ -0,0 +1,72 @@ +skipOpenApiValidations(); + + $this->setAuthToken(AuthKeysFixture::ADMIN_API_KEY); + + $url = sprintf('%s/%s', self::ENDPOINT, FeedsFixture::FEED_1_ID); + + $this->put( + $url, + [ + "name" => "feed-osint", + "provider" => "CIRCL", + "url" => "https://www.circl.lu/doc/misp/feed-osint", + "rules" => [ + "tags" => [ + "OR" => [], + "NOT" => [] + ], + "orgs" => [ + "OR" => [], + "NOT" => [] + ], + "url_params" => "" + ], + "settings" => [ + "csv" => [ + "value" => "", + "delimiter" => "" + ], + "common" => [ + "excluderegex" => "" + ] + ] + ] + ); + + $this->assertResponseOk(); + $this->assertDbRecordExists( + 'Feeds', + [ + 'id' => FeedsFixture::FEED_1_ID, + 'name' => 'feed-osint', + ] + ); + } +} diff --git a/tests/TestCase/Api/Feeds/IndexFeedsApiTest.php b/tests/TestCase/Api/Feeds/IndexFeedsApiTest.php new file mode 100644 index 000000000..89f08e6d4 --- /dev/null +++ b/tests/TestCase/Api/Feeds/IndexFeedsApiTest.php @@ -0,0 +1,38 @@ +skipOpenApiValidations(); + + $this->setAuthToken(AuthKeysFixture::ADMIN_API_KEY); + + $this->get(self::ENDPOINT); + + $this->assertResponseOk(); + $this->assertResponseContains(sprintf('"name": "%s"', FeedsFixture::FEED_1_NAME)); + $this->assertResponseContains(sprintf('"name": "%s"', FeedsFixture::FEED_2_NAME)); + } +}