From 7f8a81e161bf6b46e45e036b6f4831b2ed45abb3 Mon Sep 17 00:00:00 2001 From: iglocska Date: Fri, 30 Dec 2016 16:16:56 +0100 Subject: [PATCH] new: Added caching and pagination to freetext/csv feeds --- .gitignore | 2 + .../Component/RestResponseComponent.php | 18 +++---- app/Controller/FeedsController.php | 23 +++++++- app/Lib/Tools/ComplexTypeTool.php | 1 + app/Model/Feed.php | 54 +++++++++++++------ app/View/Feeds/add.ctp | 16 +++++- app/View/Feeds/edit.ctp | 15 +++++- app/View/Feeds/freetext_index.ctp | 46 +++++++++++++--- app/tmp/cache/feeds/empty | 0 app/webroot/js/misp2.4.58.js | 1 + 10 files changed, 138 insertions(+), 38 deletions(-) create mode 100644 app/tmp/cache/feeds/empty diff --git a/.gitignore b/.gitignore index abd17fd9a..50a6bc7df 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ /app/tmp/cache/persistent/myapp* /app/tmp/cache/views/myapp* /app/files/* +/app/tmp/cache/feeds/*.cache !/app/files/empty !/app/files/scripts/ !/app/files/warninglists @@ -64,6 +65,7 @@ /app/tmp/cached_exports/csv_sig/* /app/tmp/cached_exports/stix/* /app/tmp/cached_exports/sha256/* +/app/tmp/cached_exports/bro/* .gnupg .smime *.swp diff --git a/app/Controller/Component/RestResponseComponent.php b/app/Controller/Component/RestResponseComponent.php index 953c8a43f..720b56928 100644 --- a/app/Controller/Component/RestResponseComponent.php +++ b/app/Controller/Component/RestResponseComponent.php @@ -1,7 +1,7 @@ array( 'admin_add' => array( @@ -15,7 +15,7 @@ class RestResponseComponent extends Component { ) ) ); - + public function saveFailResponse($controller, $action, $id = false, $validationErrors, $format = false) { $this->autoRender = false; $response = array(); @@ -24,9 +24,9 @@ class RestResponseComponent extends Component { $response['message'] = $response['name']; $response['url'] = $this->__generateURL($action, $controller, $id); $response['errors'] = $validationErrors; - return $this->__sendResponse($response, 403, $format); + return $this->__sendResponse($response, 403, $format); } - + public function saveSuccessResponse($controller, $action, $id = false, $format = false, $message = false) { $action = $this->__dissectAdminRouting($action); if (!$message) { @@ -37,7 +37,7 @@ class RestResponseComponent extends Component { $response['url'] = $this->__generateURL($action, $controller, $id); return $this->__sendResponse($response, 200, $format); } - + private function __sendResponse($response, $code, $format = false) { if (strtolower($format) === 'application/xml') { $response = Xml::build($response); @@ -48,11 +48,11 @@ class RestResponseComponent extends Component { } return new CakeResponse(array('body'=> $response,'status' => $code, 'type' => $type)); } - + private function __generateURL($action, $controller, $id) { return ($action['admin'] ? '/admin' : '') . '/' . strtolower($controller) . '/' . $action['action'] . ($id ? '/' . $id : ''); } - + private function __dissectAdminRouting($action) { $admin = false; if (strlen($action) > 6 && substr($action, 0, 6) == 'admin_') { @@ -61,11 +61,11 @@ class RestResponseComponent extends Component { } return array('action' => $action, 'admin' => $admin); } - + public function viewData($data, $format = false) { return $this->__sendResponse($data, 200, $format); } - + public function describe($controller, $action, $id = false, $format = false) { $actionArray = $this->__dissectAdminRouting($action); $response['name'] = $this->__generateURL($actionArray, $controller, false) . ' API description'; diff --git a/app/Controller/FeedsController.php b/app/Controller/FeedsController.php index f99919e10..175d5a4af 100644 --- a/app/Controller/FeedsController.php +++ b/app/Controller/FeedsController.php @@ -70,6 +70,9 @@ class FeedsController extends AppController { if (!isset($this->request->data['Feed']['settings'])) { $this->request->data['Feed']['settings'] = array(); } + if (isset($this->request->data['Feed']['settings']['separator']) && empty($this->request->data['Feed']['settings']['separator'])) { + $this->request->data['Feed']['settings']['separator'] = ','; + } if (empty($this->request->data['Feed']['target_event'])) { $this->request->data['Feed']['target_event'] = 0; } @@ -119,6 +122,9 @@ class FeedsController extends AppController { if (!isset($this->request->data['Feed']['settings'])) { $this->request->data['Feed']['settings'] = array(); } + if (isset($this->request->data['Feed']['settings']['separator']) && empty($this->request->data['Feed']['settings']['separator'])) { + $this->request->data['Feed']['settings']['separator'] = ','; + } $this->request->data['Feed']['settings'] = json_encode($this->request->data['Feed']['settings']); $fields = array('id', 'name', 'provider', 'enabled', 'rules', 'url', 'distribution', 'sharing_group_id', 'tag_id', 'fixed_event', 'event_id', 'publish', 'delta_merge', 'override_ids', 'settings'); $feed = array(); @@ -129,6 +135,10 @@ class FeedsController extends AppController { } $result = $this->Feed->save($feed); if ($result) { + $feedCache = APP . 'tmp' . DS . 'cache' . DS . 'feeds' . DS . intval($feed['Feed']['id']) . '.cache'; + if (file_exists($feedCache)) { + unlink($feedCache); + } $this->Session->setFlash('Feed updated.'); $this->redirect(array('controller' => 'feeds', 'action' => 'index')); } else { @@ -286,11 +296,18 @@ class FeedsController extends AppController { } private function __previewFreetext($feed) { + if (isset($this->passedArgs['page'])) $currentPage = $this->passedArgs['page']; + else if (isset($this->passedArgs['page'])) $currentPage = $this->passedArgs['page']; + else $currentPage = 1; + $urlparams = ''; App::uses('SyncTool', 'Tools'); $syncTool = new SyncTool(); if (!in_array($feed['Feed']['source_format'], array('freetext', 'csv'))) throw new MethodNotAllowedException('Invalid feed type.'); $HttpSocket = $syncTool->setupHttpSocketFeed($feed); - $resultArray = $this->Feed->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format']); + $params = array(); + // params is passed as reference here, the pagination happens in the method, which isn't ideal but considering the performance gains here it's worth it + $resultArray = $this->Feed->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format'], $currentPage, 60, $params); + $this->params->params['paging'] = array($this->modelClass => $params); $resultArray = $this->Feed->getFreetextFeedCorrelations($resultArray); // remove all duplicates foreach ($resultArray as $k => $v) { @@ -307,11 +324,13 @@ class FeedsController extends AppController { } private function __previewCSV($feed) { + if (isset($this->passedArgs['pages'])) $currentPage = $this->passedArgs['pages']; + else $currentPage = 1; App::uses('SyncTool', 'Tools'); $syncTool = new SyncTool(); if ($feed['Feed']['source_format'] != 'csv') throw new MethodNotAllowedException('Invalid feed type.'); $HttpSocket = $syncTool->setupHttpSocketFeed($feed); - $resultArray = $this->Feed->getFreetextFeed($feed, $HttpSocket); + $resultArray = $this->Feed->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format'], $currentPage); $resultArray = $this->Feed->getFreetextFeedCorrelations($resultArray); // remove all duplicates foreach ($resultArray as $k => $v) { diff --git a/app/Lib/Tools/ComplexTypeTool.php b/app/Lib/Tools/ComplexTypeTool.php index df0795b52..c4f9cf67f 100644 --- a/app/Lib/Tools/ComplexTypeTool.php +++ b/app/Lib/Tools/ComplexTypeTool.php @@ -99,6 +99,7 @@ class ComplexTypeTool { public function checkCSV($input, $settings = array()) { $delimiter = isset($settings['delimiter']) ? $settings['delimiter'] : ","; $lines = explode("\n", $input); + unset($input); $values = !empty($settings['value']) ? $settings['value'] : array(); if (!is_array($values)) { $values = explode(',', $values); diff --git a/app/Model/Feed.php b/app/Model/Feed.php index e3e26f6f3..b5aad2d71 100644 --- a/app/Model/Feed.php +++ b/app/Model/Feed.php @@ -33,7 +33,7 @@ class Feed extends AppModel { 'message' => 'Please enter a numeric event ID or leave this field blank.', ) ); - + // currently we only have an internal name and a display name, but later on we can expand this with versions, default settings, etc public $feed_types = array( 'misp' => array( @@ -46,7 +46,7 @@ class Feed extends AppModel { 'name' => 'Simple CSV Parsed Feed' ) ); - + public function getFeedTypesOptions() { $result = array(); foreach ($this->feed_types as $key => $value) { @@ -95,24 +95,46 @@ class Feed extends AppModel { $events = $this->__filterEventsIndex($events, $feed); return $events; } - - public function getFreetextFeed($feed, $HttpSocket, $type = 'freetext') { + + public function getFreetextFeed($feed, $HttpSocket, $type = 'freetext', $page = 1, $limit = 60, &$params = array()) { $result = array(); - $response = $HttpSocket->get($feed['Feed']['url'], '', array()); - if ($response->code == 200) { - App::uses('ComplexTypeTool', 'Tools'); - $complexTypeTool = new ComplexTypeTool(); - $this->Warninglist = ClassRegistry::init('Warninglist'); - $complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists()); - $resultArray = $complexTypeTool->checkComplexRouter($response->body, $type, isset($feed['Feed']['settings'][$type]) ? $feed['Feed']['settings'][$type] : array()); + $feedCache = APP . 'tmp' . DS . 'cache' . DS . 'feeds' . DS . intval($feed['Feed']['id']) . '.cache'; + $doFetch = true; + if (file_exists($feedCache)) { + $file = new File($feedCache); + if (time() - $file->lastChange() < 600) { + $doFetch = false; + $data = file_get_contents($feedCache); + } } + if ($doFetch) { + $response = $HttpSocket->get($feed['Feed']['url'], '', array()); + if ($response->code == 200) { + $data = $response->body; + file_put_contents($feedCache, $data); + } + } + $data = explode("\n", $data); + App::uses('CustomPaginationTool', 'Tools'); + $customPagination = new CustomPaginationTool(); + $params = $customPagination->createPaginationRules($data, array('page' => $page, 'limit' => $limit), 'Feed', $sort = false); + if (!empty($page) && $page != 'all') { + $start = ($page - 1) * $limit; + $data = array_slice($data, $start, $limit); + } + $data = implode("\n", $data); + App::uses('ComplexTypeTool', 'Tools'); + $complexTypeTool = new ComplexTypeTool(); + $this->Warninglist = ClassRegistry::init('Warninglist'); + $complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists()); + $resultArray = $complexTypeTool->checkComplexRouter($data, $type, isset($feed['Feed']['settings'][$type]) ? $feed['Feed']['settings'][$type] : array()); $this->Attribute = ClassRegistry::init('Attribute'); foreach ($resultArray as $key => $value) { $resultArray[$key]['category'] = $this->Attribute->typeDefinitions[$value['default_type']]['default_category']; } return $resultArray; } - + public function getFreetextFeedCorrelations($data) { $values = array(); foreach ($data as $key => $value) { @@ -129,7 +151,7 @@ class Feed extends AppModel { $data[$key]['correlations'] = array_values($correlations[$value['value']]); } } - return $data; + return $data; } public function downloadFromFeed($actions, $feed, $HttpSocket, $user, $jobId = false) { @@ -189,9 +211,9 @@ class Feed extends AppModel { try { $commit = trim(shell_exec('git log --pretty="%H" -n1 HEAD')); } catch (Exception $e) { - $commit = false; + $commit = false; } - + $result = array( 'header' => array( 'Accept' => 'application/json', @@ -457,7 +479,7 @@ class Feed extends AppModel { } return $result; } - + public function saveFreetextFeedData($feed, $data, $user) { $this->Event = ClassRegistry::init('Event'); $event = false; diff --git a/app/View/Feeds/add.ctp b/app/View/Feeds/add.ctp index 1b8394fd8..2c7bd8f61 100644 --- a/app/View/Feeds/add.ctp +++ b/app/View/Feeds/add.ctp @@ -28,7 +28,7 @@ )); ?>
- Form->input('fixed_event', array( 'label' => 'Target Event', 'div' => 'input clear', @@ -58,6 +58,18 @@ )); ?>
+
+ Form->input('Feed.settings.csv.separator', array( + 'label' => 'Separator', + 'title' => 'Set the default CSV separator (default = ",")', + 'div' => 'input clear', + 'placeholder' => ',', + 'class' => 'form-control span6', + 'value' => ',' + )); + ?> +
Form->input('publish', array( @@ -88,7 +100,7 @@ )); ?>
- Form->input('distribution', array( 'options' => array($distributionLevels), 'div' => 'input clear', diff --git a/app/View/Feeds/edit.ctp b/app/View/Feeds/edit.ctp index 6b509e20d..d4032a9fd 100644 --- a/app/View/Feeds/edit.ctp +++ b/app/View/Feeds/edit.ctp @@ -28,7 +28,7 @@ )); ?>
- Form->input('fixed_event', array( 'label' => 'Target Event', 'div' => 'input clear', @@ -58,6 +58,17 @@ )); ?>
+
+ Form->input('Feed.settings.csv.separator', array( + 'label' => 'Separator', + 'title' => 'Set the default CSV separator (default = ",")', + 'div' => 'input clear', + 'placeholder' => ',', + 'class' => 'form-control span6' + )); + ?> +
Form->input('publish', array( @@ -87,7 +98,7 @@ )); ?>
- Form->input('distribution', array( 'options' => array($distributionLevels), 'div' => 'input clear', diff --git a/app/View/Feeds/freetext_index.ctp b/app/View/Feeds/freetext_index.ctp index 924283614..aa359de02 100644 --- a/app/View/Feeds/freetext_index.ctp +++ b/app/View/Feeds/freetext_index.ctp @@ -1,6 +1,6 @@

Parsed attributes from feed

- Form->create('Feed', array('url' => array('controller' => 'feeds', 'action' => 'fetchSelectedFromFreetextIndex', $feed['Feed']['id']))); echo $this->Form->input('data', array('style' => 'display:none;', 'label' => false, 'div' => false)); ?> @@ -8,6 +8,23 @@ Form->end(); ?> + @@ -18,7 +35,7 @@ - $attribute): ?> @@ -30,12 +47,12 @@
Correlations Distribution
- -   @@ -45,7 +62,7 @@ if ($feed['Feed']['distribution'] == 4): ?> -
- +
element('side_menu', array('menuList' => 'feeds', 'menuItem' => 'add')); @@ -90,4 +122,4 @@ function freetextFeedFetchSelected() { } - \ No newline at end of file + diff --git a/app/tmp/cache/feeds/empty b/app/tmp/cache/feeds/empty new file mode 100644 index 000000000..e69de29bb diff --git a/app/webroot/js/misp2.4.58.js b/app/webroot/js/misp2.4.58.js index 0afa61ec3..147a0f450 100644 --- a/app/webroot/js/misp2.4.58.js +++ b/app/webroot/js/misp2.4.58.js @@ -2578,6 +2578,7 @@ function feedFormUpdate() { $('#DeltaMergeDiv').show(); } $('#settingsCsvValueDiv').show(); + $('#settingsCsvSeparatorDiv').show(); break; } }