new: Added caching and pagination to freetext/csv feeds

.gitignore

@@ -18,6 +18,7 @@
 /app/tmp/cache/persistent/myapp*
 /app/tmp/cache/views/myapp*
 /app/files/*
+/app/tmp/cache/feeds/*.cache
 !/app/files/empty
 !/app/files/scripts/
 !/app/files/warninglists
@@ -64,6 +65,7 @@
 /app/tmp/cached_exports/csv_sig/*
 /app/tmp/cached_exports/stix/*
 /app/tmp/cached_exports/sha256/*
+/app/tmp/cached_exports/bro/*
 .gnupg
 .smime
 *.swp

app/Controller/Component/RestResponseComponent.php

@@ -1,7 +1,7 @@
 <?php
 
 class RestResponseComponent extends Component {
-	
+
 	private $__descriptions = array(
 			'User' => array(
 				'admin_add' => array(
@@ -15,7 +15,7 @@ class RestResponseComponent extends Component {
 				)
 			)
 	);
-	
+
 	public function saveFailResponse($controller, $action, $id = false, $validationErrors, $format = false) {
 		$this->autoRender = false;
 		$response = array();
@@ -24,9 +24,9 @@ class RestResponseComponent extends Component {
 		$response['message'] = $response['name'];
 		$response['url'] = $this->__generateURL($action, $controller, $id);
 		$response['errors'] = $validationErrors;
-		return $this->__sendResponse($response, 403, $format);	
+		return $this->__sendResponse($response, 403, $format);
 	}
-	
+
 	public function saveSuccessResponse($controller, $action, $id = false, $format = false, $message = false) {
 		$action = $this->__dissectAdminRouting($action);
 		if (!$message) {
@@ -37,7 +37,7 @@ class RestResponseComponent extends Component {
 		$response['url'] = $this->__generateURL($action, $controller, $id);
 		return $this->__sendResponse($response, 200, $format);
 	}
-	
+
 	private function __sendResponse($response, $code, $format = false) {
 		if (strtolower($format) === 'application/xml') {
 			$response = Xml::build($response);
@@ -48,11 +48,11 @@ class RestResponseComponent extends Component {
 		}
 		return new CakeResponse(array('body'=> $response,'status' => $code, 'type' => $type));
 	}
-	
+
 	private function __generateURL($action, $controller, $id) {
 		return ($action['admin'] ? '/admin' : '') . '/' . strtolower($controller) . '/' . $action['action'] . ($id ? '/' . $id : '');
 	}
-	
+
 	private function __dissectAdminRouting($action) {
 		$admin = false;
 		if (strlen($action) > 6 && substr($action, 0, 6) == 'admin_') {
@@ -61,11 +61,11 @@ class RestResponseComponent extends Component {
 		}
 		return array('action' => $action, 'admin' => $admin);
 	}
-	
+
 	public function viewData($data, $format = false) {
 		return $this->__sendResponse($data, 200, $format);
 	}
-	
+
 	public function describe($controller, $action, $id = false, $format = false) {
 		$actionArray = $this->__dissectAdminRouting($action);
 		$response['name'] = $this->__generateURL($actionArray, $controller, false) . ' API description';

app/Controller/FeedsController.php

@@ -70,6 +70,9 @@ class FeedsController extends AppController {
 			if (!isset($this->request->data['Feed']['settings'])) {
 				$this->request->data['Feed']['settings'] = array();
 			}
+			if (isset($this->request->data['Feed']['settings']['separator']) && empty($this->request->data['Feed']['settings']['separator'])) {
+				$this->request->data['Feed']['settings']['separator'] = ',';
+			}
 			if (empty($this->request->data['Feed']['target_event'])) {
 				$this->request->data['Feed']['target_event'] = 0;
 			}
@@ -119,6 +122,9 @@ class FeedsController extends AppController {
 			if (!isset($this->request->data['Feed']['settings'])) {
 				$this->request->data['Feed']['settings'] = array();
 			}
+			if (isset($this->request->data['Feed']['settings']['separator']) && empty($this->request->data['Feed']['settings']['separator'])) {
+				$this->request->data['Feed']['settings']['separator'] = ',';
+			}
 			$this->request->data['Feed']['settings'] = json_encode($this->request->data['Feed']['settings']);
 			$fields = array('id', 'name', 'provider', 'enabled', 'rules', 'url', 'distribution', 'sharing_group_id', 'tag_id', 'fixed_event', 'event_id', 'publish', 'delta_merge', 'override_ids', 'settings');
 			$feed = array();
@@ -129,6 +135,10 @@ class FeedsController extends AppController {
 			}
 			$result = $this->Feed->save($feed);
 			if ($result) {
+				$feedCache = APP . 'tmp' . DS . 'cache' . DS . 'feeds' . DS . intval($feed['Feed']['id']) . '.cache';
+				if (file_exists($feedCache)) {
+					unlink($feedCache);
+				}
 				$this->Session->setFlash('Feed updated.');
 				$this->redirect(array('controller' => 'feeds', 'action' => 'index'));
 			} else {
@@ -286,11 +296,18 @@ class FeedsController extends AppController {
 	}
 
 	private function __previewFreetext($feed) {
+		if (isset($this->passedArgs['page'])) $currentPage = $this->passedArgs['page'];
+		else if (isset($this->passedArgs['page'])) $currentPage = $this->passedArgs['page'];
+		else $currentPage = 1;
+		$urlparams = '';
 		App::uses('SyncTool', 'Tools');
 		$syncTool = new SyncTool();
 		if (!in_array($feed['Feed']['source_format'], array('freetext', 'csv'))) throw new MethodNotAllowedException('Invalid feed type.');
 		$HttpSocket = $syncTool->setupHttpSocketFeed($feed);
-		$resultArray = $this->Feed->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format']);
+		$params = array();
+		// params is passed as reference here, the pagination happens in the method, which isn't ideal but considering the performance gains here it's worth it
+		$resultArray = $this->Feed->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format'], $currentPage, 60, $params);
+		$this->params->params['paging'] = array($this->modelClass => $params);
 		$resultArray = $this->Feed->getFreetextFeedCorrelations($resultArray);
 		// remove all duplicates
 		foreach ($resultArray as $k => $v) {
@@ -307,11 +324,13 @@ class FeedsController extends AppController {
 	}
 
 	private function __previewCSV($feed) {
+		if (isset($this->passedArgs['pages'])) $currentPage = $this->passedArgs['pages'];
+		else $currentPage = 1;
 		App::uses('SyncTool', 'Tools');
 		$syncTool = new SyncTool();
 		if ($feed['Feed']['source_format'] != 'csv') throw new MethodNotAllowedException('Invalid feed type.');
 		$HttpSocket = $syncTool->setupHttpSocketFeed($feed);
-		$resultArray = $this->Feed->getFreetextFeed($feed, $HttpSocket);
+		$resultArray = $this->Feed->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format'], $currentPage);
 		$resultArray = $this->Feed->getFreetextFeedCorrelations($resultArray);
 		// remove all duplicates
 		foreach ($resultArray as $k => $v) {

app/Lib/Tools/ComplexTypeTool.php

@@ -99,6 +99,7 @@ class ComplexTypeTool {
 	public function checkCSV($input, $settings = array()) {
 		$delimiter = isset($settings['delimiter']) ? $settings['delimiter'] : ",";
 		$lines = explode("\n", $input);
+		unset($input);
 		$values = !empty($settings['value']) ? $settings['value'] : array();
 		if (!is_array($values)) {
 			$values = explode(',', $values);

app/Model/Feed.php

@@ -33,7 +33,7 @@ class Feed extends AppModel {
 			'message' => 'Please enter a numeric event ID or leave this field blank.',
 		)
 	);
-	
+
 	// currently we only have an internal name and a display name, but later on we can expand this with versions, default settings, etc
 	public $feed_types = array(
 		'misp' => array(
@@ -46,7 +46,7 @@ class Feed extends AppModel {
 				'name' => 'Simple CSV Parsed Feed'
 		)
 	);
-	
+
 	public function getFeedTypesOptions() {
 		$result = array();
 		foreach ($this->feed_types as $key => $value) {
@@ -95,24 +95,46 @@ class Feed extends AppModel {
 		$events = $this->__filterEventsIndex($events, $feed);
 		return $events;
 	}
-	
-	public function getFreetextFeed($feed, $HttpSocket, $type = 'freetext') {
+
+	public function getFreetextFeed($feed, $HttpSocket, $type = 'freetext', $page = 1, $limit = 60, &$params = array()) {
 		$result = array();
-		$response = $HttpSocket->get($feed['Feed']['url'], '', array());
-		if ($response->code == 200) {
-			App::uses('ComplexTypeTool', 'Tools');
-			$complexTypeTool = new ComplexTypeTool();
-			$this->Warninglist = ClassRegistry::init('Warninglist');
-			$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
-			$resultArray = $complexTypeTool->checkComplexRouter($response->body, $type, isset($feed['Feed']['settings'][$type]) ? $feed['Feed']['settings'][$type] : array());
+		$feedCache = APP . 'tmp' . DS . 'cache' . DS . 'feeds' . DS . intval($feed['Feed']['id']) . '.cache';
+		$doFetch = true;
+		if (file_exists($feedCache)) {
+			$file = new File($feedCache);
+			if (time() - $file->lastChange() < 600) {
+				$doFetch = false;
+				$data = file_get_contents($feedCache);
+			}
 		}
+		if ($doFetch) {
+			$response = $HttpSocket->get($feed['Feed']['url'], '', array());
+			if ($response->code == 200) {
+				$data = $response->body;
+				file_put_contents($feedCache, $data);
+			}
+		}
+		$data = explode("\n", $data);
+		App::uses('CustomPaginationTool', 'Tools');
+		$customPagination = new CustomPaginationTool();
+		$params = $customPagination->createPaginationRules($data, array('page' => $page, 'limit' => $limit), 'Feed', $sort = false);
+		if (!empty($page) && $page != 'all') {
+			$start = ($page - 1) * $limit;
+			$data = array_slice($data, $start, $limit);
+		}
+		$data = implode("\n", $data);
+		App::uses('ComplexTypeTool', 'Tools');
+		$complexTypeTool = new ComplexTypeTool();
+		$this->Warninglist = ClassRegistry::init('Warninglist');
+		$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
+		$resultArray = $complexTypeTool->checkComplexRouter($data, $type, isset($feed['Feed']['settings'][$type]) ? $feed['Feed']['settings'][$type] : array());
 		$this->Attribute = ClassRegistry::init('Attribute');
 		foreach ($resultArray as $key => $value) {
 			$resultArray[$key]['category'] = $this->Attribute->typeDefinitions[$value['default_type']]['default_category'];
 		}
 		return $resultArray;
 	}
-	
+
 	public function getFreetextFeedCorrelations($data) {
 		$values = array();
 		foreach ($data as $key => $value) {
@@ -129,7 +151,7 @@ class Feed extends AppModel {
 				$data[$key]['correlations'] = array_values($correlations[$value['value']]);
 			}
 		}
-		return $data;		
+		return $data;
 	}
 
 	public function downloadFromFeed($actions, $feed, $HttpSocket, $user, $jobId = false) {
@@ -189,9 +211,9 @@ class Feed extends AppModel {
 		try {
 			$commit = trim(shell_exec('git log --pretty="%H" -n1 HEAD'));
 		} catch (Exception $e) {
-			$commit = false;			
+			$commit = false;
 		}
-		
+
 		$result = array(
 			'header' => array(
 					'Accept' => 'application/json',
@@ -457,7 +479,7 @@ class Feed extends AppModel {
 		}
 		return $result;
 	}
-	
+
 	public function saveFreetextFeedData($feed, $data, $user) {
 		$this->Event = ClassRegistry::init('Event');
 		$event = false;

app/View/Feeds/add.ctp

@@ -28,7 +28,7 @@
 		));
 	?>
 		<div id="TargetDiv" class="optionalField">
-	<?php 
+	<?php
 		echo $this->Form->input('fixed_event', array(
 				'label' => 'Target Event',
 				'div' => 'input clear',
@@ -58,6 +58,18 @@
 				));
 			?>
 		</div>
+		<div id="settingsCsvSeparatorDiv" class="optionalField">
+			<?php
+				echo $this->Form->input('Feed.settings.csv.separator', array(
+						'label' => 'Separator',
+						'title' => 'Set the default CSV separator (default = ",")',
+						'div' => 'input clear',
+						'placeholder' => ',',
+						'class' => 'form-control span6',
+						'value' => ','
+				));
+			?>
+		</div>
 		<div id="PublishDiv" class="input clear optionalField">
 		<?php
 			echo $this->Form->input('publish', array(
@@ -88,7 +100,7 @@
 			));
 		?>
 		</div>
-	<?php 
+	<?php
 		echo $this->Form->input('distribution', array(
 				'options' => array($distributionLevels),
 				'div' => 'input clear',

app/View/Feeds/edit.ctp

@@ -28,7 +28,7 @@
 			));
 		?>
 		<div id="TargetDiv" class="optionalField">
-		<?php 
+		<?php
 			echo $this->Form->input('fixed_event', array(
 					'label' => 'Target Event',
 					'div' => 'input clear',
@@ -58,6 +58,17 @@
 				));
 			?>
 		</div>
+		<div id="settingsCsvSeparatorDiv" class="optionalField">
+			<?php
+				echo $this->Form->input('Feed.settings.csv.separator', array(
+						'label' => 'Separator',
+						'title' => 'Set the default CSV separator (default = ",")',
+						'div' => 'input clear',
+						'placeholder' => ',',
+						'class' => 'form-control span6'
+				));
+			?>
+		</div>
 		<div id="PublishDiv" class="input clear optionalField">
 		<?php
 			echo $this->Form->input('publish', array(
@@ -87,7 +98,7 @@
 			));
 		?>
 		</div>
-		<?php 
+		<?php
 			echo $this->Form->input('distribution', array(
 					'options' => array($distributionLevels),
 					'div' => 'input clear',

app/View/Feeds/freetext_index.ctp

@@ -1,6 +1,6 @@
 <div class="attributes index">
 	<h2>Parsed attributes from feed <?php echo h($feed['Feed']['name']);?></h2>
-	<?php 
+	<?php
 		echo $this->Form->create('Feed', array('url' => array('controller' => 'feeds', 'action' => 'fetchSelectedFromFreetextIndex', $feed['Feed']['id'])));
 		echo $this->Form->input('data', array('style' => 'display:none;', 'label' => false, 'div' => false));
 	?>
@@ -8,6 +8,23 @@
 	<?php
 		echo $this->Form->end();
 	?>
+	<div class="pagination">
+				<ul>
+				<?php
+					$url = array_merge(array('controller' => 'feeds', 'action' => 'previewIndex', $feed['Feed']['id']), $this->request->named);
+					$this->Paginator->options(array(
+							'url' => $url,
+							'update' => '.span12',
+							'evalScripts' => true,
+							'before' => '$(".progress").show()',
+							'complete' => '$(".progress").hide()',
+					));
+						echo $this->Paginator->prev('&laquo; ' . __('previous'), array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'prev disabled', 'escape' => false, 'disabledTag' => 'span'));
+						echo $this->Paginator->numbers(array('modulus' => 20, 'separator' => '', 'tag' => 'li', 'currentClass' => 'active', 'currentTag' => 'span'));
+						echo $this->Paginator->next(__('next') . ' &raquo;', array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'next disabled', 'escape' => false, 'disabledTag' => 'span'));
+				?>
+				</ul>
+		</div>
 	<table class="table table-striped table-hover table-condensed">
 	<tr>
 		<th><input class="select_all" type="checkbox" onClick="toggleAllAttributeCheckboxes();" /></th>
@@ -18,7 +35,7 @@
 		<th>Correlations</th>
 		<th>Distribution</th>
 	</tr>
-	<?php 
+	<?php
 		foreach ($attributes as $key => $attribute):
 	?>
 	<tr>
@@ -30,12 +47,12 @@
 		<td id="<?php echo h($key);?>_value"><?php echo h($attribute['value']);?></td>
 		<td class="short" id="<?php echo h($key);?>_to_ids" data-value="<?php echo h($attribute['to_ids']); ?>"><span class="icon-<?php echo $attribute['to_ids'] ? 'ok' : 'remove';?>"></span></td>
 		<td class="shortish">
-			<?php 
+			<?php
 				if (isset($attribute['correlations'])):
 					foreach ($attribute['correlations'] as $correlation):
 			?>
 						<a href="<?php echo $baseurl; ?>/events/view/<?php echo h($correlation); ?>"><?php echo h($correlation); ?></a>
-			<?php 
+			<?php
 					endforeach;
 				endif;
 			?>&nbsp;
@@ -45,7 +62,7 @@
 				if ($feed['Feed']['distribution'] == 4):
 			?>
 					<a href="<?php echo $baseurl; ?>/sharing_groups/view/<?php echo h($feed['Feed']['sharing_group_id']); ?>"><?php echo h($feed['SharingGroup']['name']); ?></a>
-			<?php 
+			<?php
 				else:
 					echo h($distributionLevels[$feed['Feed']['distribution']]);
 				endif;
@@ -56,7 +73,22 @@
 		endforeach;
 	?>
 	</table>
-
+	<div class="pagination">
+			<ul>
+			<?php
+				$this->Paginator->options(array(
+						'url' => $url,
+						'update' => '.span12',
+						'evalScripts' => true,
+						'before' => '$(".progress").show()',
+						'complete' => '$(".progress").hide()',
+				));
+					echo $this->Paginator->prev('&laquo; ' . __('previous'), array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'prev disabled', 'escape' => false, 'disabledTag' => 'span'));
+					echo $this->Paginator->numbers(array('modulus' => 20, 'separator' => '', 'tag' => 'li', 'currentClass' => 'active', 'currentTag' => 'span'));
+					echo $this->Paginator->next(__('next') . ' &raquo;', array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'next disabled', 'escape' => false, 'disabledTag' => 'span'));
+			?>
+			</ul>
+	</div>
 </div>
 <?php
 	echo $this->element('side_menu', array('menuList' => 'feeds', 'menuItem' => 'add'));
@@ -90,4 +122,4 @@ function freetextFeedFetchSelected() {
 }
 
 
-</script>
+</script>

app/webroot/js/misp2.4.58.js

@@ -2578,6 +2578,7 @@ function feedFormUpdate() {
 				$('#DeltaMergeDiv').show();
 			}
 			$('#settingsCsvValueDiv').show();
+			$('#settingsCsvSeparatorDiv').show();
 			break;
 	}
 }