MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

new: Added caching and pagination to freetext/csv feeds

pull/1856/head
iglocska 2016-12-30 16:16:56 +01:00
parent 0db59a4bff
commit 7f8a81e161
10 changed files with 138 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -18,6 +18,7 @@
/app/tmp/cache/persistent/myapp*
/app/tmp/cache/views/myapp*
/app/files/*
/app/tmp/cache/feeds/*.cache
!/app/files/empty
!/app/files/scripts/
!/app/files/warninglists
@ -64,6 +65,7 @@
/app/tmp/cached_exports/csv_sig/*
/app/tmp/cached_exports/stix/*
/app/tmp/cached_exports/sha256/*
/app/tmp/cached_exports/bro/*
.gnupg
.smime
*.swp

23
app/Controller/FeedsController.php
View File

@ -70,6 +70,9 @@ class FeedsController extends AppController {
if (!isset($this->request->data['Feed']['settings'])) {
$this->request->data['Feed']['settings'] = array();
}
if (isset($this->request->data['Feed']['settings']['separator']) && empty($this->request->data['Feed']['settings']['separator'])) {
$this->request->data['Feed']['settings']['separator'] = ',';
}
if (empty($this->request->data['Feed']['target_event'])) {
$this->request->data['Feed']['target_event'] = 0;
}
@ -119,6 +122,9 @@ class FeedsController extends AppController {
if (!isset($this->request->data['Feed']['settings'])) {
$this->request->data['Feed']['settings'] = array();
}
if (isset($this->request->data['Feed']['settings']['separator']) && empty($this->request->data['Feed']['settings']['separator'])) {
$this->request->data['Feed']['settings']['separator'] = ',';
}
$this->request->data['Feed']['settings'] = json_encode($this->request->data['Feed']['settings']);
$fields = array('id', 'name', 'provider', 'enabled', 'rules', 'url', 'distribution', 'sharing_group_id', 'tag_id', 'fixed_event', 'event_id', 'publish', 'delta_merge', 'override_ids', 'settings');
$feed = array();
@ -129,6 +135,10 @@ class FeedsController extends AppController {
}
$result = $this->Feed->save($feed);
if ($result) {
$feedCache = APP . 'tmp' . DS . 'cache' . DS . 'feeds' . DS . intval($feed['Feed']['id']) . '.cache';
if (file_exists($feedCache)) {
unlink($feedCache);
}
$this->Session->setFlash('Feed updated.');
$this->redirect(array('controller' => 'feeds', 'action' => 'index'));
} else {
@ -286,11 +296,18 @@ class FeedsController extends AppController {
}
private function __previewFreetext($feed) {
if (isset($this->passedArgs['page'])) $currentPage = $this->passedArgs['page'];
else if (isset($this->passedArgs['page'])) $currentPage = $this->passedArgs['page'];
else $currentPage = 1;
$urlparams = '';
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
if (!in_array($feed['Feed']['source_format'], array('freetext', 'csv'))) throw new MethodNotAllowedException('Invalid feed type.');
$HttpSocket = $syncTool->setupHttpSocketFeed($feed);
$resultArray = $this->Feed->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format']);
$params = array();
// params is passed as reference here, the pagination happens in the method, which isn't ideal but considering the performance gains here it's worth it
$resultArray = $this->Feed->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format'], $currentPage, 60, $params);
$this->params->params['paging'] = array($this->modelClass => $params);
$resultArray = $this->Feed->getFreetextFeedCorrelations($resultArray);
// remove all duplicates
foreach ($resultArray as $k => $v) {
@ -307,11 +324,13 @@ class FeedsController extends AppController {
}
private function __previewCSV($feed) {
if (isset($this->passedArgs['pages'])) $currentPage = $this->passedArgs['pages'];
else $currentPage = 1;
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
if ($feed['Feed']['source_format'] != 'csv') throw new MethodNotAllowedException('Invalid feed type.');
$HttpSocket = $syncTool->setupHttpSocketFeed($feed);
$resultArray = $this->Feed->getFreetextFeed($feed, $HttpSocket);
$resultArray = $this->Feed->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format'], $currentPage);
$resultArray = $this->Feed->getFreetextFeedCorrelations($resultArray);
// remove all duplicates
foreach ($resultArray as $k => $v) {

1
app/Lib/Tools/ComplexTypeTool.php
View File

@ -99,6 +99,7 @@ class ComplexTypeTool {
public function checkCSV($input, $settings = array()) {
$delimiter = isset($settings['delimiter']) ? $settings['delimiter'] : ",";
$lines = explode("\n", $input);
unset($input);
$values = !empty($settings['value']) ? $settings['value'] : array();
if (!is_array($values)) {
$values = explode(',', $values);

28
app/Model/Feed.php
View File

@ -96,16 +96,38 @@ class Feed extends AppModel {
return $events;
}
public function getFreetextFeed($feed, $HttpSocket, $type = 'freetext') {
public function getFreetextFeed($feed, $HttpSocket, $type = 'freetext', $page = 1, $limit = 60, &$params = array()) {
$result = array();
$feedCache = APP . 'tmp' . DS . 'cache' . DS . 'feeds' . DS . intval($feed['Feed']['id']) . '.cache';
$doFetch = true;
if (file_exists($feedCache)) {
$file = new File($feedCache);
if (time() - $file->lastChange() < 600) {
$doFetch = false;
$data = file_get_contents($feedCache);
}
}
if ($doFetch) {
$response = $HttpSocket->get($feed['Feed']['url'], '', array());
if ($response->code == 200) {
$data = $response->body;
file_put_contents($feedCache, $data);
}
}
$data = explode("\n", $data);
App::uses('CustomPaginationTool', 'Tools');
$customPagination = new CustomPaginationTool();
$params = $customPagination->createPaginationRules($data, array('page' => $page, 'limit' => $limit), 'Feed', $sort = false);
if (!empty($page) && $page != 'all') {
$start = ($page - 1) * $limit;
$data = array_slice($data, $start, $limit);
}
$data = implode("\n", $data);
App::uses('ComplexTypeTool', 'Tools');
$complexTypeTool = new ComplexTypeTool();
$this->Warninglist = ClassRegistry::init('Warninglist');
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
$resultArray = $complexTypeTool->checkComplexRouter($response->body, $type, isset($feed['Feed']['settings'][$type]) ? $feed['Feed']['settings'][$type] : array());
}
$resultArray = $complexTypeTool->checkComplexRouter($data, $type, isset($feed['Feed']['settings'][$type]) ? $feed['Feed']['settings'][$type] : array());
$this->Attribute = ClassRegistry::init('Attribute');
foreach ($resultArray as $key => $value) {
$resultArray[$key]['category'] = $this->Attribute->typeDefinitions[$value['default_type']]['default_category'];

12
app/View/Feeds/add.ctp
View File

@ -58,6 +58,18 @@
));
?>
</div>
<div id="settingsCsvSeparatorDiv" class="optionalField">
<?php
echo $this->Form->input('Feed.settings.csv.separator', array(
'label' => 'Separator',
'title' => 'Set the default CSV separator (default = ",")',
'div' => 'input clear',
'placeholder' => ',',
'class' => 'form-control span6',
'value' => ','
));
?>
</div>
<div id="PublishDiv" class="input clear optionalField">
<?php
echo $this->Form->input('publish', array(

11
app/View/Feeds/edit.ctp
View File

@ -58,6 +58,17 @@
));
?>
</div>
<div id="settingsCsvSeparatorDiv" class="optionalField">
<?php
echo $this->Form->input('Feed.settings.csv.separator', array(
'label' => 'Separator',
'title' => 'Set the default CSV separator (default = ",")',
'div' => 'input clear',
'placeholder' => ',',
'class' => 'form-control span6'
));
?>
</div>
<div id="PublishDiv" class="input clear optionalField">
<?php
echo $this->Form->input('publish', array(

34
app/View/Feeds/freetext_index.ctp
View File

@ -8,6 +8,23 @@
<?php
echo $this->Form->end();
?>
<div class="pagination">
<ul>
<?php
$url = array_merge(array('controller' => 'feeds', 'action' => 'previewIndex', $feed['Feed']['id']), $this->request->named);
$this->Paginator->options(array(
'url' => $url,
'update' => '.span12',
'evalScripts' => true,
'before' => '$(".progress").show()',
'complete' => '$(".progress").hide()',
));
echo $this->Paginator->prev('&laquo; ' . __('previous'), array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'prev disabled', 'escape' => false, 'disabledTag' => 'span'));
echo $this->Paginator->numbers(array('modulus' => 20, 'separator' => '', 'tag' => 'li', 'currentClass' => 'active', 'currentTag' => 'span'));
echo $this->Paginator->next(__('next') . ' &raquo;', array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'next disabled', 'escape' => false, 'disabledTag' => 'span'));
?>
</ul>
</div>
<table class="table table-striped table-hover table-condensed">
<tr>
<th><input class="select_all" type="checkbox" onClick="toggleAllAttributeCheckboxes();" /></th>
@ -56,7 +73,22 @@
endforeach;
?>
</table>
<div class="pagination">
<ul>
<?php
$this->Paginator->options(array(
'url' => $url,
'update' => '.span12',
'evalScripts' => true,
'before' => '$(".progress").show()',
'complete' => '$(".progress").hide()',
));
echo $this->Paginator->prev('&laquo; ' . __('previous'), array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'prev disabled', 'escape' => false, 'disabledTag' => 'span'));
echo $this->Paginator->numbers(array('modulus' => 20, 'separator' => '', 'tag' => 'li', 'currentClass' => 'active', 'currentTag' => 'span'));
echo $this->Paginator->next(__('next') . ' &raquo;', array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'next disabled', 'escape' => false, 'disabledTag' => 'span'));
?>
</ul>
</div>
</div>
<?php
echo $this->element('side_menu', array('menuList' => 'feeds', 'menuItem' => 'add'));

0
app/tmp/cache/feeds/empty vendored Normal file
View File

1
app/webroot/js/misp2.4.58.js
View File

@ -2578,6 +2578,7 @@ function feedFormUpdate() {
$('#DeltaMergeDiv').show();
}
$('#settingsCsvValueDiv').show();
$('#settingsCsvSeparatorDiv').show();
break;
}
}