mirror of https://github.com/MISP/MISP
new: Various new feed features
- import feed descriptor json pastes to add a list of pre-defined feeds - improvements to the feed pull (a single non validating attribute shouldn't break the process) - altered the saving of the attributes to happen in chunks during a feed pull to avoid very large feeds from stalling the process - split the feeds into 3 tabs: default, custom, allpull/1856/head
iglocska
parent
7f8a81e161
commit
76e9398df9
|
|
@ -25,6 +25,12 @@ class FeedsController extends AppController {
|
|||
}
|
||||
|
||||
public function index() {
|
||||
$scope = isset($this->passedArgs['scope']) ? $this->passedArgs['scope'] : 'default';
|
||||
if ($scope !== 'all') {
|
||||
$this->paginate['conditions'][] = array(
|
||||
'Feed.default' => $scope == 'custom' ? 0 : 1
|
||||
);
|
||||
}
|
||||
$data = $this->paginate();
|
||||
$this->loadModel('Event');
|
||||
foreach ($data as $key => $value) {
|
||||
|
|
@ -44,6 +50,7 @@ class FeedsController extends AppController {
|
|||
}
|
||||
return $this->RestResponse->viewData($data, $this->response->type());
|
||||
}
|
||||
$this->set('scope', $scope);
|
||||
$this->set('feeds', $data);
|
||||
$this->loadModel('Event');
|
||||
$this->set('feed_types', $this->Feed->feed_types);
|
||||
|
|
@ -54,6 +61,42 @@ class FeedsController extends AppController {
|
|||
$feed = $this->Feed->find('first', array('conditions' => array('Feed.id' => $feedId)));
|
||||
}
|
||||
|
||||
public function importFeeds() {
|
||||
if ($this->request->is('post')) {
|
||||
$feeds = json_decode($this->request->data['Feed']['json'], true);
|
||||
if (empty($feeds)) throw new NotFoundException('No valid ');
|
||||
$existingFeeds = $this->Feed->find('all', array());
|
||||
$fail = $success = 0;
|
||||
foreach ($feeds as $feed) {
|
||||
$found = false;
|
||||
foreach ($existingFeeds as $existingFeed) {
|
||||
if ($existingFeed['Feed']['url'] == $feed['Feed']['url']) {
|
||||
$found = true;
|
||||
}
|
||||
}
|
||||
if (!$found) {
|
||||
$this->Feed->create();
|
||||
if (!$this->Feed->save($feed, true, array('name', 'provider', 'url', 'rules', 'source_format', 'fixed_event', 'delta_merge', 'override_ids', 'publish', 'settings'))) {
|
||||
$fail++;
|
||||
$this->Session->setFlash('Could not save feeds. Reason: ' . json_encode($this->Feed->validationErros));
|
||||
} else {
|
||||
$success++;
|
||||
}
|
||||
}
|
||||
}
|
||||
$message = $success . ' new feeds added.';
|
||||
if ($fail) {
|
||||
$message .= ' ' . $fail . ' feeds could not be added (possibly because they already exist)';
|
||||
}
|
||||
if ($this->_isRest()) {
|
||||
|
||||
} else {
|
||||
$this->Session->setFlash($message);
|
||||
$this->redirect(array('controller' => 'Feeds', 'action' => 'index', 'all'));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function add() {
|
||||
if ($this->request->is('post')) {
|
||||
$error = false;
|
||||
|
|
@ -135,7 +178,7 @@ class FeedsController extends AppController {
|
|||
}
|
||||
$result = $this->Feed->save($feed);
|
||||
if ($result) {
|
||||
$feedCache = APP . 'tmp' . DS . 'cache' . DS . 'feeds' . DS . intval($feed['Feed']['id']) . '.cache';
|
||||
$feedCache = APP . 'tmp' . DS . 'cache' . DS . 'feeds' . DS . intval($feedId) . '.cache';
|
||||
if (file_exists($feedCache)) {
|
||||
unlink($feedCache);
|
||||
}
|
||||
|
|
@ -307,6 +350,10 @@ class FeedsController extends AppController {
|
|||
$params = array();
|
||||
// params is passed as reference here, the pagination happens in the method, which isn't ideal but considering the performance gains here it's worth it
|
||||
$resultArray = $this->Feed->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format'], $currentPage, 60, $params);
|
||||
// we want false as a valid option for the split fetch, but we don't want it for the preview
|
||||
if ($resultArray == false) {
|
||||
$resultArray = array();
|
||||
}
|
||||
$this->params->params['paging'] = array($this->modelClass => $params);
|
||||
$resultArray = $this->Feed->getFreetextFeedCorrelations($resultArray);
|
||||
// remove all duplicates
|
||||
|
|
@ -331,6 +378,10 @@ class FeedsController extends AppController {
|
|||
if ($feed['Feed']['source_format'] != 'csv') throw new MethodNotAllowedException('Invalid feed type.');
|
||||
$HttpSocket = $syncTool->setupHttpSocketFeed($feed);
|
||||
$resultArray = $this->Feed->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format'], $currentPage);
|
||||
// we want false as a valid option for the split fetch, but we don't want it for the preview
|
||||
if ($resultArray == false) {
|
||||
$resultArray = array();
|
||||
}
|
||||
$resultArray = $this->Feed->getFreetextFeedCorrelations($resultArray);
|
||||
// remove all duplicates
|
||||
foreach ($resultArray as $k => $v) {
|
||||
|
|
|
|||
|
|
@ -173,7 +173,7 @@ class ComplexTypeTool {
|
|||
if (count($compositeParts) == 2) {
|
||||
if ($this->__resolveFilename($compositeParts[0])) {
|
||||
foreach ($this->__hexHashTypes as $k => $v) {
|
||||
if (strlen($compositeParts[1]) == $k && preg_match("#[0-9a-f]{" . $k . "}$#i", $compositeParts[1])) return array('types' => $v['composite'], 'to_ids' => true, 'default_type' => $v['composite'][0]);
|
||||
if (strlen($compositeParts[1]) == $k && preg_match("#[0-9a-f]{" . $k . "}$#i", $compositeParts[1])) return array('types' => $v['composite'], 'to_ids' => true, 'default_type' => $v['composite'][0], 'value' => $input);
|
||||
}
|
||||
if (preg_match('#^[0-9]+:[0-9a-zA-Z\/\+]+:[0-9a-zA-Z\/\+]+$#', $compositeParts[1]) && !preg_match('#^[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$#', $compositeParts[1])) {
|
||||
return array('types' => array('ssdeep'), 'to_ids' => true, 'default_type' => 'filename|ssdeep', 'value' => $input);
|
||||
|
|
@ -184,7 +184,7 @@ class ComplexTypeTool {
|
|||
|
||||
// check for hashes
|
||||
foreach ($this->__hexHashTypes as $k => $v) {
|
||||
if (strlen($input) == $k && preg_match("#[0-9a-f]{" . $k . "}$#i", $input)) return array('types' => $v['single'], 'to_ids' => true, 'default_type' => $v['single'][0]);
|
||||
if (strlen($input) == $k && preg_match("#[0-9a-f]{" . $k . "}$#i", $input)) return array('types' => $v['single'], 'to_ids' => true, 'default_type' => $v['single'][0], 'value' => $input);
|
||||
}
|
||||
if (preg_match('#^[0-9]+:[0-9a-zA-Z\/\+]+:[0-9a-zA-Z\/\+]+$#', $input) && !preg_match('#^[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$#', $input)) return array('types' => array('ssdeep'), 'to_ids' => true, 'default_type' => 'ssdeep', 'value' => $input);
|
||||
$inputRefanged = $input;
|
||||
|
|
|
|||
|
|
@ -120,6 +120,9 @@ class Feed extends AppModel {
|
|||
$params = $customPagination->createPaginationRules($data, array('page' => $page, 'limit' => $limit), 'Feed', $sort = false);
|
||||
if (!empty($page) && $page != 'all') {
|
||||
$start = ($page - 1) * $limit;
|
||||
if ($start > count($data)) {
|
||||
return false;
|
||||
}
|
||||
$data = array_slice($data, $start, $limit);
|
||||
}
|
||||
$data = implode("\n", $data);
|
||||
|
|
@ -459,28 +462,33 @@ class Feed extends AppModel {
|
|||
$job->id = $jobId;
|
||||
$job->saveField('message', 'Fetching data.');
|
||||
}
|
||||
$data = $this->getFreetextFeed($this->data, $HttpSocket, $this->data['Feed']['source_format']);
|
||||
foreach ($data as $key => $value) {
|
||||
$data[$key] = array(
|
||||
$temp = $this->getFreetextFeed($this->data, $HttpSocket, $this->data['Feed']['source_format'], 'all');
|
||||
foreach ($temp as $key => $value) {
|
||||
$data[] = array(
|
||||
'category' => $value['category'],
|
||||
'type' => $value['default_type'],
|
||||
'value' => $value['value'],
|
||||
'to_ids' => $value['to_ids']
|
||||
);
|
||||
}
|
||||
if ($jobId) {
|
||||
$job->saveField('progress', 50);
|
||||
$job->saveField('message', 'Saving data.');
|
||||
}
|
||||
$result = $this->saveFreetextFeedData($this->data, $data, $user);
|
||||
$message = 'Job complete.';
|
||||
if ($result !== true) {
|
||||
return false;
|
||||
}
|
||||
if ($jobId) {
|
||||
$job->saveField('progress', '100');
|
||||
$job->saveField('message', 'Job complete.');
|
||||
}
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
|
||||
public function saveFreetextFeedData($feed, $data, $user) {
|
||||
public function saveFreetextFeedData($feed, $data, $user, $jobId = false) {
|
||||
$this->Event = ClassRegistry::init('Event');
|
||||
$event = false;
|
||||
if ($feed['Feed']['fixed_event'] && $feed['Feed']['event_id']) {
|
||||
|
|
@ -515,7 +523,7 @@ class Feed extends AppModel {
|
|||
$to_delete = array();
|
||||
foreach ($data as $k => $dataPoint) {
|
||||
foreach ($event['Attribute'] as $attribute_key => $attribute) {
|
||||
if ($dataPoint['value'] == $attribute['value']) {
|
||||
if ($dataPoint['value'] == $attribute['value'] && $dataPoint['type'] == $attribute['type'] && $attribute['category'] == $dataPoint['category']) {
|
||||
unset($data[$k]);
|
||||
unset($event['Attribute'][$attribute_key]);
|
||||
}
|
||||
|
|
@ -540,8 +548,16 @@ class Feed extends AppModel {
|
|||
$data[$key]['sharing_group_id'] = $feed['Feed']['sharing_group_id'];
|
||||
$data[$key]['to_ids'] = $feed['Feed']['override_ids'] ? 0 : $data[$key]['to_ids'];
|
||||
}
|
||||
if (!$this->Event->Attribute->saveMany($data)) {
|
||||
return 'Could not save the parsed attributes.';
|
||||
if ($jobId) {
|
||||
$job = ClassRegistry::init('Job');
|
||||
$job->id = $jobId;
|
||||
}
|
||||
$data = array_chunk($data, 100);
|
||||
foreach ($data as $k => $chunk) {
|
||||
$this->Event->Attribute->saveMany($chunk);
|
||||
if ($jobId) {
|
||||
$job->saveField('progress', 50 + round((50 * ((($k + 1) * 100) / count($data)))));
|
||||
}
|
||||
}
|
||||
if ($feed['Feed']['publish']) {
|
||||
$this->Event->publishRouter($event['Event']['id'], null, $user);
|
||||
|
|
|
|||
|
|
@ -312,6 +312,7 @@
|
|||
case 'feeds': ?>
|
||||
<li id='liindex'><a href="<?php echo $baseurl;?>/feeds/index">List Feeds</a></li>
|
||||
<li id='liadd'><a href="<?php echo $baseurl;?>/feeds/add">Add Feed</a></li>
|
||||
<li id='liadd'><a href="<?php echo $baseurl;?>/feeds/importFeeds">Import Feeds from JSON</a></li>
|
||||
<?php if ($menuItem === 'edit'): ?>
|
||||
<li class="active"><a href="#">Edit Feed</a></li>
|
||||
<?php elseif ($menuItem === 'previewIndex'): ?>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,25 @@
|
|||
<div class="feed form">
|
||||
<?php echo $this->Form->create('Feed');?>
|
||||
<fieldset>
|
||||
<legend>Paste feed data</legend>
|
||||
<p>Paste a MISP feed metadata JSON below to add feeds.</p>
|
||||
<div>
|
||||
<?php
|
||||
echo $this->Form->input('json', array(
|
||||
'div' => 'input clear',
|
||||
'placeholder' => 'Feed metadata JSON',
|
||||
'class' => 'form-control span6',
|
||||
'type' => 'textarea',
|
||||
'rows' => 18
|
||||
));
|
||||
?>
|
||||
</div>
|
||||
</fieldset>
|
||||
<?php
|
||||
echo $this->Form->button('Add', array('class' => 'btn btn-primary'));
|
||||
echo $this->Form->end();
|
||||
?>
|
||||
</div>
|
||||
<?php
|
||||
echo $this->element('side_menu', array('menuList' => 'feeds', 'menuItem' => 'import'));
|
||||
?>
|
||||
|
|
@ -15,6 +15,11 @@
|
|||
echo $this->Paginator->next(__('next') . ' »', array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'next disabled', 'escape' => false, 'disabledTag' => 'span'));
|
||||
?>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="tabMenuFixedContainer" style="display:inline-block;">
|
||||
<span class="tabMenuFixed tabMenuFixedCenter tabMenuSides useCursorPointer <?php echo $scope == 'default' ? 'tabMenuActive' : ''; ?>" onclick="window.location='/feeds/index/scope:default'">Default feeds</span>
|
||||
<span class="tabMenuFixed tabMenuFixedCenter tabMenuSides useCursorPointer <?php echo $scope == 'custom' ? 'tabMenuActive' : ''; ?> " onclick="window.location='/feeds/index/scope:custom'">Custom Feeds</span>
|
||||
<span class="tabMenuFixed tabMenuFixedCenter tabMenuSides useCursorPointer <?php echo $scope == 'all' ? 'tabMenuActive' : ''; ?> " onclick="window.location='/feeds/index/scope:all'">All Feeds</span>
|
||||
</div>
|
||||
<table class="table table-striped table-hover table-condensed">
|
||||
<tr>
|
||||
|
|
|
|||
Loading…
Reference in New Issue