MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

new: various improvements 

- use the feed uuid caches to link directly to affected MISP events
- various UI improvements
- Feed preview pagination / POSTed event ID filters added
pull/2706/head
iglocska 2017-12-05 00:05:11 +01:00
parent e760ba7b6a
commit 4f6dba5f35
9 changed files with 133 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/Controller/AppController.php
View File

@ -46,7 +46,7 @@ class AppController extends Controller {
public $helpers = array('Utility');
private $__queryVersion = '24';
private $__queryVersion = '25';
public $pyMispVersion = '2.4.82';
public $phpmin = '5.6.5';
public $phprec = '7.0.16';

19
app/Controller/EventsController.php
View File

@ -700,15 +700,14 @@ class EventsController extends AppController {
$rules = array('published', 'eventid', 'tag', 'date', 'eventinfo', 'threatlevel', 'distribution', 'analysis', 'attribute', 'hasproposal');
if ($this->_isSiteAdmin()) $rules[] = 'email';
if (Configure::read('MISP.showorg')) {
$orgs = $this->Event->find('list', array(
'recursive' => -1,
'fields' => array('Orgc.name'),
'contain' => array('Orgc'),
'conditions' => $conditions,
'group' => array('LOWER(Orgc.name)','Event.id', 'Orgc.name')
$orgs = $this->Event->Orgc->find('list', array(
'conditions' => array(),
'recursive' => -1,
'fields' => array('Orgc.id', 'Orgc.name'),
'sort' => array('lower(Orgc.name) asc')
));
$this->set('showorg', true);
$this->set('orgs', $this->_arrayToValuesIndexArray($orgs));
$this->set('orgs', $orgs);
$rules[] = 'org';
} else {
$this->set('showorg', false);
@ -2173,8 +2172,8 @@ class EventsController extends AppController {
// Usage: csv($key, $eventid) - key can be a valid auth key or the string 'download'. Download requires the user to be logged in interactively and will generate a .csv file
// $eventid can be one of 3 options: left empty it will get all the visible to_ids attributes,
// $ignore is a flag that allows the export tool to ignore the ids flag. 0 = only IDS signatures, 1 = everything.
public function csv($key, $eventid = false, $ignore = false, $tags = false, $category = false, $type = false, $includeContext = false, $from = false, $to = false, $last = false, $headerless = false, $enforceWarninglist = false) {
$paramArray = array('eventid', 'ignore', 'tags', 'category', 'type', 'includeContext', 'from', 'to', 'last', 'headerless', 'enforceWarninglist');
public function csv($key, $eventid = false, $ignore = false, $tags = false, $category = false, $type = false, $includeContext = false, $from = false, $to = false, $last = false, $headerless = false, $enforceWarninglist = false, $value = false) {
$paramArray = array('eventid', 'ignore', 'tags', 'category', 'type', 'includeContext', 'from', 'to', 'last', 'headerless', 'enforceWarninglist', 'value');
if ($this->request->is('post')) {
if (empty($this->request->data)) {
return $this->RestResponse->throwException(400, 'Either specify the search terms in the url, or POST a json or xml with the filter parameters.', 'csv', true);
@ -2265,7 +2264,7 @@ class EventsController extends AppController {
if (isset($data['request']['obj_attributes'])) $requested_obj_attributes = $data['request']['obj_attributes'];
if (isset($events)) {
foreach ($events as $eventid) {
$attributes = $this->Event->csv($user, $eventid, $ignore, $list, false, $category, $type, $includeContext, $enforceWarninglist);
$attributes = $this->Event->csv($user, $eventid, $ignore, $list, false, $category, $type, $includeContext, false, false, false, $enforceWarninglist, $value);
$attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true);
foreach ($attributes as $attribute) {
$line1 = '';

32
app/Controller/FeedsController.php
View File

@ -21,6 +21,7 @@ class FeedsController extends AppController {
public function beforeFilter() {
parent::beforeFilter();
$this->Security->unlockedActions = array('previewIndex');
if (!$this->_isSiteAdmin()) throw new MethodNotAllowedException('You don\'t have the required privileges to do that.');
}
@ -335,22 +336,45 @@ class FeedsController extends AppController {
if (!empty($this->Feed->data['Feed']['settings'])) {
$this->Feed->data['Feed']['settings'] = json_decode($this->Feed->data['Feed']['settings'], true);
}
$params = array();
if ($this->request->is('post')) {
$params = $this->request->data['Feed'];
}
if ($this->Feed->data['Feed']['source_format'] == 'misp') {
return $this->__previewIndex($this->Feed->data);
return $this->__previewIndex($this->Feed->data, $params);
} else if (in_array($this->Feed->data['Feed']['source_format'], array('freetext', 'csv'))) {
return $this->__previewFreetext($this->Feed->data);
}
}
private function __previewIndex($feed) {
private function __previewIndex($feed, $filterParams = array()) {
if (isset($this->passedArgs['pages'])) $currentPage = $this->passedArgs['pages'];
else $currentPage = 1;
$urlparams = '';
App::uses('CustomPaginationTool', 'Tools');
$customPagination = new CustomPaginationTool();
$passedArgs = array();
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocketFeed($feed);
$events = $this->Feed->getManifest($feed, $HttpSocket);
foreach ($filterParams as $k => $filter) {
if (!empty($filter)) {
$filterParams[$k] = json_decode($filter);
}
}
if (!empty($filterParams['eventid'])) {
foreach ($events as $k => $event) {
if (!in_array($k, $filterParams['eventid'])) {
unset($events[$k]);
continue;
}
}
}
$params = $customPagination->createPaginationRules($events, $this->passedArgs, $this->alias);
$this->params->params['paging'] = array($this->modelClass => $params);
$events = $customPagination->sortArray($events, $params, true);
if (is_array($events)) $customPagination->truncateByPagination($events, $params);
if ($this->_isRest()) {
return $this->RestResponse->viewData($events, $this->response->type());
}
@ -363,10 +387,6 @@ class FeedsController extends AppController {
$this->passedArgs['page'] = 0;
}
}
$params = $customPagination->createPaginationRules($events, $this->passedArgs, $this->alias);
$this->params->params['paging'] = array($this->modelClass => $params);
if (is_array($events)) $customPagination->truncateByPagination($events, $params);
else ($events = array());
$this->set('events', $events);
$this->loadModel('Event');
$threat_levels = $this->Event->ThreatLevel->find('all');

11
app/Lib/Tools/CustomPaginationTool.php
View File

@ -43,8 +43,7 @@ class CustomPaginationTool {
$items = array_slice($items, $params['current'] - 1, $params['limit']);
}
function applyRulesOnArray(&$items, $options, $model, $sort = 'id', $focusKey = 'uuid') {
$params = $this->createPaginationRules($items, $options, $model, $sort, $focusKey);
function sortArray($items, $params, $escapeReindex = false) {
if (isset($params['sort'])) {
$sortArray = array();
foreach ($items as $k => $item) {
@ -62,7 +61,13 @@ class CustomPaginationTool {
$items = array();
$items = $sortArray;
}
$items = array_values($items);
if (!$escapeReindex) $items = array_values($items);
return $items;
}
function applyRulesOnArray(&$items, $options, $model, $sort = 'id', $focusKey = 'uuid') {
$params = $this->createPaginationRules($items, $options, $model, $sort, $focusKey);
$items = $this->sortArray($items, $params);
if (!empty($params['options']['focus'])) {
foreach ($items as $k => $item) {
if ($item[$focusKey] == $params['options']['focus']) {

12
app/Model/Event.php
View File

@ -1791,7 +1791,7 @@ class Event extends AppModel {
$field = '"' . $field . '"';
}
public function csv($user, $eventid=false, $ignore=false, $attributeIDList = array(), $tags = false, $category = false, $type = false, $includeContext = false, $from = false, $to = false, $last = false, $enforceWarninglist = false) {
public function csv($user, $eventid=false, $ignore=false, $attributeIDList = array(), $tags = false, $category = false, $type = false, $includeContext = false, $from = false, $to = false, $last = false, $enforceWarninglist = false, $value = false) {
$this->recursive = -1;
$conditions = array();
// If we are not in the search result csv download function then we need to check what can be downloaded. CSV downloads are already filtered by the search function.
@ -1825,6 +1825,16 @@ class Event extends AppModel {
if (!$ignore) $conditions['AND']['Attribute.to_ids'] = 1;
if ($type) $conditions['AND']['Attribute.type'] = $type;
if ($category) $conditions['AND']['Attribute.category'] = $category;
if ($value) {
$temp = array(
'OR' => array(
'Attribute.value1' => $value,
'Attribute.value2' => $value
)
);
$conditions['AND'][] = $temp;
}
}
if ($eventid === 'search') {

44
app/Model/Feed.php
View File

@ -279,12 +279,17 @@ class Feed extends AppModel {
$objectsWithFeedHits = array();
$hashTable = array();
$hitIds = array();
$this->Event = ClassRegistry::init('Event');
foreach ($objects as $k => $object) {
$hashTable[$k] = md5($object['value']);
if (in_array($object['type'], $this->Event->Attribute->getCompositeTypes())) {
$value = explode('|', $object['value']);
$hashTable[$k] = md5($value[0]);
} else {
$hashTable[$k] = md5($object['value']);
}
$redis->sismember('misp:feed_cache:combined', $hashTable[$k]);
}
$results = $pipe->exec();
if (!$overrideLimit && count($objects) > 10000) {
foreach ($results as $k => $result) {
if ($result) {
@ -313,6 +318,41 @@ class Feed extends AppModel {
$objects[$hitIds[$k4]]['Feed'][] = $feed['Feed'];
}
}
if ($feed['Feed']['source_format'] == 'misp') {
$pipe = $redis->multi(Redis::PIPELINE);
$eventUuidHitPosition = array();
$i = 0;
foreach ($objects as $k => $object) {
if (isset($object['Feed'])) {
foreach ($object['Feed'] as $currentFeed) {
if ($feed['Feed']['id'] == $currentFeed['id']) {
$eventUuidHitPosition[$i] = $k;
$i++;
if (in_array($object['type'], $this->Event->Attribute->getCompositeTypes())) {
$value = explode('|', $object['value']);
$redis->smembers('misp:feed_cache:event_uuid_lookup:' . md5($value[0]));
} else {
$redis->smembers('misp:feed_cache:event_uuid_lookup:' . md5($object['value']));
}
}
}
}
}
$mispFeedHits = $pipe->exec();
foreach ($mispFeedHits as $feedhitPos => $f) {
foreach ($f as $url) {
$urlParts = explode('/', $url);
if (empty($event['Feed'][$urlParts[0]]['event_uuids']) || !in_array($urlParts[1], $event['Feed'][$urlParts[0]]['event_uuids'])) {
$event['Feed'][$urlParts[0]]['event_uuids'][] = $urlParts[1];
}
foreach ($objects[$eventUuidHitPosition[$feedhitPos]]['Feed'] as $tempKey => $tempFeed) {
if ($tempFeed['id'] == $urlParts[0]) {
$objects[$eventUuidHitPosition[$feedhitPos]]['Feed'][$tempKey]['event_uuids'][] = $urlParts[1];
}
}
}
}
}
}
}
}

23
app/View/Elements/Events/View/row_attribute.ctp
View File

@ -145,13 +145,30 @@
$popover = '';
foreach ($feed as $k => $v):
if ($k == 'id') continue;
$popover .= '<span class=\'bold black\'>' . Inflector::humanize(h($k)) . '</span>: <span class="blue">' . h($v) . '</span><br />';
if (is_array($v)) {
foreach ($v as $k2 => $v2) {
$v[$k2] = h($v2);
}
$v = implode('<br />', $v);
} else {
$v = h($v);
}
$popover .= '<span class=\'bold black\'>' . Inflector::humanize(h($k)) . '</span>: <span class="blue">' . $v . '</span><br />';
endforeach;
?>
<li style="padding-right: 0px; padding-left:0px;" data-toggle="popover" data-content="<?php echo h($popover);?>" data-trigger="hover"><span>
<li style="padding-right: 0px; padding-left:0px;"><span>
<?php
if ($isSiteAdmin):
echo $this->Html->link($feed['id'], array('controller' => 'feeds', 'action' => 'previewIndex', $feed['id']), array('style' => 'margin-right:3px;'));
if ($feed['source_format'] == 'misp'):
?>
<form action="<?php echo $baseurl; ?>/feeds/previewIndex/1" method="post" style="width=0px;">
<input type="hidden" name="data[Feed][eventid]" value="<?php echo h(json_encode($feed['event_uuids'], true)); ?>">
<input type="submit" class="linkButton useCursorPointer" value="<?php echo h($feed['id']); ?>" data-toggle="popover" data-content="<?php echo h($popover);?>" data-trigger="hover" />
</form>
<?php
else:
echo $this->Html->link($feed['id'], array('controller' => 'feeds', 'action' => 'previewIndex', $feed['id']), array('style' => 'margin-right:3px;'));
endif;
else:
?>
<span style="margin-right:3px;"><?php echo h($feed['id']);?></span>

5
app/View/Events/view.ctp
View File

@ -271,7 +271,10 @@
}
?>
<span data-toggle="popover" data-content="<?php echo h($popover); ?>" data-trigger="hover" style="white-space: nowrap;">
<a href="<?php echo $baseurl . 'feeds/view/' . h($relatedFeed['id']); ?>"><?php echo h($relatedFeed['name']) . ' (' . $relatedFeed['id'] . ')'; ?></a>
<form action="<?php echo $baseurl; ?>/feeds/previewIndex/<?php h($relatedFeed['id']); ?>" method="post">
<input type="hidden" name="data[Feed][eventid]" value="<?php echo h(json_encode($relatedFeed['event_uuids'], true)); ?>">
<input type="submit" class="linkButton useCursorPointer" value="<?php echo h($relatedFeed['name']) . ' (' . $relatedFeed['id'] . ')'; ?>" />
</form>
</span>
<?php
endforeach;

12
app/webroot/css/main.css
View File

@ -1970,6 +1970,18 @@ table tr:hover .down-expand-button {
color:black;
}
.linkButton {
background: none;
border: none;
padding: 0px;
color: red;
cursor: pointer;
margin: 0;
font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
font-size: 12px;
line-height: 20px;
}
@-webkit-keyframes rotation {
from {-webkit-transform: rotate(0deg);}
to {-webkit-transform: rotate(359deg);}