MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'feature/sync/timestamp' of https://github.com/MISP/MISP into feature/sync/timestamp

pull/217/head
iglocska 2013-06-19 16:22:55 +02:00
parent c436d8ed67 8fe83c333f
commit 20f8d18cc8
8 changed files with 134 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
app/Controller/AppController.php
View File

@ -363,4 +363,63 @@ class AppController extends Controller {
//}
return false;
}
public function reportValidationIssuesEvents() {
// search for validation problems in the events
if (!self::_isAdmin()) throw new NotFoundException();
print ("<h2>Listing invalid event validations</h2>");
$this->loadModel('Event');
// first remove executing some Behaviors because of Noud's crappy code
$this->Event->Behaviors->detach('Regexp');
$this->Event->Behaviors->detach('Blacklist');
// get all events..
$events = $this->Event->find('all', array('recursive' => -1));
// for all events..
foreach ($events as $event) {
$this->Event->set($event);
if ($this->Event->validates()) {
// validates
} else {
$errors = $this->Event->validationErrors;
print ("<h3>Validation errors for event: " . $event['Event']['id'] . "</h3><pre>");
print_r($errors);
print ("</pre><p>Event details:</p><pre>");
print_r($event);
print ("</pre><br/>");
}
}
}
public function reportValidationIssuesAttributes() {
// TODO improve performance of this function by eliminating the additional SQL query per attribute
// search for validation problems in the attributes
if (!self::_isAdmin()) throw new NotFoundException();
print ("<h2>Listing invalid attribute validations</h2>");
$this->loadModel('Attribute');
// first remove executing some Behaviors because of Noud's crappy code
$this->Attribute->Behaviors->detach('Regexp');
$this->Attribute->Behaviors->detach('Blacklist');
// for efficiency reasons remove the unique requirement
$this->Attribute->validator()->remove('value', 'unique');
// get all attributes..
$attributes = $this->Attribute->find('all', array('recursive' => -1));
// for all attributes..
foreach ($attributes as $attribute) {
$this->Attribute->set($attribute);
if ($this->Attribute->validates()) {
// validates
} else {
$errors = $this->Attribute->validationErrors;
print ("<h3>Validation errors for attribute: " . $attribute['Attribute']['id'] . "</h3><pre>");
print_r($errors);
print ("</pre><p>Attribute details:</p><pre>");
print_r($attribute);
print ("</pre><br/>");
}
}
}
}

42
app/Controller/EventsController.php
View File

@ -436,9 +436,6 @@ class EventsController extends AppController {
// Moved out of if (!$fromXML), since we might get a restful event without the orgc/timestamp set
if (!isset ($data['Event']['orgc'])) $data['Event']['orgc'] = $data['Event']['org'];
if ($fromXml) {
// FIXME FIXME chri: temporary workaround for unclear org, orgc, from
//$data['Event']['orgc'] = $data['Event']['org'];
//$data['Event']['from'] = $data['Event']['org'];
// Workaround for different structure in XML/array than what CakePHP expects
$this->Event->cleanupEventArrayFromXML($data);
// the event_id field is not set (normal) so make sure no validation errors are thrown
@ -496,27 +493,27 @@ class EventsController extends AppController {
} else {
return 'Event exists and is the same or newer.';
}
$fieldList = array(
'Event' => array('date', 'risk', 'analysis', 'info', 'published', 'uuid', 'from', 'distribution', 'timestamp'),
'Attribute' => array('event_id', 'category', 'type', 'value', 'value1', 'value2', 'to_ids', 'uuid', 'revision', 'distribution', 'timestamp')
$fieldList = array(
'Event' => array('date', 'risk', 'analysis', 'info', 'published', 'uuid', 'from', 'distribution', 'timestamp'),
'Attribute' => array('event_id', 'category', 'type', 'value', 'value1', 'value2', 'to_ids', 'uuid', 'revision', 'distribution', 'timestamp')
);
if (isset($data['Event']['Attribute'])) {
if (isset($data['Event']['Attribute'])) {
foreach ($data['Event']['Attribute'] as $k => &$attribute) {
$existingAttribute = $this->Event->Attribute->findByUuid($attribute['uuid']);
if (count($existingAttribute)) {
$data['Event']['Attribute'][$k]['id'] = $existingAttribute['Attribute']['id'];
// Check if the attribute's timestamp is bigger than the one that already exists.
// If yes, it means that it's newer, so insert it. If no, it means that it's the same attribute or older - don't insert it, insert the old attribute.
// Alternatively, we could unset this attribute from the request, but that could lead with issues if we decide that we want to start deleting attributes that don't exist in a pushed event.
if ($data['Event']['Attribute'][$k]['timestamp'] > $existingAttribute['Attribute']['timestamp']) {
} else {
unset($data['Event']['Attribute'][$k]);
}
$data['Event']['Attribute'][$k]['id'] = $existingAttribute['Attribute']['id'];
// Check if the attribute's timestamp is bigger than the one that already exists.
// If yes, it means that it's newer, so insert it. If no, it means that it's the same attribute or older - don't insert it, insert the old attribute.
// Alternatively, we could unset this attribute from the request, but that could lead with issues if we decide that we want to start deleting attributes that don't exist in a pushed event.
if ($data['Event']['Attribute'][$k]['timestamp'] > $existingAttribute['Attribute']['timestamp']) {
} else {
unset($data['Event']['Attribute'][$k]);
}
} else {
unset($data['Event']['Attribute'][$k]['id']);
}
}
}
}
}
$this->Event->cleanupEventArrayFromXML($data);
$saveResult = $this->Event->saveAssociated($data, array('validate' => true, 'fieldList' => $fieldList));
@ -607,7 +604,7 @@ class EventsController extends AppController {
} else {
$message = 'This would be a downgrade...';
$this->set('event', $existingEvent);
$this->view($existingEvent['Event']['id']);
$this->view($existingEvent['Event']['id']);
$this->render('view');
return true;
}
@ -1765,13 +1762,6 @@ class EventsController extends AppController {
return $toReturn;
}
/**
* generateAllFor<FieldName>
**/
public function generateAllFor($field) {
parent::generateAllFor($field);
}
public function downloadSearchResult() {
$idList = $this->Session->read('search_find_idlist');
$this->Session->write('search_find_idlist', '');

56
app/Controller/ServersController.php
View File

@ -137,12 +137,21 @@ class ServersController extends AppController {
$this->redirect(array('action' => 'index'));
}
public function pull($id = null, $full=false) {
/**
* Pull one or more events with attributes from a remote instance.
* Set $technique to
* full - download everything
* incremental - only new events
* <int> - specific id of the event to pull
* For example to download event 10 from server 2 to /servers/pull/2/5
* @param int $id The id of the server
* @param unknown_type $technique
* @throws MethodNotAllowedException
* @throws NotFoundException
*/
public function pull($id = null, $technique=false) {
// TODO should we de-activate data validation for type and category / and or mapping? Maybe other instances have other configurations that are incompatible.
if (!$this->_IsSiteAdmin() && !($this->Server->organization == $this->Auth->user('org') && $this->checkAction('perm_sync'))) $this->redirect(array('controller' => 'servers', 'action' => 'index'));
if (!$this->request->is('post')) {
throw new MethodNotAllowedException();
}
$this->Server->id = $id;
if (!$this->Server->exists()) {
throw new NotFoundException(__('Invalid server'));
@ -155,7 +164,8 @@ class ServersController extends AppController {
$this->redirect(array('action' => 'index'));
}
if ("full" == $full) {
$eventIds = array();
if ("full" == $technique) {
// get a list of the event_ids on the server
$eventIds = $this->Event->getEventIdsFromServer($this->Server->data);
// FIXME this is not clean at all ! needs to be refactored with try catch error handling/communication
@ -169,6 +179,16 @@ class ServersController extends AppController {
// reverse array of events, to first get the old ones, and then the new ones
$eventIds = array_reverse($eventIds);
} elseif ("incremental" == $technique) {
// TODO incremental pull
throw new NotFoundException('Sorry, this is not yet implemented');
} elseif (true == $technique) {
$eventIds[] = intval($technique);
}
// now process the $eventIds to pull each of the events sequentially
if (!empty($eventIds)) {
$successes = array();
$fails = array();
// download each event
@ -255,16 +275,11 @@ class ServersController extends AppController {
// add data for newly imported events
$event['Event']['info'] .= "\n Imported from " . $this->Server->data['Server']['url'];
$passAlong = $this->Server->data['Server']['url'];
try {
$result = $eventsController->_add($event, $fromXml = true, $this->Server->data['Server']['organization'], $passAlong, true);
} catch (MethodNotAllowedException $e) {
if ($e->getMessage() == 'Event already exists') {
//$successes[] = $eventId; // commented given it's in a catch..
continue;
}
}
$result = $eventsController->_add($event, $fromXml = true, $this->Server->data['Server']['organization'], $passAlong, true);
if ($result) $successes[] = $eventId;
else $fails[$eventId] = 'failed';
else {
$fails[$eventId] = 'Failed (partially?) because of validation errors: '. print_r($eventsController->Event->validationErrors, true);
}
} else {
$result = $eventsController->_edit($event, $existingEvent['Event']['id']);
if ($result === 'success') $successes[] = $eventId;
@ -272,7 +287,7 @@ class ServersController extends AppController {
}
} else {
// error
$fails[$eventId] = 'failed';
$fails[$eventId] = 'failed downloading the event';
}
}
if (count($fails) > 0) {
@ -287,20 +302,13 @@ class ServersController extends AppController {
$this->Server->save($event, array('fieldList' => array('lastpulledid', 'url')));
}
} else {
// TODO incremental pull
// lastpulledid
throw new NotFoundException('Sorry, this is not yet implemented');
// increment lastid based on the highest ID seen
}
$this->set('successes', $successes);
$this->set('fails', $fails);
}
public function push($id = null, $full=false) {
public function push($id = null, $technique=false) {
if ($this->Auth->user('org') != 'ADMIN' && !($this->Server->organization == $this->Auth->user('org') && $this->checkAction('perm_sync'))) $this->redirect(array('controller' => 'servers', 'action' => 'index'));
if (!$this->request->is('post')) {
throw new MethodNotAllowedException();
@ -318,7 +326,7 @@ class ServersController extends AppController {
$this->redirect(array('action' => 'index'));
}
if ("full" == $full) $lastpushedid = 0;
if ("full" == $technique) $lastpushedid = 0;
else $lastpushedid = $this->Server->data['Server']['lastpushedid'];
$findParams = array(

29
app/View/Attributes/index.ctp
View File

@ -69,21 +69,20 @@ foreach ($attributes as $attribute):
<?php echo $attribute['Attribute']['category']; ?>&nbsp;</td>
<td title="<?php echo $typeDefinitions[$attribute['Attribute']['type']]['desc'];?>" class="short" onclick="document.location='/events/view/<?php echo $attribute['Event']['id'];?>';">
<?php echo $attribute['Attribute']['type']; ?>&nbsp;</td>
<td onclick="document.location='/events/view/<?php echo $attribute['Event']['id'];?>';">
<?php
$sigDisplay = nl2br(h($attribute['Attribute']['value']));
if ($isSearch == 1 && !empty($replacePairs)) {
// highlight the keywords if there are any
$sigDisplay = $this->Highlight->highlighter($sigDisplay, $replacePairs);
}
if ('attachment' == $attribute['Attribute']['type'] || 'malware-sample' == $attribute['Attribute']['type']) {
?><a href="/attributes/download/<?php echo $attribute['Attribute']['id'];?>"><?php echo $sigDisplay; ?></a><?php
} elseif ('link' == $attribute['Attribute']['type']) {
?><a href="<?php echo nl2br(h($attribute['Attribute']['value']));?>"><?php echo $sigDisplay; ?></a><?php
} else {
echo $sigDisplay;
}
?>&nbsp;</td>
<td class="showspaces" onclick="document.location='/events/view/<?php echo $attribute['Event']['id'];?>';"><?php
$sigDisplay = nl2br(h($attribute['Attribute']['value']));
if ($isSearch == 1 && !empty($replacePairs)) {
// highlight the keywords if there are any
$sigDisplay = $this->Highlight->highlighter($sigDisplay, $replacePairs);
}
if ('attachment' == $attribute['Attribute']['type'] || 'malware-sample' == $attribute['Attribute']['type']) {
?><a href="/attributes/download/<?php echo $attribute['Attribute']['id'];?>"><?php echo $sigDisplay; ?></a><?php
} elseif ('link' == $attribute['Attribute']['type']) {
?><a href="<?php echo nl2br(h($attribute['Attribute']['value']));?>"><?php echo $sigDisplay; ?></a><?php
} else {
echo $sigDisplay;
}
?></td>
<td class="short" onclick="document.location ='document.location ='/events/view/<?php echo $attribute['Event']['id'];?>';">
<?php echo $attribute['Attribute']['to_ids'] ? 'Yes' : 'No'; ?>&nbsp;
</td>

4
app/View/Elements/global_menu.ctp
View File

@ -1,9 +1,9 @@
<div class="navbar-wrapper">
<div class="navbar navbar-inverse">
<div class="navbar navbar-inverse navbar-fixed-top">
<div class="navbar-inner">
<?php if ($me != false ):?>
<div class="nav-collapse collapse">
<ul class="nav" style="position:fixed; width:100%; background:black;">
<ul class="nav">
<li class="active"><a href="/">home</a></li>

14
app/View/Events/view.ctp
View File

@ -171,7 +171,7 @@ if (!empty($event['Attribute'])):?>
<?php echo h($attribute['type']);?>
</td>
<td class="<?php echo $extra; ?>"><?php
<td class="showspaces <?php echo $extra; ?>"><?php
$sigDisplay = $attribute['value'];
if ('attachment' == $attribute['type'] || 'malware-sample' == $attribute['type'] ) {
$filenameHash = explode('|', $attribute['value']);
@ -249,8 +249,7 @@ if (!empty($event['Attribute'])):?>
if ($shadowAttribute['type'] != $attribute['type']) echo h($shadowAttribute['type']);
?>
</td>
<td class = "highlight2">
<?php
<td class="showspaces highlight2"><?php
if ($shadowAttribute['value'] != $attribute['value']) {
$sigDisplay = $shadowAttribute['value'];
if ('attachment' == $shadowAttribute['type'] || 'malware-sample' == $shadowAttribute['type'] ) {
@ -276,8 +275,7 @@ if (!empty($event['Attribute'])):?>
echo h($sigDisplay);
}
}
?>
</td>
?></td>
<td class="short highlight2">
</td>
<td class="short highlight2">
@ -325,8 +323,7 @@ if (!empty($event['Attribute'])):?>
echo h($remain['type']);
?>
</td>
<td class = "short highlight2">
<?php
<td class="showspaces short highlight2"><?php
$sigDisplay = nl2br(h($remain['value']));
if ('attachment' == $remain['type'] || 'malware-sample' == $remain['type'] ) {
$filenameHash = explode('|', $remain['value']);
@ -350,8 +347,7 @@ if (!empty($event['Attribute'])):?>
} else {
echo h($sigDisplay);
}
?>
</td>
?></td>
<td class="short highlight2">
</td>
<td class="short highlight2">

6
app/View/Servers/index.ctp
View File

@ -48,8 +48,10 @@ foreach ($servers as $server): ?>
<td class="short"><?php echo $server['Server']['lastpushedid']; ?></td>
<td class="short action-links">
<?php
if ($server['Server']['pull'] && $me['org'] == 'ADMIN') echo $this->Form->postLink('', array('action' => 'pull', $server['Server']['id'], 'full'), array('class' => 'icon-download', 'title' => 'Pull all'));
if ($server['Server']['push'] && $me['org'] == 'ADMIN') echo $this->Form->postLink('', array('action' => 'push', $server['Server']['id'], 'full'), array('class' => 'icon-upload', 'title' => 'Push all') );
if ($server['Server']['pull'] && $me['org'] == 'ADMIN')
echo $this->Html->link('', array('action' => 'pull', $server['Server']['id'], 'full'), array('class' => 'icon-download', 'title' => 'Pull all'));
if ($server['Server']['push'] && $me['org'] == 'ADMIN')
echo $this->Form->postLink('', array('action' => 'push', $server['Server']['id'], 'full'), array('class' => 'icon-upload', 'title' => 'Push all') );
?>
&nbsp;
<?php

24
app/webroot/css/main.css
View File

@ -284,29 +284,9 @@ td.action-links {
text-align: right;
}
/*
.view_icon{
background-position: 0 0;
}
.edit_icon{
background-position: -17px 0;
.showspaces {
white-space: pre-wrap;
}
.delete_icon{
background-position: -34px 0;
}
.tick_icon{
width: 16px;
height: 16px;
display: block;
background: url('../img/icons/icons-trans.png') -51px 0 no-repeat;
}
.cross_icon{
width: 16px;
height: 16px;
display: block;
background: url('../img/icons/icons-trans.png') -67px 0 no-repeat;
}
*/
/** Scaffold View **/
dl {