MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Work on the background job and the proposals 

- Proposals now get synced on pull
- several bug fixes
- new startup script for the background workers
pull/217/head
iglocska 2014-01-05 21:30:39 +01:00
parent b2bc0460c6
commit a380ab444a
10 changed files with 184 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/Console/worker/start.sh
View File

@ -1,3 +1,4 @@
../cake CakeResque.CakeResque stop --all
../cake CakeResque.CakeResque start --interval 5 --queue default
../cake CakeResque.CakeResque start --interval 5 --queue cache
../cake CakeResque.CakeResque start --interval 5 --queue email

94
app/Controller/EventsController.php
View File

@ -248,54 +248,55 @@ class EventsController extends AppController {
$this->__setDeletable($pivot, $id, true);
$this->set('pivot', $pivot);
$this->set('currentEvent', $id);
}
$this->set('allPivots', $this->Session->read('pivot_thread'));
// Show the discussion
$this->loadModel('Thread');
$params = array('conditions' => array('event_id' => $id),
'recursive' => -1,
'fields' => array('id', 'event_id', 'distribution', 'title')
);
$thread = $this->Thread->find('first', $params);
if (empty($thread)) {
$newThread = array(
'date_created' => date('Y/m/d H:i:s'),
'date_modified' => date('Y/m/d H:i:s'),
'user_id' => $this->Auth->user('id'),
'event_id' => $id,
'title' => 'Discussion about Event #' . $result['Event']['id'] . ' (' . $result['Event']['info'] . ')',
'distribution' => $result['Event']['distribution'],
'post_count' => 0,
'org' => $result['Event']['orgc']
$this->set('allPivots', $this->Session->read('pivot_thread'));
// Show the discussion
$this->loadModel('Thread');
$params = array('conditions' => array('event_id' => $id),
'recursive' => -1,
'fields' => array('id', 'event_id', 'distribution', 'title')
);
$this->Thread->save($newThread);
$thread = ($this->Thread->read());
} else {
if ($thread['Thread']['distribution'] != $result['Event']['distribution']) {
$this->Thread->saveField('distribution', $result['Event']['distribution']);
$thread = $this->Thread->find('first', $params);
if (empty($thread)) {
$newThread = array(
'date_created' => date('Y/m/d H:i:s'),
'date_modified' => date('Y/m/d H:i:s'),
'user_id' => $this->Auth->user('id'),
'event_id' => $id,
'title' => 'Discussion about Event #' . $result['Event']['id'] . ' (' . $result['Event']['info'] . ')',
'distribution' => $result['Event']['distribution'],
'post_count' => 0,
'org' => $result['Event']['orgc']
);
$this->Thread->save($newThread);
$thread = ($this->Thread->read());
} else {
if ($thread['Thread']['distribution'] != $result['Event']['distribution']) {
$this->Thread->saveField('distribution', $result['Event']['distribution']);
}
}
$this->loadModel('Post');
$this->paginate['Post'] = array(
'limit' => 5,
'conditions' => array('Post.thread_id' => $thread['Thread']['id']),
'contain' => 'User'
);
$posts = $this->paginate('Post');
// Show the discussion
$this->set('posts', $posts);
$this->set('thread_id', $thread['Thread']['id']);
$this->set('myuserid', $this->Auth->user('id'));
$this->set('thread_title', $thread['Thread']['title']);
if ($this->request->is('ajax')) {
$this->disableCache();
$this->layout = 'ajax';
$this->render('/Elements/eventdiscussion');
}
$pivot = $this->Session->read('pivot_thread');
$this->__arrangePivotVertical($pivot);
$this->__setDeletable($pivot, $id, true);
$this->set('pivot', $pivot);
}
$this->loadModel('Post');
$this->paginate['Post'] = array(
'limit' => 5,
'conditions' => array('Post.thread_id' => $thread['Thread']['id']),
'contain' => 'User'
);
$posts = $this->paginate('Post');
// Show the discussion
$this->set('posts', $posts);
$this->set('thread_id', $thread['Thread']['id']);
$this->set('myuserid', $this->Auth->user('id'));
$this->set('thread_title', $thread['Thread']['title']);
if ($this->request->is('ajax')) {
$this->disableCache();
$this->layout = 'ajax';
$this->render('/Elements/eventdiscussion');
}
$pivot = $this->Session->read('pivot_thread');
$this->__arrangePivotVertical($pivot);
$this->__setDeletable($pivot, $id, true);
$this->set('pivot', $pivot);
$this->set('currentEvent', $id);
}
@ -1968,6 +1969,9 @@ class EventsController extends AppController {
foreach ($result as $eventId) {
$conditions['OR'][] = array('Event.id =' => $eventId['ShadowAttribute']['event_id']);
}
if (empty($result)) {
$conditions['OR'][] = array('Event.id =' => -1);
}
$this->paginate = array(
'conditions' => $conditions,
'contain' => array(

35
app/Controller/ServersController.php
View File

@ -157,7 +157,7 @@ class ServersController extends AppController {
}
App::uses('HttpSocket', 'Network/Http');
$this->Server->read(null, $id);
$s = $this->Server->read(null, $id);
if (false == $this->Server->data['Server']['pull']) {
$this->Session->setFlash(__('Pull setting not enabled for this server.'));
$this->redirect(array('action' => 'index'));
@ -187,11 +187,11 @@ class ServersController extends AppController {
} else {
$this->redirect(array('action' => 'index'));
}
// now process the $eventIds to pull each of the events sequentially
if (!empty($eventIds)) {
$successes = array();
$fails = array();
$pulledProposals = array();
// download each event
if (null != $eventIds) {
App::import('Controller', 'Events');
@ -298,12 +298,41 @@ class ServersController extends AppController {
// increment lastid based on the highest ID seen
$this->Server->set('lastpulledid', $lastpulledid);
$this->Server->save($event, array('fieldList' => array('lastpulledid', 'url')));
// grab all of the shadow attributes that are relevant to us
$events = $this->Event->find('all', array(
'fields' => array('id', 'uuid'),
'recursive' => -1,
));
$this->loadModel('ShadowAttribute');
$this->ShadowAttribute->recursive = -1;
foreach ($events as &$event) {
$proposals = $this->Event->downloadEventFromServer($event['Event']['uuid'], $s, null, true);
if (null != $proposals) {
if (isset($proposals['ShadowAttribute']['id'])) {
$temp = $proposals['ShadowAttribute'];
$proposals['ShadowAttribute'] = array(0 => $temp);
}
foreach($proposals['ShadowAttribute'] as &$proposal) {
unset($proposal['id']);
$proposal['event_id'] = $event['Event']['id'];
if (!$this->ShadowAttribute->findByUuid($proposal['uuid'])) {
if (isset($pulledProposals[$event['Event']['id']])) {
$pulledProposals[$event['Event']['id']]++;
} else {
$pulledProposals[$event['Event']['id']] = 1;
}
$this->ShadowAttribute->create();
$this->ShadowAttribute->save($proposal);
}
}
}
}
}
}
$this->set('successes', $successes);
$this->set('fails', $fails);
$this->set('pulledProposals', $pulledProposals);
}
public function push($id = null, $technique=false) {

49
app/Controller/ShadowAttributesController.php
View File

@ -198,13 +198,15 @@ class ShadowAttributesController extends AppController {
*/
public function add($eventId = null) {
if ($this->request->is('post')) {
// Give error if someone tried to submit a attribute with attachment or malware-sample type.
// TODO change behavior attachment options - this is bad ... it should rather by a messagebox or should be filtered out on the view level
if (isset($this->request->data['ShadowAttribute']['type']) && $this->ShadowAttribute->typeIsAttachment($this->request->data['ShadowAttribute']['type'])) {
$this->Session->setFlash(__('Attribute has not been added: attachments are added by "Add attachment" button', true), 'default', array(), 'error');
$this->redirect(array('controller' => 'events', 'action' => 'view', $this->request->data['ShadowAttribute']['event_id']));
}
$temp = $this->_getEventData($this->request->data['ShadowAttribute']['event_id']);
$event_uuid = $temp['uuid'];
$event_org = $temp['orgc'];
//
// multiple attributes in batch import
//
@ -224,6 +226,8 @@ class ShadowAttributesController extends AppController {
$this->request->data['ShadowAttribute']['value'] = $attribute; // set the value as the content of the single line
$this->request->data['ShadowAttribute']['email'] = $this->Auth->user('email');
$this->request->data['ShadowAttribute']['org'] = $this->Auth->user('org');
$this->request->data['ShadowAttribute']['event_uuid'] = $event_uuid;
$this->request->data['ShadowAttribute']['event_org'] = $event_org;
// TODO loop-holes,
// there seems to be a loop-hole in misp here
// be it an create and not an update
@ -265,8 +269,10 @@ class ShadowAttributesController extends AppController {
$savedId = $this->ShadowAttribute->getId();
$this->request->data['ShadowAttribute']['email'] = $this->Auth->user('email');
$this->request->data['ShadowAttribute']['org'] = $this->Auth->user('org');
$this->request->data['ShadowAttribute']['event_uuid'] = $event_uuid;
$this->request->data['ShadowAttribute']['event_org'] = $event_org;
if ($this->ShadowAttribute->save($this->request->data)) {
$this->__sendProposalAlertEmail($eventId);
$this->__sendProposalAlertEmail($this->request->data['ShadowAttribute']['event_id']);
// inform the user and redirect
$this->Session->setFlash(__('The proposal has been saved'));
$this->redirect(array('controller' => 'events', 'action' => 'view', $this->request->data['ShadowAttribute']['event_id']));
@ -333,7 +339,6 @@ class ShadowAttributesController extends AppController {
*/
public function add_attachment($eventId = null) {
if ($this->request->is('post')) {
$this->loadModel('Event');
// Check if there were problems with the file upload
// only keep the last part of the filename, this should prevent directory attacks
$filename = basename($this->request->data['ShadowAttribute']['value']['name']);
@ -347,8 +352,9 @@ class ShadowAttributesController extends AppController {
$this->Session->setFlash(__('There was a problem to upload the file.', true), 'default', array(), 'error');
$this->redirect(array('controller' => 'events', 'action' => 'view', $this->request->data['ShadowAttribute']['event_id']));
}
$this->Event->id = $this->request->data['ShadowAttribute']['event_id'];
$temp = $this->_getEventData($this->request->data['ShadowAttribute']['event_id']);
$event_uuid = $temp['uuid'];
$event_org = $temp['orgc'];
// save the file-info in the database
$this->ShadowAttribute->create();
if ($this->request->data['ShadowAttribute']['malware']) {
@ -368,6 +374,8 @@ class ShadowAttributesController extends AppController {
$this->request->data['ShadowAttribute']['batch_import'] = 0;
$this->request->data['ShadowAttribute']['email'] = $this->Auth->user('email');
$this->request->data['ShadowAttribute']['org'] = $this->Auth->user('org');
$this->request->data['ShadowAttribute']['event_uuid'] = $event_uuid;
$this->request->data['ShadowAttribute']['event_org'] = $event_org;
if ($this->ShadowAttribute->save($this->request->data)) {
$this->__sendProposalAlertEmail($eventId);
} else {
@ -434,8 +442,6 @@ class ShadowAttributesController extends AppController {
} else {
// set the event_id in the form
$this->request->data['ShadowAttribute']['event_id'] = $eventId;
$this->loadModel('Event');
$events = $this->Event->findById($eventId);
}
// combobox for categories
@ -488,7 +494,7 @@ class ShadowAttributesController extends AppController {
}
$uuid = $this->Attribute->data['Attribute']['uuid'];
if (!$this->_isSiteAdmin()) {
if (($this->Attribute->data['Attribute']['distribution'] == 0) || ($this->Attribute->data['Event']['org'] == $this->Auth->user('org'))) {
if (($this->Attribute->data['Attribute']['distribution'] == 0) || ($this->Attribute->data['Event']['orgc'] == $this->Auth->user('org'))) {
$this->Session->setFlash(__('Invalid Attribute.'));
$this->redirect(array('controller' => 'events', 'action' => 'index'));
}
@ -506,9 +512,14 @@ class ShadowAttributesController extends AppController {
if ($this->request->is('post') || $this->request->is('put')) {
$existingAttribute = $this->Attribute->findByUuid($uuid);
$temp = $this->_getEventData($eventId);
$event_uuid = $temp['uuid'];
$event_org = $temp['orgc'];
$this->request->data['ShadowAttribute']['old_id'] = $existingAttribute['Attribute']['id'];
$this->request->data['ShadowAttribute']['uuid'] = $existingAttribute['Attribute']['uuid'];
$this->request->data['ShadowAttribute']['event_id'] = $existingAttribute['Attribute']['event_id'];
$this->request->data['ShadowAttribute']['event_uuid'] = $event_uuid;
$this->request->data['ShadowAttribute']['event_org'] = $event_org;
if ($attachment) $this->request->data['ShadowAttribute']['value'] = $existingAttribute['Attribute']['value'];
if ($attachment) $this->request->data['ShadowAttribute']['type'] = $existingAttribute['Attribute']['type'];
$this->request->data['ShadowAttribute']['org'] = $this->Auth->user('org');
@ -683,4 +694,26 @@ class ShadowAttributesController extends AppController {
}
}
private function _getEventData($event_id) {
$this->loadModel('Event');
$this->Event->recursive = -1;
$this->Event->read(array('id', 'uuid', 'orgc'), $event_id);
return $this->Event->data['Event'];
}
// takes a uuid and finds all proposals that belong to an event with the given uuid. These are then returned.
public function getProposalsByUuid($uuid) {
if (!$this->_isRest()) {
throw new MethodNotAllowedException(__('This feature is only available for REST users'));
}
if (strlen($uuid) != 36) {
throw new NotFoundException(__('Invalid UUID'));
}
$this->ShadowAttribute->recursive = -1;
$temp = $this->ShadowAttribute->findAllByEventUuid($uuid);
if ($temp == null) throw new NotFoundException(__('Invalid event'));
$this->set('proposal', $temp);
$this->render('get_proposals_by_uuid');
}
}

10
app/Model/Event.php
View File

@ -672,7 +672,7 @@ class Event extends AppModel {
* TODO move this to a component
* @return array|NULL
*/
public function downloadEventFromServer($eventId, $server, $HttpSocket=null) {
public function downloadEventFromServer($eventId, $server, $HttpSocket=null, $propsalDownload = false) {
$url = $server['Server']['url'];
$authkey = $server['Server']['authkey'];
if (null == $HttpSocket) {
@ -690,7 +690,11 @@ class Event extends AppModel {
//'Connection' => 'keep-alive' // LATER followup cakephp ticket 2854 about this problem http://cakephp.lighthouseapp.com/projects/42648-cakephp/tickets/2854
)
);
$uri = $url . '/events/' . $eventId;
if (!$propsalDownload) {
$uri = $url . '/events/' . $eventId;
} else {
$uri = $url . '/shadow_attributes/getProposalsByUuid/' . $eventId;
}
$response = $HttpSocket->get($uri, $data = '', $request);
if ($response->isOk()) {
$xmlArray = Xml::toArray(Xml::build($response->body));
@ -831,7 +835,7 @@ class Event extends AppModel {
// do not expose all the data ...
$fields = array('Event.id', 'Event.org', 'Event.date', 'Event.threat_level_id', 'Event.info', 'Event.published', 'Event.uuid', 'Event.attribute_count', 'Event.analysis', 'Event.timestamp', 'Event.distribution', 'Event.proposal_email_lock', 'Event.orgc', 'Event.user_id', 'Event.locked');
$fieldsAtt = array('Attribute.id', 'Attribute.type', 'Attribute.category', 'Attribute.value', 'Attribute.to_ids', 'Attribute.uuid', 'Attribute.event_id', 'Attribute.distribution', 'Attribute.timestamp', 'Attribute.comment');
$fieldsShadowAtt = array('ShadowAttribute.id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.value', 'ShadowAttribute.to_ids', 'ShadowAttribute.uuid', 'ShadowAttribute.event_id', 'ShadowAttribute.old_id');
$fieldsShadowAtt = array('ShadowAttribute.id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.value', 'ShadowAttribute.to_ids', 'ShadowAttribute.uuid', 'ShadowAttribute.event_id', 'ShadowAttribute.old_id', 'ShadowAttribute.comment');
$params = array('conditions' => $conditions,
'recursive' => 0,

2
app/View/Elements/side_menu.ctp
View File

@ -107,7 +107,7 @@
<li class="divider"></li>
<?php endif; ?>
<li <?php if ($menuItem === 'index') echo 'class="active";'?>><?php echo $this->Html->link('List Servers', array('controller' => 'servers', 'action' => 'index'));?></li>
<li <?php if ($menuItem === 'add') echo 'class="active";'?>><?php if ($isAclAdd && $me['org'] == 'ADMIN') echo $this->Html->link(__('New Server'), array('controller' => 'servers', 'action' => 'add')); ?></li>
<li <?php if ($menuItem === 'add') echo 'class="active";'?>><?php if ($isSiteAdmin) echo $this->Html->link(__('New Server'), array('controller' => 'servers', 'action' => 'add')); ?></li>
<?php
break;

8
app/View/Events/view.ctp
View File

@ -1,5 +1,5 @@
<?php
$mayModify = (($isAclModify && $event['Event']['user_id'] == $me['id']) || ($isAclModifyOrg && $event['Event']['orgc'] == $me['org']));
$mayModify = (($isAclModify && $event['Event']['user_id'] == $me['id'] && $event['Event']['orgc'] == $me['org']) || ($isAclModifyOrg && $event['Event']['orgc'] == $me['org']));
$mayPublish = ($isAclPublish && $event['Event']['orgc'] == $me['org']);
?>
<?php
@ -293,6 +293,9 @@ if (!empty($event['Attribute'])):?>
}
?></td>
<td class="short highlight2">
<?php
echo h($shadowAttribute['comment']);
?>
</td>
<td class="short highlight2">
</td>
@ -368,6 +371,9 @@ if (!empty($event['Attribute'])):?>
}
?></td>
<td class="short highlight2">
<?php
echo h($remain['comment']);
?>
</td>
<td class="short highlight2">
</td>

12
app/View/Servers/pull.ctp
View File

@ -21,6 +21,18 @@ else:?>
</ul>
<?php
endif;?>
<h2>Proposals pulled</h2>
<?php
if (0 == count($pulledProposals)):?>
<p>No proposals pulled</p>
<?php
else:?>
<ul>
<?php foreach ($pulledProposals as $e => $p) echo '<li>Event ' . $e . ' : ' . $p . ' proposal(s).</li>'; ?>
</ul>
<?php
endif;?>
</div>
<?php
echo $this->element('side_menu', array('menuList' => 'sync', 'menuItem' => 'pull'));

28
app/View/ShadowAttributes/xml/get_proposals_by_uuid.ctp Normal file
View File

@ -0,0 +1,28 @@
<?php
$xmlArray = array();
//
// cleanup the array from things we do not want to expose
//
if (isset($proposal['ShadowAttribute']['id'])) {
$temp = $proposal['ShadowAttribute'];
unset ($proposal['ShadowAttribute']);
$proposal['ShadowAttribute'][0] = $temp;
unset ($temp);
}
$xmlArray['response']['ShadowAttribute'] = array();
foreach ($proposal as &$temp) {
unset($temp['ShadowAttribute']['email']);
unset($temp['ShadowAttribute']['category_order']);
unset($temp['ShadowAttribute']['value1']);
unset($temp['ShadowAttribute']['value2']);
// hide the org field is we are not in showorg mode
if ('true' != Configure::read('CyDefSIG.showorg') && !$isAdmin) {
unset($temp['ShadowAttribute']['org']);
unset($temp['ShadowAttribute']['event_org']);
}
$xmlArray['response']['ShadowAttribute'][] = $temp['ShadowAttribute'];
}
// display the XML to the user
$xmlObject = Xml::fromArray($xmlArray, array('format' => 'tags'));
echo $xmlObject->asXML();

6
app/webroot/js/jquery-2.0.3.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long