Work on the background job and the proposals

- Proposals now get synced on pull
- several bug fixes
- new startup script for the background workers

app/Console/worker/start.sh

@@ -1,3 +1,4 @@
+../cake CakeResque.CakeResque stop --all
 ../cake CakeResque.CakeResque start --interval 5 --queue default
 ../cake CakeResque.CakeResque start --interval 5 --queue cache
 ../cake CakeResque.CakeResque start --interval 5 --queue email

app/Controller/EventsController.php

@@ -248,54 +248,55 @@ class EventsController extends AppController {
 			$this->__setDeletable($pivot, $id, true);
 			$this->set('pivot', $pivot);
 			$this->set('currentEvent', $id);
-		}
+
-		$this->set('allPivots', $this->Session->read('pivot_thread'));
+			$this->set('allPivots', $this->Session->read('pivot_thread'));
-		// Show the discussion
+			// Show the discussion
-		$this->loadModel('Thread');
+			$this->loadModel('Thread');
-		$params = array('conditions' => array('event_id' => $id),
+			$params = array('conditions' => array('event_id' => $id),
-				'recursive' => -1,
+					'recursive' => -1,
-				'fields' => array('id', 'event_id', 'distribution', 'title')
+					'fields' => array('id', 'event_id', 'distribution', 'title')
-		);
-		$thread = $this->Thread->find('first', $params);
-		if (empty($thread)) {
-			$newThread = array(
-					'date_created' => date('Y/m/d H:i:s'),
-					'date_modified' => date('Y/m/d H:i:s'),
-					'user_id' => $this->Auth->user('id'),
-					'event_id' => $id,
-					'title' => 'Discussion about Event #' . $result['Event']['id'] . ' (' . $result['Event']['info'] . ')',
-					'distribution' => $result['Event']['distribution'],
-					'post_count' => 0,
-					'org' => $result['Event']['orgc']
 			);
-			$this->Thread->save($newThread);
+			$thread = $this->Thread->find('first', $params);
-			$thread = ($this->Thread->read());
+			if (empty($thread)) {
-		} else {
+				$newThread = array(
-			if ($thread['Thread']['distribution'] != $result['Event']['distribution']) {
+						'date_created' => date('Y/m/d H:i:s'),
-				$this->Thread->saveField('distribution', $result['Event']['distribution']);
+						'date_modified' => date('Y/m/d H:i:s'),
+						'user_id' => $this->Auth->user('id'),
+						'event_id' => $id,
+						'title' => 'Discussion about Event #' . $result['Event']['id'] . ' (' . $result['Event']['info'] . ')',
+						'distribution' => $result['Event']['distribution'],
+						'post_count' => 0,
+						'org' => $result['Event']['orgc']
+				);
+				$this->Thread->save($newThread);
+				$thread = ($this->Thread->read());
+			} else {
+				if ($thread['Thread']['distribution'] != $result['Event']['distribution']) {
+					$this->Thread->saveField('distribution', $result['Event']['distribution']);
+				}
 			}
+			$this->loadModel('Post');
+			$this->paginate['Post'] = array(
+					'limit' => 5,
+					'conditions' => array('Post.thread_id' => $thread['Thread']['id']),
+					'contain' => 'User'
+			);
+			$posts = $this->paginate('Post');
+			// Show the discussion
+			$this->set('posts', $posts);
+			$this->set('thread_id', $thread['Thread']['id']);
+			$this->set('myuserid', $this->Auth->user('id'));
+			$this->set('thread_title', $thread['Thread']['title']);
+			if ($this->request->is('ajax')) {
+				$this->disableCache();
+				$this->layout = 'ajax';
+				$this->render('/Elements/eventdiscussion');
+			}
+			$pivot = $this->Session->read('pivot_thread');
+			$this->__arrangePivotVertical($pivot);
+			$this->__setDeletable($pivot, $id, true);
+			$this->set('pivot', $pivot);
 		}
-		$this->loadModel('Post');
-		$this->paginate['Post'] = array(
-				'limit' => 5,
-				'conditions' => array('Post.thread_id' => $thread['Thread']['id']),
-				'contain' => 'User'
-		);
-		$posts = $this->paginate('Post');
-		// Show the discussion
-		$this->set('posts', $posts);
-		$this->set('thread_id', $thread['Thread']['id']);
-		$this->set('myuserid', $this->Auth->user('id'));
-		$this->set('thread_title', $thread['Thread']['title']);
-		if ($this->request->is('ajax')) {
-			$this->disableCache();
-			$this->layout = 'ajax';
-			$this->render('/Elements/eventdiscussion');
-		}
-		$pivot = $this->Session->read('pivot_thread');
-		$this->__arrangePivotVertical($pivot);
-		$this->__setDeletable($pivot, $id, true);
-		$this->set('pivot', $pivot);
 		$this->set('currentEvent', $id);
 	}
 
@@ -1968,6 +1969,9 @@ class EventsController extends AppController {
 		foreach ($result as $eventId) {
 				$conditions['OR'][] = array('Event.id =' => $eventId['ShadowAttribute']['event_id']);
 		}
+		if (empty($result)) {
+			$conditions['OR'][] = array('Event.id =' => -1);
+		}
 		$this->paginate = array(
 				'conditions' => $conditions,
 				'contain' => array(

app/Controller/ServersController.php

@@ -157,7 +157,7 @@ class ServersController extends AppController {
 		}
 
 		App::uses('HttpSocket', 'Network/Http');
-		$this->Server->read(null, $id);
+		$s = $this->Server->read(null, $id);
 		if (false == $this->Server->data['Server']['pull']) {
 			$this->Session->setFlash(__('Pull setting not enabled for this server.'));
 			$this->redirect(array('action' => 'index'));
@@ -187,11 +187,11 @@ class ServersController extends AppController {
 		} else {
 			$this->redirect(array('action' => 'index'));
 		}
-
 		// now process the $eventIds to pull each of the events sequentially
 		if (!empty($eventIds)) {
 			$successes = array();
 			$fails = array();
+			$pulledProposals = array();
 			// download each event
 			if (null != $eventIds) {
 				App::import('Controller', 'Events');
@@ -298,12 +298,41 @@ class ServersController extends AppController {
 				// increment lastid based on the highest ID seen
 				$this->Server->set('lastpulledid', $lastpulledid);
 				$this->Server->save($event, array('fieldList' => array('lastpulledid', 'url')));
+				// grab all of the shadow attributes that are relevant to us
 
+				$events = $this->Event->find('all', array(
+					'fields' => array('id', 'uuid'),
+					'recursive' => -1,
+				));
+				$this->loadModel('ShadowAttribute');
+				$this->ShadowAttribute->recursive = -1;
+				foreach ($events as &$event) {
+					$proposals = $this->Event->downloadEventFromServer($event['Event']['uuid'], $s, null, true);
+					if (null != $proposals) {
+						if (isset($proposals['ShadowAttribute']['id'])) {
+							$temp = $proposals['ShadowAttribute'];
+							$proposals['ShadowAttribute'] = array(0 => $temp);
+						}
+						foreach($proposals['ShadowAttribute'] as &$proposal) {
+							unset($proposal['id']);
+							$proposal['event_id'] = $event['Event']['id'];
+							if (!$this->ShadowAttribute->findByUuid($proposal['uuid'])) {
+								if (isset($pulledProposals[$event['Event']['id']])) {
+									$pulledProposals[$event['Event']['id']]++;
+								} else {
+									$pulledProposals[$event['Event']['id']] = 1;
+								}
+								$this->ShadowAttribute->create();
+								$this->ShadowAttribute->save($proposal);
+							}
+						}
+					}
+				}
 			}
 		}
-
 		$this->set('successes', $successes);
 		$this->set('fails', $fails);
+		$this->set('pulledProposals', $pulledProposals);
 	}
 
 	public function push($id = null, $technique=false) {

app/Controller/ShadowAttributesController.php

@@ -198,13 +198,15 @@ class ShadowAttributesController extends AppController {
  */
 	public function add($eventId = null) {
 		if ($this->request->is('post')) {
-
 			// Give error if someone tried to submit a attribute with attachment or malware-sample type.
 			// TODO change behavior attachment options - this is bad ... it should rather by a messagebox or should be filtered out on the view level
 			if (isset($this->request->data['ShadowAttribute']['type']) && $this->ShadowAttribute->typeIsAttachment($this->request->data['ShadowAttribute']['type'])) {
 				$this->Session->setFlash(__('Attribute has not been added: attachments are added by "Add attachment" button', true), 'default', array(), 'error');
 				$this->redirect(array('controller' => 'events', 'action' => 'view', $this->request->data['ShadowAttribute']['event_id']));
 			}
+			$temp = $this->_getEventData($this->request->data['ShadowAttribute']['event_id']);
+			$event_uuid = $temp['uuid'];
+			$event_org = $temp['orgc'];
 			//
 			// multiple attributes in batch import
 			//
@@ -224,6 +226,8 @@ class ShadowAttributesController extends AppController {
 					$this->request->data['ShadowAttribute']['value'] = $attribute; // set the value as the content of the single line
 					$this->request->data['ShadowAttribute']['email'] = $this->Auth->user('email');
 					$this->request->data['ShadowAttribute']['org'] = $this->Auth->user('org');
+					$this->request->data['ShadowAttribute']['event_uuid'] = $event_uuid;
+					$this->request->data['ShadowAttribute']['event_org'] = $event_org;
 					// TODO loop-holes,
 					// there seems to be a loop-hole in misp here
 					// be it an create and not an update
@@ -265,8 +269,10 @@ class ShadowAttributesController extends AppController {
 				$savedId = $this->ShadowAttribute->getId();
 				$this->request->data['ShadowAttribute']['email'] = $this->Auth->user('email');
 				$this->request->data['ShadowAttribute']['org'] = $this->Auth->user('org');
+				$this->request->data['ShadowAttribute']['event_uuid'] = $event_uuid;
+				$this->request->data['ShadowAttribute']['event_org'] = $event_org;
 				if ($this->ShadowAttribute->save($this->request->data)) {
-					$this->__sendProposalAlertEmail($eventId);
+					$this->__sendProposalAlertEmail($this->request->data['ShadowAttribute']['event_id']);
 					// inform the user and redirect
 					$this->Session->setFlash(__('The proposal has been saved'));
 					$this->redirect(array('controller' => 'events', 'action' => 'view', $this->request->data['ShadowAttribute']['event_id']));
@@ -333,7 +339,6 @@ class ShadowAttributesController extends AppController {
  */
 	public function add_attachment($eventId = null) {
 		if ($this->request->is('post')) {
-			$this->loadModel('Event');
 			// Check if there were problems with the file upload
 			// only keep the last part of the filename, this should prevent directory attacks
 			$filename = basename($this->request->data['ShadowAttribute']['value']['name']);
@@ -347,8 +352,9 @@ class ShadowAttributesController extends AppController {
 				$this->Session->setFlash(__('There was a problem to upload the file.', true), 'default', array(), 'error');
 				$this->redirect(array('controller' => 'events', 'action' => 'view', $this->request->data['ShadowAttribute']['event_id']));
 			}
-
+			$temp = $this->_getEventData($this->request->data['ShadowAttribute']['event_id']);
-			$this->Event->id = $this->request->data['ShadowAttribute']['event_id'];
+			$event_uuid = $temp['uuid'];
+			$event_org = $temp['orgc'];
 			// save the file-info in the database
 			$this->ShadowAttribute->create();
 			if ($this->request->data['ShadowAttribute']['malware']) {
@@ -368,6 +374,8 @@ class ShadowAttributesController extends AppController {
 			$this->request->data['ShadowAttribute']['batch_import'] = 0;
 			$this->request->data['ShadowAttribute']['email'] = $this->Auth->user('email');
 			$this->request->data['ShadowAttribute']['org'] = $this->Auth->user('org');
+			$this->request->data['ShadowAttribute']['event_uuid'] = $event_uuid;
+			$this->request->data['ShadowAttribute']['event_org'] = $event_org;
 			if ($this->ShadowAttribute->save($this->request->data)) {
 				$this->__sendProposalAlertEmail($eventId);
 			} else {
@@ -434,8 +442,6 @@ class ShadowAttributesController extends AppController {
 		} else {
 			// set the event_id in the form
 			$this->request->data['ShadowAttribute']['event_id'] = $eventId;
-			$this->loadModel('Event');
-			$events = $this->Event->findById($eventId);
 		}
 
 		// combobox for categories
@@ -488,7 +494,7 @@ class ShadowAttributesController extends AppController {
 		}
 		$uuid = $this->Attribute->data['Attribute']['uuid'];
 		if (!$this->_isSiteAdmin()) {
-			if (($this->Attribute->data['Attribute']['distribution'] == 0) || ($this->Attribute->data['Event']['org'] == $this->Auth->user('org'))) {
+			if (($this->Attribute->data['Attribute']['distribution'] == 0) || ($this->Attribute->data['Event']['orgc'] == $this->Auth->user('org'))) {
 				$this->Session->setFlash(__('Invalid Attribute.'));
 				$this->redirect(array('controller' => 'events', 'action' => 'index'));
 			}
@@ -506,9 +512,14 @@ class ShadowAttributesController extends AppController {
 
 		if ($this->request->is('post') || $this->request->is('put')) {
 			$existingAttribute = $this->Attribute->findByUuid($uuid);
+			$temp = $this->_getEventData($eventId);
+			$event_uuid = $temp['uuid'];
+			$event_org = $temp['orgc'];
 			$this->request->data['ShadowAttribute']['old_id'] = $existingAttribute['Attribute']['id'];
 			$this->request->data['ShadowAttribute']['uuid'] = $existingAttribute['Attribute']['uuid'];
 			$this->request->data['ShadowAttribute']['event_id'] = $existingAttribute['Attribute']['event_id'];
+			$this->request->data['ShadowAttribute']['event_uuid'] = $event_uuid;
+			$this->request->data['ShadowAttribute']['event_org'] = $event_org;
 			if ($attachment) $this->request->data['ShadowAttribute']['value'] = $existingAttribute['Attribute']['value'];
 			if ($attachment) $this->request->data['ShadowAttribute']['type'] = $existingAttribute['Attribute']['type'];
 			$this->request->data['ShadowAttribute']['org'] =  $this->Auth->user('org');
@@ -683,4 +694,26 @@ class ShadowAttributesController extends AppController {
 			
 		}
 	}
+	
+	private function _getEventData($event_id) {
+		$this->loadModel('Event');
+		$this->Event->recursive = -1;
+		$this->Event->read(array('id', 'uuid', 'orgc'), $event_id);
+		return $this->Event->data['Event'];
+	} 
+	
+	// takes a uuid and finds all proposals that belong to an event with the given uuid. These are then returned. 
+	public function getProposalsByUuid($uuid) {
+		if (!$this->_isRest()) {
+			throw new MethodNotAllowedException(__('This feature is only available for REST users'));
+		}
+		if (strlen($uuid) != 36) {
+			throw new NotFoundException(__('Invalid UUID'));
+		}
+		$this->ShadowAttribute->recursive = -1;
+		$temp = $this->ShadowAttribute->findAllByEventUuid($uuid);
+		if ($temp == null) throw new NotFoundException(__('Invalid event'));
+		$this->set('proposal', $temp);
+		$this->render('get_proposals_by_uuid');
+	}
 }

app/Model/Event.php

@@ -672,7 +672,7 @@ class Event extends AppModel {
  * TODO move this to a component
  * @return array|NULL
  */
-	public function downloadEventFromServer($eventId, $server, $HttpSocket=null) {
+	public function downloadEventFromServer($eventId, $server, $HttpSocket=null, $propsalDownload = false) {
 		$url = $server['Server']['url'];
 		$authkey = $server['Server']['authkey'];
 		if (null == $HttpSocket) {
@@ -690,7 +690,11 @@ class Event extends AppModel {
 						//'Connection' => 'keep-alive' // LATER followup cakephp ticket 2854 about this problem http://cakephp.lighthouseapp.com/projects/42648-cakephp/tickets/2854
 				)
 		);
-		$uri = $url . '/events/' . $eventId;
+		if (!$propsalDownload) {
+			$uri = $url . '/events/' . $eventId;
+		} else {
+			$uri = $url . '/shadow_attributes/getProposalsByUuid/' . $eventId;
+		}
 		$response = $HttpSocket->get($uri, $data = '', $request);
 		if ($response->isOk()) {
 			$xmlArray = Xml::toArray(Xml::build($response->body));
@@ -831,7 +835,7 @@ class Event extends AppModel {
 		// do not expose all the data ...
 		$fields = array('Event.id', 'Event.org', 'Event.date', 'Event.threat_level_id', 'Event.info', 'Event.published', 'Event.uuid', 'Event.attribute_count', 'Event.analysis', 'Event.timestamp', 'Event.distribution', 'Event.proposal_email_lock', 'Event.orgc', 'Event.user_id', 'Event.locked');
 		$fieldsAtt = array('Attribute.id', 'Attribute.type', 'Attribute.category', 'Attribute.value', 'Attribute.to_ids', 'Attribute.uuid', 'Attribute.event_id', 'Attribute.distribution', 'Attribute.timestamp', 'Attribute.comment');
-		$fieldsShadowAtt = array('ShadowAttribute.id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.value', 'ShadowAttribute.to_ids', 'ShadowAttribute.uuid', 'ShadowAttribute.event_id', 'ShadowAttribute.old_id');
+		$fieldsShadowAtt = array('ShadowAttribute.id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.value', 'ShadowAttribute.to_ids', 'ShadowAttribute.uuid', 'ShadowAttribute.event_id', 'ShadowAttribute.old_id', 'ShadowAttribute.comment');
 			
 		$params = array('conditions' => $conditions,
 			'recursive' => 0,

app/View/Elements/side_menu.ctp

@@ -107,7 +107,7 @@
 					<li class="divider"></li>
 					<?php endif; ?>
 					<li <?php if ($menuItem === 'index') echo 'class="active";'?>><?php echo $this->Html->link('List Servers', array('controller' => 'servers', 'action' => 'index'));?></li>
-					<li <?php if ($menuItem === 'add') echo 'class="active";'?>><?php if ($isAclAdd && $me['org'] == 'ADMIN') echo $this->Html->link(__('New Server'), array('controller' => 'servers', 'action' => 'add')); ?></li>
+					<li <?php if ($menuItem === 'add') echo 'class="active";'?>><?php if ($isSiteAdmin) echo $this->Html->link(__('New Server'), array('controller' => 'servers', 'action' => 'add')); ?></li>
 					<?php 
 				break;	
 				

app/View/Events/view.ctp

@@ -1,5 +1,5 @@
 <?php
-$mayModify = (($isAclModify && $event['Event']['user_id'] == $me['id']) || ($isAclModifyOrg && $event['Event']['orgc'] == $me['org']));
+$mayModify = (($isAclModify && $event['Event']['user_id'] == $me['id'] && $event['Event']['orgc'] == $me['org']) || ($isAclModifyOrg && $event['Event']['orgc'] == $me['org']));
 $mayPublish = ($isAclPublish && $event['Event']['orgc'] == $me['org']);
 ?>
 <?php
@@ -293,6 +293,9 @@ if (!empty($event['Attribute'])):?>
 							}
 						?></td>
 					<td class="short highlight2">
+					<?php 
+						echo h($shadowAttribute['comment']);
+					?>
 					</td>
 					<td class="short highlight2">
 					</td>
@@ -368,6 +371,9 @@ if (!empty($event['Attribute'])):?>
 										}
 									?></td>
 								<td class="short highlight2">
+								<?php 
+									echo h($remain['comment']);
+								?>
 								</td>
 								<td class="short highlight2">
 								</td>

app/View/Servers/pull.ctp

@@ -21,6 +21,18 @@ else:?>
 	</ul>
 	<?php
 endif;?>
+	<h2>Proposals pulled</h2>
+	<?php
+if (0 == count($pulledProposals)):?>
+	<p>No proposals pulled</p>
+	<?php
+else:?>
+	<ul>
+	<?php foreach ($pulledProposals as $e => $p) echo '<li>Event ' . $e . ' : ' . $p . ' proposal(s).</li>'; ?>
+	</ul>
+	<?php
+endif;?>
+
 </div>
 <?php 
 	echo $this->element('side_menu', array('menuList' => 'sync', 'menuItem' => 'pull'));

app/View/ShadowAttributes/xml/get_proposals_by_uuid.ctp

@@ -0,0 +1,28 @@
+<?php
+$xmlArray = array();
+//
+// cleanup the array from things we do not want to expose
+//
+if (isset($proposal['ShadowAttribute']['id'])) {
+	$temp = $proposal['ShadowAttribute'];
+	unset ($proposal['ShadowAttribute']);
+	$proposal['ShadowAttribute'][0] = $temp;
+	unset ($temp);
+}
+$xmlArray['response']['ShadowAttribute'] = array();
+foreach ($proposal as &$temp) {
+	unset($temp['ShadowAttribute']['email']);
+	unset($temp['ShadowAttribute']['category_order']);
+	unset($temp['ShadowAttribute']['value1']);
+	unset($temp['ShadowAttribute']['value2']);
+	// hide the org field is we are not in showorg mode
+	if ('true' != Configure::read('CyDefSIG.showorg') && !$isAdmin) {
+	    unset($temp['ShadowAttribute']['org']);
+	    unset($temp['ShadowAttribute']['event_org']);
+	}
+	$xmlArray['response']['ShadowAttribute'][] = $temp['ShadowAttribute'];
+}
+
+// display the XML to the user
+$xmlObject = Xml::fromArray($xmlArray, array('format' => 'tags'));
+echo $xmlObject->asXML();