Merge branch '2.4' into objects_wip

2017-09-04 17:38:06 +02:00 · 2017-09-04 17:38:06 +02:00 · 40ea22a272
parent 58bfabfa73 73b47eda82
commit 40ea22a272
23 changed files with 1046 additions and 126 deletions
--- a/.gitignore
+++ b/.gitignore
@ -78,3 +78,5 @@
 *.swp
 *.iml
 .ropeproject/
+vagrant/.vagrant/
+vagrant/*.log
--- a/2
+++ b/2
@ -1 +1 @@
-Subproject commit 394c312d0d5acf1295ab921602c42647c4ad76ff
+Subproject commit bfa5b67c1d17467ae18b29cb1e328a19698a3146
--- a/VERSION.json
+++ b/VERSION.json
@ -1 +1 @@
-{"major":2, "minor":4, "hotfix":78}
+{"major":2, "minor":4, "hotfix":79}
--- a/app/Console/Command/ServerShell.php
+++ b/app/Console/Command/ServerShell.php
@ -185,6 +185,85 @@ class ServerShell extends AppShell
 		$this->Task->saveField('message', count($servers) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '. Failed jobs: ' . $failCount . '/' . $count);
 	}

+	public function enqueueFeed() {
+		$timestamp = $this->args[0];
+		$userId = $this->args[1];
+		$taskId = $this->args[2];
+		// action options:
+		// 0 = pull
+		// 1 = cache
+		$action = $this->args[3];
+		$task = $this->Task->read(null, $taskId);
+		if ($timestamp != $task['Task']['next_execution_time']) {
+			return;
+		}
+		if ($task['Task']['timer'] > 0)	$this->Task->reQueue($task, 'default', 'ServerShell', 'enqueueCachePull', $userId, $taskId, $action);
+		$user = $this->User->getAuthUser($userId);
+		$count = count($feeds);
+		$failCount = 0;
+		if ($action == 0) {
+			$feeds = $this->Feed->find('all', array(
+				'recursive' => -1,
+				'conditions' => array('enabled' => 1)
+			));
+			foreach ($feeds as $k => $feed) {
+				$this->Job->create();
+				$data = array(
+						'worker' => 'default',
+						'job_type' => 'feed_pull',
+						'job_input' => 'Feed: ' . $feed['Feed']['id'],
+						'retries' => 0,
+						'org' => $user['Organisation']['name'],
+						'org_id' => $user['org_id'],
+						'process_id' => 'Part of scheduled feed pull',
+						'message' => 'Pulling.',
+				);
+				$this->Job->save($data);
+				$jobId = $this->Job->id;
+				App::uses('SyncTool', 'Tools');
+				$result = $this->Feed->downloadFromFeedInitiator($feed['Feed']['id'], $user, $jobId);
+				$this->Job->save(array(
+						'id' => $jobId,
+						'message' => 'Job done.',
+						'progress' => 100,
+						'status' => 4
+				));
+				if ($result !== true) {
+					$this->Job->saveField('message', 'Could not pull feed.');
+					$failCount++;
+				}
+			}
+			$this->Task->id = $task['Task']['id'];
+			$this->Task->saveField('message', count($feeds) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '. Failed jobs: ' . $failCount . '/' . count($feeds));
+		} else {
+			$this->Job->create();
+			$data = array(
+					'worker' => 'default',
+					'job_type' => 'feed_cache',
+					'job_input' => 'Feed: ' . $feed['Feed']['id'],
+					'retries' => 0,
+					'org' => $user['Organisation']['name'],
+					'org_id' => $user['org_id'],
+					'process_id' => 'Part of scheduled feed caching',
+					'message' => 'Caching.',
+			);
+			$this->Job->save($data);
+			$jobId = $this->Job->id;
+			$result = $this->Feed->cacheFeedInitiator($user, $jobId, 'all');
+			$this->Job->save(array(
+					'id' => $jobId,
+					'message' => 'Job done.',
+					'progress' => 100,
+					'status' => 4
+			));
+			$this->Task->id = $task['Task']['id'];
+			$this->Task->saveField('message', 'Job completed at ' . date('d/m/Y - H:i:s'));
+		}
+	}
+
+
+
+
 	public function enqueuePush() {
 		$timestamp = $this->args[0];
 		$taskId = $this->args[1];
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@ -47,7 +47,7 @@ class AppController extends Controller {
 	public $helpers = array('Utility');

 	private $__queryVersion = '18';
-	public $pyMispVersion = '2.4.77';
+	public $pyMispVersion = '2.4.79';
 	public $phpmin = '5.6.5';
 	public $phprec = '7.0.16';

--- a/app/Controller/EventsController.php
+++ b/app/Controller/EventsController.php
@ -63,8 +63,7 @@ class EventsController extends AppController {

 		// if not admin or own org, check private as well..
 		if (!$this->_isSiteAdmin() && in_array($this->action, $this->paginationFunctions)) {
-			$sgids = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user());
-			if (empty($sgids)) $sgids = array(-1);
+			$sgids = $this->Event->cacheSgids($this->Auth->user(), true);
 			$conditions = array(
 				'AND' => array(
 					array(
@ -920,7 +919,11 @@ class EventsController extends AppController {
 		// find the id of the event, change $id to it and proceed to read the event as if the ID was entered.
 		if (Validation::uuid($id)) {
 			$this->Event->recursive = -1;
-			$temp = $this->Event->findByUuid($id);
+			$temp = $this->Event->find('first', array(
+				'recursive' => -1,
+				'conditions' => array('Event.uuid' => $id),
+				'fields' => array('Event.id', 'Event.uuid')
+			));
 			if ($temp == null) throw new NotFoundException('Invalid event');
 			$id = $temp['Event']['id'];
 		} else if (!is_numeric($id)) {
@ -2600,7 +2603,7 @@ class EventsController extends AppController {
 			$key = strtolower($key);
 			if (!$this->Auth->user()) throw new UnauthorizedException('You are not authorized. Please send the Authorization header with your auth key along with an Accept header for application/xml.');
 		}
-		if (!is_array($value)) $value = str_replace('|', '/', $value);
+		if (!is_array($value) && $value !== false) $value = str_replace('|', '/', $value);
 		// request handler for POSTed queries. If the request is a post, the parameters (apart from the key) will be ignored and replaced by the terms defined in the posted json or xml object.
 		// The correct format for both is a "request" root element, as shown by the examples below:
 		// For Json: {"request":{"value": "7.7.7.7&&1.1.1.1","type":"ip-src"}}
@ -2648,8 +2651,13 @@ class EventsController extends AppController {
 			$eventIds = $this->__quickFilter($value);
 		} else {
 			$parameters = array('value', 'type', 'category', 'org', 'eventid', 'uuid');
+			$attributeLevelFilters = array('value', 'type', 'category', 'uuid');
+			$preFilterLevel = 'event';
 			foreach ($parameters as $k => $param) {
-				if (isset(${$parameters[$k]})) {
+				if (${$parameters[$k]} !== false) {
+					if (in_array($param, $attributeLevelFilters)) {
+						$preFilterLevel = 'attribute';
+					}
 					$conditions = $this->Event->setSimpleConditions($parameters[$k], ${$parameters[$k]}, $conditions);
 				}
 			}
@ -2660,22 +2668,28 @@ class EventsController extends AppController {
 			if ($publish_timestamp) $conditions = $this->Event->Attribute->setPublishTimestampConditions($publish_timestamp, $conditions);
 			if ($timestamp) $conditions = $this->Event->Attribute->setTimestampConditions($timestamp, $conditions);
 			if ($last) $conditions['AND'][] = array('Event.publish_timestamp >=' => $last);
-			if ($published !== null) $conditions['AND'][] = array('Event.published' => $published);
-			$params = array(
-					'conditions' => $conditions,
-					'fields' => array('DISTINCT(Attribute.event_id)'),
-					'contain' => array(),
-					'recursive' => -1,
-					'list' => true
-			);
-			$attributes = $this->Event->Attribute->fetchAttributes($this->Auth->user(), $params);
-			$eventIds = array();
-			// Add event ID if specifically specified to allow for the export of an empty event
-			if (isset($eventid) && $eventid) $eventIds[] = $eventid;
-			foreach ($attributes as $attribute) {
-				if (!in_array($attribute, $eventIds)) $eventIds[] = $attribute;
+			if ($published !== null && $published !== false) $conditions['AND'][] = array('Event.published' => $published);
+			if ($preFilterLevel == 'event') {
+				$params = array(
+					'conditions' => $conditions
+				);
+				$eventIds = $this->Event->fetchSipleEventIds($this->Auth->user(), $params);
+			} else {
+				$params = array(
+						'conditions' => $conditions,
+						'fields' => array('DISTINCT(Attribute.event_id)'),
+						'contain' => array(),
+						'recursive' => -1,
+						'list' => true,
+						'event_ids' => true
+				);
+				$attributes = $this->Event->Attribute->fetchAttributes($this->Auth->user(), $params);
+				$eventIds = array();
+				// Add event ID if specifically specified to allow for the export of an empty event
+				if (isset($eventid) && $eventid) $eventIds[] = $eventid;
+				$eventIds = array_merge($eventIds, array_values($attributes));
+				unset($attributes);
 			}
-			unset($attributes);
 		}
 		$this->loadModel('Whitelist');
 		$responseType = 'xml';
@ -2698,13 +2712,17 @@ class EventsController extends AppController {
 		$i = 0;
 		foreach ($eventIds as $k => $currentEventId) {
 			$i++;
-			$result = $this->Event->fetchEvent($this->Auth->user(), array(
-				'eventid' => $currentEventId,
-				'includeAttachments' => $withAttachments,
-				'metadata' => $metadata,
-				'enforceWarninglist' => $enforceWarninglist,
-				'sgReferenceOnly' => $sgReferenceOnly
-			));
+			$result = $this->Event->fetchEvent(
+				$this->Auth->user(),
+				array(
+					'eventid' => $currentEventId,
+					'includeAttachments' => $withAttachments,
+					'metadata' => $metadata,
+					'enforceWarninglist' => $enforceWarninglist,
+					'sgReferenceOnly' => $sgReferenceOnly
+				),
+				true
+			);
 			if (!empty($result)) {
 				$result = $this->Whitelist->removeWhitelistedFromArray($result, false);
 				$final .= $converter->convert($result[0]);
--- a/app/Controller/JobsController.php
+++ b/app/Controller/JobsController.php
@ -84,20 +84,35 @@ class JobsController extends AppController {
 		$org_id = $this->Auth->user('org_id');
 		if ($this->_isSiteAdmin()) $org_id = 0;

-		$progress = $this->Job->find('first', array(
-			'conditions' => array(
-				'job_type' => $type,
-				'org_id' => $org_id
-			),
-			'fields' => array('id', 'progress'),
-			'order' => array('Job.id' => 'desc'),
-		));
+		if (is_numeric($type)) {
+			$progress = $this->Job->find('first', array(
+				'conditions' => array(
+					'Job.id' => $type,
+					'org_id' => $org_id
+				),
+				'fields' => array('id', 'progress'),
+				'order' => array('Job.id' => 'desc'),
+			));
+		} else {
+			$progress = $this->Job->find('first', array(
+				'conditions' => array(
+					'job_type' => $type,
+					'org_id' => $org_id
+				),
+				'fields' => array('id', 'progress'),
+				'order' => array('Job.id' => 'desc'),
+			));
+		}
 		if (!$progress) {
 			$progress = 0;
 		} else {
 			$progress = $progress['Job']['progress'];
 		}
-		return new CakeResponse(array('body' => json_encode($progress), 'type' => 'json'));
+		if ($this->_isRest()) {
+			return $this->RestResponse->viewData(array('progress' => $progress . '%'), $this->response->type());
+		} else {
+			return new CakeResponse(array('body' => json_encode($progress), 'type' => 'json'));
+		}
 	}

 	public function cache($type) {
@ -110,6 +125,10 @@ class JobsController extends AppController {
 			$target = 'Events visible to: '.$this->Auth->user('Organisation')['name'];
 		}
 		$id = $this->Job->cache($type, $this->Auth->user());
-		return new CakeResponse(array('body' => json_encode($id), 'type' => 'json'));
+		if ($this->_isRest()) {
+			return $this->RestResponse->viewData(array('job_id' => $id), $this->response->type());
+		} else {
+			return new CakeResponse(array('body' => json_encode($id), 'type' => 'json'));
+		}
 	}
 }
--- a/app/Controller/TasksController.php
+++ b/app/Controller/TasksController.php
@ -81,6 +81,8 @@ class TasksController extends AppController {
 		if ($type === 'cache_exports') $this->_cacheScheduler($timestamp, $id);
 		if ($type === 'pull_all') $this->_pullScheduler($timestamp, $id);
 		if ($type === 'push_all') $this->_pushScheduler($timestamp, $id);
+		if ($type === 'cache_feeds') $this->_feedScheduler($timestamp, $id, 1);
+		if ($type === 'pull_feeds') $this->_feedScheduler($timestamp, $id, 0);
 	}

 	private function _cacheScheduler($timestamp, $id) {
@ -119,4 +121,40 @@ class TasksController extends AppController {
 		$this->Task->saveField('process_id', $process_id);
 	}

+	private function _feedScheduler($timestamp, $id, $type) {
+		$process_id = CakeResque::enqueueAt(
+				$timestamp,
+				'default',
+				'ServerShell',
+				array('enqueueFeed', $timestamp, $this->Auth->user('id'),  $id, $type),
+				true
+		);
+		$this->Task->id = $id;
+		$this->Task->saveField('process_id', $process_id);
+	}
+
+	private function _feedPullScheduler($timestamp, $id) {
+		$process_id = CakeResque::enqueueAt(
+				$timestamp,
+				'default',
+				'ServerShell',
+				array('enqueuePull', $timestamp, $this->Auth->user('id'),  $id),
+				true
+		);
+		$this->Task->id = $id;
+		$this->Task->saveField('process_id', $process_id);
+	}
+
+	private function _feedCacheScheduler($timestamp, $id) {
+		$process_id = CakeResque::enqueueAt(
+				$timestamp,
+				'default',
+				'ServerShell',
+				array('enqueuePull', $timestamp, $this->Auth->user('id'),  $id),
+				true
+		);
+		$this->Task->id = $id;
+		$this->Task->saveField('process_id', $process_id);
+	}
+
 }
--- a/app/Lib/Export/NidsSuricataExport.php
+++ b/app/Lib/Export/NidsSuricataExport.php
@ -136,6 +136,7 @@ class NidsSuricataExport extends NidsExport {
 				    $content = 'flow:to_server; app-layer-protocol:tls;';
 				}
 				# Domain: rule on https certificate subject
+				/*
 				else {
 				    $suricata_protocol = 'tls';
 				    $suricata_src_ip = '$EXTERNAL_NET';
@ -144,6 +145,7 @@ class NidsSuricataExport extends NidsExport {
 				    $suricata_dst_port = 'any';
 				    $content = 'tls_cert_subject; content:"' . $data['host'] . '"; nocase; pcre:"/' . $data['host'] . '$/";';
 				}
+				*/
 				break;
 	
 		    case "ssh":
--- a/app/Model/Attribute.php
+++ b/app/Model/Attribute.php
@ -155,6 +155,7 @@ class Attribute extends AppModel {
 			'bin' => array('desc' => 'Bank Identification Number', 'default_category' => 'Financial fraud', 'to_ids' => 1),
 			'cc-number' => array('desc' => 'Credit-Card Number', 'default_category' => 'Financial fraud', 'to_ids' => 1),
 			'prtn' => array('desc' => 'Premium-Rate Telephone Number', 'default_category' => 'Financial fraud', 'to_ids' => 1),
+			'phone-number' => array('desc' => 'Telephone Number', 'default_category' => 'Person', 'to_ids' => 0),
 			'threat-actor' => array('desc' => 'A string identifying the threat actor', 'default_category' => 'Attribution', 'to_ids' => 0),
 			'campaign-name' => array('desc' => 'Associated campaign name', 'default_category' => 'Attribution', 'to_ids' => 0),
 			'campaign-id' => array('desc' => 'Associated campaign ID', 'default_category' => 'Attribution', 'to_ids' => 0),
@ -305,7 +306,7 @@ class Attribute extends AppModel {
 			'Financial fraud' => array(
 					'desc' => 'Financial Fraud indicators',
 					'formdesc' => 'Financial Fraud indicators, for example: IBAN Numbers, BIC codes, Credit card numbers, etc.',
-					'types' => array('btc', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'comment', 'text', 'other', 'hex'),
+					'types' => array('btc', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'phone-number', 'comment', 'text', 'other', 'hex'),
 					),
 			'Support Tool' => array(
 					'desc' => 'Tools supporting analysis or detection of the event',
@ -318,11 +319,11 @@ class Attribute extends AppModel {
 			),
 			'Person' => array(
 					'desc' => 'A human being - natural person',
-					'types' => array('first-name', 'middle-name', 'last-name', 'date-of-birth', 'place-of-birth', 'gender', 'passport-number', 'passport-country', 'passport-expiration', 'redress-number', 'nationality', 'visa-number', 'issue-date-of-the-visa', 'primary-residence', 'country-of-residence', 'special-service-request', 'frequent-flyer-number', 'travel-details', 'payment-details', 'place-port-of-original-embarkation', 'place-port-of-clearance', 'place-port-of-onward-foreign-destination', 'passenger-name-record-locator-number', 'comment', 'text', 'other')
+					'types' => array('first-name', 'middle-name', 'last-name', 'date-of-birth', 'place-of-birth', 'gender', 'passport-number', 'passport-country', 'passport-expiration', 'redress-number', 'nationality', 'visa-number', 'issue-date-of-the-visa', 'primary-residence', 'country-of-residence', 'special-service-request', 'frequent-flyer-number', 'travel-details', 'payment-details', 'place-port-of-original-embarkation', 'place-port-of-clearance', 'place-port-of-onward-foreign-destination', 'passenger-name-record-locator-number', 'comment', 'text', 'other', 'phone-number')
 			),
 			'Other' => array(
 					'desc' => 'Attributes that are not part of any other category or are meant to be used as a component in MISP objects in the future',
-					'types' => array('comment', 'text', 'other', 'size-in-bytes', 'counter', 'datetime', 'cpe', 'port', 'float', 'hex')
+					'types' => array('comment', 'text', 'other', 'size-in-bytes', 'counter', 'datetime', 'cpe', 'port', 'float', 'hex', 'phone-number')
 					)
 	);

@ -366,7 +367,7 @@ class Attribute extends AppModel {
 	public $typeGroupings = array(
 		'file' => array('attachment', 'pattern-in-file', 'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'impfuzzy','authentihash', 'pehash', 'tlsh', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|authentihash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|pehash', 'malware-sample', 'x509-fingerprint-sha1'),
 		'network' => array('ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port', 'hostname', 'hostname|port', 'domain', 'domain|ip', 'email-dst', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'pattern-in-traffic', 'x509-fingerprint-sha1'),
-		'financial' => array('btc', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn')
+		'financial' => array('btc', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'phone-number')
 	);

 	public $order = array("Attribute.event_id" => "DESC");
@ -1061,6 +1062,7 @@ class Attribute extends AppModel {
 			case 'bank-account-nr':
 			case 'aba-rtn':
 			case 'prtn':
+			case 'phone-number':
 			case 'whois-registrant-phone':
 				if (is_numeric($value)) {
 					$returnValue = true;
@ -1140,6 +1142,7 @@ class Attribute extends AppModel {
 				break;
 			case 'prtn':
 			case 'whois-registrant-phone':
+			case 'phone-number':
 				if (substr($value, 0, 1) == '+') $value = '00' . substr($value, 1);
 				$value = preg_replace('/[^0-9]+/', '', $value);
 				break;
@ -2316,11 +2319,20 @@ class Attribute extends AppModel {
 		if (isset($options['group'])) $params['group'] = array_merge(array('Attribute.id'), $options['group']);
 		if (Configure::read('MISP.unpublishedprivate')) $params['conditions']['AND'][] = array('OR' => array('Event.published' => 1, 'Event.orgc_id' => $user['org_id']));
 		if (!empty($options['list'])) {
+			if (!empty($options['event_ids'])) {
+				$fields = array('Attribute.event_id', 'Attribute.event_id');
+				$group = array('Attribute.event_id');
+			} else {
+				$fields = array('Attribute.event_id');
+				$group = false;
+			}
+			$start = microtime(true);
 			$results = $this->find('list', array(
 				'conditions' => $params['conditions'],
 				'recursive' => -1,
 				'contain' => array('Event'),
-				'fields' => array('Attribute.event_id'),
+				'fields' => $fields,
+				'group' => $group,
 				'sort' => false
 			));
 			return $results;
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@ -57,6 +57,8 @@ class Event extends AppModel {

 	public $shortDist = array(0 => 'Organisation', 1 => 'Community', 2 => 'Connected', 3 => 'All', 4 => ' sharing Group');

+	private $__assetCache = array();
+
 	public $export_types = array(
 			'json' => array(
 					'extension' => '.json',
@ -465,10 +467,10 @@ class Event extends AppModel {
 		$eventIds = Set::extract('/Event/id', $events);
 		$conditionsCorrelation = $this->__buildEventConditionsCorrelation($user, $eventIds, $sgids);
 		$correlations = $this->Correlation->find('all',array(
-				'fields' => array('Correlation.1_event_id', 'count(distinct(Correlation.event_id)) as count'),
-				'conditions' => $conditionsCorrelation,
-				'recursive' => -1,
-				'group' => array('Correlation.1_event_id'),
+			'fields' => array('Correlation.1_event_id', 'count(distinct(Correlation.event_id)) as count'),
+			'conditions' => $conditionsCorrelation,
+			'recursive' => -1,
+			'group' => array('Correlation.1_event_id'),
 		));
 		$correlations = Hash::combine($correlations, '{n}.Correlation.1_event_id', '{n}.0.count');
 		foreach ($events as &$event) $event['Event']['correlation_count'] = (isset($correlations[$event['Event']['id']])) ? $correlations[$event['Event']['id']] : 0;
@ -1191,6 +1193,38 @@ class Event extends AppModel {
 		}
 	}

+	public function fetchSimpleEventIds($user, $params = array()) {
+		$conditions = array();
+		if (!$user['Role']['perm_site_admin']) {
+			$sgids = $this->cacheSgids($user, true);
+			$conditions['AND']['OR'] = array(
+				'Event.org_id' => $user['org_id'],
+				array(
+					'AND' => array(
+						'Event.distribution >' => 0,
+						'Event.distribution <' => 4,
+						Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(),
+					),
+				),
+				array(
+					'AND' => array(
+						'Event.sharing_group_id' => $sgids,
+						'Event.distribution' => 4,
+						Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(),
+					)
+				)
+			);
+		}
+		$conditions['AND'][] = $params['conditions'];
+		$fields = array('Event.id', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id');
+		$results = array_values($this->find('list', array(
+			'conditions' => $conditions,
+			'recursive' => -1,
+			'fields' => array('Event.id')
+		)));
+		return $results;
+	}
+
 	public function fetchEventIds($user, $from = false, $to = false, $last = false, $list = false, $timestamp = false, $publish_timestamp = false, $eventIdList = false) {
 		$conditions = array();
 		// restricting to non-private or same org if the user is not a site-admin.
@ -1249,7 +1283,7 @@ class Event extends AppModel {
 	// to: date string (YYYY-MM-DD)
 	// includeAllTags: true will include the tags
 	// includeAttachments: true will attach the attachments to the attributes in the data field
-	public function fetchEvent($user, $options = array()) {
+	public function fetchEvent($user, $options = array(), $useCache = false) {
 		if (isset($options['Event.id'])) $options['eventid'] = $options['Event.id'];
 		$possibleOptions = array('eventid', 'idList', 'tags', 'from', 'to', 'last', 'to_ids', 'includeAllTags', 'includeAttachments', 'event_uuid', 'distribution', 'sharing_group_id', 'disableSiteAdmin', 'metadata', 'includeGalaxy', 'enforceWarninglist', 'sgReferenceOnly');
 		if (!isset($options['excludeGalaxy']) || !$options['excludeGalaxy']) {
@ -1275,11 +1309,9 @@ class Event extends AppModel {
 		} else {
 			$flattenObjects = false;
 		}
-
+		$sgids = $this->cacheSgids($user, $useCache);
 		// restricting to non-private or same org if the user is not a site-admin.
 		if (!$isSiteAdmin) {
-			$sgids = $this->SharingGroup->fetchAllAuthorised($user);
-			if (empty($sgids)) $sgids = array(-1);
 			$conditions['AND']['OR'] = array(
 				'Event.org_id' => $user['org_id'],
 				array(
@ -1299,14 +1331,9 @@ class Event extends AppModel {
 			);
 			// if delegations are enabled, check if there is an event that the current user might see because of the request itself
 			if (Configure::read('MISP.delegation')) {
-				$this->EventDelegation = ClassRegistry::init('EventDelegation');
-				$delegatedEventIDs = $this->EventDelegation->find('list', array(
-					'conditions' => array('EventDelegation.org_id' => $user['org_id']),
-					'fields' => array('event_id')
-				));
+				$delegatedEventIDs = $this->__cachedelegatedEventIDs($user, $useCache);
 				$conditions['AND']['OR']['Event.id'] = $delegatedEventIDs;
 			}
-
 			$conditionsAttributes['AND'][0]['OR'] = array(
 				array('AND' => array(
 					'Attribute.distribution >' => 0,
@ -1368,19 +1395,10 @@ class Event extends AppModel {
 		}
 		// If we sent any tags along, load the associated tag names for each attribute
 		if ($options['tags']) {
-			$tag = ClassRegistry::init('Tag');
-			$args = $this->Attribute->dissectArgs($options['tags']);
-			$tagArray = $tag->fetchEventTagIds($args[0], $args[1]);
-			$temp = array();
-			foreach ($tagArray[0] as $accepted) {
-				$temp['OR'][] = array('Event.id' => $accepted);
+			$temp = $this->__generateCachedTagFilters($tagRules);
+			foreach ($temp as $rules) {
+				$conditions['AND'][] = $rules;
 			}
-			$conditions['AND'][] = $temp;
-			$temp = array();
-			foreach ($tagArray[1] as $rejected) {
-				$temp['AND'][] = array('Event.id !=' => $rejected);
-			}
-			$conditions['AND'][] = $temp;
 		}

 		if ($options['to_ids']) {
@ -1400,29 +1418,9 @@ class Event extends AppModel {
 		$fieldsShadowAtt = array('ShadowAttribute.id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.value', 'ShadowAttribute.to_ids', 'ShadowAttribute.uuid', 'ShadowAttribute.event_uuid', 'ShadowAttribute.event_id', 'ShadowAttribute.old_id', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.proposal_to_delete', 'ShadowAttribute.timestamp');
 		$fieldsOrg = array('id', 'name', 'uuid');
 		$fieldsServer = array('id', 'url', 'name');
-		$fieldsSharingGroup = array(
-			array('fields' => array('SharingGroup.id','SharingGroup.name', 'SharingGroup.releasability', 'SharingGroup.description')),
-			array(
-				'fields' => array('SharingGroup.*'),
-					'Organisation' => array('fields' => $fieldsOrg),
-					'SharingGroupOrg' => array(
-						'Organisation' => array('fields' => $fieldsOrg, 'order' => false),
-					),
-					'SharingGroupServer' => array(
-						'Server' => array('fields' => $fieldsServer, 'order' => false),
-				),
-			),
-		);
 		if (!$options['includeAllTags']) $tagConditions = array('exportable' => 1);
 		else $tagConditions = array();
-
-		$sharingGroupDataTemp = $this->SharingGroup->find('all', array(
-			'fields' => $fieldsSharingGroup[(($user['Role']['perm_site_admin'] || $user['Role']['perm_sync']) ? 1 : 0)]['fields'],
-		));
-		$sharingGroupData = array();
-		foreach ($sharingGroupDataTemp as $k => $v) {
-			$sharingGroupData[$v['SharingGroup']['id']] = $v;
-		}
+		$sharingGroupData = $this->__cacheSharingGroupData($user, $useCache);
 		$params = array(
 			'conditions' => $conditions,
 			'recursive' => 0,
@ -1477,7 +1475,6 @@ class Event extends AppModel {
 		if (empty($results)) return array();

 		// Do some refactoring with the event
-		$sgsids = $this->SharingGroup->fetchAllAuthorised($user);
 		if (Configure::read('Plugin.Sightings_enable') !== false) {
 			$this->Sighting = ClassRegistry::init('Sighting');
 		}
@ -1565,10 +1562,10 @@ class Event extends AppModel {
 				$event['EventTag'] = array_values($event['EventTag']);
 			}
 			// Let's find all the related events and attach it to the event itself
-			$results[$eventKey]['RelatedEvent'] = $this->getRelatedEvents($user, $event['Event']['id'], $sgsids);
+			$results[$eventKey]['RelatedEvent'] = $this->getRelatedEvents($user, $event['Event']['id'], $sgids);
 			// Let's also find all the relations for the attributes - this won't be in the xml export though
-			$results[$eventKey]['RelatedAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], $sgsids);
-			$results[$eventKey]['RelatedShadowAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], $sgsids, true);
+			$results[$eventKey]['RelatedAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], $sgids);
+			$results[$eventKey]['RelatedShadowAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], $sgids, true);
 			if (isset($event['ShadowAttribute']) && !empty($event['ShadowAttribute']) && isset($options['includeAttachments']) && $options['includeAttachments']) {
 				foreach ($event['ShadowAttribute'] as $k => $sa) {
 					if ($this->ShadowAttribute->typeIsAttachment($sa['type'])) {
@ -1583,8 +1580,8 @@ class Event extends AppModel {
 					$warninglists = $this->Warninglist->fetchForEventView();
 				}
 				if (isset($options['includeFeedCorrelations']) && $options['includeFeedCorrelations']) {
-					$this->Feed = ClassRegistry::init('Feed');
-					$event['Attribute'] = $this->Feed->attachFeedCorrelations($event['Attribute'], $user);
+				$this->Feed = ClassRegistry::init('Feed');
+				$event['Attribute'] = $this->Feed->attachFeedCorrelations($event['Attribute'], $user);
 				}
 				foreach ($event['Attribute'] as $key => $attribute) {
 					if (!$options['sgReferenceOnly'] && $event['Attribute'][$key]['sharing_group_id']) {
@ -1650,7 +1647,7 @@ class Event extends AppModel {
 				}
 				$event['Attribute'] = array_values($event['Attribute']);
 			}
-			if (isset($event['ShadowAttribute'])) {
+			if (!empty($event['ShadowAttribute'])) {
 				if ($isSiteAdmin && isset($options['includeFeedCorrelations']) && $options['includeFeedCorrelations']) {
 					$this->Feed = ClassRegistry::init('Feed');
 					$event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user);
@ -3651,4 +3648,93 @@ class Event extends AppModel {
 			}
 		}
 	}
+
+	public function cacheSgids($user, $useCache = false) {
+		if ($useCache && isset($this->__assetCache['sgids'])) {
+			return $this->__assetCache['sgids'];
+		} else {
+			$sgids = $this->SharingGroup->fetchAllAuthorised($user);
+			if (empty($sgids)) $sgids = array(-1);
+			if ($useCache) $this->__assetCache['sgids'] = $sgids;
+			return $sgids;
+		}
+	}
+
+	private function __cacheSharingGroupData($user, $useCache = false) {
+		if ($useCache && isset($this->__assetCache['sharingGroupData'])) {
+			return $this->__assetCache['sharingGroupData'];
+		} else {
+			$fieldsOrg = array('id', 'name', 'uuid');
+			$fieldsServer = array('id', 'url', 'name');
+			$fieldsSharingGroup = array(
+				array('fields' => array('SharingGroup.id','SharingGroup.name', 'SharingGroup.releasability', 'SharingGroup.description')),
+				array(
+					'fields' => array('SharingGroup.*'),
+						'Organisation' => array('fields' => $fieldsOrg),
+						'SharingGroupOrg' => array(
+							'Organisation' => array('fields' => $fieldsOrg, 'order' => false),
+						),
+						'SharingGroupServer' => array(
+							'Server' => array('fields' => $fieldsServer, 'order' => false),
+					),
+				)
+			);
+			$containsSharingGroup = array(
+				array(),
+				array('Organisation', 'SharingGroupOrg' => array('Organisation'), 'SharingGroupServer' => array('Server'))
+			);
+			$sharingGroupDataTemp = $this->SharingGroup->find('all', array(
+				'fields' => $fieldsSharingGroup[(($user['Role']['perm_site_admin'] || $user['Role']['perm_sync']) ? 1 : 0)]['fields'],
+				'contain' => $containsSharingGroup[(($user['Role']['perm_site_admin'] || $user['Role']['perm_sync']) ? 1 : 0)],
+				'recursive' => -1
+			));
+			$sharingGroupData = array();
+			foreach ($sharingGroupDataTemp as $k => $v) {
+				$sharingGroupData[$v['SharingGroup']['id']] = $v;
+			}
+			if ($useCache) $this->__assetCache['sharingGroupData'] = $sharingGroupData;
+			return $sharingGroupData;
+		}
+	}
+
+	private function __cachedelegatedEventIDs($user, $useCache = false) {
+		if ($useCache && isset($this->__assetCache['delegatedEventIDs'])) {
+			return $this->__assetCache['delegatedEventIDs'];
+		} else {
+			$this->EventDelegation = ClassRegistry::init('EventDelegation');
+			$delegatedEventIDs = $this->EventDelegation->find('list', array(
+				'conditions' => array('EventDelegation.org_id' => $user['org_id']),
+				'fields' => array('event_id')
+			));
+			if ($useCache) $this->__assetCache['delegationEventIDs'] = $delegatedEventIDs;
+			return $delegatedEventIDs;
+		}
+	}
+
+	private function __generateCachedTagFilters($tagRules, $useCache = false) {
+		if ($useCache && isset($this->__assetCache['tagFilters'])) {
+			return $this->__assetCache['tagFilters'];
+		} else {
+			$filters = array();
+			$tag = ClassRegistry::init('Tag');
+			$args = $this->Attribute->dissectArgs($tagRules);
+			$tagArray = $this->EventTag->Tag->fetchEventTagIds($args[0], $args[1]);
+			$temp = array();
+			foreach ($tagArray[0] as $accepted) {
+				$temp['OR'][] = array('Event.id' => $accepted);
+			}
+			$filters[] = $temp;
+			$temp = array();
+			foreach ($tagArray[1] as $rejected) {
+				$temp['AND'][] = array('Event.id !=' => $rejected);
+			}
+			$filters[] = $temp;
+			if ($useCache) $this->__assetCache['tagFilters'] = $filters;
+			return $filters;
+		}
+	}
+
+	private function __destroyCaches() {
+		$this->__assetCache = array();
+	}
 }
--- a/app/Model/Feed.php
+++ b/app/Model/Feed.php
@ -43,7 +43,7 @@ class Feed extends AppModel {
 			'name' => 'Freetext Parsed Feed'
 		),
 		'csv' => array(
-				'name' => 'Simple CSV Parsed Feed'
+			'name' => 'Simple CSV Parsed Feed'
 		)
 	);

@ -172,6 +172,10 @@ class Feed extends AppModel {
 				} catch (Exception $e) {
 					return false;
 				}
+				if ($response->code == 302) {
+					$HttpSocket = $this->__setupHttpSocket(false);
+					$response = $HttpSocket->get($response['header']['Location'], '', array());
+				}
 				if ($response->code == 200) {
 					$redis = $this->setupRedis();
 					if ($redis === false) {
@ -321,9 +325,9 @@ class Feed extends AppModel {
 		}
 		if (isset($actions['edit']) && !empty($actions['edit'])) {
 			foreach ($actions['edit'] as $editTarget) {
+				if ($result === 'blocked') continue;
 				$result = $this->__updateEventFromFeed($HttpSocket, $feed, $editTarget['uuid'], $editTarget['id'], $user, $filterRules);
 				$this->__cleanupFile($feed, '/' . $uuid . '.json');
-				if ($result === 'blocked') continue;
 				if ($result === true) {
 					$results['edit']['success'] = $uuid;
 				} else {
@ -625,7 +629,7 @@ class Feed extends AppModel {
 				return true;
 			}
 			$result = $this->downloadFromFeed($actions, $this->data, $HttpSocket, $user, $jobId);
-			$this->__cleanupFile($feed, '/manifest.json');
+			$this->__cleanupFile($this->data, '/manifest.json');
 			if ($jobId) {
 				$job->id = $jobId;
 				$job->saveField('message', 'Job complete.');
--- a/app/Model/Task.php
+++ b/app/Model/Task.php
@ -32,7 +32,25 @@ class Task extends AppModel {
 					'description' => 'Initiates a full push for all eligible instances.',
 					'next_execution_time' => 1391601600,
 					'message' => 'Not scheduled yet.'
-			)
+			),
+			'cache_feeds' => array(
+					'type' => 'cache_feeds',
+					'timer' => 0,
+					'scheduled_time' => '12:00',
+					'process_id' => '',
+					'description' => 'Initiates the caching of all feeds.',
+					'next_execution_time' => 1391601600,
+					'message' => 'Not scheduled yet.'
+			),
+			'pull_feeds' => array(
+					'type' => 'pull_feeds',
+					'timer' => 0,
+					'scheduled_time' => '12:00',
+					'process_id' => '',
+					'description' => 'Initiates the pull of all feeds.',
+					'next_execution_time' => 1391601600,
+					'message' => 'Not scheduled yet.'
+			),
 	);

 	// takes a time in the 24h format (13:49) and an integer representing the number of hours
--- a/app/Model/User.php
+++ b/app/Model/User.php
@ -157,7 +157,7 @@ class User extends AppModel {
 			),
 		),
 		'newsread' => array(
-			'boolean' => array(
+			'numeric' => array(
 				'rule' => array('numeric')
 			),
 		),
@ -245,6 +245,9 @@ class User extends AppModel {
 		if (!isset($this->data['User']['certif_public']) || empty($this->data['User']['certif_public'])) $this->data['User']['certif_public'] = '';
 		if (!isset($this->data['User']['authkey']) || empty($this->data['User']['authkey'])) $this->data['User']['authkey'] = $this->generateAuthKey();
 		if (!isset($this->data['User']['nids_sid']) || empty($this->data['User']['nids_sid'])) $this->data['User']['nids_sid'] = mt_rand(1000000, 9999999);
+		if (isset($this->data['User']['newsread']) && $this->data['User']['newsread'] === null) {
+			$this->data['User']['newsread'] = 0;
+		}
 		return true;
 	}

--- a/app/View/Elements/Events/eventIndexTable.ctp
+++ b/app/View/Elements/Events/eventIndexTable.ctp
@ -1,9 +1,11 @@
 <table class="table table-striped table-hover table-condensed">
 	<tr>
-		<?php if ($isSiteAdmin && !empty($events)): ?>
+		<?php if ($isSiteAdmin): ?>
 			<th>
 				<input class="select_all select" type="checkbox" title="Select all" role="button" tabindex="0" aria-label="Select all eventson current page" onClick="toggleAllCheckboxes();" />&nbsp;
 			</th>
+		<?php else: ?>
+			<th style="padding-left:0px;padding-right:0px;">&nbsp;</th>
 		<?php endif;?>
 		<th class="filter">
 			<?php echo $this->Paginator->sort('published');?>
@ -64,17 +66,19 @@
 	</tr>
 	<?php foreach ($events as $event): ?>
 	<tr <?php if ($event['Event']['distribution'] == 0) echo 'class = "privateRed"'?>>
-		<?php if ($me['Role']['perm_modify']): ?>
-			<td style="width:10px;" data-id="<?php echo h($event['Event']['id']); ?>">
-				<?php
-					if ($isSiteAdmin || ($event['Event']['orgc_id'] == $me['org_id'])):
-				?>
+			<?php
+				if ($isSiteAdmin || ($event['Event']['orgc_id'] == $me['org_id'])):
+			?>
+					<td data-id="<?php echo h($event['Event']['id']); ?>">
 						<input class="select" type="checkbox" data-id="<?php echo $event['Event']['id'];?>" />
-				<?php
-					endif;
-				?>
-			</td>
-		<?php endif;?>
+					</td>
+			<?php
+				else:
+			?>
+					<td style="padding-left:0px;padding-right:0px;"></td>
+			<?php
+				endif;
+			?>
 		<td class="short" ondblclick="document.location.href ='<?php echo $baseurl."/events/view/".$event['Event']['id'];?>'">
 			<?php
 			if ($event['Event']['published'] == 1) {
--- a/app/View/Events/index.ctp
+++ b/app/View/Events/index.ctp
@ -41,7 +41,7 @@
 				<?php echo $this->Html->link('', array('controller' => 'events', 'action' => 'index'), array('class' => 'icon-remove', 'title' => 'Remove filters'));?>
 			</span>
 			<?php endif;?>
-			<span role="button" tabindex="0" aria-label="Quickfilter" title="Quickfilter" id="quickFilterButton" class="tabMenuFilterFieldButton useCursorPointer" onClick='quickFilter(<?php echo h($passedArgs);?>, "/events/index");'>Filter</span>
+			<span role="button" tabindex="0" aria-label="Quickfilter" title="Quickfilter" id="quickFilterButton" class="tabMenuFilterFieldButton useCursorPointer" onClick="quickFilter(<?php echo h($passedArgs); ?>, '<?php echo $baseurl . '/events/index'; ?>');">Filter</span>
 			<input class="tabMenuFilterField" type="text" id="quickFilterField"></input>
 			<?php
 				$tempArgs = json_decode($passedArgs, true);
--- a/app/View/Helper/CommandHelper.php
+++ b/app/View/Helper/CommandHelper.php
@ -6,14 +6,88 @@ App::uses('AppHelper', 'View/Helper');


 	class CommandHelper extends AppHelper {
-		public function convertQuotes($string) {
-			$string = str_ireplace('[QUOTE]', '<div class="quote">', $string);
-			$string = str_ireplace('[/QUOTE]', '</div>', $string);
-			$string = preg_replace('%\[event\]\s*(\d*)\s*\[/event\]%isU', '<a href="' . h(Configure::read('MISP.baseurl')). '/events/view/$1> Event $1</a>', $string);
-			$string = preg_replace('%\[thread\]\s*(\d*)\s*\[/thread\]%isU', '<a href="' . h(Configure::read('MISP.baseurl')). '/threads/view/$1> Thread $1</a>', $string);
-			$string = preg_replace('%\[link\]\s*(http|https|ftp|git|ftps)(.*)\s*\[/link\]%isU', '<a href="$1$2">$1$2</a>', $string);
-			$string = preg_replace('%\[code\](.*)\[/code\]%isU', '<pre>$1</pre>', $string);
+		var $helpers = array('Html');

+		private $__replacement;
+
+		private function __buildReplacements() {
+			$this->__replacement = array(
+				'link' => array('type' => 'url', 'url' => '$1', 'text' => '$1'),
+				'thread' => array('type' => 'url', 'url' => h(Configure::read('MISP.baseurl')). '/threads/view/$1', 'text' => ' Thread $1'),
+				'event' => array('type' => 'url', 'url' => h(Configure::read('MISP.baseurl')). '/events/view/$1', 'text' => ' Event $1'),
+				'code' => array('type' => 'replace', 'text' => '<pre>$1</pre>'),
+				'quote' => array('type' => 'replace', 'text' => '<div class="quote">$1</div>')
+			);
+		}
+
+
+		public function convertQuotes($string) {
+			$this->__buildReplacements();
+			foreach ($this->__replacement as $trigger => $replacement) {
+				$result = $this->__handleLinks($string, $trigger);
+				if (!$result) return 'Malformed syntax.';
+			}
 			return $string;
 		}
+
+		private function __handleLinks(&$string, $trigger) {
+			$opening = preg_match('%\[' . $trigger . '\]%isU', $string, $opening_matches, PREG_OFFSET_CAPTURE);
+			$closing = preg_match('%\[/' . $trigger . '\]%isU', $string, $closing_matches, PREG_OFFSET_CAPTURE);
+			$opening_len = strlen($trigger) + 2;
+			$closing_len = $opening_len + 1;
+			if ((count($opening) !== count($closing))) return false;
+			$pairs = array();
+			$rearrangedTags = array();
+			foreach ($opening_matches as $opening_tag) {
+				$rearrangedTags[$opening_tag[1]] = 'open';
+			}
+			foreach ($closing_matches as $closing_tag) {
+				$rearrangedTags[$closing_tag[1]] = 'close';
+			}
+			foreach ($opening_matches as $opening_tag) {
+				$counter = 1;
+				foreach ($rearrangedTags as $pos => $type) {
+					if ($opening_tag[1] == $pos) continue;
+					if ($type == 'close') $counter--;
+					else $counter++;
+					if ($counter == 0) {
+						$pairs[] = array($opening_tag[1], $pos);
+						continue 2;
+					}
+				}
+			}
+			foreach ($pairs as $pair) {
+				$temp = substr($string, 0, $pair[0]);
+				if ($this->__replacement[$trigger]['type'] == 'url') {
+					$data = substr($string, $pair[0] + $opening_len, $pair[1] - ($pair[0] + $opening_len));
+					if (empty($data)) {
+						$replacement = '';
+					} else {
+						if (!is_numeric($data) && ($trigger == 'event' || $trigger == 'thread')) {
+							$replacement = '%MALFORMED URL%';
+						} else {
+							if (filter_var(str_replace('$1', $data, $this->__replacement[$trigger]['url']), FILTER_VALIDATE_URL)) {
+								$replacement = $this->Html->link(
+									str_replace('$1', $data, $this->__replacement[$trigger]['text']),
+									str_replace('$1', $data, $this->__replacement[$trigger]['url'])
+								);
+							} else {
+								$replacement = '%MALFORMED URL%';
+							}
+						}
+					}
+				} else {
+					$data = substr($string, $pair[0] + $opening_len, $pair[1] - ($pair[0] + $opening_len));
+					if (empty($data)) {
+						$replacement = '';
+					} else {
+						$replacement = str_replace('$1', $data, $this->__replacement[$trigger]['text']);
+					}
+				}
+				$temp .= $replacement;
+				$temp .= substr($string, $pair[1] + $closing_len, strlen($string));
+				$string = $temp;
+			}
+			return true;
+		}
 	}
--- a/app/View/Organisations/admin_add.ctp
+++ b/app/View/Organisations/admin_add.ctp
@ -1,5 +1,5 @@
 <div class="users form">
-<?php echo $this->Form->create('Organisation');?>
+<?php echo $this->Form->create('Organisation', array('enctype' => 'multipart/form-data'));?>
 	<fieldset>
 		<legend><?php echo __('New Organisation'); ?></legend>
 		<p style="font-weight:bold;">If the organisation should have access to this instance, make sure that the Local organisation setting is checked. <br />If you would only like to add a known external organisation for inclusion in sharing groups, uncheck the Local organisation setting.</p>
@ -28,6 +28,14 @@
 		?>
 		<hr />
 		<p style="font-weight:bold;">The following fields are all optional.</p>
+	<?php
+		echo $this->Form->input('logo', array(
+			'error' => array('escape' => false),
+			'type' => 'file',
+			'label' => 'Logo (48x48 png)'
+		));
+	?>
+		<div class="clear"></div>
 	<?php
 		echo $this->Form->input('nationality', array('options' => $countries));
 		echo $this->Form->input('sector', array('placeholder' => 'For example "financial".', 'style' => 'width:300px;'));
--- a/app/files/misp-galaxy
+++ b/app/files/misp-galaxy
@ -1 +1 @@
-Subproject commit cf780290bedcca5f6818a31e33eb072bdc44622c
+Subproject commit 6bb2a0738365c3b62154e9ccd2adccc012a64a35
--- a/app/files/taxonomies
+++ b/app/files/taxonomies
@ -1 +1 @@
-Subproject commit c062f77feabafd880f6c00ffce83125c52608bb9
+Subproject commit 6a6168b4a5235ce74053a3319a4e30c0ded54c61
--- a/vagrant/README.rst
+++ b/vagrant/README.rst
@ -0,0 +1,42 @@
+Development environment for MISP
+================================
+
+Vagrant is convenient to use in order to setup your development environment.
+
+This VM uses `synced folders <https://www.vagrantup.com/docs/synced-folders/>`_
+feature of Vagrant in order to let you work on the MISP source code on your
+host machine while the softwares (Apache, PHP, MariaDB, etc.) and libraries
+will be installed on the guest Vagrant machine.
+
+
+Installation of VirtualBox and Vagrant
+--------------------------------------
+
+.. code-block:: bash
+
+    $ sudo apt-get install virtualbox vagrant
+
+
+Deployment of MISP
+------------------
+
+MISP will be automatically deployed in an Ubuntu Zesty Server.
+
+.. code-block:: bash
+
+    $ git clone https://github.com/MISP/MISP.git
+    $ cd MISP/vagrant/
+    $ vagrant up
+
+Once the VM will be configured by Vagrant, go to the address
+http://127.0.0.1:5000.
+
+You can now edit the source code with your favorite editor and test it in your
+browser. The only thing is to not forget to restart Apache in the VM after a
+modification.
+
+Modules activated by default in the VM:
+
+* `MISP galaxy <https://github.com/MISP/misp-galaxy>`_ (http://127.0.0.1:5000/taxonomies/index)
+* `MISP taxonomies <https://github.com/MISP/misp-taxonomies>`_ (http://127.0.0.1:5000/galaxies.json)
+* `MISP modules <https://github.com/MISP/misp-modules>`_ (curl -s http://127.0.0.1:6666/modules)
--- a/vagrant/Vagrantfile
+++ b/vagrant/Vagrantfile
@ -0,0 +1,128 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
+VAGRANTFILE_API_VERSION = "2"
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+  # All Vagrant configuration is done here. The most common configuration
+  # options are documented and commented below. For a complete reference,
+  # please see the online documentation at vagrantup.com.
+
+  # Every Vagrant virtual environment requires a box to build off of.
+  #config.vm.box = "bento/ubuntu-16.04"
+  config.vm.box = "ubuntu/zesty64"
+  #config.vm.box_url = "https://atlas.hashicorp.com/ubuntu/boxes/zesty64/versions/20170412.1.0"
+  config.vm.provision :shell, path: "bootstrap.sh"
+
+  # Disable automatic box update checking. If you disable this, then
+  # boxes will only be checked for updates when the user runs
+  # `vagrant box outdated`. This is not recommended.
+  # config.vm.box_check_update = false
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine. In the example below,
+  # accessing "localhost:8080" will access port 80 on the guest machine.
+  config.vm.network :forwarded_port, guest: 80, host: 5000
+  config.vm.network :forwarded_port, guest: 6666, host: 6666
+
+  # Create a private network, which allows host-only access to the machine
+  # using a specific IP.
+  # config.vm.network "private_network", ip: "192.168.33.10"
+
+  # Create a public network, which generally matched to bridged network.
+  # Bridged networks make the machine appear as another physical device on
+  # your network.
+  # config.vm.network "public_network"
+
+  # If true, then any SSH connections made will enable agent forwarding.
+  # Default value: false
+  # config.ssh.forward_agent = true
+
+  # Share an additional folder to the guest VM. The first argument is
+  # the path on the host to the actual folder. The second argument is
+  # the path on the guest to mount the folder. And the optional third
+  # argument is a set of non-required options.
+  config.vm.synced_folder "..", "/var/www/MISP",
+                        owner: "www-data", group: "www-data", disabled: false
+
+  # Provider-specific configuration so you can fine-tune various
+  # backing providers for Vagrant. These expose provider-specific options.
+  # Example for VirtualBox:
+  #
+  config.vm.provider "virtualbox" do |vb|
+  #   # Don't boot with headless mode
+  #   vb.gui = true
+  #
+  #   # Use VBoxManage to customize the VM. For example to change memory:
+    vb.customize ["modifyvm", :id, "--memory", "4096"]
+    vb.customize ["modifyvm", :id, "--name", "MISP - Ubuntu 17.04 - DEV"]
+  end
+  #
+  # View the documentation for the provider you're using for more
+  # information on available options.
+
+  # Enable provisioning with CFEngine. CFEngine Community packages are
+  # automatically installed. For example, configure the host as a
+  # policy server and optionally a policy file to run:
+  #
+  # config.vm.provision "cfengine" do |cf|
+  #   cf.am_policy_hub = true
+  #   # cf.run_file = "motd.cf"
+  # end
+  #
+  # You can also configure and bootstrap a client to an existing
+  # policy server:
+  #
+  # config.vm.provision "cfengine" do |cf|
+  #   cf.policy_server_address = "10.0.2.15"
+  # end
+
+  # Enable provisioning with Puppet stand alone.  Puppet manifests
+  # are contained in a directory path relative to this Vagrantfile.
+  # You will need to create the manifests directory and a manifest in
+  # the file default.pp in the manifests_path directory.
+  #
+  # config.vm.provision "puppet" do |puppet|
+  #   puppet.manifests_path = "manifests"
+  #   puppet.manifest_file  = "site.pp"
+  # end
+
+  # Enable provisioning with chef solo, specifying a cookbooks path, roles
+  # path, and data_bags path (all relative to this Vagrantfile), and adding
+  # some recipes and/or roles.
+  #
+  # config.vm.provision "chef_solo" do |chef|
+  #   chef.cookbooks_path = "../my-recipes/cookbooks"
+  #   chef.roles_path = "../my-recipes/roles"
+  #   chef.data_bags_path = "../my-recipes/data_bags"
+  #   chef.add_recipe "mysql"
+  #   chef.add_role "web"
+  #
+  #   # You may also specify custom JSON attributes:
+  #   chef.json = { :mysql_password => "foo" }
+  # end
+
+  # Enable provisioning with chef server, specifying the chef server URL,
+  # and the path to the validation key (relative to this Vagrantfile).
+  #
+  # The Opscode Platform uses HTTPS. Substitute your organization for
+  # ORGNAME in the URL and validation key.
+  #
+  # If you have your own Chef Server, use the appropriate URL, which may be
+  # HTTP instead of HTTPS depending on your configuration. Also change the
+  # validation key to validation.pem.
+  #
+  # config.vm.provision "chef_client" do |chef|
+  #   chef.chef_server_url = "https://api.opscode.com/organizations/ORGNAME"
+  #   chef.validation_key_path = "ORGNAME-validator.pem"
+  # end
+  #
+  # If you're using the Opscode platform, your validator client is
+  # ORGNAME-validator, replacing ORGNAME with your organization name.
+  #
+  # If you have your own Chef Server, the default validation client name is
+  # chef-validator, unless you changed the configuration.
+  #
+  #   chef.validation_client_name = "ORGNAME-validator"
+end
--- a/vagrant/bootstrap.sh
+++ b/vagrant/bootstrap.sh
@ -0,0 +1,383 @@
+#! /usr/bin/env bash
+
+# Database configuration
+DBHOST='localhost'
+DBNAME='misp'
+DBUSER_ADMIN='root'
+DBPASSWORD_ADMIN="$(openssl rand -hex 32)"
+DBUSER_MISP='misp'
+DBPASSWORD_MISP="$(openssl rand -hex 32)"
+
+# Webserver configuration
+PATH_TO_MISP='/var/www/MISP'
+MISP_BASEURL='http://127.0.0.1:5000'
+MISP_LIVE='1'
+FQDN='localhost'
+
+# OpenSSL configuration
+OPENSSL_C='LU'
+OPENSSL_ST='State'
+OPENSSL_L='Location'
+OPENSSL_O='Organization'
+OPENSSL_OU='Organizational Unit'
+OPENSSL_CN='Common Name'
+OPENSSL_EMAILADDRESS='info@localhost'
+
+# GPG configuration
+GPG_REAL_NAME='Real name'
+GPG_EMAIL_ADDRESS='info@localhost'
+GPG_KEY_LENGTH='2048'
+GPG_PASSPHRASE=''
+
+
+
+
+echo -e "\n--- Installing MISP... ---\n"
+
+
+echo -e "\n--- Updating packages list ---\n"
+apt-get -qq update
+
+
+echo -e "\n--- Install base packages ---\n"
+apt-get -y install curl gcc git gnupg-agent make python openssl redis-server sudo vim zip > /dev/null 2>&1
+
+# To prevent a random error when cloning with Git: 'RPC failed; curl 56 GnuTLS recv error (-54): Error in the pull function.'
+git config --global http.postBuffer 1048576000
+git config --global https.postBuffer 1048576000
+
+echo -e "\n--- Installing and configuring Postfix ---\n"
+# # Postfix Configuration: Satellite system
+# # change the relay server later with:
+# sudo postconf -e 'relayhost = example.com'
+# sudo postfix reload
+echo "postfix postfix/mailname string `hostname`.ourdomain.org" | debconf-set-selections
+echo "postfix postfix/main_mailer_type string 'Satellite system'" | debconf-set-selections
+apt-get install -y postfix > /dev/null 2>&1
+
+
+echo -e "\n--- Installing MariaDB specific packages and settings ---\n"
+apt-get install -y mariadb-client mariadb-server > /dev/null 2>&1
+# Secure the MariaDB installation (especially by setting a strong root password)
+sleep 7 # give some time to the DB to launch...
+apt-get install -y expect > /dev/null 2>&1
+expect -f - <<-EOF
+  set timeout 10
+  spawn mysql_secure_installation
+  expect "Enter current password for root (enter for none):"
+  send -- "\r"
+  expect "Set root password?"
+  send -- "y\r"
+  expect "New password:"
+  send -- "${DBPASSWORD_ADMIN}\r"
+  expect "Re-enter new password:"
+  send -- "${DBPASSWORD_ADMIN}\r"
+  expect "Remove anonymous users?"
+  send -- "y\r"
+  expect "Disallow root login remotely?"
+  send -- "y\r"
+  expect "Remove test database and access to it?"
+  send -- "y\r"
+  expect "Reload privilege tables now?"
+  send -- "y\r"
+  expect eof
+EOF
+apt-get purge -y expect > /dev/null 2>&1
+
+
+echo -e "\n--- Installing Apache2 ---\n"
+apt-get install -y apache2 apache2-doc apache2-utils > /dev/null 2>&1
+a2dismod status > /dev/null 2>&1
+a2enmod ssl > /dev/null 2>&1
+a2enmod rewrite > /dev/null 2>&1
+a2dissite 000-default > /dev/null 2>&1
+a2ensite default-ssl > /dev/null 2>&1
+
+
+echo -e "\n--- Installing PHP-specific packages ---\n"
+apt-get install -y libapache2-mod-php php php-cli php-crypt-gpg php-dev php-json php-mysql php-opcache php-readline php-redis php-xml > /dev/null 2>&1
+
+
+echo -e "\n--- Restarting Apache ---\n"
+systemctl restart apache2 > /dev/null 2>&1
+
+
+echo -e "\n--- Retrieving MISP ---\n"
+mkdir $PATH_TO_MISP
+chown www-data:www-data $PATH_TO_MISP
+cd $PATH_TO_MISP
+#git clone https://github.com/MISP/MISP.git $PATH_TO_MISP
+#git checkout tags/$(git describe --tags `git rev-list --tags --max-count=1`)
+git config core.filemode false
+# chown -R www-data $PATH_TO_MISP
+# chgrp -R www-data $PATH_TO_MISP
+# chmod -R 700 $PATH_TO_MISP
+
+
+echo -e "\n--- Installing Mitre's STIX ---\n"
+apt-get install -y python-dev python-pip libxml2-dev libxslt1-dev zlib1g-dev python-setuptools > /dev/null 2>&1
+cd $PATH_TO_MISP/app/files/scripts
+git clone https://github.com/CybOXProject/python-cybox.git
+git clone https://github.com/STIXProject/python-stix.git
+cd $PATH_TO_MISP/app/files/scripts/python-cybox
+git checkout v2.1.0.12
+python setup.py install > /dev/null 2>&1
+cd $PATH_TO_MISP/app/files/scripts/python-stix
+git checkout v1.1.1.4
+python setup.py install > /dev/null 2>&1
+# install mixbox to accomodate the new STIX dependencies:
+cd $PATH_TO_MISP/app/files/scripts/
+git clone https://github.com/CybOXProject/mixbox.git
+cd $PATH_TO_MISP/app/files/scripts/mixbox
+git checkout v1.0.2
+python setup.py install > /dev/null 2>&1
+
+
+echo -e "\n--- Retrieving CakePHP... ---\n"
+# CakePHP is included as a submodule of MISP, execute the following commands to let git fetch it:
+cd $PATH_TO_MISP
+git submodule init
+git submodule update
+# Once done, install CakeResque along with its dependencies if you intend to use the built in background jobs:
+cd $PATH_TO_MISP/app
+php composer.phar require kamisama/cake-resque:4.1.2
+php composer.phar config vendor-dir Vendor
+php composer.phar install
+# Enable CakeResque with php-redis
+phpenmod redis
+# To use the scheduler worker for scheduled tasks, do the following:
+cp -fa $PATH_TO_MISP/INSTALL/setup/config.php $PATH_TO_MISP/app/Plugin/CakeResque/Config/config.php
+
+
+echo -e "\n--- Setting the permissions... ---\n"
+chown -R www-data:www-data $PATH_TO_MISP
+chmod -R 750 $PATH_TO_MISP
+chmod -R g+ws $PATH_TO_MISP/app/tmp
+chmod -R g+ws $PATH_TO_MISP/app/files
+chmod -R g+ws $PATH_TO_MISP/app/files/scripts/tmp
+
+
+echo -e "\n--- Creating a database user... ---\n"
+mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "create database $DBNAME;"
+mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "grant usage on *.* to $DBNAME@localhost identified by '$DBPASSWORD_MISP';"
+mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "grant all privileges on $DBNAME.* to '$DBUSER_MISP'@'localhost';"
+mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "flush privileges;"
+# Import the empty MISP database from MYSQL.sql
+mysql -u $DBUSER_MISP -p$DBPASSWORD_MISP $DBNAME < /var/www/MISP/INSTALL/MYSQL.sql
+
+
+echo -e "\n--- Configuring Apache... ---\n"
+# !!! apache.24.misp.ssl seems to be missing
+#cp $PATH_TO_MISP/INSTALL/apache.24.misp.ssl /etc/apache2/sites-available/misp-ssl.conf
+# If a valid SSL certificate is not already created for the server, create a self-signed certificate:
+sudo openssl req -newkey rsa:4096 -days 365 -nodes -x509 -subj "/C=$OPENSSL_C/ST=$OPENSSL_ST/L=$OPENSSL_L/O=<$OPENSSL_O/OU=$OPENSSL_OU/CN=$OPENSSL_CN/emailAddress=$OPENSSL_EMAILADDRESS" -keyout /etc/ssl/private/misp.local.key -out /etc/ssl/private/misp.local.crt
+
+
+echo -e "\n--- Add a VirtualHost for MISP ---\n"
+cat > /etc/apache2/sites-available/misp-ssl.conf <<EOF
+<VirtualHost *:80>
+        ServerAdmin me@me.local
+        ServerName misp.local
+        DocumentRoot $PATH_TO_MISP/app/webroot
+
+        <Directory $PATH_TO_MISP/app/webroot>
+            Options -Indexes
+            AllowOverride all
+            Require all granted
+        </Directory>
+
+        LogLevel warn
+        ErrorLog /var/log/apache2/misp.local_error.log
+        CustomLog /var/log/apache2/misp.local_access.log combined
+        ServerSignature Off
+</VirtualHost>
+EOF
+# cat > /etc/apache2/sites-available/misp-ssl.conf <<EOF
+# <VirtualHost *:80>
+#         ServerName misp.local
+#
+#         Redirect permanent / https://$FQDN
+#
+#         LogLevel warn
+#         ErrorLog /var/log/apache2/misp.local_error.log
+#         CustomLog /var/log/apache2/misp.local_access.log combined
+#         ServerSignature Off
+# </VirtualHost>
+#
+# <VirtualHost *:443>
+#         ServerAdmin me@me.local
+#         ServerName misp.local
+#         DocumentRoot $PATH_TO_MISP/app/webroot
+#
+#         <Directory $PATH_TO_MISP/app/webroot>
+#             Options -Indexes
+#             AllowOverride all
+#             Require all granted
+#         </Directory>
+#
+#         SSLEngine On
+#         SSLCertificateFile /etc/ssl/private/misp.local.crt
+#         SSLCertificateKeyFile /etc/ssl/private/misp.local.key
+#         #SSLCertificateChainFile /etc/ssl/private/misp-chain.crt
+#
+#         LogLevel warn
+#         ErrorLog /var/log/apache2/misp.local_error.log
+#         CustomLog /var/log/apache2/misp.local_access.log combined
+#         ServerSignature Off
+# </VirtualHost>
+# EOF
+# activate new vhost
+a2dissite default-ssl
+a2ensite misp-ssl
+
+
+echo -e "\n--- Restarting Apache ---\n"
+systemctl restart apache2 > /dev/null 2>&1
+
+
+echo -e "\n--- Configuring log rotation ---\n"
+cp $PATH_TO_MISP/INSTALL/misp.logrotate /etc/logrotate.d/misp
+
+
+echo -e "\n--- MISP configuration ---\n"
+# There are 4 sample configuration files in /var/www/MISP/app/Config that need to be copied
+cp -a $PATH_TO_MISP/app/Config/bootstrap.default.php /var/www/MISP/app/Config/bootstrap.php
+cp -a $PATH_TO_MISP/app/Config/database.default.php /var/www/MISP/app/Config/database.php
+cp -a $PATH_TO_MISP/app/Config/core.default.php /var/www/MISP/app/Config/core.php
+cp -a $PATH_TO_MISP/app/Config/config.default.php /var/www/MISP/app/Config/config.php
+cat > $PATH_TO_MISP/app/Config/database.php <<EOF
+<?php
+class DATABASE_CONFIG {
+        public \$default = array(
+                'datasource' => 'Database/Mysql',
+                //'datasource' => 'Database/Postgres',
+                'persistent' => false,
+                'host' => '$DBHOST',
+                'login' => '$DBUSER_MISP',
+                'port' => 3306, // MySQL & MariaDB
+                //'port' => 5432, // PostgreSQL
+                'password' => '$DBPASSWORD_MISP',
+                'database' => '$DBNAME',
+                'prefix' => '',
+                'encoding' => 'utf8',
+        );
+}
+EOF
+# and make sure the file permissions are still OK
+chown -R www-data:www-data $PATH_TO_MISP/app/Config
+chmod -R 750 $PATH_TO_MISP/app/Config
+# Set some MISP directives with the command line tool
+$PATH_TO_MISP/app/Console/cake Baseurl $MISP_BASEURL
+$PATH_TO_MISP/app/Console/cake Live $MISP_LIVE
+
+
+echo -e "\n--- Generating a GPG encryption key... ---\n"
+apt-get install -y rng-tools haveged
+mkdir $PATH_TO_MISP/.gnupg
+chmod 700 $PATH_TO_MISP/.gnupg
+cat >gen-key-script <<EOF
+    %echo Generating a default key
+    Key-Type: default
+    Key-Length: $GPG_KEY_LENGTH
+    Subkey-Type: default
+    Name-Real: $GPG_REAL_NAME
+    Name-Comment: no comment
+    Name-Email: $GPG_EMAIL_ADDRESS
+    Expire-Date: 0
+    Passphrase: '$GPG_PASSPHRASE'
+    # Do a commit here, so that we can later print "done"
+    %commit
+    %echo done
+EOF
+gpg --homedir $PATH_TO_MISP/.gnupg --batch --gen-key gen-key-script
+rm gen-key-script
+# And export the public key to the webroot
+gpg --homedir $PATH_TO_MISP/.gnupg --export --armor $EMAIL_ADDRESS > $PATH_TO_MISP/app/webroot/gpg.asc
+
+
+echo -e "\n--- Making the background workers start on boot... ---\n"
+chmod 755 $PATH_TO_MISP/app/Console/worker/start.sh
+cat > /etc/systemd/system/workers.service  <<EOF
+[Unit]
+Description=Start the background workers at boot
+
+[Service]
+Type=forking
+User=www-data
+ExecStart=$PATH_TO_MISP/app/Console/worker/start.sh
+
+[Install]
+WantedBy=multi-user.target
+EOF
+systemctl enable workers.service > /dev/null
+systemctl restart workers.service > /dev/null
+
+
+echo -e "\n--- Installing MISP modules... ---\n"
+apt-get install -y python3-dev python3-pip libpq5 libjpeg-dev > /dev/null 2>&1
+cd /usr/local/src/
+git clone https://github.com/MISP/misp-modules.git
+cd misp-modules
+pip3 install -I -r REQUIREMENTS > /dev/null 2>&1
+pip3 install -I . > /dev/null 2>&1
+cat > /etc/systemd/system/misp-modules.service  <<EOF
+[Unit]
+Description=Start the misp modules server at boot
+
+[Service]
+Type=forking
+User=www-data
+ExecStart=/bin/sh -c 'misp-modules -l 0.0.0.0 -s &'
+
+[Install]
+WantedBy=multi-user.target
+EOF
+systemctl enable misp-modules.service > /dev/null
+systemctl restart misp-modules.service > /dev/null
+
+
+echo -e "\n--- Restarting Apache... ---\n"
+systemctl restart apache2 > /dev/null 2>&1
+sleep 5
+
+echo -e "\n--- Updating the galaxies... ---\n"
+sudo -E $PATH_TO_MISP/app/Console/cake userInit -q > /dev/null
+AUTH_KEY=$(mysql -u $DBUSER_MISP -p$DBPASSWORD_MISP misp -e "SELECT authkey FROM users;" | tail -1)
+curl -k -X POST -H "Authorization: $AUTH_KEY" -H "Accept: application/json" -v http://127.0.0.1/galaxies/update > /dev/null 2>&1
+
+
+echo -e "\n--- Updating the taxonomies... ---\n"
+curl -k -X POST -H "Authorization: $AUTH_KEY" -H "Accept: application/json" -v http://127.0.0.1/taxonomies/update > /dev/null 2>&1
+
+
+# echo -e "\n--- Enabling MISP new pub/sub feature (ZeroMQ)... ---\n"
+# # ZeroMQ depends on the Python client for Redis
+# pip install redis > /dev/null 2>&1
+# ## Install ZeroMQ and prerequisites
+# apt-get install -y pkg-config > /dev/null 2>&1
+# cd /usr/local/src/
+# git clone git://github.com/jedisct1/libsodium.git > /dev/null 2>&1
+# cd libsodium
+# /autogen.sh > /dev/null 2>&1
+# ./configure > /dev/null 2>&1
+# make check > /dev/null 2>&1
+# make > /dev/null 2>&1
+# make install > /dev/null 2>&1
+# ldconfig > /dev/null 2>&1
+# cd /usr/local/src/
+# wget https://archive.org/download/zeromq_4.1.5/zeromq-4.1.5.tar.gz > /dev/null 2>&1
+# tar -xvf zeromq-4.1.5.tar.gz > /dev/null 2>&1
+# cd zeromq-4.1.5/
+# ./autogen.sh > /dev/null 2>&1
+# ./configure > /dev/null 2>&1
+# make check > /dev/null 2>&1
+# make > /dev/null 2>&1
+# make install > /dev/null 2>&1
+# ldconfig > /dev/null 2>&1
+# ## install pyzmq
+# pip install pyzmq > /dev/null 2>&1
+
+
+echo -e "\e[32mMISP is ready\e[0m"
+echo -e "\e[0mPoint your Web browser to \e[33m$MISP_BASEURL\e[0m"
+echo -e "\e[0mDefault user/pass = \e[33madmin@admin.test/admin\e[0m"