Merge branch 'master' into feature/sg

Conflicts:
	app/Controller/EventsController.php
	app/Controller/UsersController.php
	app/Model/Event.php

.gitignore

@@ -37,6 +37,7 @@
 /app/Config/database.php
 /app/Config/core.php
 /app/Config/config.php
-/cakephp
+/app/Lib/cakephp
 /app/webroot/gpg.asc
 /app/tmp/logs
+*.swp

INSTALL/INSTALL.centos6.txt

@@ -26,7 +26,7 @@ rpm -Uvh epel.rpm
 yum install vim
 
 # Install the dependencies:
-yum install gcc git zip redis mysql-server php-mysql python-devel python-pip libxslt-devel zlib-devel php-devel 
+yum install gcc git zip redis mysql-server php-mysql python-devel python-pip libxslt-devel zlib-devel php-devel php-xml
 yum install php-pear php-pecl-geoip
 
 pear channel-update pear.php.net
@@ -77,7 +77,7 @@ git submodule update
 # Once done, install CakeResque along with its dependencies if you intend to use the built in background jobs:
 cd /var/www/MISP/app
 curl -s https://getcomposer.org/installer | php
-php composer.phar require --no-update kamisama/cake-resque:4.1.0
+php composer.phar require kamisama/cake-resque:4.1.2
 php composer.phar config vendor-dir Vendor
 php composer.phar install
 
@@ -123,8 +123,9 @@ mysql> grant usage on *.* to misp@localhost identified by 'XXXXXXXXX';
 mysql> grant all privileges on misp.* to misp@localhost ;
 mysql> exit
 
-# Import the empty MySQL database from MYSQL.sql
 cd /var/www/MISP
+
+# Import the empty MySQL database from MYSQL.sql
 mysql -u misp -p misp < INSTALL/MYSQL.sql
 
 
@@ -191,7 +192,8 @@ chmod 700 /var/www/MISP/.gnupg
 gpg --homedir /var/www/MISP/.gnupg --gen-key
 chown -R apache:apache /var/www/MISP/.gnupg
 # Recommended key type: RSA
-# The email address should match the one set int he config.php configuration file
+# The email address should match the one set in the config.php configuration file
+# Make sure that you use the same settings in the MISP Server Settings tool (Described on line 212)
 
 # And export the public key to the webroot
 sudo -u apache gpg --homedir /var/www/MISP/.gnupg --export --armor YOUR-EMAIL > /var/www/MISP/app/webroot/gpg.asc
@@ -222,6 +224,11 @@ chown -R apache:apache /var/www/MISP/<directory path with an indicated issue>
 
 # Make sure that the STIX libraries and GnuPG work as intended, if not, refer to INSTALL.txt's paragraphs dealing with these two items
 
+# If anything goes wrong, make sure that you check MISP's logs for errors:
+# /var/www/MISP/app/tmp/logs/error.log
+# /var/www/MISP/app/tmp/logs/resque-worker-error.log
+# /var/www/MISP/app/tmp/logs/resque-scheduler-error.log
+# /var/www/MISP/app/tmp/logs/resque-2015-01-01.log //where the actual date is the current date
 
 Recommended actions
 -------------------

INSTALL/INSTALL.centos7.txt

@@ -26,7 +26,7 @@ rpm -Uvh epel.rpm
 yum install vim
 
 # Install the dependencies:
-yum install git httpd zip php redis mysql-server php-mysql python-devel python-pip libxslt-devel zlib-devel php-devel 
+yum install git httpd zip php redis mysql-server php-mysql python-devel python-pip libxslt-devel zlib-devel php-devel php-xml
 yum install php-pear php-pecl-geoip
 
 pear channel-update pear.php.net
@@ -77,7 +77,7 @@ git submodule update
 # Once done, install CakeResque along with its dependencies if you intend to use the built in background jobs:
 cd /var/www/MISP/app
 curl -s https://getcomposer.org/installer | php
-php composer.phar require --no-update kamisama/cake-resque:4.1.0
+php composer.phar require kamisama/cake-resque:4.1.2
 php composer.phar config vendor-dir Vendor
 php composer.phar install
 
@@ -127,8 +127,9 @@ MariaDB [(none)]> grant usage on *.* to misp@localhost identified by 'XXXXXXXXX'
 MariaDB [(none)]> grant all privileges on misp.* to misp@localhost ;
 MariaDB [(none)]> exit
 
-# Import the empty MySQL database from MYSQL.sql
 cd /var/www/MISP
+
+# Import the empty MySQL database from MYSQL.sql
 mysql -u misp -p misp < INSTALL/MYSQL.sql
 
 
@@ -205,7 +206,8 @@ chmod 700 /var/www/MISP/.gnupg
 gpg --homedir /var/www/MISP/.gnupg --gen-key
 chown -R apache:apache /var/www/MISP/.gnupg
 # Recommended key type: RSA
-# The email address should match the one set int he config.php configuration file
+# The email address should match the one set in the config.php configuration file
+# Make sure that you use the same settings in the MISP Server Settings tool (Described on line 226)
 
 # And export the public key to the webroot
 sudo -u apache gpg --homedir /var/www/MISP/.gnupg --export --armor YOUR-EMAIL > /var/www/MISP/app/webroot/gpg.asc
@@ -236,6 +238,11 @@ chown -R apache:apache /var/www/MISP/<directory path with an indicated issue>
 
 # Make sure that the STIX libraries and GnuPG work as intended, if not, refer to INSTALL.txt's paragraphs dealing with these two items
 
+# If anything goes wrong, make sure that you check MISP's logs for errors:
+# /var/www/MISP/app/tmp/logs/error.log
+# /var/www/MISP/app/tmp/logs/resque-worker-error.log
+# /var/www/MISP/app/tmp/logs/resque-scheduler-error.log
+# /var/www/MISP/app/tmp/logs/resque-2015-01-01.log //where the actual date is the current date
 
 Recommended actions
 -------------------

INSTALL/INSTALL.ubuntu1404.txt

@@ -14,6 +14,7 @@ Install a minimal ubuntu 14.04-server system with the software:
 - Mail server
 
 You will get some questions, you will probably want to set:
+sudo apt-get install postfix
 - Postfix Configuration: Satellite system
 
 Make sure your system is up2date:
@@ -64,7 +65,7 @@ git submodule update
 # Once done, install CakeResque along with its dependencies if you intend to use the built in background jobs:
 cd /var/www/MISP/app
 curl -s https://getcomposer.org/installer | php
-php composer.phar require --no-update kamisama/cake-resque:4.1.0
+php composer.phar require kamisama/cake-resque:4.1.2
 php composer.phar config vendor-dir Vendor
 php composer.phar install
 
@@ -102,7 +103,8 @@ mysql> exit
 cd /var/www/MISP
 
 # Import the empty MySQL database from MYSQL.sql 
-mysql -u misp -p<password> misp < INSTALL/MYSQL.sql
+mysql -u misp -p misp < INSTALL/MYSQL.sql
+#enter the password you set previously
 
 
 7/ Apache configuration
@@ -163,8 +165,8 @@ chown www-data:www-data /var/www/MISP/.gnupg
 chmod 700 /var/www/MISP/.gnupg
 sudo -u www-data gpg --homedir /var/www/MISP/.gnupg --gen-key
 # Recommended key type: RSA
-# The email address should match the one set int he bootstrap.php configuration file
+# The email address should match the one set in the config.php configuration file
-# Make sure that you use the same settings in the MISP Server Settings tool tool (Described on line 184)
+# Make sure that you use the same settings in the MISP Server Settings tool (Described on line 184)
 
 # And export the public key to the webroot
 sudo -u www-data gpg --homedir /var/www/MISP/.gnupg --export --armor YOUR-EMAIL > /var/www/MISP/app/webroot/gpg.asc

INSTALL/MYSQL.sql

@@ -221,7 +221,7 @@ CREATE TABLE IF NOT EXISTS `servers` (
   `url` varchar(255) COLLATE utf8_bin NOT NULL,
   `authkey` varchar(40) COLLATE utf8_bin NOT NULL,
   `org` varchar(255) COLLATE utf8_bin NOT NULL,
-  `organization` varchar(10) COLLATE utf8_bin NOT NULL,
+  `organization` varchar(255) COLLATE utf8_bin NOT NULL,
   `push` tinyint(1) NOT NULL,
   `pull` tinyint(1) NOT NULL,
   `lastpulledid` int(11) NOT NULL,

INSTALL/UPDATE.txt

@@ -0,0 +1,54 @@
+# After installing MISP you can keep it up to date by periodically running the commands below.
+
+# 1. Update the MISP code to the latest hotfix. If a new major version (2.4.x) has been released, refer to UPGRADE.txt instead.
+
+cd /var/www/MISP
+git pull
+
+# 2. Update CakePHP to the latest 2.6 code
+
+cd /var/www/MISP/Lib/cakephp
+git fetch origin
+git checkout 2.6
+
+# 3. Update Mitre's STIX and its dependencies
+
+cd /var/www/MISP/app/files/scripts/python-cybox
+git pull
+python setup.py install
+cd /var/www/MISP/app/files/scripts/python-stix
+git pull
+python setup.py install
+
+# 4. Update CakeResque and it's dependencies
+
+cd /var/www/MISP/app
+
+# Edit composer.json so that cake-resque is allowed to be updated
+# "kamisama/cake-resque": ">=4.1.2"
+
+vim composer.json
+php composer.phar self-update
+php composer.phar update
+
+# To use the scheduler worker for scheduled tasks, do the following:
+cp -fa /var/www/MISP/INSTALL/setup/config.php /var/www/MISP/app/Plugin/CakeResque/Config/config.php
+
+# 5. Make sure all file permissions are set correctly
+
+chown -R root:www-data /var/www/MISP
+find /var/www/MISP -type d -exec chmod g=rx {} \;
+chmod -R g+r,o= /var/www/MISP
+chown www-data:www-data /var/www/MISP/app/files
+chown www-data:www-data /var/www/MISP/app/files/terms
+chown www-data:www-data /var/www/MISP/app/files/scripts/tmp
+chown www-data:www-data /var/www/MISP/app/Plugin/CakeResque/tmp
+chown -R www-data:www-data /var/www/MISP/app/tmp
+chown -R www-data:www-data /var/www/MISP/app/webroot/img/orgs
+chown -R www-data:www-data /var/www/MISP/app/webroot/img/custom
+
+# 6. Restart the CakeResque workers
+
+su www-data -c 'bash /var/www/MISP/app/Console/worker/start.sh'
+
+# You can also do this using the MISP application by navigating to the workers tab in the server settings and clicking on the "Restart all workers" button.

INSTALL/UPGRADE.txt

@@ -32,7 +32,7 @@ python setup.py install
 # install / update CakeResque (using the background workers is optional buy highly recommended)
 cd /var/www/MISP/app
 curl -s https://getcomposer.org/installer | php
-php composer.phar require --no-update kamisama/cake-resque:4.1.0
+php composer.phar require kamisama/cake-resque:4.1.2
 php composer.phar config vendor-dir Vendor
 php composer.phar install
 
@@ -78,4 +78,4 @@ chown -R www-data:www-data /var/www/MISP/<directory path with an indicated issue
 # Amend your rc.local using the following command to always start the background workers using the apache user (substitute www-data with your apache user)
 su www-data -c 'bash /var/www/MISP/app/Console/worker/start.sh'
 
-# Start the background workers using the application by navigating to the workers tab in the server settings and clicking on the "Restart all workers" button.
+# Start the background workers using the application by navigating to the workers tab in the server settings and clicking on the "Restart all workers" button.

INSTALL/get-misp-automation.py

@@ -0,0 +1,23 @@
+#!/usr/bin/python
+
+'''
+Example file on how to get the exported IDS data from MISP
+
+Add your API key, set the MISP host and define the output file.
+'''
+
+import urllib2
+
+MISP_HOST="http:/"
+API_KEY=""
+EXPORT_DATA="events/nids/suricata/download"
+OUTPUT_FILE="misp-suricata"
+
+URL="%s/%s" % (MISP_HOST, EXPORT_DATA)
+request = urllib2.Request(URL)
+f = open(OUTPUT_FILE,'w')
+request.add_header('Authorization', API_KEY)
+data = urllib2.urlopen(request).read()
+f.write(data)
+f.close()
+

VERSION.json

@@ -1 +1 @@
-{"major":2, "minor":3, "hotfix":53}
+{"major":2, "minor":3, "hotfix":58}

app/Config/config.default.php

@@ -25,7 +25,8 @@ $config = array (
     'tagging' => true,
     'full_tags_on_event_index' => true,
     'footer_logo' => '',
-  	'take_ownership_xml_import' => false,
+    'take_ownership_xml_import' => false,
+    'unpublishedprivate' => false,
   ),
   'GnuPG' => 
   array (
@@ -34,6 +35,14 @@ $config = array (
     'homedir' => '',
     'password' => '',
   ),
+  'Proxy' =>
+  array (
+    'host' => '',
+    'port' => '',
+    'method' => '',
+    'user' => '',
+    'password' => '',
+  ),
   'SecureAuth' => 
   array (
     'amount' => 5,

app/Console/Command/EventShell.php

@@ -45,7 +45,7 @@ class EventShell extends AppShell
 		$eventIds = $this->Event->fetchEventIds($org, $isSiteAdmin);
 		$result = array();
 		$eventCount = count($eventIds);
-		$dir = new Folder(APP . DS . '/tmp/cached_exports/xml');
+		$dir = new Folder(APP . 'tmp/cached_exports/xml');
 		if ($isSiteAdmin) {
 			$file = new File($dir->pwd() . DS . 'misp.xml' . '.ADMIN.xml');
 		} else {
@@ -192,7 +192,7 @@ class EventShell extends AppShell
 		$eventIds = $this->Event->fetchEventIds($org, $isSiteAdmin);
 		$eventCount = count($eventIds);
 		$attributes = array();
-		$dir = new Folder(APP . DS . '/tmp/cached_exports/' . $extra);
+		$dir = new Folder(APP . 'tmp/cached_exports/' . $extra);
 		if ($isSiteAdmin) {
 			$file = new File($dir->pwd() . DS . 'misp.' . $extra . '.ADMIN.csv');
 		} else {
@@ -200,11 +200,13 @@ class EventShell extends AppShell
 		}
 		$file->write('uuid,event_id,category,type,value,to_ids,date' . PHP_EOL);
 		foreach ($eventIds as $k => $eventId) {
+			$chunk = "";
 			$attributes = $this->Event->csv($org, $isSiteAdmin, $eventId['Event']['id'], $ignore);
 			$attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true);
 			foreach ($attributes as $attribute) {
-				$file->append($attribute['Attribute']['uuid'] . ',' . $attribute['Attribute']['event_id'] . ',' . $attribute['Attribute']['category'] . ',' . $attribute['Attribute']['type'] . ',' . $attribute['Attribute']['value'] . ',' . intval($attribute['Attribute']['to_ids']) . ',' . $attribute['Attribute']['timestamp'] . PHP_EOL);
+				$chunk .= $attribute['Attribute']['uuid'] . ',' . $attribute['Attribute']['event_id'] . ',' . $attribute['Attribute']['category'] . ',' . $attribute['Attribute']['type'] . ',' . $attribute['Attribute']['value'] . ',' . intval($attribute['Attribute']['to_ids']) . ',' . $attribute['Attribute']['timestamp'] . PHP_EOL;
 			}
+			$file->append($chunk);
 			if ($k % 10 == 0) {
 				$this->Job->saveField('progress', $k / $eventCount * 80);
 			}
@@ -333,7 +335,7 @@ class EventShell extends AppShell
 			// Now that we have figured out when the next execution should happen, it's time to enqueue it.
 			$process_id = CakeResque::enqueueAt(
 					$task['Task']['next_execution_time'],
-					'default',
+					'cache',
 					'EventShell',
 					array('enqueueCaching', $task['Task']['next_execution_time']),
 					true

app/Controller/AppController.php

@@ -301,4 +301,23 @@ class AppController extends Controller {
 		$this->Session->setFlash(__('All done. attribute_count generated from scratch for ' . $k . ' events.'));
 		$this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
 	}
+	
+	public function updateDatabase($command) {
+		if (!$this->_isSiteAdmin()) throw new MethodNotAllowedException();
+		$sql = '';
+		switch ($command) {
+			case 'extendServerOrganizationLength':
+				$sql = 'ALTER TABLE servers MODIFY COLUMN organization varchar(255) NOT NULL';
+				$controller = 'Servers';
+				break;
+			default:
+				$this->Session->setFlash('Invalid command.');
+				$this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
+				break;
+		}
+		$this->loadModel($controller);
+		$this->$controller->query($sql);
+		$this->Session->setFlash('Done.');
+		$this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
+	}
 }

app/Controller/AttributesController.php

@@ -58,6 +58,7 @@ class AttributesController extends AppController {
 								'AND' => array(
 										'Attribute.distribution >' => 0,
 										'Event.distribution >' => 0,
+										Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(),
 			)))));
 		}
 
@@ -1281,9 +1282,17 @@ class AttributesController extends AppController {
 						// merge in private conditions
 						$this->paginate = Set::merge($this->paginate, array(
 							'conditions' =>
-								array("OR" => array(
+								array("OR" =>
-								array('Event.org =' => $this->Auth->user('org')),
+									array(
-								array("AND" => array('Event.org !=' => $this->Auth->user('org')), array('Event.distribution !=' => 0), array('Attribute.distribution !=' => 0)))),
+										array('Event.org =' => $this->Auth->user('org')),
+										array("AND" =>
+											array('Event.org !=' => $this->Auth->user('org')),
+											array('Event.distribution !=' => 0),
+											array('Attribute.distribution !=' => 0),
+											Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(),
+										)
+									)
+								)
 							)
 						);
 					}
@@ -1369,8 +1378,14 @@ class AttributesController extends AppController {
 	public function searchAlternate($data) {
 		$data['AND'][] = array(
 				"OR" => array(
-					array('Event.org =' => $this->Auth->user('org')),
+						array('Event.org =' => $this->Auth->user('org')),
-					array("AND" => array('Event.org !=' => $this->Auth->user('org')), array('Event.distribution !=' => 0), array('Attribute.distribution !=' => 0))));
+						array("AND" => array('Event.org !=' => $this->Auth->user('org')),
+							array('Event.distribution !=' => 0),
+							array('Attribute.distribution !=' => 0),
+							Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(),
+						)
+					)
+				);
 		$attributes = $this->Attribute->find('all', array(
 			'conditions' => $data,
 			'fields' => array(
@@ -1555,7 +1570,11 @@ class AttributesController extends AppController {
 
 		if (!$user['User']['siteAdmin']) {
 			$temp = array();
-			$temp['AND'] = array('Event.distribution >' => 0, 'Attribute.distribution >' => 0);
+			$temp['AND'] = 	array(
+						'Event.distribution >' => 0,
+						'Attribute.distribution >' => 0,
+						Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array()
+					);
 			$subcondition['OR'][] = $temp;
 			$subcondition['OR'][] = array('Event.org' => $user['User']['org']);
 			array_push($conditions['AND'], $subcondition);

app/Controller/EventsController.php

@@ -76,21 +76,23 @@ class EventsController extends AppController {
 			$this->paginate = Set::merge($this->paginate,array(
 					'conditions' =>
 					array("OR" => array(
-							array(
+						array(
-								'Event.org_id' => $this->Auth->user('organisation_id')
+							'Event.org_id' => $this->Auth->user('organisation_id')
+						),
+						array(
+							'AND' => array(
+								'Event.distribution >' => 0,
+								'Event.distribution <' => 4,
+								Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(),
 							),
-							array(
+						),
-								'AND' => array(
+						array(
-									'Event.distribution >' => 0,
+							'AND' => array(
-									'Event.distribution <' => 4,
+								'Event.distribution' => 4,
-								),
+								'Event.sharing_group_id' => $sgids,
-							),
+								Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(),
-							array(
+							),	
-								'AND' => array(
+						)
-									'Event.distribution' => 4,
-									'Event.sharing_group_id' => $sgids
-								),	
-							)
 			))));
 		}
 	}
@@ -2474,7 +2476,11 @@ class EventsController extends AppController {
 	
 			if (!$user['User']['siteAdmin']) {
 				$temp = array();
-				$temp['AND'] = array('Event.distribution >' => 0, 'Attribute.distribution >' => 0);
+				$temp['AND'] = array(
+							'Event.distribution >' => 0,
+							'Attribute.distribution >' => 0,
+							Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array()
+						);
 				$subcondition['OR'][] = $temp;
 				$subcondition['OR'][] = array('Event.org' => $user['User']['org']);
 				array_push($conditions['AND'], $subcondition);
@@ -3033,7 +3039,7 @@ class EventsController extends AppController {
 				'fields' => array('Event.uuid', 'Event.timestamp', 'Event.locked'),
 			));
 			foreach ($events as $k => $v) {
-				if (!$v['Event']['timestamp'] < $incomingEvents[$v['Event']['uuid']]) {
+				if ($v['Event']['timestamp'] >= $incomingEvents[$v['Event']['uuid']]) {
 					unset($incomingEvents[$v['Event']['uuid']]);
 					continue;
 				}

app/Controller/LogsController.php

@@ -164,19 +164,19 @@ class LogsController extends AppController {
 
 				// search the db
 				$conditions = array();
-				if ($email) {
+				if (isset($email) && !empty($email)) {
 					$conditions['LOWER(Log.email) LIKE'] = '%' . strtolower($email) . '%';
 				}
-				if (isset($org)) {
+				if (isset($org) && !empty($org)) {
 					$conditions['LOWER(Log.org) LIKE'] = '%' . strtolower($org) . '%';
 				}
 				if ($action != 'ALL') {
 					$conditions['Log.action ='] = $action;
 				}
-				if (isset($title)) {
+				if (isset($title) && !empty($title)) {
 					$conditions['LOWER(Log.title) LIKE'] = '%' . strtolower($title) . '%';
 				}
-				if (isset($change)) {
+				if (isset($change) && !empty($change)) {
 					$conditions['LOWER(Log.change) LIKE'] = '%' . strtolower($change) . '%';
 				}
 				$this->{$this->defaultModel}->recursive = 0;

app/Controller/ServersController.php

@@ -289,11 +289,13 @@ class ServersController extends AppController {
 			$tabs = array(
 					'MISP' => array('count' => 0, 'errors' => 0, 'severity' => 5),
 					'GnuPG' => array('count' => 0, 'errors' => 0, 'severity' => 5),
+					'Proxy' => array('count' => 0, 'errors' => 0, 'severity' => 5),
 					'Security' => array('count' => 0, 'errors' => 0, 'severity' => 5),
 					'misc' => array('count' => 0, 'errors' => 0, 'severity' => 5)
 			);
 			$writeableErrors = array(0 => 'OK', 1 => 'Directory doesn\'t exist', 2 => 'Directory is not writeable');
 			$gpgErrors = array(0 => 'OK', 1 => 'FAIL: settings not set', 2 => 'FAIL: bad GnuPG.*', 3 => 'FAIL: encrypt failed');
+			$proxyErrors = array(0 => 'OK', 1 => 'not configured (so not tested)', 2 => 'Getting URL via proxy failed');
 			$stixErrors = array(0 => 'ERROR', 1 => 'OK');
 			
 			$results = $this->Server->serverSettingsRead();
@@ -394,7 +396,29 @@ class ServersController extends AppController {
 				$gpgStatus = 1;
 			}
 			if ($gpgStatus != 0) $diagnostic_errors++;
+
+			// if Proxy is set up in the settings, try to connect to a test URL
+			$proxyStatus = 0;
+			$proxy = Configure::read('Proxy');
+			if(!empty($proxy['host'])) {
+				App::uses('SyncTool', 'Tools');
+				$syncTool = new SyncTool();
+				try {
+					$HttpSocket = $syncTool->setupHttpSocket();
+					$proxyResponse = $HttpSocket->get('http://www.example.com/');
+				} catch (Exception $e) {
+					$proxyStatus = 2;
+				}
+				if(empty($proxyResponse) || $proxyResponse->code > 399) {
+					$proxyStatus = 2;
+				}
+			} else {
+					$proxyStatus = 1;
+			}
+			if ($proxyStatus > 1) $diagnostic_errors++;
+
 			$this->set('gpgStatus', $gpgStatus);
+			$this->set('proxyStatus', $proxyStatus);
 			$this->set('diagnostic_errors', $diagnostic_errors);
 			$this->set('tab', $tab);
 			$this->set('tabs', $tabs);
@@ -403,6 +427,7 @@ class ServersController extends AppController {
 			
 			$this->set('writeableErrors', $writeableErrors);
 			$this->set('gpgErrors', $gpgErrors);
+			$this->set('proxyErrors', $proxyErrors);
 			$this->set('stixErrors', $stixErrors);
 			
 			if (Configure::read('MISP.background_jobs')) {
@@ -437,7 +462,7 @@ class ServersController extends AppController {
 				foreach ($dumpResults as &$dr) {
 					unset($dr['description']);
 				}
-				$dump = array('gpgStatus' => $gpgErrors[$gpgStatus], 'stix' => $stixErrors[$stix], 'writeableDirs' => $writeableDirs, 'finalSettings' => $dumpResults);
+				$dump = array('gpgStatus' => $gpgErrors[$gpgStatus], 'proxyStatus' => $proxyErrors[$proxyStatus], 'stix' => $stixErrors[$stix], 'writeableDirs' => $writeableDirs, 'finalSettings' => $dumpResults);
 				$this->response->body(json_encode($dump, JSON_PRETTY_PRINT));
 				$this->response->type('json');
 				$this->response->download('MISP.report.json');
@@ -453,12 +478,16 @@ class ServersController extends AppController {
 	
 	private function __checkVersion() {
 		if (!$this->_isSiteAdmin()) throw new MethodNotAllowedException();
-		set_error_handler(function() {});
+		App::uses('SyncTool', 'Tools');
-		$options  = array('http' => array('user_agent'=> $_SERVER['HTTP_USER_AGENT']));
+		$syncTool = new SyncTool();
-		$context  = stream_context_create($options);
+		try {
-		$tags = file_get_contents('https://api.github.com/repos/MISP/MISP/tags', false, $context);
+			$HttpSocket = $syncTool->setupHttpSocket();
-		restore_error_handler();
+			$response = $HttpSocket->get('https://api.github.com/repos/MISP/MISP/tags');
-		if ($tags != false) {
+			$tags = $response->body;
+		} catch (Exception $e) {
+			return false;
+		}
+		if ($response->isOK() && !empty($tags)) {
 			$json_decoded_tags = json_decode($tags);
 	
 			// find the latest version tag in the v[major].[minor].[hotfix] format

app/Controller/ShadowAttributesController.php

@@ -50,6 +50,7 @@ class ShadowAttributesController extends AppController {
 								'AND' => array(
 									'ShadowAttribute.org =' => $this->Auth->user('org'),
 									'Event.distribution >' => 0,
+									Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(),
 								),
 							)
 			)));
@@ -271,7 +272,7 @@ class ShadowAttributesController extends AppController {
 				'recursive' => -1,
 				'fields' => array('id', 'orgc', 'distribution', 'org'),
 		));
-		if ((($event['Event']['distribution'] == 0 && $event['Event']['org'] != $this->Auth->user('org'))) || ($event['Event']['orgc'] == $this->Auth->user('org'))) {
+		if (!$this->_isSiteAdmin() && (($event['Event']['distribution'] == 0 && $event['Event']['org'] != $this->Auth->user('org'))) || ($event['Event']['orgc'] == $this->Auth->user('org'))) {
 			$this->Session->setFlash(__('Invalid Event.'));
 			$this->redirect(array('controller' => 'events', 'action' => 'index'));
 		}

app/Controller/TagsController.php

@@ -37,7 +37,15 @@ class TagsController extends AppController {
 					$eventIDs[] = $eventTag['event_id'];
 				}
 				$conditions = array('Event.id' => $eventIDs);
-				if (!$this->_isSiteAdmin()) $conditions = array_merge($conditions, array('OR' => array(array('Event.distribution >' => 0),  array('Event.orgc' => $this->Auth->user('org')))));
+				if (!$this->_isSiteAdmin()) $conditions = array_merge(
+					$conditions,
+					array('OR' => array(
+						array('AND' => array(
+							array('Event.distribution >' => 0),
+							array('Event.published =' => 1)
+						)),
+						array('Event.orgc' => $this->Auth->user('org'))
+					)));
 				$events = $this->Event->find('all', array(
 					'fields' => array('Event.id', 'Event.distribution', 'Event.orgc'),
 					'conditions' => $conditions
@@ -136,4 +144,4 @@ class TagsController extends AppController {
 		$this->set('id', $id);
 		$this->render('ajax/view_tag');
 	}
-}
+}

app/Controller/UsersController.php

@@ -344,8 +344,17 @@ class UsersController extends AppController {
 		if ($this->request->is('post')) {
 			$this->User->create();
 			// set invited by
+			$this->loadModel('Role');
+			$this->Role->recursive = -1;
+			$chosenRole = $this->Role->findById($this->request->data['User']['role_id']);
 			$this->request->data['User']['invited_by'] = $this->Auth->user('id');
-			$this->request->data['User']['change_pw'] = 1;
+			if ($chosenRole['Role']['perm_sync']) {
+				$this->request->data['User']['change_pw'] = 0;
+				$this->request->data['User']['termsaccepted'] = 1;
+			} else {
+				$this->request->data['User']['change_pw'] = 1;
+				$this->request->data['User']['termsaccepted'] = 0;
+			}
 			$this->request->data['User']['newsread'] = '2000-01-01';
 			if (!$this->_isSiteAdmin()) {
 				$this->request->data['User']['organisation_id'] = $this->Auth->User('organisation_id');

app/Lib/Tools/SyncTool.php

@@ -2,12 +2,18 @@
 
 class SyncTool {
 	// take a server as parameter and return a HttpSocket object using the ssl options defined in the server settings
-	public function setupHttpSocket($server) {
+	public function setupHttpSocket($server = null) {
 		$params = array();
 		App::uses('HttpSocket', 'Network/Http');
-		if ($server['Server']['cert_file'])	$params['ssl_cafile'] = APP . "files" . DS . "certs" . DS . $server['Server']['id'] . '.pem';
+		if(!empty($server)) {
-		if ($server['Server']['self_signed']) $params['ssl_allow_self_signed'] = $server['Server']['self_signed'];
+			if ($server['Server']['cert_file']) $params['ssl_cafile'] = APP . "files" . DS . "certs" . DS . $server['Server']['id'] . '.pem';
+			if ($server['Server']['self_signed']) $params['ssl_allow_self_signed'] = $server['Server']['self_signed'];
+		}
 		$HttpSocket = new HttpSocket($params);
+
+		$proxy = Configure::read('Proxy');
+		$HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']);
+
 		return $HttpSocket;
 	}
 }

app/Lib/cakephp

@@ -1 +1 @@
-Subproject commit a0aac5cfa9acf2ccbde65c45ad6ee06ecd32fb54
+Subproject commit 4c61b579c40863ada019e1e08c6195ae612c589f

app/Model/Attribute.php

@@ -924,7 +924,7 @@ class Attribute extends AppModel {
 			// TODO check if CakePHP has no easy/safe wrapper to execute commands
 			$execRetval = '';
 			$execOutput = array();
-			exec("zip -j -P infected " . $zipfile->path . ' "' . addslashes($fileInZip->path) . '"', $execOutput, $execRetval);
+			exec("zip -j -P infected " . $zipfile->path . ' \'' . addslashes($fileInZip->path) . '\'', $execOutput, $execRetval);
 			if ($execRetval != 0) { // not EXIT_SUCCESS
 				// do some?
 			};

app/Model/Event.php

@@ -922,7 +922,10 @@ class Event extends AppModel {
 		$conditions = array();
 		if (!$isSiteAdmin) {
 			$conditions['OR'] = array(
-					'Event.distribution >' => 0,
+					"AND" => array(
+						'Event.distribution >' => 0,
+						Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array()
+					),
 					'Event.org LIKE' => $org
 			);
 		}
@@ -959,19 +962,21 @@ class Event extends AppModel {
 		if (!$user['Role']['perm_site_admin']) {
 			$sgids = $this->SharingGroup->fetchAllAuthorised($user);
 			$conditions['AND']['OR'] = array(
-					'Event.org_id' => $user['organisation_id'],
+				'Event.org_id' => $user['organisation_id'],
-					array(
+				array(
-						'AND' => array(
+					'AND' => array(
-							'Event.distribution >' => 0,
+						'Event.distribution >' => 0,
-							'Event.distribution <' => 4
+						'Event.distribution <' => 4,
-						),
+						Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(),
 					),
-					array(
+				),
-						'AND' => array(
+				array(
-							'Event.sharing_group_id' => $sgids,
+					'AND' => array(
-							'Event.distribution' => 4,
+						'Event.sharing_group_id' => $sgids,
-						)
+						'Event.distribution' => 4,
+						Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(),
 					)
+				)
 			);
 			$conditionsAttributes['OR'] = array(
 				'Attribute.distribution >' => 0,
@@ -1078,7 +1083,13 @@ class Event extends AppModel {
 	 		if ($from) $econditions['AND'][] = array('Event.date >=' => $from);
 	 		if ($to) $econditions['AND'][] = array('Event.date <=' => $to);
 	 		// This is for both single event downloads and for full downloads. Org has to be the same as the user's or distribution not org only - if the user is no siteadmin
-	 		if(!$isSiteAdmin) $econditions['AND']['OR'] = array('Event.distribution >' => 0, 'Event.org =' => $org);
+			if(!$isSiteAdmin) $econditions['AND']['OR'] = array(
+				"AND" => array(
+					'Event.distribution >' => 0,
+					Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(),
+				),
+				'Event.org =' => $org
+			);
 	 		if ($eventid == 0 && $ignore == 0) $econditions['AND'][] = array('Event.published =' => 1);
 	 		
 	 		// If it's a full download (eventid == false) and the user is not a site admin, we need to first find all the events that the user can see and save the IDs
@@ -1196,7 +1207,7 @@ class Event extends AppModel {
 	 		$job = ClassRegistry::init('Job');
 	 		$job->create();
 	 		$data = array(
-	 				'worker' => 'default',
+	 				'worker' => 'email',
 	 				'job_type' => 'publish_alert_email',
 	 				'job_input' => 'Event: ' . $id,
 	 				'status' => 0,
@@ -1207,7 +1218,7 @@ class Event extends AppModel {
 	 		$job->save($data);
 	 		$jobId = $job->id;
 	 		$process_id = CakeResque::enqueue(
-	 				'default',
+	 				'email',
 	 				'EventShell',
 	 				array('alertemail', $user['org'], $jobId, $id)
 	 		);
@@ -1285,7 +1296,7 @@ class Event extends AppModel {
 		
 		if (Configure::read('MISP.extended_alert_subject')) {
 			$subject = preg_replace( "/\r|\n/", "", $event['Event']['info']);
-			if (strlen($subject) > 55) {
+			if (strlen($subject) > 58) {
 				$subject = substr($subject, 0, 55) . '... - ';
 			} else {
 				$subject .= " - ";
@@ -1778,7 +1789,7 @@ class Event extends AppModel {
 			$job = ClassRegistry::init('Job');
 			$job->create();
 			$data = array(
-					'worker' => 'default',
+					'worker' => 'email',
 					'job_type' => 'contact_alert',
 					'job_input' => 'To entire org: ' . $all,
 					'status' => 0,
@@ -1789,7 +1800,7 @@ class Event extends AppModel {
 			$job->save($data);
 			$jobId = $job->id;
 			$process_id = CakeResque::enqueue(
-					'default',
+					'email',
 					'EventShell',
 					array('contactemail', $id, $message, $all, $user['id'], $isSiteAdmin, $jobId)
 			);
@@ -1986,7 +1997,7 @@ class Event extends AppModel {
 		$scriptFile = APP . "files" . DS . "scripts" . DS . "misp2stix.py";
 		
 		// Execute the python script and point it to the temporary filename
-		$result = shell_exec('python ' . $scriptFile . ' ' . $randomFileName . ' ' . $returnType);
+		$result = shell_exec('python ' . $scriptFile . ' ' . $randomFileName . ' ' . $returnType . ' ' . Configure::read('MISP.baseurl') . ' "' . Configure::read('MISP.org') . '"');
 		
 		// The result of the script will be a returned JSON object with 2 variables: success (boolean) and message
 		// If success = 1 then the temporary output file was successfully written, otherwise an error message is passed along

app/Model/Server.php

@@ -381,6 +381,14 @@ class Server extends AppModel {
 							'test' => 'testBool',
 							'type' => 'boolean'
 					),
+					'unpublishedprivate' => array(
+							'level' => 2,
+							'description' => 'True will deny access to unpublished events to users outside the organization of the submitter except site admins.',
+							'value' => '',
+							'errorMessage' => '',
+							'test' => 'testBool',
+							'type' => 'boolean'
+					),
 			),
 			'GnuPG' => array(
 					'branch' => 1,
@@ -417,6 +425,49 @@ class Server extends AppModel {
 							'type' => 'string',
 					),
 			),
+			'Proxy' => array(
+					'branch' => 1,
+					'host' => array(
+							'level' => 2,
+							'description' => 'The hostname of an HTTP proxy for outgoing sync requests. Leave empty to not use a proxy.',
+							'value' => '',
+							'errorMessage' => '',
+							'test' => 'testForEmpty',
+							'type' => 'string',
+					),
+					'port' => array(
+							'level' => 2,
+							'description' => 'The TCP port for the HTTP proxy.',
+							'value' => '',
+							'errorMessage' => '',
+							'test' => 'testForNumeric',
+							'type' => 'numeric',
+					),
+					'method' => array(
+							'level' => 2,
+							'description' => 'The authentication method for the HTTP proxy. Currently supported are Basic or Digest. Leave empty for no proxy authentication.',
+							'value' => '',
+							'errorMessage' => '',
+							'test' => 'testForEmpty',
+							'type' => 'string',
+					),
+					'user' => array(
+							'level' => 2,
+							'description' => 'The authentication username for the HTTP proxy.',
+							'value' => '',
+							'errorMessage' => '',
+							'test' => 'testForEmpty',
+							'type' => 'string',
+					),
+					'password' => array(
+							'level' => 2,
+							'description' => 'The authentication password for the HTTP proxy.',
+							'value' => '',
+							'errorMessage' => '',
+							'test' => 'testForEmpty',
+							'type' => 'string',
+					),
+			),
 			'Security' => array(
 					'branch' => 1,
 					'salt' => array(
@@ -1008,7 +1059,7 @@ class Server extends AppModel {
 	
 	public function serverSettingsSaveValue($setting, $value) {
 		Configure::write($setting, $value);
-		Configure::dump('config.php', 'default', array('MISP', 'GnuPG', 'SecureAuth', 'Security', 'debug'));
+		Configure::dump('config.php', 'default', array('MISP', 'GnuPG', 'Proxy', 'SecureAuth', 'Security', 'debug'));
 	}
 	
 	public function checkVersion($newest) {

app/Model/ShadowAttribute.php

@@ -510,7 +510,7 @@ class ShadowAttribute extends AppModel {
 			// TODO check if CakePHP has no easy/safe wrapper to execute commands
 			$execRetval = '';
 			$execOutput = array();
-			exec("zip -j -P infected " . $zipfile->path . ' "' . addslashes($fileInZip->path) . '"', $execOutput, $execRetval);
+			exec("zip -j -P infected " . $zipfile->path . ' \'' . addslashes($fileInZip->path) . '\'', $execOutput, $execRetval);
 			if ($execRetval != 0) { // not EXIT_SUCCESS
 				// do some?
 			};

app/Plugin/CakeResque

@@ -1 +0,0 @@
-Subproject commit 34f9a7cd370bfb2706b9bd6b54fb8ede17e5e5bd

app/View/Attributes/alternate_search_result.ctp

@@ -27,7 +27,7 @@
 				</td>
 				<td>
 					<?php 
-						if (strlen(h($event['Event']['info'])) > 60) { 
+						if (strlen(h($event['Event']['info'])) > 63) { 
 							echo (substr(h($event['Event']['info']), 0, 60)) . '...'; 
 						} else 	echo h($event['Event']['info']);
 					?>
@@ -52,4 +52,4 @@
 </div>
 <?php 
 	echo $this->element('side_menu', array('menuList' => 'event-collection', 'menuItem' => 'searchAttributes'));
-?>
+?>

app/View/Elements/eventattribute.ctp

@@ -228,15 +228,16 @@
 					<td class="short action-links <?php echo $extra;?>">
 						<?php
 							if ($object['objectType'] == 0) {
+								if ($isSiteAdmin || !$mayModify)  {
+						?>
+									<a href="/shadow_attributes/edit/<?php echo $object['id']; ?>" title="Propose Edit" class="icon-share useCursorPointer"></a>
+						<?php 
+								}
 								if ($isSiteAdmin || $mayModify) {
 						?>
 							<a href="/attributes/edit/<?php echo $object['id']; ?>" title="Edit" class="icon-edit useCursorPointer"></a>
 							<span class="icon-trash useCursorPointer" onClick="deleteObject('attributes', 'delete', '<?php echo $object['id']; ?>', '<?php echo $event['Event']['id']; ?>');"></span>
 						<?php 			
-								} else {
-						?>
-									<a href="/shadow_attributes/edit/<?php echo $object['id']; ?>" title="Propose Edit" class="icon-edit useCursorPointer"></a>
-						<?php 
 								}
 							} else {
 								if (($event['Event']['orgc'] == $me['org'] && $mayModify) || $isSiteAdmin) {

app/View/Elements/healthElements/diagnostics.ctp

@@ -86,4 +86,18 @@
 			echo 'GnuPG installation and settings....<span style="color:' . $colour . ';">' . $message . '</span>';
 		?>
 	</div>
-</div>
+	<h3>
+	Proxy
+	</h3>
+	<p>This tool tests whether your HTTP proxy settings are correct.</p>
+	<div style="background-color:#f7f7f9;width:300px;">
+		<?php
+			$colour = 'green';
+			$message = $proxyErrors[$proxyStatus];
+			if ($proxyStatus > 1) {
+				$colour = 'red';
+			}
+			echo 'Proxy settings....<span style="color:' . $colour . ';">' . $message . '</span>';
+		?>
+	</div>
+</div>

app/View/Elements/side_menu.ctp

@@ -16,19 +16,19 @@
 					?>
 					<li <?php if ($menuItem === 'viewEvent') echo 'class="active"';?>><a href="/events/view/<?php echo $event['Event']['id'];?>">View Event</a></li>
 					<li <?php if ($menuItem === 'eventLog') echo 'class="active"';?>><a href="/logs/event_index/<?php echo $event['Event']['id'];?>">View Event History</a></li>
+					<li class="divider"></li>
 					<?php if ($isSiteAdmin || (isset($mayModify) && $mayModify)): ?>
 					<li <?php if ($menuItem === 'editEvent') echo 'class="active"';?>><a href="/events/edit/<?php echo $event['Event']['id'];?>">Edit Event</a></li>
 					<li><?php echo $this->Form->postLink('Delete Event', array('action' => 'delete', $event['Event']['id']), null, __('Are you sure you want to delete # %s?', $event['Event']['id'])); ?></li>
-					<li class="divider"></li>
 					<li <?php if ($menuItem === 'addAttribute') echo 'class="active"';?>><a href="/attributes/add/<?php echo $event['Event']['id'];?>">Add Attribute</a></li>
 					<li <?php if ($menuItem === 'addAttachment') echo 'class="active"';;?>><a href="/attributes/add_attachment/<?php echo $event['Event']['id'];?>">Add Attachment</a></li>
 					<li <?php if ($menuItem === 'addIOC') echo 'class="active"';?>><a href="/events/addIOC/<?php echo $event['Event']['id'];?>">Populate from OpenIOC</a></li>
 					<li <?php if ($menuItem === 'addThreatConnect') echo 'class="active"';?>><a href="/attributes/add_threatconnect/<?php echo $event['Event']['id']; ?>">Populate from ThreatConnect</a></li>
-					<?php if ($menuItem === 'populateFromtemplate'): ?>
+						<?php if ($menuItem === 'populateFromtemplate'): ?>
-						<li class="active"><a href="/templates/populateEventFromTemplate/<?php echo $template_id . '/' . $event['Event']['id']; ?>">Populate From Template</a></li>
+							<li class="active"><a href="/templates/populateEventFromTemplate/<?php echo $template_id . '/' . $event['Event']['id']; ?>">Populate From Template</a></li>
+						<?php endif; ?>
 					<?php endif; ?>
-					<?php elseif (!isset($mayModify) || !$mayModify): ?>
+					<?php if (($isSiteAdmin && (!isset($mayModify) || !$mayModify)) || (!isset($mayModify) || !$mayModify)): ?>
-					<li class="divider"></li>
 					<li <?php if ($menuItem === 'proposeAttribute') echo 'class="active"';?>><a href="/shadow_attributes/add/<?php echo $event['Event']['id'];?>">Propose Attribute</a></li>
 					<li <?php if ($menuItem === 'proposeAttachment') echo 'class="active"';?>><a href="/shadow_attributes/add_attachment/<?php echo $event['Event']['id'];?>">Propose Attachment</a></li>
 					<?php endif; ?>

app/View/Events/json/view.ctp

@@ -1,4 +1,4 @@
 <?php
 App::uses('JSONConverterTool', 'Tools');
 $converter = new JSONConverterTool();
-echo json_encode($converter->event2JSON($event));
+echo $converter->event2JSON($event);

app/View/Events/view.ctp

@@ -12,7 +12,7 @@ $mayPublish = ($isAclPublish && $event['Event']['orgc'] == $me['org']);
 			$left = true;
 		}
 		$title = $event['Event']['info'];
-		if (strlen($title) > 55) $title = substr($title, 0, 55) . '...';
+		if (strlen($title) > 58) $title = substr($title, 0, 55) . '...';
 	?>
 	<div class="row-fluid">
 		<div class="span8">
@@ -252,4 +252,4 @@ $(document).ready(function () {
 		$('#addTagButton').hide();
 	});
 });
-</script>
+</script>

app/View/Helper/PivotHelper.php

@@ -9,7 +9,7 @@ App::uses('AppHelper', 'View/Helper');
 			$active = '';
 			$pivot['info'] = h($pivot['info']);
 			// Truncate string if longer than (11 - length of event id) chars to fit the pivot bubble
-			if (strlen($pivot['info']) > (11 - strlen((string)$pivot['id']))) {
+			if (strlen($pivot['info']) > (11 - strlen((string)$pivot['id'])) && strlen($pivot['info']) > 9) {
 				$text .= substr($pivot['info'], 0, 6) . '...';				
 			} else {
 				$text .= $pivot['info'];

app/View/Pages/administration.ctp

@@ -16,6 +16,7 @@ if (!$isSiteAdmin) exit();
 <li><a href="/events/generateLocked">generateLocked</a> (This is for upgrading to hotfix 2.1.8 or later, all events that were created by an organisation that doesn't have users on this instance, or only has a single sync user will have their locked setting set to 1)</li>
 <li><a href="/users/verifyGPG">Verify GPG keys</a> (Check whether every user's GPG key is usable)</li>
 <li><a href="/events/generateThreatLevelFromRisk">Upgrade Risk to Threat Level</a> (As of version 2.2 the risk field is replaced by Threat Level. This script will convert all values in risk to threat level.)</li>
+<li><a href="/servers/updateDatabase/extendServerOrganizationLength">Extend Organization length</a> (Hotfix 2.3.57: Increase the max length of the organization field when adding a new server connection.)</li>
 </ul>
 </div>
 <?php 

app/View/Pages/doc/administration.ctp

@@ -31,6 +31,7 @@
 		<li><b>Overview</b>: General overview of the current state of your MISP installation</li>
 		<li><b>MISP settings</b>: Basic MISP settings. This includes the way MISP handles the default settings for distribution settings, whether background jobs are enabled, etc</li>
 		<li><b>GnuPG settings</b>: GPG related settings.</li>
+		<li><b>Proxy settings</b>: HTTP proxy related settings.</li>
 		<li><b>Security settings</b>: Settings controlling the brute-force protection and the application's salt key.</li>
 		<li><b>Misc settings</b>: You change the debug options here, but make sure that debug is always disabled on a production system.</li>
 		<li><b>Diagnostics</b>: The diagnostics tool checks if all directories that MISP uses to store data are writeable by the apache user. Also, the tool checks whether the STIX libraries and GPG are working as intended.</li>
@@ -240,4 +241,4 @@
 		<li><b>Message</b>: This field shows when the job was queued by the scheduler for execution. </li>
 	</ul>
 	<br /><img src="/img/doc/schedule.png" alt = "" title = "Site administrators can schedule reccuring tasks on this page."/><br />
-</div>
+</div>

app/View/Servers/server_settings.ctp

@@ -2,7 +2,7 @@
 	<h2>Server settings</h2>
 	<?php
 		echo $this->element('healthElements/tabs');
-		if (in_array($tab, array('MISP', 'Security', 'GnuPG', 'misc'))) {
+		if (in_array($tab, array('MISP', 'Security', 'GnuPG', 'Proxy', 'misc'))) {
 			echo $this->element('healthElements/settings_tab');
 		} else if ($tab == 'diagnostics') {
 			echo $this->element('healthElements/diagnostics');
@@ -17,4 +17,4 @@
 </div>
 <?php 
 	echo $this->element('side_menu', array('menuList' => 'admin', 'menuItem' => 'serverSettings'));
-?>
+?>

app/files/scripts/misp2ciq.py

@@ -1,7 +1,7 @@
 from stix.extensions.identity.ciq_identity_3_0 import (CIQIdentity3_0Instance, STIXCIQIdentity3_0, OrganisationInfo, PartyName, Address, ElectronicAddressIdentifier, FreeTextAddress)
 from stix.common import Identity
 
-def resolveIdentityAttribute(incident, attribute):
+def resolveIdentityAttribute(incident, attribute, namespace):
     ciq_identity = CIQIdentity3_0Instance()
     identity_spec = STIXCIQIdentity3_0()
     if attribute["type"] == 'target-user':
@@ -20,6 +20,6 @@ def resolveIdentityAttribute(incident, attribute):
     ciq_identity.id_ = "example:Identity-" + attribute["uuid"]
 
     # is this a good idea?
-    ciq_identity.name = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"]
+    ciq_identity.name = attribute["type"] + ": " + attribute["value"] + " (MISP Attribute #" + attribute["id"] + ")"
     incident.add_victim(ciq_identity)
     return incident

app/files/scripts/misp2cybox.py

@@ -1,4 +1,4 @@
-from cybox.core import Observable, ObservableComposition
+from cybox.core import Object, Observable, ObservableComposition
 from cybox.objects.file_object import File
 from cybox.objects.address_object import Address
 from cybox.objects.hostname_object import Hostname
@@ -40,7 +40,11 @@ def generateObservable(indicator, attribute):
         if (attribute["type"] in simple_type_to_method.keys()):
             action = getattr(this_module, simple_type_to_method[attribute["type"]], None)
             if (action != None):
-                observable = action(attribute)
+                property = action(attribute)
+                object = Object(property)
+                object.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":" + property.__class__.__name__ + "-" + attribute["uuid"]
+                observable = Observable(object)
+                observable.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-" + attribute["uuid"]
                 indicator.add_observable(observable)
 
 def resolveFileObservable(attribute):
@@ -177,30 +181,44 @@ def resolvePatternObservable(attribute):
 # create an artifact object for the malware-sample type.
 def createArtifactObject(indicator, attribute):
     artifact = Artifact(data = attribute["data"])
-    indicator.add_observable(artifact)
+    artifact.parent.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":artifact-" + attribute["uuid"]
+    observable = Observable(artifact)
+    observable.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-artifact-" + attribute["uuid"]
+    indicator.add_observable(observable)
 
 # return either a composition if data is set in attribute, or just an observable with a filename if it's not set
 def returnAttachmentComposition(attribute):
     file_object = File()
     file_object.file_name = attribute["value"]
+    file_object.parent.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":file-" + attribute["uuid"]
     observable = Observable()
     if "data" in attribute:
         artifact = Artifact(data = attribute["data"])
-        composition = ObservableComposition(observables = [artifact, file_object])
+        artifact.parent.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":artifact-" + attribute["uuid"]
+        observable_artifact = Observable(artifact)
+        observable_artifact.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-artifact-" + attribute["uuid"]
+        observable_file = Observable(file_object)
+        observable_file.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-file-" + attribute["uuid"]
+        composition = ObservableComposition(observables = [observable_artifact, observable_file])
         observable.observable_composition = composition
     else:
         observable = Observable(file_object)
+    observable.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-" + attribute["uuid"]
     if attribute["comment"] != "":
         observable.description = attribute["comment"]
     return observable
 
 # email-attachment are mapped to an email message observable that contains the attachment as a file object
-def generateEmailAttachmentObject(indicator, filename):
+def generateEmailAttachmentObject(indicator, attribute):
     file_object = File()
-    file_object.file_name = filename
+    file_object.file_name = attribute["value"]
     email = EmailMessage()
     email.attachments = Attachments()
     email.add_related(file_object, "Contains", inline=True)
+    file_object.parent.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":file-" + attribute["uuid"]
     email.attachments.append(file_object.parent.id_)
-    indicator.observable = email
+    email.parent.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":EmailMessage-" + attribute["uuid"]
+    observable = Observable(email)
+    observable.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-" + attribute["uuid"]
+    indicator.observable = observable
 

app/files/scripts/misp2stix.py

@@ -20,10 +20,6 @@ from cybox.utils import Namespace
 
 namespace = ['https://github.com/MISP/MISP', 'MISP']
 
-cybox.utils.idgen.set_id_namespace(Namespace(namespace[0], namespace[1]))
-stix.utils.idgen.set_id_namespace({namespace[0]: namespace[1]})
-
-
 NS_DICT = {
 	"http://cybox.mitre.org/common-2" : 'cyboxCommon',
 	"http://cybox.mitre.org/cybox-2" : 'cybox',	
@@ -55,7 +51,6 @@ NS_DICT = {
 	"urn:oasis:names:tc:ciq:xal:3" : 'xal',
 	"urn:oasis:names:tc:ciq:xnl:3" : 'xnl',
 	"urn:oasis:names:tc:ciq:xpil:3" : 'xpil',
-        namespace[0] : namespace[1]
 }
 
 SCHEMALOC_DICT = {
@@ -126,7 +121,7 @@ def saveFile(args, pathname, package):
 def generateMainPackage(events):
     stix_package = STIXPackage()
     stix_header = STIXHeader()
-    stix_header.title="Export from MISP"
+    stix_header.title="Export from " + namespace[1] + " MISP"
     stix_header.package_intents="Threat Report"
     stix_package.stix_header = stix_header
     return stix_package
@@ -134,9 +129,10 @@ def generateMainPackage(events):
 # generate a package for each event
 def generateEventPackage(event):
     package_name = namespace[1] + ':STIXPackage-' + event["Event"]["uuid"]
-    stix_package = STIXPackage(id_=package_name)
+    timestamp = getDateFromTimestamp(int(event["Event"]["timestamp"]))
+    stix_package = STIXPackage(id_=package_name, timestamp=timestamp)
     stix_header = STIXHeader()
-    stix_header.title="MISP event #" + event["Event"]["id"] + " uuid: " + event["Event"]["uuid"]
+    stix_header.title=event["Event"]["info"] + " (MISP Event #" + event["Event"]["id"] + ")"
     stix_header.package_intents="Threat Report"
     stix_package.stix_header = stix_header
     objects = generateSTIXObjects(event)
@@ -189,9 +185,8 @@ def resolveAttributes(incident, ttps, attributes):
 # Create the indicator and pass the attribute further for observable creation - this can be called from resolveattributes directly or from handleNonindicatorAttribute, for some special cases
 def handleIndicatorAttribute(incident, ttps, attribute):
     indicator = generateIndicator(attribute)
-    indicator.title = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"]
     if attribute["type"] == "email-attachment":
-        generateEmailAttachmentObject(indicator, attribute["value"])
+        generateEmailAttachmentObject(indicator, attribute)
     else:
         generateObservable(indicator, attribute)
     if "data" in attribute:
@@ -227,7 +222,7 @@ def handleNonIndicatorAttribute(incident, ttps, attribute):
         else:
             addReference(incident, attribute["value"])
     elif attribute["type"].startswith('target-'):
-        resolveIdentityAttribute(incident, attribute)
+        resolveIdentityAttribute(incident, attribute, namespace[1])
     elif attribute["type"] == "attachment":
         observable = returnAttachmentComposition(attribute)
         related_observable = RelatedObservable(observable, relationship=attribute["category"])
@@ -236,14 +231,14 @@ def handleNonIndicatorAttribute(incident, ttps, attribute):
 
 # TTPs are only used to describe malware names currently (attribute with category Payload Type and type text/comment/other)
 def generateTTP(incident, attribute):
-    ttp = TTP()
+    ttp = TTP(timestamp=getDateFromTimestamp(int(attribute["timestamp"])))
     ttp.id_= namespace[1] + ":ttp-" + attribute["uuid"]
     setTLP(ttp, attribute["distribution"])
-    ttp.title = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"]
+    ttp.title = attribute["category"] + ": " + attribute["value"] + " (MISP Attribute #" + attribute["id"] + ")"
     if attribute["type"] == "vulnerability":
         vulnerability = Vulnerability()
         vulnerability.cve_id = attribute["value"]
-        et = ExploitTarget()
+        et = ExploitTarget(timestamp=getDateFromTimestamp(int(attribute["timestamp"])))
         et.add_vulnerability(vulnerability)
         ttp.exploit_targets.append(et)
     else:
@@ -258,9 +253,9 @@ def generateTTP(incident, attribute):
 
 # Threat actors are currently only used for the category:attribution / type:(text|comment|other) attributes 
 def generateThreatActor(attribute):
-    ta = ThreatActor()
+    ta = ThreatActor(timestamp=getDateFromTimestamp(int(attribute["timestamp"])))
     ta.id_= namespace[1] + ":threatactor-" + attribute["uuid"]
-    ta.title = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"]
+    ta.title = attribute["category"] + ": " + attribute["value"] + " (MISP Attribute #" + attribute["id"] + ")"
     if attribute["comment"] != "":
         ta.description = attribute["value"] + " (" + attribute["comment"] + ")"
     else:
@@ -269,17 +264,17 @@ def generateThreatActor(attribute):
 
 # generate the indicator and add the relevant information
 def generateIndicator(attribute):
-    indicator = Indicator()
+    indicator = Indicator(timestamp=getDateFromTimestamp(int(attribute["timestamp"])))
     indicator.id_= namespace[1] + ":indicator-" + attribute["uuid"]
     if attribute["comment"] != "":
         indicator.description = attribute["comment"]
     setTLP(indicator, attribute["distribution"])
-    indicator.title = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"]
+    indicator.title = attribute["category"] + ": " + attribute["value"] + " (MISP Attribute #" + attribute["id"] + ")"
     confidence_description = "Derived from MISP's IDS flag. If an attribute is marked for IDS exports, the confidence will be high, otherwise none"
     confidence_value = confidence_mapping.get(attribute["to_ids"], None)
     if confidence_value is None:
         return indicator
-    indicator.confidence = Confidence(value=confidence_value, description=confidence_description)
+    indicator.confidence = Confidence(value=confidence_value, description=confidence_description, timestamp=getDateFromTimestamp(int(attribute["timestamp"])))
     return indicator
 
 # converts timestamp to the format used by STIX
@@ -323,6 +318,13 @@ def addJournalEntry(incident, entry_line):
 # main
 def main(args):
     pathname = os.path.dirname(sys.argv[0])
+    if len(sys.argv) > 3:
+        namespace[0] = sys.argv[3]
+    if len(sys.argv) > 4:
+        namespace[1] = sys.argv[4].replace(" ", "_")
+    NS_DICT[namespace[0]]=namespace[1]
+    cybox.utils.idgen.set_id_namespace(Namespace(namespace[0], namespace[1]))
+    stix.utils.idgen.set_id_namespace({namespace[0]: namespace[1]})
     events = loadEvent(args, pathname)
     stix_package = generateMainPackage(events)
     for event in events: