From 253e579307492a3e4cf8a110fc5ef1faa9c2c1e0 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Tue, 27 Jan 2015 10:10:22 +0100 Subject: [PATCH 01/44] Disallow unpublished events --- app/Controller/AttributesController.php | 7 ++++--- app/Controller/EventsController.php | 8 +++++--- app/Controller/ShadowAttributesController.php | 1 + app/Controller/TagsController.php | 4 ++-- app/Model/Event.php | 10 +++++++--- 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 58dfc30e0..60ea35bc0 100755 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -58,6 +58,7 @@ class AttributesController extends AppController { 'AND' => array( 'Attribute.distribution >' => 0, 'Event.distribution >' => 0, + 'Event.published =' => 1, ))))); } @@ -1283,7 +1284,7 @@ class AttributesController extends AppController { 'conditions' => array("OR" => array( array('Event.org =' => $this->Auth->user('org')), - array("AND" => array('Event.org !=' => $this->Auth->user('org')), array('Event.distribution !=' => 0), array('Attribute.distribution !=' => 0)))), + array("AND" => array('Event.org !=' => $this->Auth->user('org')), array('Event.distribution !=' => 0), array('Event.published =' => 1), array('Attribute.distribution !=' => 0)))), ) ); } @@ -1370,7 +1371,7 @@ class AttributesController extends AppController { $data['AND'][] = array( "OR" => array( array('Event.org =' => $this->Auth->user('org')), - array("AND" => array('Event.org !=' => $this->Auth->user('org')), array('Event.distribution !=' => 0), array('Attribute.distribution !=' => 0)))); + array("AND" => array('Event.org !=' => $this->Auth->user('org')), array('Event.distribution !=' => 0), array('Event.distribution =' => 1), array('Attribute.distribution !=' => 0)))); $attributes = $this->Attribute->find('all', array( 'conditions' => $data, 'fields' => array( @@ -1555,7 +1556,7 @@ class AttributesController extends AppController { if (!$user['User']['siteAdmin']) { $temp = array(); - $temp['AND'] = array('Event.distribution >' => 0, 'Attribute.distribution >' => 0); + $temp['AND'] = array('Event.distribution >' => 0, 'Event.published =' => 1, 'Attribute.distribution >' => 0); $subcondition['OR'][] = $temp; $subcondition['OR'][] = array('Event.org' => $user['User']['org']); array_push($conditions['AND'], $subcondition); diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 162f89142..1a8470493 100755 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -71,8 +71,10 @@ class EventsController extends AppController { 'conditions' => array("OR" => array( array('Event.org =' => $this->Auth->user('org')), - array('Event.distribution >' => 0), - )))); + "AND" => array( + array('Event.distribution >' => 0), + array('Event.published =' => 1), + ))))); } } @@ -2437,7 +2439,7 @@ class EventsController extends AppController { if (!$user['User']['siteAdmin']) { $temp = array(); - $temp['AND'] = array('Event.distribution >' => 0, 'Attribute.distribution >' => 0); + $temp['AND'] = array('Event.distribution >' => 0, 'Event.published = ' => 1, 'Attribute.distribution >' => 0); $subcondition['OR'][] = $temp; $subcondition['OR'][] = array('Event.org' => $user['User']['org']); array_push($conditions['AND'], $subcondition); diff --git a/app/Controller/ShadowAttributesController.php b/app/Controller/ShadowAttributesController.php index 09bfcb2a0..8bcf2fe1e 100644 --- a/app/Controller/ShadowAttributesController.php +++ b/app/Controller/ShadowAttributesController.php @@ -50,6 +50,7 @@ class ShadowAttributesController extends AppController { 'AND' => array( 'ShadowAttribute.org =' => $this->Auth->user('org'), 'Event.distribution >' => 0, + 'Event.published =' => 1, ), ) ))); diff --git a/app/Controller/TagsController.php b/app/Controller/TagsController.php index 527661f6d..414697d74 100644 --- a/app/Controller/TagsController.php +++ b/app/Controller/TagsController.php @@ -37,7 +37,7 @@ class TagsController extends AppController { $eventIDs[] = $eventTag['event_id']; } $conditions = array('Event.id' => $eventIDs); - if (!$this->_isSiteAdmin()) $conditions = array_merge($conditions, array('OR' => array(array('Event.distribution >' => 0), array('Event.orgc' => $this->Auth->user('org'))))); + if (!$this->_isSiteAdmin()) $conditions = array_merge($conditions, array('OR' => array(array('AND' => array(array('Event.distribution >' => 0), array('Event.published =' => 1))), array('Event.orgc' => $this->Auth->user('org'))))); $events = $this->Event->find('all', array( 'fields' => array('Event.id', 'Event.distribution', 'Event.orgc'), 'conditions' => $conditions @@ -136,4 +136,4 @@ class TagsController extends AppController { $this->set('id', $id); $this->render('ajax/view_tag'); } -} \ No newline at end of file +} diff --git a/app/Model/Event.php b/app/Model/Event.php index a55831750..cd8309b86 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -799,7 +799,9 @@ class Event extends AppModel { $conditions = array(); if (!$isSiteAdmin) { $conditions['OR'] = array( - 'Event.distribution >' => 0, + "AND" => array( + 'Event.distribution >' => 0, + 'Event.published =' => 1), 'Event.org LIKE' => $org ); } @@ -831,7 +833,9 @@ class Event extends AppModel { //restricting to non-private or same org if the user is not a site-admin. if (!$isSiteAdmin) { $conditions['AND']['OR'] = array( - 'Event.distribution >' => 0, + "AND" => array( + 'Event.distribution >' => 0, + 'Event.published =' => 1), 'Event.org LIKE' => $org ); $conditionsAttributes['OR'] = array( @@ -923,7 +927,7 @@ class Event extends AppModel { if ($eventid !== 'search') { // This is for both single event downloads and for full downloads. Org has to be the same as the user's or distribution not org only - if the user is no siteadmin if(!$isSiteAdmin) { - $econditions['AND']['OR'] = array('Event.distribution >' => 0, 'Event.org =' => $org); + $econditions['AND']['OR'] = array("AND" => array('Event.distribution >' => 0, 'Event.published =' => 1), 'Event.org =' => $org); } if ($eventid == 0 && $ignore == 0) { $econditions['AND'][] = array('Event.published =' => 1); From 12dc5b657976370e0b4dfdee37c6181ecbf1b9c4 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Thu, 5 Feb 2015 17:16:40 +0100 Subject: [PATCH 02/44] Make unpublished events private if MISP.unpublishedprivate == true --- app/Config/config.default.php | 3 +- app/Controller/AttributesController.php | 28 +++++++++++++++---- app/Controller/EventsController.php | 17 +++++++---- app/Controller/ShadowAttributesController.php | 2 +- app/Controller/TagsController.php | 10 ++++++- app/Model/Event.php | 14 ++++++++-- app/Model/Server.php | 8 ++++++ 7 files changed, 65 insertions(+), 17 deletions(-) diff --git a/app/Config/config.default.php b/app/Config/config.default.php index c9f6a61f9..bb59db51a 100644 --- a/app/Config/config.default.php +++ b/app/Config/config.default.php @@ -25,7 +25,8 @@ $config = array ( 'tagging' => true, 'full_tags_on_event_index' => true, 'footer_logo' => '', - 'take_ownership_xml_import' => false, + 'take_ownership_xml_import' => false, + 'unpublishedprivate' => false, ), 'GnuPG' => array ( diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 2172f8d29..a33f311d9 100755 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -58,7 +58,7 @@ class AttributesController extends AppController { 'AND' => array( 'Attribute.distribution >' => 0, 'Event.distribution >' => 0, - 'Event.published =' => 1, + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(), ))))); } @@ -1283,8 +1283,14 @@ class AttributesController extends AppController { $this->paginate = Set::merge($this->paginate, array( 'conditions' => array("OR" => array( - array('Event.org =' => $this->Auth->user('org')), - array("AND" => array('Event.org !=' => $this->Auth->user('org')), array('Event.distribution !=' => 0), array('Event.published =' => 1), array('Attribute.distribution !=' => 0)))), + array('Event.org =' => $this->Auth->user('org')), + array("AND" => + array('Event.org !=' => $this->Auth->user('org')), + array('Event.distribution !=' => 0), + array('Attribute.distribution !=' => 0), + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(), + ) + ) ) ); } @@ -1370,8 +1376,14 @@ class AttributesController extends AppController { public function searchAlternate($data) { $data['AND'][] = array( "OR" => array( - array('Event.org =' => $this->Auth->user('org')), - array("AND" => array('Event.org !=' => $this->Auth->user('org')), array('Event.distribution !=' => 0), array('Event.distribution =' => 1), array('Attribute.distribution !=' => 0)))); + array('Event.org =' => $this->Auth->user('org')), + array("AND" => array('Event.org !=' => $this->Auth->user('org')), + array('Event.distribution !=' => 0), + array('Attribute.distribution !=' => 0), + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(), + ) + ) + ); $attributes = $this->Attribute->find('all', array( 'conditions' => $data, 'fields' => array( @@ -1556,7 +1568,11 @@ class AttributesController extends AppController { if (!$user['User']['siteAdmin']) { $temp = array(); - $temp['AND'] = array('Event.distribution >' => 0, 'Event.published =' => 1, 'Attribute.distribution >' => 0); + $temp['AND'] = array( + 'Event.distribution >' => 0, + 'Attribute.distribution >' => 0, + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array() + ); $subcondition['OR'][] = $temp; $subcondition['OR'][] = array('Event.org' => $user['User']['org']); array_push($conditions['AND'], $subcondition); diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 939e6baec..cbb973a48 100755 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -72,9 +72,12 @@ class EventsController extends AppController { array("OR" => array( array('Event.org =' => $this->Auth->user('org')), "AND" => array( - array('Event.distribution >' => 0), - array('Event.published =' => 1), - ))))); + array('Event.distribution >' => 0), + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(), + ) + ) + ) + ); } } @@ -2454,7 +2457,11 @@ class EventsController extends AppController { if (!$user['User']['siteAdmin']) { $temp = array(); - $temp['AND'] = array('Event.distribution >' => 0, 'Event.published = ' => 1, 'Attribute.distribution >' => 0); + $temp['AND'] = array( + 'Event.distribution >' => 0, + 'Attribute.distribution >' => 0, + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array() + ); $subcondition['OR'][] = $temp; $subcondition['OR'][] = array('Event.org' => $user['User']['org']); array_push($conditions['AND'], $subcondition); @@ -3147,4 +3154,4 @@ class EventsController extends AppController { $this->set('id', $id); $this->render('ajax/exportChoice'); } -} \ No newline at end of file +} diff --git a/app/Controller/ShadowAttributesController.php b/app/Controller/ShadowAttributesController.php index 8bcf2fe1e..a6b9410b5 100644 --- a/app/Controller/ShadowAttributesController.php +++ b/app/Controller/ShadowAttributesController.php @@ -50,7 +50,7 @@ class ShadowAttributesController extends AppController { 'AND' => array( 'ShadowAttribute.org =' => $this->Auth->user('org'), 'Event.distribution >' => 0, - 'Event.published =' => 1, + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(), ), ) ))); diff --git a/app/Controller/TagsController.php b/app/Controller/TagsController.php index 414697d74..7c0a00627 100644 --- a/app/Controller/TagsController.php +++ b/app/Controller/TagsController.php @@ -37,7 +37,15 @@ class TagsController extends AppController { $eventIDs[] = $eventTag['event_id']; } $conditions = array('Event.id' => $eventIDs); - if (!$this->_isSiteAdmin()) $conditions = array_merge($conditions, array('OR' => array(array('AND' => array(array('Event.distribution >' => 0), array('Event.published =' => 1))), array('Event.orgc' => $this->Auth->user('org'))))); + if (!$this->_isSiteAdmin()) $conditions = array_merge( + $conditions, + array('OR' => array( + array('AND' => array( + array('Event.distribution >' => 0), + array('Event.published =' => 1) + )), + array('Event.orgc' => $this->Auth->user('org')) + ))); $events = $this->Event->find('all', array( 'fields' => array('Event.id', 'Event.distribution', 'Event.orgc'), 'conditions' => $conditions diff --git a/app/Model/Event.php b/app/Model/Event.php index 61f892bdd..2557e2bc7 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -801,7 +801,8 @@ class Event extends AppModel { $conditions['OR'] = array( "AND" => array( 'Event.distribution >' => 0, - 'Event.published =' => 1), + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array() + ), 'Event.org LIKE' => $org ); } @@ -839,7 +840,8 @@ class Event extends AppModel { $conditions['AND']['OR'] = array( "AND" => array( 'Event.distribution >' => 0, - 'Event.published =' => 1), + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(), + ), 'Event.org LIKE' => $org ); $conditionsAttributes['OR'] = array( @@ -936,7 +938,13 @@ class Event extends AppModel { if ($from) $econditions['AND'][] = array('Event.date >=' => $from); if ($to) $econditions['AND'][] = array('Event.date <=' => $to); // This is for both single event downloads and for full downloads. Org has to be the same as the user's or distribution not org only - if the user is no siteadmin - if(!$isSiteAdmin) $econditions['AND']['OR'] = array("AND" => array('Event.distribution >' => 0, 'Event.published =' => 1), 'Event.org =' => $org); + if(!$isSiteAdmin) $econditions['AND']['OR'] = array( + "AND" => array( + 'Event.distribution >' => 0, + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(), + ), + 'Event.org =' => $org + ); if ($eventid == 0 && $ignore == 0) $econditions['AND'][] = array('Event.published =' => 1); // If it's a full download (eventid == false) and the user is not a site admin, we need to first find all the events that the user can see and save the IDs diff --git a/app/Model/Server.php b/app/Model/Server.php index 83c5fd9ea..d3cb703e6 100755 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -381,6 +381,14 @@ class Server extends AppModel { 'test' => 'testBool', 'type' => 'boolean' ), + 'unpublishedprivate' => array( + 'level' => 2, + 'description' => 'True will deny access to unpublished events to users outside the organization of the submitter except site admins.', + 'value' => '', + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), ), 'GnuPG' => array( 'branch' => 1, From d5e556ee1ec18b9c55ef52ccce11f694eb3f786c Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Thu, 5 Feb 2015 17:40:56 +0100 Subject: [PATCH 03/44] =?UTF-8?q?Fixed=20missing=20parentheses=E2=80=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/Controller/AttributesController.php | 16 +++++++++------- app/Controller/EventsController.php | 11 ++++++----- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index a33f311d9..e3fbe3ee8 100755 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -1282,13 +1282,15 @@ class AttributesController extends AppController { // merge in private conditions $this->paginate = Set::merge($this->paginate, array( 'conditions' => - array("OR" => array( - array('Event.org =' => $this->Auth->user('org')), - array("AND" => - array('Event.org !=' => $this->Auth->user('org')), - array('Event.distribution !=' => 0), - array('Attribute.distribution !=' => 0), - Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(), + array("OR" => + array( + array('Event.org =' => $this->Auth->user('org')), + array("AND" => + array('Event.org !=' => $this->Auth->user('org')), + array('Event.distribution !=' => 0), + array('Attribute.distribution !=' => 0), + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(), + ) ) ) ) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index cbb973a48..28ecf9b9a 100755 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -70,13 +70,14 @@ class EventsController extends AppController { $this->paginate = Set::merge($this->paginate,array( 'conditions' => array("OR" => array( - array('Event.org =' => $this->Auth->user('org')), - "AND" => array( - array('Event.distribution >' => 0), - Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(), - ) + array('Event.org =' => $this->Auth->user('org')), + "AND" => array( + array('Event.distribution >' => 0), + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(), ) + ) ) + ) ); } } From 4eca9d72b6764b0921ac7578f08264f96f23f473 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Tue, 10 Feb 2015 09:56:54 +0100 Subject: [PATCH 04/44] Documentation changes - Added changes from 9378837f39a52e246fb1c11aac18343c8c8992a0 for CentOS - Fixed some typos --- INSTALL/INSTALL.centos6.txt | 11 +++++++++-- INSTALL/INSTALL.centos7.txt | 11 +++++++++-- INSTALL/INSTALL.ubuntu1404.txt | 4 ++-- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/INSTALL/INSTALL.centos6.txt b/INSTALL/INSTALL.centos6.txt index b89243bff..39b2b62ec 100644 --- a/INSTALL/INSTALL.centos6.txt +++ b/INSTALL/INSTALL.centos6.txt @@ -123,8 +123,9 @@ mysql> grant usage on *.* to misp@localhost identified by 'XXXXXXXXX'; mysql> grant all privileges on misp.* to misp@localhost ; mysql> exit -# Import the empty MySQL database from MYSQL.sql cd /var/www/MISP + +# Import the empty MySQL database from MYSQL.sql mysql -u misp -p misp < INSTALL/MYSQL.sql @@ -191,7 +192,8 @@ chmod 700 /var/www/MISP/.gnupg gpg --homedir /var/www/MISP/.gnupg --gen-key chown -R apache:apache /var/www/MISP/.gnupg # Recommended key type: RSA -# The email address should match the one set int he config.php configuration file +# The email address should match the one set in the config.php configuration file +# Make sure that you use the same settings in the MISP Server Settings tool (Described on line 212) # And export the public key to the webroot sudo -u apache gpg --homedir /var/www/MISP/.gnupg --export --armor YOUR-EMAIL > /var/www/MISP/app/webroot/gpg.asc @@ -222,6 +224,11 @@ chown -R apache:apache /var/www/MISP/ # Make sure that the STIX libraries and GnuPG work as intended, if not, refer to INSTALL.txt's paragraphs dealing with these two items +# If anything goes wrong, make sure that you check MISP's logs for errors: +# /var/www/MISP/app/tmp/logs/error.log +# /var/www/MISP/app/tmp/logs/resque-worker-error.log +# /var/www/MISP/app/tmp/logs/resque-scheduler-error.log +# /var/www/MISP/app/tmp/logs/resque-2015-01-01.log //where the actual date is the current date Recommended actions ------------------- diff --git a/INSTALL/INSTALL.centos7.txt b/INSTALL/INSTALL.centos7.txt index 83b5c3df6..fafeb6d0c 100644 --- a/INSTALL/INSTALL.centos7.txt +++ b/INSTALL/INSTALL.centos7.txt @@ -127,8 +127,9 @@ MariaDB [(none)]> grant usage on *.* to misp@localhost identified by 'XXXXXXXXX' MariaDB [(none)]> grant all privileges on misp.* to misp@localhost ; MariaDB [(none)]> exit -# Import the empty MySQL database from MYSQL.sql cd /var/www/MISP + +# Import the empty MySQL database from MYSQL.sql mysql -u misp -p misp < INSTALL/MYSQL.sql @@ -205,7 +206,8 @@ chmod 700 /var/www/MISP/.gnupg gpg --homedir /var/www/MISP/.gnupg --gen-key chown -R apache:apache /var/www/MISP/.gnupg # Recommended key type: RSA -# The email address should match the one set int he config.php configuration file +# The email address should match the one set in the config.php configuration file +# Make sure that you use the same settings in the MISP Server Settings tool (Described on line 226) # And export the public key to the webroot sudo -u apache gpg --homedir /var/www/MISP/.gnupg --export --armor YOUR-EMAIL > /var/www/MISP/app/webroot/gpg.asc @@ -236,6 +238,11 @@ chown -R apache:apache /var/www/MISP/ # Make sure that the STIX libraries and GnuPG work as intended, if not, refer to INSTALL.txt's paragraphs dealing with these two items +# If anything goes wrong, make sure that you check MISP's logs for errors: +# /var/www/MISP/app/tmp/logs/error.log +# /var/www/MISP/app/tmp/logs/resque-worker-error.log +# /var/www/MISP/app/tmp/logs/resque-scheduler-error.log +# /var/www/MISP/app/tmp/logs/resque-2015-01-01.log //where the actual date is the current date Recommended actions ------------------- diff --git a/INSTALL/INSTALL.ubuntu1404.txt b/INSTALL/INSTALL.ubuntu1404.txt index ab4dc9d7a..7e9923a73 100644 --- a/INSTALL/INSTALL.ubuntu1404.txt +++ b/INSTALL/INSTALL.ubuntu1404.txt @@ -163,8 +163,8 @@ chown www-data:www-data /var/www/MISP/.gnupg chmod 700 /var/www/MISP/.gnupg sudo -u www-data gpg --homedir /var/www/MISP/.gnupg --gen-key # Recommended key type: RSA -# The email address should match the one set int he bootstrap.php configuration file -# Make sure that you use the same settings in the MISP Server Settings tool tool (Described on line 184) +# The email address should match the one set in the config.php configuration file +# Make sure that you use the same settings in the MISP Server Settings tool (Described on line 184) # And export the public key to the webroot sudo -u www-data gpg --homedir /var/www/MISP/.gnupg --export --armor YOUR-EMAIL > /var/www/MISP/app/webroot/gpg.asc From 622800a4b2a9e8fc48258811f8f90ba0ef77237c Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Thu, 19 Feb 2015 14:49:40 +0100 Subject: [PATCH 05/44] php-xml is needed for DOMDocument class --- INSTALL/INSTALL.centos6.txt | 3 ++- INSTALL/INSTALL.centos7.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/INSTALL/INSTALL.centos6.txt b/INSTALL/INSTALL.centos6.txt index 39b2b62ec..8c93af3cf 100644 --- a/INSTALL/INSTALL.centos6.txt +++ b/INSTALL/INSTALL.centos6.txt @@ -26,7 +26,8 @@ rpm -Uvh epel.rpm yum install vim # Install the dependencies: -yum install gcc git zip redis mysql-server php-mysql python-devel python-pip libxslt-devel zlib-devel php-devel +yum install gcc git zip redis mysql-server php-mysql python-devel python-pip +libxslt-devel zlib-devel php-devel php-xml yum install php-pear php-pecl-geoip pear channel-update pear.php.net diff --git a/INSTALL/INSTALL.centos7.txt b/INSTALL/INSTALL.centos7.txt index fafeb6d0c..4cdfb4d17 100644 --- a/INSTALL/INSTALL.centos7.txt +++ b/INSTALL/INSTALL.centos7.txt @@ -26,7 +26,8 @@ rpm -Uvh epel.rpm yum install vim # Install the dependencies: -yum install git httpd zip php redis mysql-server php-mysql python-devel python-pip libxslt-devel zlib-devel php-devel +yum install git httpd zip php redis mysql-server php-mysql python-devel +python-pip libxslt-devel zlib-devel php-devel php-xml yum install php-pear php-pecl-geoip pear channel-update pear.php.net From fc84de0cc6eaf5e42693695691a0e17901cecd26 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Thu, 19 Feb 2015 14:54:03 +0100 Subject: [PATCH 06/44] Fix line breaks --- INSTALL/INSTALL.centos6.txt | 3 +-- INSTALL/INSTALL.centos7.txt | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/INSTALL/INSTALL.centos6.txt b/INSTALL/INSTALL.centos6.txt index 8c93af3cf..ed46189ea 100644 --- a/INSTALL/INSTALL.centos6.txt +++ b/INSTALL/INSTALL.centos6.txt @@ -26,8 +26,7 @@ rpm -Uvh epel.rpm yum install vim # Install the dependencies: -yum install gcc git zip redis mysql-server php-mysql python-devel python-pip -libxslt-devel zlib-devel php-devel php-xml +yum install gcc git zip redis mysql-server php-mysql python-devel python-pip libxslt-devel zlib-devel php-devel php-xml yum install php-pear php-pecl-geoip pear channel-update pear.php.net diff --git a/INSTALL/INSTALL.centos7.txt b/INSTALL/INSTALL.centos7.txt index 4cdfb4d17..0bfa02235 100644 --- a/INSTALL/INSTALL.centos7.txt +++ b/INSTALL/INSTALL.centos7.txt @@ -26,8 +26,7 @@ rpm -Uvh epel.rpm yum install vim # Install the dependencies: -yum install git httpd zip php redis mysql-server php-mysql python-devel -python-pip libxslt-devel zlib-devel php-devel php-xml +yum install git httpd zip php redis mysql-server php-mysql python-devel python-pip libxslt-devel zlib-devel php-devel php-xml yum install php-pear php-pecl-geoip pear channel-update pear.php.net From 3c205c66f76c96b6239a6a9f3f8ac0fe14be1877 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Thu, 19 Feb 2015 12:25:57 +0100 Subject: [PATCH 07/44] More informative STIX titles --- app/files/scripts/misp2stix.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/app/files/scripts/misp2stix.py b/app/files/scripts/misp2stix.py index 28b0659d1..c608fd907 100644 --- a/app/files/scripts/misp2stix.py +++ b/app/files/scripts/misp2stix.py @@ -136,7 +136,7 @@ def generateEventPackage(event): package_name = namespace[1] + ':STIXPackage-' + event["Event"]["uuid"] stix_package = STIXPackage(id_=package_name) stix_header = STIXHeader() - stix_header.title="MISP event #" + event["Event"]["id"] + " uuid: " + event["Event"]["uuid"] + stix_header.title=event["Event"]["info"] + " (MISP Event #" + event["Event"]["id"] + ")" stix_header.package_intents="Threat Report" stix_package.stix_header = stix_header objects = generateSTIXObjects(event) @@ -189,7 +189,6 @@ def resolveAttributes(incident, ttps, attributes): # Create the indicator and pass the attribute further for observable creation - this can be called from resolveattributes directly or from handleNonindicatorAttribute, for some special cases def handleIndicatorAttribute(incident, ttps, attribute): indicator = generateIndicator(attribute) - indicator.title = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"] if attribute["type"] == "email-attachment": generateEmailAttachmentObject(indicator, attribute["value"]) else: @@ -239,7 +238,7 @@ def generateTTP(incident, attribute): ttp = TTP() ttp.id_= namespace[1] + ":ttp-" + attribute["uuid"] setTLP(ttp, attribute["distribution"]) - ttp.title = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"] + ttp.title = attribute["category"] + ": " + attribute["value"] + " (MISP Attribute #" + attribute["id"] + ")" if attribute["type"] == "vulnerability": vulnerability = Vulnerability() vulnerability.cve_id = attribute["value"] @@ -260,7 +259,7 @@ def generateTTP(incident, attribute): def generateThreatActor(attribute): ta = ThreatActor() ta.id_= namespace[1] + ":threatactor-" + attribute["uuid"] - ta.title = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"] + ta.title = attribute["category"] + ": " + attribute["value"] + " (MISP Attribute #" + attribute["id"] + ")" if attribute["comment"] != "": ta.description = attribute["value"] + " (" + attribute["comment"] + ")" else: @@ -274,7 +273,7 @@ def generateIndicator(attribute): if attribute["comment"] != "": indicator.description = attribute["comment"] setTLP(indicator, attribute["distribution"]) - indicator.title = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"] + indicator.title = attribute["category"] + ": " + attribute["value"] + " (MISP Attribute #" + attribute["id"] + ")" confidence_description = "Derived from MISP's IDS flag. If an attribute is marked for IDS exports, the confidence will be high, otherwise none" confidence_value = confidence_mapping.get(attribute["to_ids"], None) if confidence_value is None: From c9bd754adcb7e6d8f7251f6ef290b03859b73550 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Thu, 19 Feb 2015 13:24:01 +0100 Subject: [PATCH 08/44] More informative CIQ titles --- app/files/scripts/misp2ciq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/scripts/misp2ciq.py b/app/files/scripts/misp2ciq.py index 47998d0e7..d08ae0621 100644 --- a/app/files/scripts/misp2ciq.py +++ b/app/files/scripts/misp2ciq.py @@ -20,6 +20,6 @@ def resolveIdentityAttribute(incident, attribute): ciq_identity.id_ = "example:Identity-" + attribute["uuid"] # is this a good idea? - ciq_identity.name = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"] + ciq_identity.name = attribute["type"] + ": " + attribute["value"] + " (MISP Attribute #" + attribute["id"] + ")" incident.add_victim(ciq_identity) return incident From d2556ff91d6186efa90c477d6cd31dfb63913a3f Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Thu, 19 Feb 2015 15:05:51 +0100 Subject: [PATCH 09/44] Use org name and baseurl in XML namespace for STIX --- app/Model/Event.php | 2 +- app/files/scripts/misp2ciq.py | 2 +- app/files/scripts/misp2stix.py | 16 +++++++++------- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index 469aa6d74..56136388c 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -1851,7 +1851,7 @@ class Event extends AppModel { $scriptFile = APP . "files" . DS . "scripts" . DS . "misp2stix.py"; // Execute the python script and point it to the temporary filename - $result = shell_exec('python ' . $scriptFile . ' ' . $randomFileName . ' ' . $returnType); + $result = shell_exec('python ' . $scriptFile . ' ' . $randomFileName . ' ' . $returnType . ' ' . Configure::read('MISP.baseurl') . ' "' . Configure::read('MISP.org') . '"'); // The result of the script will be a returned JSON object with 2 variables: success (boolean) and message // If success = 1 then the temporary output file was successfully written, otherwise an error message is passed along diff --git a/app/files/scripts/misp2ciq.py b/app/files/scripts/misp2ciq.py index d08ae0621..98704adc2 100644 --- a/app/files/scripts/misp2ciq.py +++ b/app/files/scripts/misp2ciq.py @@ -1,7 +1,7 @@ from stix.extensions.identity.ciq_identity_3_0 import (CIQIdentity3_0Instance, STIXCIQIdentity3_0, OrganisationInfo, PartyName, Address, ElectronicAddressIdentifier, FreeTextAddress) from stix.common import Identity -def resolveIdentityAttribute(incident, attribute): +def resolveIdentityAttribute(incident, attribute, namespace): ciq_identity = CIQIdentity3_0Instance() identity_spec = STIXCIQIdentity3_0() if attribute["type"] == 'target-user': diff --git a/app/files/scripts/misp2stix.py b/app/files/scripts/misp2stix.py index c608fd907..5d4d5eb7e 100644 --- a/app/files/scripts/misp2stix.py +++ b/app/files/scripts/misp2stix.py @@ -20,10 +20,6 @@ from cybox.utils import Namespace namespace = ['https://github.com/MISP/MISP', 'MISP'] -cybox.utils.idgen.set_id_namespace(Namespace(namespace[0], namespace[1])) -stix.utils.idgen.set_id_namespace({namespace[0]: namespace[1]}) - - NS_DICT = { "http://cybox.mitre.org/common-2" : 'cyboxCommon', "http://cybox.mitre.org/cybox-2" : 'cybox', @@ -55,7 +51,6 @@ NS_DICT = { "urn:oasis:names:tc:ciq:xal:3" : 'xal', "urn:oasis:names:tc:ciq:xnl:3" : 'xnl', "urn:oasis:names:tc:ciq:xpil:3" : 'xpil', - namespace[0] : namespace[1] } SCHEMALOC_DICT = { @@ -126,7 +121,7 @@ def saveFile(args, pathname, package): def generateMainPackage(events): stix_package = STIXPackage() stix_header = STIXHeader() - stix_header.title="Export from MISP" + stix_header.title="Export from " + namespace[1] + " MISP" stix_header.package_intents="Threat Report" stix_package.stix_header = stix_header return stix_package @@ -226,7 +221,7 @@ def handleNonIndicatorAttribute(incident, ttps, attribute): else: addReference(incident, attribute["value"]) elif attribute["type"].startswith('target-'): - resolveIdentityAttribute(incident, attribute) + resolveIdentityAttribute(incident, attribute, namespace[1]) elif attribute["type"] == "attachment": observable = returnAttachmentComposition(attribute) related_observable = RelatedObservable(observable, relationship=attribute["category"]) @@ -322,6 +317,13 @@ def addJournalEntry(incident, entry_line): # main def main(args): pathname = os.path.dirname(sys.argv[0]) + if len(sys.argv) > 3: + namespace[0] = sys.argv[3] + if len(sys.argv) > 4: + namespace[1] = sys.argv[4].replace(" ", "_") + NS_DICT[namespace[0]]=namespace[1] + cybox.utils.idgen.set_id_namespace(Namespace(namespace[0], namespace[1])) + stix.utils.idgen.set_id_namespace({namespace[0]: namespace[1]}) events = loadEvent(args, pathname) stix_package = generateMainPackage(events) for event in events: From 66db71e835b89b10428c15764d06073361c5b3fe Mon Sep 17 00:00:00 2001 From: iglocska Date: Tue, 24 Feb 2015 15:46:42 +0100 Subject: [PATCH 10/44] json view fixed, fixes #411 --- app/View/Events/json/view.ctp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/View/Events/json/view.ctp b/app/View/Events/json/view.ctp index 48fc62226..093151213 100644 --- a/app/View/Events/json/view.ctp +++ b/app/View/Events/json/view.ctp @@ -1,4 +1,4 @@ event2JSON($event)); \ No newline at end of file +echo $converter->event2JSON($event); \ No newline at end of file From a6b603752ba6a41fe28f1cc4ac528769d2d08d77 Mon Sep 17 00:00:00 2001 From: iglocska Date: Tue, 24 Feb 2015 15:57:30 +0100 Subject: [PATCH 11/44] Version bump --- VERSION.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.json b/VERSION.json index a416b8601..091e370f4 100644 --- a/VERSION.json +++ b/VERSION.json @@ -1 +1 @@ -{"major":2, "minor":3, "hotfix":53} +{"major":2, "minor":3, "hotfix":54} From 8b89caf5b9714c90b418576679303ff9042d619b Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Tue, 24 Feb 2015 18:02:51 +0100 Subject: [PATCH 12/44] Use attribute uuid for cybox id's --- app/files/scripts/misp2cybox.py | 17 ++++++++++++----- app/files/scripts/misp2stix.py | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/app/files/scripts/misp2cybox.py b/app/files/scripts/misp2cybox.py index 4579e3415..df3f5069b 100644 --- a/app/files/scripts/misp2cybox.py +++ b/app/files/scripts/misp2cybox.py @@ -1,4 +1,4 @@ -from cybox.core import Observable, ObservableComposition +from cybox.core import Object, Observable, ObservableComposition from cybox.objects.file_object import File from cybox.objects.address_object import Address from cybox.objects.hostname_object import Hostname @@ -40,7 +40,10 @@ def generateObservable(indicator, attribute): if (attribute["type"] in simple_type_to_method.keys()): action = getattr(this_module, simple_type_to_method[attribute["type"]], None) if (action != None): - observable = action(attribute) + object = Object(action(attribute)) + object.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":" + attribute["type"] + "-" + attribute["uuid"] + observable = Observable(object) + observable.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-" + attribute["uuid"] indicator.add_observable(observable) def resolveFileObservable(attribute): @@ -195,12 +198,16 @@ def returnAttachmentComposition(attribute): return observable # email-attachment are mapped to an email message observable that contains the attachment as a file object -def generateEmailAttachmentObject(indicator, filename): +def generateEmailAttachmentObject(indicator, attribute): file_object = File() - file_object.file_name = filename + file_object.file_name = attribute["value"] email = EmailMessage() email.attachments = Attachments() email.add_related(file_object, "Contains", inline=True) + file_object.parent.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":file-" + attribute["uuid"] email.attachments.append(file_object.parent.id_) - indicator.observable = email + email.parent.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":EmailMessage-" + attribute["uuid"] + observable = Observable(email) + observable.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-" + attribute["uuid"] + indicator.observable = observable diff --git a/app/files/scripts/misp2stix.py b/app/files/scripts/misp2stix.py index 28b0659d1..80bed3ce1 100644 --- a/app/files/scripts/misp2stix.py +++ b/app/files/scripts/misp2stix.py @@ -191,7 +191,7 @@ def handleIndicatorAttribute(incident, ttps, attribute): indicator = generateIndicator(attribute) indicator.title = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"] if attribute["type"] == "email-attachment": - generateEmailAttachmentObject(indicator, attribute["value"]) + generateEmailAttachmentObject(indicator, attribute) else: generateObservable(indicator, attribute) if "data" in attribute: From a4fd3b957a7b2e9ceaa09c3f4fc8aa560a7f4b3b Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Wed, 25 Feb 2015 09:50:00 +0100 Subject: [PATCH 13/44] Use property class name in object ID --- app/files/scripts/misp2cybox.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/files/scripts/misp2cybox.py b/app/files/scripts/misp2cybox.py index df3f5069b..046bd5aa5 100644 --- a/app/files/scripts/misp2cybox.py +++ b/app/files/scripts/misp2cybox.py @@ -40,8 +40,9 @@ def generateObservable(indicator, attribute): if (attribute["type"] in simple_type_to_method.keys()): action = getattr(this_module, simple_type_to_method[attribute["type"]], None) if (action != None): - object = Object(action(attribute)) - object.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":" + attribute["type"] + "-" + attribute["uuid"] + property = action(attribute) + object = Object(property) + object.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":" + property.__class__.__name__ + "-" + attribute["uuid"] observable = Observable(object) observable.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-" + attribute["uuid"] indicator.add_observable(observable) From a565f14d0c9eebbbcb882769b693da48347a8ef1 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Wed, 4 Mar 2015 15:54:23 +0100 Subject: [PATCH 14/44] Fix cakephp path in .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3586ce29c..8b49b0d5f 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,6 @@ /app/Config/database.php /app/Config/core.php /app/Config/config.php -/cakephp +/app/Lib/cakephp /app/webroot/gpg.asc /app/tmp/logs From bff42361ef7e23600ecfbea77075700601d342ef Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Wed, 4 Mar 2015 17:07:32 +0100 Subject: [PATCH 15/44] Consistent id's for observable compositions --- app/files/scripts/misp2cybox.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/files/scripts/misp2cybox.py b/app/files/scripts/misp2cybox.py index 046bd5aa5..bbda17ffd 100644 --- a/app/files/scripts/misp2cybox.py +++ b/app/files/scripts/misp2cybox.py @@ -187,13 +187,20 @@ def createArtifactObject(indicator, attribute): def returnAttachmentComposition(attribute): file_object = File() file_object.file_name = attribute["value"] + file_object.parent.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":file-" + attribute["uuid"] observable = Observable() if "data" in attribute: artifact = Artifact(data = attribute["data"]) - composition = ObservableComposition(observables = [artifact, file_object]) + artifact.parent.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":artifact-" + attribute["uuid"] + observable_artifact = Observable(artifact) + observable_artifact.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-artifact-" + attribute["uuid"] + observable_file = Observable(file_object) + observable_file.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-file-" + attribute["uuid"] + composition = ObservableComposition(observables = [observable_artifact, observable_file]) observable.observable_composition = composition else: observable = Observable(file_object) + observable.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-" + attribute["uuid"] if attribute["comment"] != "": observable.description = attribute["comment"] return observable From 2a56a00e88825d0d05d09524ca12aca7bf2afc35 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Wed, 4 Mar 2015 17:28:44 +0100 Subject: [PATCH 16/44] Consistent id's for malware-sample artifacts --- app/files/scripts/misp2cybox.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/files/scripts/misp2cybox.py b/app/files/scripts/misp2cybox.py index bbda17ffd..5e12e7f17 100644 --- a/app/files/scripts/misp2cybox.py +++ b/app/files/scripts/misp2cybox.py @@ -181,7 +181,10 @@ def resolvePatternObservable(attribute): # create an artifact object for the malware-sample type. def createArtifactObject(indicator, attribute): artifact = Artifact(data = attribute["data"]) - indicator.add_observable(artifact) + artifact.parent.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":artifact-" + attribute["uuid"] + observable = Observable(artifact) + observable.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-artifact-" + attribute["uuid"] + indicator.add_observable(observable) # return either a composition if data is set in attribute, or just an observable with a filename if it's not set def returnAttachmentComposition(attribute): From 7f201fdf817084630fd7c9bbe2b8a157d384a3c9 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Thu, 5 Mar 2015 13:26:13 +0100 Subject: [PATCH 17/44] Consistent timestamps for STIX objects --- app/files/scripts/misp2stix.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/app/files/scripts/misp2stix.py b/app/files/scripts/misp2stix.py index 80bed3ce1..e748eaee9 100644 --- a/app/files/scripts/misp2stix.py +++ b/app/files/scripts/misp2stix.py @@ -134,7 +134,8 @@ def generateMainPackage(events): # generate a package for each event def generateEventPackage(event): package_name = namespace[1] + ':STIXPackage-' + event["Event"]["uuid"] - stix_package = STIXPackage(id_=package_name) + timestamp = getDateFromTimestamp(int(event["Event"]["timestamp"])) + stix_package = STIXPackage(id_=package_name, timestamp=timestamp) stix_header = STIXHeader() stix_header.title="MISP event #" + event["Event"]["id"] + " uuid: " + event["Event"]["uuid"] stix_header.package_intents="Threat Report" @@ -236,14 +237,14 @@ def handleNonIndicatorAttribute(incident, ttps, attribute): # TTPs are only used to describe malware names currently (attribute with category Payload Type and type text/comment/other) def generateTTP(incident, attribute): - ttp = TTP() + ttp = TTP(timestamp=getDateFromTimestamp(int(attribute["timestamp"]))) ttp.id_= namespace[1] + ":ttp-" + attribute["uuid"] setTLP(ttp, attribute["distribution"]) ttp.title = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"] if attribute["type"] == "vulnerability": vulnerability = Vulnerability() vulnerability.cve_id = attribute["value"] - et = ExploitTarget() + et = ExploitTarget(timestamp=getDateFromTimestamp(int(attribute["timestamp"]))) et.add_vulnerability(vulnerability) ttp.exploit_targets.append(et) else: @@ -258,7 +259,7 @@ def generateTTP(incident, attribute): # Threat actors are currently only used for the category:attribution / type:(text|comment|other) attributes def generateThreatActor(attribute): - ta = ThreatActor() + ta = ThreatActor(timestamp=getDateFromTimestamp(int(attribute["timestamp"]))) ta.id_= namespace[1] + ":threatactor-" + attribute["uuid"] ta.title = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"] if attribute["comment"] != "": @@ -269,7 +270,7 @@ def generateThreatActor(attribute): # generate the indicator and add the relevant information def generateIndicator(attribute): - indicator = Indicator() + indicator = Indicator(timestamp=getDateFromTimestamp(int(attribute["timestamp"]))) indicator.id_= namespace[1] + ":indicator-" + attribute["uuid"] if attribute["comment"] != "": indicator.description = attribute["comment"] @@ -279,7 +280,7 @@ def generateIndicator(attribute): confidence_value = confidence_mapping.get(attribute["to_ids"], None) if confidence_value is None: return indicator - indicator.confidence = Confidence(value=confidence_value, description=confidence_description) + indicator.confidence = Confidence(value=confidence_value, description=confidence_description, timestamp=getDateFromTimestamp(int(attribute["timestamp"]))) return indicator # converts timestamp to the format used by STIX From 6763159e87d97fdc24114772476d233e3efd7f54 Mon Sep 17 00:00:00 2001 From: iglocska Date: Tue, 10 Mar 2015 10:56:29 +0100 Subject: [PATCH 18/44] Security fix - filenames are now enclosed by quotes instead of double quotes while executing the zip command via exec --- VERSION.json | 2 +- app/Model/Attribute.php | 2 +- app/Model/ShadowAttribute.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/VERSION.json b/VERSION.json index 091e370f4..8692bfe50 100644 --- a/VERSION.json +++ b/VERSION.json @@ -1 +1 @@ -{"major":2, "minor":3, "hotfix":54} +{"major":2, "minor":3, "hotfix":55} diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 2cbef4f32..0747c2645 100755 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -925,7 +925,7 @@ class Attribute extends AppModel { // TODO check if CakePHP has no easy/safe wrapper to execute commands $execRetval = ''; $execOutput = array(); - exec("zip -j -P infected " . $zipfile->path . ' "' . addslashes($fileInZip->path) . '"', $execOutput, $execRetval); + exec("zip -j -P infected " . $zipfile->path . ' \'' . addslashes($fileInZip->path) . '\'', $execOutput, $execRetval); if ($execRetval != 0) { // not EXIT_SUCCESS // do some? }; diff --git a/app/Model/ShadowAttribute.php b/app/Model/ShadowAttribute.php index 245bec946..237de28cb 100644 --- a/app/Model/ShadowAttribute.php +++ b/app/Model/ShadowAttribute.php @@ -510,7 +510,7 @@ class ShadowAttribute extends AppModel { // TODO check if CakePHP has no easy/safe wrapper to execute commands $execRetval = ''; $execOutput = array(); - exec("zip -j -P infected " . $zipfile->path . ' "' . addslashes($fileInZip->path) . '"', $execOutput, $execRetval); + exec("zip -j -P infected " . $zipfile->path . ' \'' . addslashes($fileInZip->path) . '\'', $execOutput, $execRetval); if ($execRetval != 0) { // not EXIT_SUCCESS // do some? }; From 4c8d1d3e2bb757bf6d3e5650228be671e47da959 Mon Sep 17 00:00:00 2001 From: Alexander J Date: Fri, 13 Mar 2015 13:42:11 +0100 Subject: [PATCH 19/44] update to install howto remove of -p password in order to avoid having the password in your bash history and the install command for postfix --- INSTALL/INSTALL.ubuntu1404.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/INSTALL/INSTALL.ubuntu1404.txt b/INSTALL/INSTALL.ubuntu1404.txt index fa0a789fb..42a6149b5 100644 --- a/INSTALL/INSTALL.ubuntu1404.txt +++ b/INSTALL/INSTALL.ubuntu1404.txt @@ -14,6 +14,7 @@ Install a minimal ubuntu 14.04-server system with the software: - Mail server You will get some questions, you will probably want to set: +sudo apt-get install postfix - Postfix Configuration: Satellite system Make sure your system is up2date: @@ -102,7 +103,8 @@ mysql> exit cd /var/www/MISP # Import the empty MySQL database from MYSQL.sql -mysql -u misp -p misp < INSTALL/MYSQL.sql +mysql -u misp -p misp < INSTALL/MYSQL.sql +#enter the password you set previously 7/ Apache configuration From 50a03e897da552a46e8b6570b921babceb1e802b Mon Sep 17 00:00:00 2001 From: Iglocska Date: Sat, 14 Mar 2015 08:29:19 +0100 Subject: [PATCH 20/44] Search in logs fixed, fixes #434 - The log search incorrectly set the search terms for empty fields, meaning that any log entries that had unfilled columns, such as it is the case with admin_email would never return results --- app/Controller/LogsController.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/Controller/LogsController.php b/app/Controller/LogsController.php index 3b2aee7c9..5b4ab3035 100755 --- a/app/Controller/LogsController.php +++ b/app/Controller/LogsController.php @@ -165,19 +165,19 @@ class LogsController extends AppController { // search the db $conditions = array(); - if ($email) { + if (isset($email) && !empty($email)) { $conditions['LOWER(Log.email) LIKE'] = '%' . strtolower($email) . '%'; } - if (isset($org)) { + if (isset($org) && !empty($org)) { $conditions['LOWER(Log.org) LIKE'] = '%' . strtolower($org) . '%'; } if ($action != 'ALL') { $conditions['Log.action ='] = $action; } - if (isset($title)) { + if (isset($title) && !empty($title)) { $conditions['LOWER(Log.title) LIKE'] = '%' . strtolower($title) . '%'; } - if (isset($change)) { + if (isset($change) && !empty($change)) { $conditions['LOWER(Log.change) LIKE'] = '%' . strtolower($change) . '%'; } $this->{$this->defaultModel}->recursive = 0; From 524871ab5251c65f59aab0913624aad5f06fd2e5 Mon Sep 17 00:00:00 2001 From: Iglocska Date: Sat, 14 Mar 2015 08:35:24 +0100 Subject: [PATCH 21/44] Sync users should default to termsaccepted and no password change required, fixes #432 --- app/Controller/UsersController.php | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php index 0cb273129..0eaa31b73 100755 --- a/app/Controller/UsersController.php +++ b/app/Controller/UsersController.php @@ -326,14 +326,20 @@ class UsersController extends AppController { if ($this->request->is('post')) { $this->User->create(); // set invited by + $this->loadModel('Role'); + $this->Role->recursive = -1; + $chosenRole = $this->Role->findById($this->request->data['User']['role_id']); $this->request->data['User']['invited_by'] = $this->Auth->user('id'); - $this->request->data['User']['change_pw'] = 1; + if ($chosenRole['Role']['perm_sync']) { + $this->request->data['User']['change_pw'] = 0; + $this->request->data['User']['termsaccepted'] = 1; + } else { + $this->request->data['User']['change_pw'] = 1; + $this->request->data['User']['termsaccepted'] = 0; + } $this->request->data['User']['newsread'] = '2000-01-01'; if (!$this->_isSiteAdmin()) { $this->request->data['User']['org'] = $this->Auth->User('org'); - $this->loadModel('Role'); - $this->Role->recursive = -1; - $chosenRole = $this->Role->findById($this->request->data['User']['role_id']); if ($chosenRole['Role']['perm_site_admin'] == 1 || $chosenRole['Role']['perm_regexp_access'] == 1 || $chosenRole['Role']['perm_sync'] == 1) { throw new Exception('You are not authorised to assign that role to a user.'); } From 150f29276b95e6a7f1cdb2451b13d2894d01996b Mon Sep 17 00:00:00 2001 From: Iglocska Date: Sat, 14 Mar 2015 08:38:00 +0100 Subject: [PATCH 22/44] Version bump --- VERSION.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.json b/VERSION.json index 8692bfe50..f0c6b4c9b 100644 --- a/VERSION.json +++ b/VERSION.json @@ -1 +1 @@ -{"major":2, "minor":3, "hotfix":55} +{"major":2, "minor":3, "hotfix":56} From b29e8fdadaed963f4aa087258b8f7c456ade6d03 Mon Sep 17 00:00:00 2001 From: Iglocska Date: Sat, 14 Mar 2015 09:01:20 +0100 Subject: [PATCH 23/44] Site admins can now create proposals, fixes #417 - site admins can now create proposals to an event / attribute as long as the event does not belong to their organisation - new icon for proposals to differentiate them from edits --- app/Controller/ShadowAttributesController.php | 2 +- app/View/Elements/eventattribute.ctp | 9 +++++---- app/View/Elements/side_menu.ctp | 10 +++++----- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/app/Controller/ShadowAttributesController.php b/app/Controller/ShadowAttributesController.php index 09bfcb2a0..156c04a63 100644 --- a/app/Controller/ShadowAttributesController.php +++ b/app/Controller/ShadowAttributesController.php @@ -271,7 +271,7 @@ class ShadowAttributesController extends AppController { 'recursive' => -1, 'fields' => array('id', 'orgc', 'distribution', 'org'), )); - if ((($event['Event']['distribution'] == 0 && $event['Event']['org'] != $this->Auth->user('org'))) || ($event['Event']['orgc'] == $this->Auth->user('org'))) { + if (!$this->_isSiteAdmin() && (($event['Event']['distribution'] == 0 && $event['Event']['org'] != $this->Auth->user('org'))) || ($event['Event']['orgc'] == $this->Auth->user('org'))) { $this->Session->setFlash(__('Invalid Event.')); $this->redirect(array('controller' => 'events', 'action' => 'index')); } diff --git a/app/View/Elements/eventattribute.ctp b/app/View/Elements/eventattribute.ctp index 2c6b5cd58..7aa363a4b 100644 --- a/app/View/Elements/eventattribute.ctp +++ b/app/View/Elements/eventattribute.ctp @@ -228,15 +228,16 @@ + + - -
  • >View Event
  • >View Event History
    • +
  • >Edit Event
  • Form->postLink('Delete Event', array('action' => 'delete', $event['Event']['id']), null, __('Are you sure you want to delete # %s?', $event['Event']['id'])); ?>
    • -
  • >Add Attribute
  • >Add Attachment
  • >Populate from OpenIOC
  • >Populate from ThreatConnect
    • - -
  • Populate From Template
    • + +
  • Populate From Template
    • + - -
  • +
  • >Propose Attribute
  • >Propose Attachment
    • From f36304e635fdd20b4801d913415b22b24cd017b2 Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Sun, 15 Mar 2015 02:20:50 +0100 Subject: [PATCH 24/44] Example file on how to get the exported IDS data from MISP --- INSTALL/.get-misp-automation.py.swp | Bin 0 -> 12288 bytes INSTALL/get-misp-automation.py | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 INSTALL/.get-misp-automation.py.swp create mode 100644 INSTALL/get-misp-automation.py diff --git a/INSTALL/.get-misp-automation.py.swp b/INSTALL/.get-misp-automation.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..9a16d6db7efca6b4f50ad448db60ef7c503616d6 GIT binary patch literal 12288 zcmeI2O>Wab6o6k5+ZF^1wo{kLiIjLsK>}6sB2t5hNNJQG)Ut`3%(%8{dyHp{8#bJQ z18@R<4#F9@01J>{kHoW^L=l^^ExeIFk3B!{jX$qOp6TxM?jd{VJOC^=0bYE$KK^wL z;GF}IDP_v$%gW4)W%L(Ad7Ky?ocfvWOy6J`tkO4A4 z2FL&zAOmE843L3;-+)Os;0?YeZuzTnXHjqbQ7;~;A_HWA43GgbKnBPF86X2>fDDiU zGC&5dpaD++eAooIyM>F#|NqPV|MwdJ-;l4!C*&g%BOwwXC&)3|Ez?)ExOyRcVb))Edb64l3pUIXyrS~bmU+F~_UFHew9o*WHF zP1ZT?9}Py6gZ8N109cFbTGg>~ooC?;U&Vs8k*+0Q$;4=$ghKP>5H3`aM9LEuELQAV z7OZl)R$xrJ&n?_2$EnK9(hY!Z+t3z*%~hV2Z_Cc)yvejQ%mk8^5m=zKVO}DbkbaoR zszc=_%? Date: Tue, 17 Mar 2015 00:06:59 +0100 Subject: [PATCH 25/44] Organization field in Servers too short to fit valid organisation identifiers, fixes #436 - updated the MYSQL.sql file for future MISP installations - added admin script to do the update from the web interface --- INSTALL/MYSQL.sql | 2 +- VERSION.json | 2 +- app/Controller/AppController.php | 19 +++++++++++++++++++ app/View/Pages/administration.ctp | 1 + 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/INSTALL/MYSQL.sql b/INSTALL/MYSQL.sql index ab756b9f7..6d414b853 100644 --- a/INSTALL/MYSQL.sql +++ b/INSTALL/MYSQL.sql @@ -221,7 +221,7 @@ CREATE TABLE IF NOT EXISTS `servers` ( `url` varchar(255) COLLATE utf8_bin NOT NULL, `authkey` varchar(40) COLLATE utf8_bin NOT NULL, `org` varchar(255) COLLATE utf8_bin NOT NULL, - `organization` varchar(10) COLLATE utf8_bin NOT NULL, + `organization` varchar(255) COLLATE utf8_bin NOT NULL, `push` tinyint(1) NOT NULL, `pull` tinyint(1) NOT NULL, `lastpulledid` int(11) NOT NULL, diff --git a/VERSION.json b/VERSION.json index f0c6b4c9b..5a6fbbaef 100644 --- a/VERSION.json +++ b/VERSION.json @@ -1 +1 @@ -{"major":2, "minor":3, "hotfix":56} +{"major":2, "minor":3, "hotfix":57} diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index b9674b3ca..b04b77d52 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -312,4 +312,23 @@ class AppController extends Controller { $this->Session->setFlash(__('All done. attribute_count generated from scratch for ' . $k . ' events.')); $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); } + + public function updateDatabase($command) { + if (!$this->_isSiteAdmin()) throw new MethodNotAllowedException(); + $sql = ''; + switch ($command) { + case 'extendServerOrganizationLength': + $sql = 'ALTER TABLE servers MODIFY COLUMN organization varchar(255) NOT NULL'; + $controller = 'Servers'; + break; + default: + $this->Session->setFlash('Invalid command.'); + $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); + break; + } + $this->loadModel($controller); + $this->$controller->query($sql); + $this->Session->setFlash('Done.'); + $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); + } } \ No newline at end of file diff --git a/app/View/Pages/administration.ctp b/app/View/Pages/administration.ctp index 69e106b6e..5f5128471 100644 --- a/app/View/Pages/administration.ctp +++ b/app/View/Pages/administration.ctp @@ -16,6 +16,7 @@ if (!$isSiteAdmin) exit();
  • generateLocked (This is for upgrading to hotfix 2.1.8 or later, all events that were created by an organisation that doesn't have users on this instance, or only has a single sync user will have their locked setting set to 1)
  • Verify GPG keys (Check whether every user's GPG key is usable)
  • Upgrade Risk to Threat Level (As of version 2.2 the risk field is replaced by Threat Level. This script will convert all values in risk to threat level.)
    • +
  • Extend Organization length (Hotfix 2.3.57: Increase the max length of the organization field when adding a new server connection.)
    • Date: Thu, 19 Mar 2015 09:23:12 +0100 Subject: [PATCH 26/44] Removed .swp file ; updated .gitignore --- .gitignore | 1 + INSTALL/.get-misp-automation.py.swp | Bin 12288 -> 0 bytes 2 files changed, 1 insertion(+) delete mode 100644 INSTALL/.get-misp-automation.py.swp diff --git a/.gitignore b/.gitignore index 3586ce29c..224a3d84f 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ /cakephp /app/webroot/gpg.asc /app/tmp/logs +*.swp diff --git a/INSTALL/.get-misp-automation.py.swp b/INSTALL/.get-misp-automation.py.swp deleted file mode 100644 index 9a16d6db7efca6b4f50ad448db60ef7c503616d6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2O>Wab6o6k5+ZF^1wo{kLiIjLsK>}6sB2t5hNNJQG)Ut`3%(%8{dyHp{8#bJQ z18@R<4#F9@01J>{kHoW^L=l^^ExeIFk3B!{jX$qOp6TxM?jd{VJOC^=0bYE$KK^wL z;GF}IDP_v$%gW4)W%L(Ad7Ky?ocfvWOy6J`tkO4A4 z2FL&zAOmE843L3;-+)Os;0?YeZuzTnXHjqbQ7;~;A_HWA43GgbKnBPF86X2>fDDiU zGC&5dpaD++eAooIyM>F#|NqPV|MwdJ-;l4!C*&g%BOwwXC&)3|Ez?)ExOyRcVb))Edb64l3pUIXyrS~bmU+F~_UFHew9o*WHF zP1ZT?9}Py6gZ8N109cFbTGg>~ooC?;U&Vs8k*+0Q$;4=$ghKP>5H3`aM9LEuELQAV z7OZl)R$xrJ&n?_2$EnK9(hY!Z+t3z*%~hV2Z_Cc)yvejQ%mk8^5m=zKVO}DbkbaoR zszc=_%? Date: Wed, 18 Mar 2015 17:58:14 +0100 Subject: [PATCH 27/44] Add proxy support to SyncTool --- app/Config/config.default.php | 8 +++++++ app/Lib/Tools/SyncTool.php | 9 ++++++++ app/Model/Server.php | 43 +++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+) diff --git a/app/Config/config.default.php b/app/Config/config.default.php index c9f6a61f9..ddbf162cc 100644 --- a/app/Config/config.default.php +++ b/app/Config/config.default.php @@ -34,6 +34,14 @@ $config = array ( 'homedir' => '', 'password' => '', ), + 'Proxy' => + array ( + 'host' => '', + 'port' => '', + 'method' => '', + 'user' => '', + 'password' => '', + ), 'SecureAuth' => array ( 'amount' => 5, diff --git a/app/Lib/Tools/SyncTool.php b/app/Lib/Tools/SyncTool.php index d41ad1938..e0528713e 100644 --- a/app/Lib/Tools/SyncTool.php +++ b/app/Lib/Tools/SyncTool.php @@ -8,6 +8,15 @@ class SyncTool { if ($server['Server']['cert_file']) $params['ssl_cafile'] = APP . "files" . DS . "certs" . DS . $server['Server']['id'] . '.pem'; if ($server['Server']['self_signed']) $params['ssl_allow_self_signed'] = $server['Server']['self_signed']; $HttpSocket = new HttpSocket($params); + + $proxy = Configure::read('Proxy'); + if(!empty($proxy['host']) && !empty($proxy['port'])) { + if(!empty($proxy['method']) && !empty($proxy['user']) && !empty($proxy['password'])) { + $HttpSocket->configProxy($proxy['host'], $proxy['port']); + } else { + $HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']); + } + } return $HttpSocket; } } diff --git a/app/Model/Server.php b/app/Model/Server.php index 83c5fd9ea..99a3d57ac 100755 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -417,6 +417,49 @@ class Server extends AppModel { 'type' => 'string', ), ), + 'Proxy' => array( + 'branch' => 1, + 'host' => array( + 'level' => 2, + 'description' => 'The hostname of an HTTP proxy for outgoing sync requests. Leave empty to not use a proxy.', + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'port' => array( + 'level' => 2, + 'description' => 'The TCP port for the HTTP proxy.', + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForNumeric', + 'type' => 'numeric', + ), + 'method' => array( + 'level' => 2, + 'description' => 'The authentication method for the HTTP proxy. Currently supported are Basic or Digest. Leave empty for no proxy authentication.', + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'user' => array( + 'level' => 2, + 'description' => 'The authentication username for the HTTP proxy.', + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'password' => array( + 'level' => 2, + 'description' => 'The authentication password for the HTTP proxy.', + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + ), 'Security' => array( 'branch' => 1, 'salt' => array( From 0d242d854930d5573903ea8036a3ccbbcff2e1ad Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Thu, 19 Mar 2015 09:55:11 +0100 Subject: [PATCH 28/44] configProxy() checks for empty arguments, no need to do it twice --- app/Lib/Tools/SyncTool.php | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/app/Lib/Tools/SyncTool.php b/app/Lib/Tools/SyncTool.php index e0528713e..b38e21965 100644 --- a/app/Lib/Tools/SyncTool.php +++ b/app/Lib/Tools/SyncTool.php @@ -10,13 +10,8 @@ class SyncTool { $HttpSocket = new HttpSocket($params); $proxy = Configure::read('Proxy'); - if(!empty($proxy['host']) && !empty($proxy['port'])) { - if(!empty($proxy['method']) && !empty($proxy['user']) && !empty($proxy['password'])) { - $HttpSocket->configProxy($proxy['host'], $proxy['port']); - } else { - $HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']); - } - } + $HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']); + return $HttpSocket; } } From 2ccab722d71d06c4e4bf27c3d5cd7114c210905c Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Thu, 19 Mar 2015 12:15:28 +0100 Subject: [PATCH 29/44] Add proxy section to server diagnostics --- app/Controller/ServersController.php | 21 ++++++++++++++++++- app/Model/Server.php | 2 +- .../Elements/healthElements/diagnostics.ctp | 16 +++++++++++++- app/View/Pages/doc/administration.ctp | 3 ++- app/View/Servers/server_settings.ctp | 4 ++-- 5 files changed, 40 insertions(+), 6 deletions(-) diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index 36ffe81d1..c3df95ca0 100755 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -289,11 +289,13 @@ class ServersController extends AppController { $tabs = array( 'MISP' => array('count' => 0, 'errors' => 0, 'severity' => 5), 'GnuPG' => array('count' => 0, 'errors' => 0, 'severity' => 5), + 'Proxy' => array('count' => 0, 'errors' => 0, 'severity' => 5), 'Security' => array('count' => 0, 'errors' => 0, 'severity' => 5), 'misc' => array('count' => 0, 'errors' => 0, 'severity' => 5) ); $writeableErrors = array(0 => 'OK', 1 => 'Directory doesn\'t exist', 2 => 'Directory is not writeable'); $gpgErrors = array(0 => 'OK', 1 => 'FAIL: settings not set', 2 => 'FAIL: bad GnuPG.*', 3 => 'FAIL: encrypt failed'); + $proxyErrors = array(0 => 'OK', 1 => 'Getting http://www.example.com/ via proxy failed'); $stixErrors = array(0 => 'ERROR', 1 => 'OK'); $results = $this->Server->serverSettingsRead(); @@ -394,7 +396,23 @@ class ServersController extends AppController { $gpgStatus = 1; } if ($gpgStatus != 0) $diagnostic_errors++; + + // if Proxy is set up in the settings, try to connect to a test URL + $proxyStatus = 0; + $proxy = Configure::read('Proxy'); + if(!empty($proxy['host'])) { + App::uses('HttpSocket', 'Network/Http'); + $HttpSocket = new HttpSocket(); + $HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']); + $proxyResult = $HttpSocket->get('http://www.example.com/' === false); + if(empty($proxyResult) { + $proxyStatus = 1; + } + } + if ($proxyStatus != 0) $diagnostic_errors++; + $this->set('gpgStatus', $gpgStatus); + $this->set('proxyStatus', $proxyStatus); $this->set('diagnostic_errors', $diagnostic_errors); $this->set('tab', $tab); $this->set('tabs', $tabs); @@ -403,6 +421,7 @@ class ServersController extends AppController { $this->set('writeableErrors', $writeableErrors); $this->set('gpgErrors', $gpgErrors); + $this->set('proxyErrors', $proxyErrors); $this->set('stixErrors', $stixErrors); if (Configure::read('MISP.background_jobs')) { @@ -437,7 +456,7 @@ class ServersController extends AppController { foreach ($dumpResults as &$dr) { unset($dr['description']); } - $dump = array('gpgStatus' => $gpgErrors[$gpgStatus], 'stix' => $stixErrors[$stix], 'writeableDirs' => $writeableDirs, 'finalSettings' => $dumpResults); + $dump = array('gpgStatus' => $gpgErrors[$gpgStatus], 'proxyStatus' => $proxyErrors[$proxyStatus], 'stix' => $stixErrors[$stix], 'writeableDirs' => $writeableDirs, 'finalSettings' => $dumpResults); $this->response->body(json_encode($dump, JSON_PRETTY_PRINT)); $this->response->type('json'); $this->response->download('MISP.report.json'); diff --git a/app/Model/Server.php b/app/Model/Server.php index 99a3d57ac..2c7b11ac1 100755 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -1051,7 +1051,7 @@ class Server extends AppModel { public function serverSettingsSaveValue($setting, $value) { Configure::write($setting, $value); - Configure::dump('config.php', 'default', array('MISP', 'GnuPG', 'SecureAuth', 'Security', 'debug')); + Configure::dump('config.php', 'default', array('MISP', 'GnuPG', 'Proxy', 'SecureAuth', 'Security', 'debug')); } public function checkVersion($newest) { diff --git a/app/View/Elements/healthElements/diagnostics.ctp b/app/View/Elements/healthElements/diagnostics.ctp index f1cd65e4b..b3f58d179 100644 --- a/app/View/Elements/healthElements/diagnostics.ctp +++ b/app/View/Elements/healthElements/diagnostics.ctp @@ -86,4 +86,18 @@ echo 'GnuPG installation and settings....' . $message . ''; ?> - \ No newline at end of file +

    + Proxy +

    +

    This tool tests whether your HTTP proxy settings are correct.

    +
    + 0) { + $colour = 'red'; + } + echo 'Proxy settings....' . $message . ''; + ?> +
    + diff --git a/app/View/Pages/doc/administration.ctp b/app/View/Pages/doc/administration.ctp index d5f3fa1e7..75119d3be 100755 --- a/app/View/Pages/doc/administration.ctp +++ b/app/View/Pages/doc/administration.ctp @@ -31,6 +31,7 @@
  • Overview: General overview of the current state of your MISP installation
  • MISP settings: Basic MISP settings. This includes the way MISP handles the default settings for distribution settings, whether background jobs are enabled, etc
  • GnuPG settings: GPG related settings.
    • +
  • Proxy settings: HTTP proxy related settings.
  • Security settings: Settings controlling the brute-force protection and the application's salt key.
  • Misc settings: You change the debug options here, but make sure that debug is always disabled on a production system.
  • Diagnostics: The diagnostics tool checks if all directories that MISP uses to store data are writeable by the apache user. Also, the tool checks whether the STIX libraries and GPG are working as intended.
    • @@ -240,4 +241,4 @@
  • Message: This field shows when the job was queued by the scheduler for execution.


    • - \ No newline at end of file + diff --git a/app/View/Servers/server_settings.ctp b/app/View/Servers/server_settings.ctp index d6f2f54de..cdb97ad23 100644 --- a/app/View/Servers/server_settings.ctp +++ b/app/View/Servers/server_settings.ctp @@ -2,7 +2,7 @@

    Server settings

    element('healthElements/tabs'); - if (in_array($tab, array('MISP', 'Security', 'GnuPG', 'misc'))) { + if (in_array($tab, array('MISP', 'Security', 'GnuPG', 'Proxy', 'misc'))) { echo $this->element('healthElements/settings_tab'); } else if ($tab == 'diagnostics') { echo $this->element('healthElements/diagnostics'); @@ -17,4 +17,4 @@ element('side_menu', array('menuList' => 'admin', 'menuItem' => 'serverSettings')); -?> \ No newline at end of file +?> From b62032fc7bba31c3f612e726cb4f8a4fe63e3318 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Thu, 19 Mar 2015 12:20:16 +0100 Subject: [PATCH 30/44] Fix typo --- app/Controller/ServersController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index c3df95ca0..35aea8e6a 100755 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -405,7 +405,7 @@ class ServersController extends AppController { $HttpSocket = new HttpSocket(); $HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']); $proxyResult = $HttpSocket->get('http://www.example.com/' === false); - if(empty($proxyResult) { + if(empty($proxyResult)) { $proxyStatus = 1; } } From 0e66ff140fb43be053dcea4e3215d6cc73ee43b2 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Thu, 19 Mar 2015 12:31:35 +0100 Subject: [PATCH 31/44] Use SyncTool for diagnostics --- app/Controller/ServersController.php | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index 35aea8e6a..39383a97e 100755 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -401,11 +401,11 @@ class ServersController extends AppController { $proxyStatus = 0; $proxy = Configure::read('Proxy'); if(!empty($proxy['host'])) { - App::uses('HttpSocket', 'Network/Http'); - $HttpSocket = new HttpSocket(); - $HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']); - $proxyResult = $HttpSocket->get('http://www.example.com/' === false); - if(empty($proxyResult)) { + App::uses('SyncTool', 'Tools'); + $syncTool = new SyncTool(); + $HttpSocket = $syncTool->setupHttpSocket('www.example.com'); + $proxyResponse = $HttpSocket->get('http://www.example.com/'); + if(empty($proxyResponse)) { $proxyStatus = 1; } } @@ -473,9 +473,11 @@ class ServersController extends AppController { private function __checkVersion() { if (!$this->_isSiteAdmin()) throw new MethodNotAllowedException(); set_error_handler(function() {}); - $options = array('http' => array('user_agent'=> $_SERVER['HTTP_USER_AGENT'])); - $context = stream_context_create($options); - $tags = file_get_contents('https://api.github.com/repos/MISP/MISP/tags', false, $context); + App::uses('SyncTool', 'Tools'); + $syncTool = new SyncTool(); + $HttpSocket = $syncTool->setupHttpSocket('api.github.com'); + $response = $HttpSocket->get('https://api.github.com/repos/MISP/MISP/tags'); + $tags = $response->body; restore_error_handler(); if ($tags != false) { $json_decoded_tags = json_decode($tags); From 05bc4c4389326eff944db3b74f0d5b69599d04be Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Thu, 19 Mar 2015 12:57:46 +0100 Subject: [PATCH 32/44] Allow SyncTool to be used for generic HTTP(S) connections --- app/Lib/Tools/SyncTool.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/Lib/Tools/SyncTool.php b/app/Lib/Tools/SyncTool.php index b38e21965..d62b34cd6 100644 --- a/app/Lib/Tools/SyncTool.php +++ b/app/Lib/Tools/SyncTool.php @@ -5,8 +5,10 @@ class SyncTool { public function setupHttpSocket($server) { $params = array(); App::uses('HttpSocket', 'Network/Http'); - if ($server['Server']['cert_file']) $params['ssl_cafile'] = APP . "files" . DS . "certs" . DS . $server['Server']['id'] . '.pem'; - if ($server['Server']['self_signed']) $params['ssl_allow_self_signed'] = $server['Server']['self_signed']; + if(!empty($server['Server'])) { + if ($server['Server']['cert_file']) $params['ssl_cafile'] = APP . "files" . DS . "certs" . DS . $server['Server']['id'] . '.pem'; + if ($server['Server']['self_signed']) $params['ssl_allow_self_signed'] = $server['Server']['self_signed']; + } $HttpSocket = new HttpSocket($params); $proxy = Configure::read('Proxy'); From 106b6cb06b6dbe478feb6b4f5c749b35d57e9a65 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Thu, 19 Mar 2015 13:01:29 +0100 Subject: [PATCH 33/44] Allow SyncTool with empty $server --- app/Controller/ServersController.php | 4 ++-- app/Lib/Tools/SyncTool.php | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index 39383a97e..9d89153cc 100755 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -403,7 +403,7 @@ class ServersController extends AppController { if(!empty($proxy['host'])) { App::uses('SyncTool', 'Tools'); $syncTool = new SyncTool(); - $HttpSocket = $syncTool->setupHttpSocket('www.example.com'); + $HttpSocket = $syncTool->setupHttpSocket(); $proxyResponse = $HttpSocket->get('http://www.example.com/'); if(empty($proxyResponse)) { $proxyStatus = 1; @@ -475,7 +475,7 @@ class ServersController extends AppController { set_error_handler(function() {}); App::uses('SyncTool', 'Tools'); $syncTool = new SyncTool(); - $HttpSocket = $syncTool->setupHttpSocket('api.github.com'); + $HttpSocket = $syncTool->setupHttpSocket(); $response = $HttpSocket->get('https://api.github.com/repos/MISP/MISP/tags'); $tags = $response->body; restore_error_handler(); diff --git a/app/Lib/Tools/SyncTool.php b/app/Lib/Tools/SyncTool.php index d62b34cd6..c55195063 100644 --- a/app/Lib/Tools/SyncTool.php +++ b/app/Lib/Tools/SyncTool.php @@ -2,10 +2,10 @@ class SyncTool { // take a server as parameter and return a HttpSocket object using the ssl options defined in the server settings - public function setupHttpSocket($server) { + public function setupHttpSocket($server = null) { $params = array(); App::uses('HttpSocket', 'Network/Http'); - if(!empty($server['Server'])) { + if(!empty($server)) { if ($server['Server']['cert_file']) $params['ssl_cafile'] = APP . "files" . DS . "certs" . DS . $server['Server']['id'] . '.pem'; if ($server['Server']['self_signed']) $params['ssl_allow_self_signed'] = $server['Server']['self_signed']; } From 2f58fdec0db2bd6c373fc9a167fc62bd71b5babf Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Thu, 19 Mar 2015 13:33:17 +0100 Subject: [PATCH 34/44] Catch invalid proxy configuration --- app/Controller/ServersController.php | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index 9d89153cc..9c9302a16 100755 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -295,7 +295,7 @@ class ServersController extends AppController { ); $writeableErrors = array(0 => 'OK', 1 => 'Directory doesn\'t exist', 2 => 'Directory is not writeable'); $gpgErrors = array(0 => 'OK', 1 => 'FAIL: settings not set', 2 => 'FAIL: bad GnuPG.*', 3 => 'FAIL: encrypt failed'); - $proxyErrors = array(0 => 'OK', 1 => 'Getting http://www.example.com/ via proxy failed'); + $proxyErrors = array(0 => 'OK', 1 => 'Getting URL via proxy failed'); $stixErrors = array(0 => 'ERROR', 1 => 'OK'); $results = $this->Server->serverSettingsRead(); @@ -403,8 +403,11 @@ class ServersController extends AppController { if(!empty($proxy['host'])) { App::uses('SyncTool', 'Tools'); $syncTool = new SyncTool(); - $HttpSocket = $syncTool->setupHttpSocket(); - $proxyResponse = $HttpSocket->get('http://www.example.com/'); + try { + $HttpSocket = $syncTool->setupHttpSocket(); + $proxyResponse = $HttpSocket->get('http://www.example.com/'); + } catch (Exception $e) { + } if(empty($proxyResponse)) { $proxyStatus = 1; } @@ -472,14 +475,15 @@ class ServersController extends AppController { private function __checkVersion() { if (!$this->_isSiteAdmin()) throw new MethodNotAllowedException(); - set_error_handler(function() {}); App::uses('SyncTool', 'Tools'); $syncTool = new SyncTool(); - $HttpSocket = $syncTool->setupHttpSocket(); - $response = $HttpSocket->get('https://api.github.com/repos/MISP/MISP/tags'); - $tags = $response->body; - restore_error_handler(); - if ($tags != false) { + try { + $HttpSocket = $syncTool->setupHttpSocket(); + $response = $HttpSocket->get('https://api.github.com/repos/MISP/MISP/tags'); + $tags = $response->body; + } catch (Exception $e) { + } + if (!empty($tags)) { $json_decoded_tags = json_decode($tags); // find the latest version tag in the v[major].[minor].[hotfix] format From 850e4bd19fb3f686233a84c73e93a619279bc5d0 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Thu, 19 Mar 2015 14:05:10 +0100 Subject: [PATCH 35/44] Catch HTTP error codes --- app/Controller/ServersController.php | 14 +++++++++----- app/View/Elements/healthElements/diagnostics.ctp | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index 9c9302a16..f999631fc 100755 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -295,7 +295,7 @@ class ServersController extends AppController { ); $writeableErrors = array(0 => 'OK', 1 => 'Directory doesn\'t exist', 2 => 'Directory is not writeable'); $gpgErrors = array(0 => 'OK', 1 => 'FAIL: settings not set', 2 => 'FAIL: bad GnuPG.*', 3 => 'FAIL: encrypt failed'); - $proxyErrors = array(0 => 'OK', 1 => 'Getting URL via proxy failed'); + $proxyErrors = array(0 => 'OK', 1 => 'not configured (so not tested)', 2 => 'Getting URL via proxy failed'); $stixErrors = array(0 => 'ERROR', 1 => 'OK'); $results = $this->Server->serverSettingsRead(); @@ -407,12 +407,15 @@ class ServersController extends AppController { $HttpSocket = $syncTool->setupHttpSocket(); $proxyResponse = $HttpSocket->get('http://www.example.com/'); } catch (Exception $e) { + $proxyStatus = 2; } - if(empty($proxyResponse)) { + if(empty($proxyResponse) || $proxyResponse->code > 399) { + $proxyStatus = 2; + } + } else { $proxyStatus = 1; - } } - if ($proxyStatus != 0) $diagnostic_errors++; + if ($proxyStatus > 1) $diagnostic_errors++; $this->set('gpgStatus', $gpgStatus); $this->set('proxyStatus', $proxyStatus); @@ -482,8 +485,9 @@ class ServersController extends AppController { $response = $HttpSocket->get('https://api.github.com/repos/MISP/MISP/tags'); $tags = $response->body; } catch (Exception $e) { + return false; } - if (!empty($tags)) { + if ($response->code < 300 && !empty($tags)) { $json_decoded_tags = json_decode($tags); // find the latest version tag in the v[major].[minor].[hotfix] format diff --git a/app/View/Elements/healthElements/diagnostics.ctp b/app/View/Elements/healthElements/diagnostics.ctp index b3f58d179..93da87ac8 100644 --- a/app/View/Elements/healthElements/diagnostics.ctp +++ b/app/View/Elements/healthElements/diagnostics.ctp @@ -94,7 +94,7 @@ 0) { + if ($proxyStatus > 1) { $colour = 'red'; } echo 'Proxy settings....' . $message . ''; From 75f93641dc51d92330ae3deafe72d47854449410 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Thu, 19 Mar 2015 15:09:54 +0100 Subject: [PATCH 36/44] Use isOK() for version check --- app/Controller/ServersController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index f999631fc..9d3920d48 100755 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -487,7 +487,7 @@ class ServersController extends AppController { } catch (Exception $e) { return false; } - if ($response->code < 300 && !empty($tags)) { + if ($response->isOK() && !empty($tags)) { $json_decoded_tags = json_decode($tags); // find the latest version tag in the v[major].[minor].[hotfix] format From fa112d77c37cb1e5d4178be8dba4ff8c5382b565 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Mon, 23 Mar 2015 17:58:41 +0100 Subject: [PATCH 37/44] Use correct CakeResque queues --- app/Console/Command/EventShell.php | 2 +- app/Model/Event.php | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/Console/Command/EventShell.php b/app/Console/Command/EventShell.php index ae031324a..e14e8119e 100755 --- a/app/Console/Command/EventShell.php +++ b/app/Console/Command/EventShell.php @@ -333,7 +333,7 @@ class EventShell extends AppShell // Now that we have figured out when the next execution should happen, it's time to enqueue it. $process_id = CakeResque::enqueueAt( $task['Task']['next_execution_time'], - 'default', + 'cache', 'EventShell', array('enqueueCaching', $task['Task']['next_execution_time']), true diff --git a/app/Model/Event.php b/app/Model/Event.php index 469aa6d74..b6eaa7f0c 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -1060,7 +1060,7 @@ class Event extends AppModel { $job = ClassRegistry::init('Job'); $job->create(); $data = array( - 'worker' => 'default', + 'worker' => 'email', 'job_type' => 'publish_alert_email', 'job_input' => 'Event: ' . $id, 'status' => 0, @@ -1071,7 +1071,7 @@ class Event extends AppModel { $job->save($data); $jobId = $job->id; $process_id = CakeResque::enqueue( - 'default', + 'email', 'EventShell', array('alertemail', $user['org'], $jobId, $id) ); @@ -1643,7 +1643,7 @@ class Event extends AppModel { $job = ClassRegistry::init('Job'); $job->create(); $data = array( - 'worker' => 'default', + 'worker' => 'email', 'job_type' => 'contact_alert', 'job_input' => 'To entire org: ' . $all, 'status' => 0, @@ -1654,7 +1654,7 @@ class Event extends AppModel { $job->save($data); $jobId = $job->id; $process_id = CakeResque::enqueue( - 'default', + 'email', 'EventShell', array('contactemail', $id, $message, $all, $user['id'], $isSiteAdmin, $jobId) ); From 056b21ff6f24adf5582467834b7e6075e0fd5b61 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Tue, 24 Mar 2015 10:03:17 +0100 Subject: [PATCH 38/44] Update cakephp to latest 2.6 branch --- app/Lib/cakephp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Lib/cakephp b/app/Lib/cakephp index a0aac5cfa..4c61b579c 160000 --- a/app/Lib/cakephp +++ b/app/Lib/cakephp @@ -1 +1 @@ -Subproject commit a0aac5cfa9acf2ccbde65c45ad6ee06ecd32fb54 +Subproject commit 4c61b579c40863ada019e1e08c6195ae612c589f From 03069122712b5c910d8fa42d1dfb9c42d7cbd04d Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Tue, 24 Mar 2015 10:16:38 +0100 Subject: [PATCH 39/44] Remove gitlink for app/Plugin/CakeResque CakeResque is installed with composer.phar Removing the gitlink gets rid of this annoying error message: No submodule mapping found in .gitmodules for path 'app/Plugin/CakeResque' --- app/Plugin/CakeResque | 1 - 1 file changed, 1 deletion(-) delete mode 160000 app/Plugin/CakeResque diff --git a/app/Plugin/CakeResque b/app/Plugin/CakeResque deleted file mode 160000 index 34f9a7cd3..000000000 --- a/app/Plugin/CakeResque +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 34f9a7cd370bfb2706b9bd6b54fb8ede17e5e5bd From cc5c32fca2302a4b1ee9b995c3e64eedc979627a Mon Sep 17 00:00:00 2001 From: Iglocska Date: Thu, 2 Apr 2015 00:14:36 +0200 Subject: [PATCH 40/44] Sync update issue fixed - attributes were not correctly updated during a manual push due to an incorrect conditional - re-publishing was unaffected --- VERSION.json | 2 +- app/Controller/EventsController.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION.json b/VERSION.json index 5a6fbbaef..517fb9647 100644 --- a/VERSION.json +++ b/VERSION.json @@ -1 +1 @@ -{"major":2, "minor":3, "hotfix":57} +{"major":2, "minor":3, "hotfix":58} diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 3ea68b144..1102109dd 100755 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -3003,7 +3003,7 @@ class EventsController extends AppController { 'fields' => array('Event.uuid', 'Event.timestamp', 'Event.locked'), )); foreach ($events as $k => $v) { - if (!$v['Event']['timestamp'] < $incomingEvents[$v['Event']['uuid']]) { + if ($v['Event']['timestamp'] >= $incomingEvents[$v['Event']['uuid']]) { unset($incomingEvents[$v['Event']['uuid']]); continue; } From eb9adae030094beac73d82197a3e6fd2f0265313 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Wed, 8 Apr 2015 13:05:05 +0200 Subject: [PATCH 41/44] Use cake-resque:4.1.2 - Remove --no-update for cake-resque - Added UPDATE.txt for keeping up2date between major releases --- INSTALL/INSTALL.centos6.txt | 2 +- INSTALL/INSTALL.centos7.txt | 2 +- INSTALL/INSTALL.ubuntu1404.txt | 2 +- INSTALL/UPDATE.txt | 53 ++++++++++++++++++++++++++++++++++ INSTALL/UPGRADE.txt | 4 +-- 5 files changed, 58 insertions(+), 5 deletions(-) create mode 100644 INSTALL/UPDATE.txt diff --git a/INSTALL/INSTALL.centos6.txt b/INSTALL/INSTALL.centos6.txt index b89243bff..3d493d374 100644 --- a/INSTALL/INSTALL.centos6.txt +++ b/INSTALL/INSTALL.centos6.txt @@ -77,7 +77,7 @@ git submodule update # Once done, install CakeResque along with its dependencies if you intend to use the built in background jobs: cd /var/www/MISP/app curl -s https://getcomposer.org/installer | php -php composer.phar require --no-update kamisama/cake-resque:4.1.0 +php composer.phar require kamisama/cake-resque:4.1.2 php composer.phar config vendor-dir Vendor php composer.phar install diff --git a/INSTALL/INSTALL.centos7.txt b/INSTALL/INSTALL.centos7.txt index 83b5c3df6..d0de7047b 100644 --- a/INSTALL/INSTALL.centos7.txt +++ b/INSTALL/INSTALL.centos7.txt @@ -77,7 +77,7 @@ git submodule update # Once done, install CakeResque along with its dependencies if you intend to use the built in background jobs: cd /var/www/MISP/app curl -s https://getcomposer.org/installer | php -php composer.phar require --no-update kamisama/cake-resque:4.1.0 +php composer.phar require kamisama/cake-resque:4.1.2 php composer.phar config vendor-dir Vendor php composer.phar install diff --git a/INSTALL/INSTALL.ubuntu1404.txt b/INSTALL/INSTALL.ubuntu1404.txt index fa0a789fb..9212246b4 100644 --- a/INSTALL/INSTALL.ubuntu1404.txt +++ b/INSTALL/INSTALL.ubuntu1404.txt @@ -64,7 +64,7 @@ git submodule update # Once done, install CakeResque along with its dependencies if you intend to use the built in background jobs: cd /var/www/MISP/app curl -s https://getcomposer.org/installer | php -php composer.phar require --no-update kamisama/cake-resque:4.1.0 +php composer.phar require kamisama/cake-resque:4.1.2 php composer.phar config vendor-dir Vendor php composer.phar install diff --git a/INSTALL/UPDATE.txt b/INSTALL/UPDATE.txt new file mode 100644 index 000000000..ba4d1b5f8 --- /dev/null +++ b/INSTALL/UPDATE.txt @@ -0,0 +1,53 @@ +# After installing MISP you can keep it up to date by periodically running the commands below. + +# 1. Update the MISP code to the latest hotfix. If a new major version (2.4.x) has been released, refer to UPGRADE.txt instead. + +cd /var/www/MISP +git pull + +# 2. Update CakePHP to the latest 2.6 code + +cd /var/www/MISP/Lib/cakephp +git fetch origin +git checkout 2.6 + +# 3. Update Mitre's STIX and its dependencies + +cd /var/www/MISP/app/files/scripts/python-cybox +git pull +python setup.py install +cd /var/www/MISP/app/files/scripts/python-stix +git pull +python setup.py install + +# 4. Update CakeResque and it's dependencies + +cd /var/www/MISP/app + +# Edit composer.json so that cake-resque is allowed to be updated +# "kamisama/cake-resque": ">=4.1.2" + +vim composer.json +php composer.phar update + +# To use the scheduler worker for scheduled tasks, do the following: +cp -fa /var/www/MISP/INSTALL/setup/config.php /var/www/MISP/app/Plugin/CakeResque/Config/config.php + +# 5. Make sure all file permissions are set correctly + +chown -R root:www-data /var/www/MISP +find /var/www/MISP -type d -exec chmod g=rx {} \; +chmod -R g+r,o= /var/www/MISP +chown www-data:www-data /var/www/MISP/app/files +chown www-data:www-data /var/www/MISP/app/files/terms +chown www-data:www-data /var/www/MISP/app/files/scripts/tmp +chown www-data:www-data /var/www/MISP/app/Plugin/CakeResque/tmp +chown -R www-data:www-data /var/www/MISP/app/tmp +chown -R www-data:www-data /var/www/MISP/app/webroot/img/orgs +chown -R www-data:www-data /var/www/MISP/app/webroot/img/custom + +# 6. Restart the CakeResque workers + +su www-data -c 'bash /var/www/MISP/app/Console/worker/start.sh' + +# You can also do this using the MISP application by navigating to the workers tab in the server settings and clicking on the "Restart all workers" button. diff --git a/INSTALL/UPGRADE.txt b/INSTALL/UPGRADE.txt index 291a1669d..b26ffe607 100755 --- a/INSTALL/UPGRADE.txt +++ b/INSTALL/UPGRADE.txt @@ -32,7 +32,7 @@ python setup.py install # install / update CakeResque (using the background workers is optional buy highly recommended) cd /var/www/MISP/app curl -s https://getcomposer.org/installer | php -php composer.phar require --no-update kamisama/cake-resque:4.1.0 +php composer.phar require kamisama/cake-resque:4.1.2 php composer.phar config vendor-dir Vendor php composer.phar install @@ -78,4 +78,4 @@ chown -R www-data:www-data /var/www/MISP/ Date: Wed, 8 Apr 2015 14:08:38 +0200 Subject: [PATCH 42/44] Only truncate string if adding ... will make it shorter --- app/Model/Event.php | 2 +- app/View/Attributes/alternate_search_result.ctp | 4 ++-- app/View/Events/view.ctp | 4 ++-- app/View/Helper/PivotHelper.php | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index 469aa6d74..12412cf6d 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -1149,7 +1149,7 @@ class Event extends AppModel { if (Configure::read('MISP.extended_alert_subject')) { $subject = preg_replace( "/\r|\n/", "", $event['Event']['info']); - if (strlen($subject) > 55) { + if (strlen($subject) > 58) { $subject = substr($subject, 0, 55) . '... - '; } else { $subject .= " - "; diff --git a/app/View/Attributes/alternate_search_result.ctp b/app/View/Attributes/alternate_search_result.ctp index 0a53e37aa..2142b05c9 100644 --- a/app/View/Attributes/alternate_search_result.ctp +++ b/app/View/Attributes/alternate_search_result.ctp @@ -27,7 +27,7 @@ 60) { + if (strlen(h($event['Event']['info'])) > 63) { echo (substr(h($event['Event']['info']), 0, 60)) . '...'; } else echo h($event['Event']['info']); ?> @@ -52,4 +52,4 @@ element('side_menu', array('menuList' => 'event-collection', 'menuItem' => 'searchAttributes')); -?> \ No newline at end of file +?> diff --git a/app/View/Events/view.ctp b/app/View/Events/view.ctp index 29cdacd35..4d9b4bac9 100755 --- a/app/View/Events/view.ctp +++ b/app/View/Events/view.ctp @@ -12,7 +12,7 @@ $mayPublish = ($isAclPublish && $event['Event']['orgc'] == $me['org']); $left = true; } $title = $event['Event']['info']; - if (strlen($title) > 55) $title = substr($title, 0, 55) . '...'; + if (strlen($title) > 58) $title = substr($title, 0, 55) . '...'; ?>
    @@ -252,4 +252,4 @@ $(document).ready(function () { $('#addTagButton').hide(); }); }); - \ No newline at end of file + diff --git a/app/View/Helper/PivotHelper.php b/app/View/Helper/PivotHelper.php index 1a735b8e7..9165628f8 100644 --- a/app/View/Helper/PivotHelper.php +++ b/app/View/Helper/PivotHelper.php @@ -9,7 +9,7 @@ App::uses('AppHelper', 'View/Helper'); $active = ''; $pivot['info'] = h($pivot['info']); // Truncate string if longer than (11 - length of event id) chars to fit the pivot bubble - if (strlen($pivot['info']) > (11 - strlen((string)$pivot['id']))) { + if (strlen($pivot['info']) > (11 - strlen((string)$pivot['id'])) && strlen($pivot['info']) > 9) { $text .= substr($pivot['info'], 0, 6) . '...'; } else { $text .= $pivot['info']; From 09171aa4155d8adc3a4b1137a80e0b732da3b16c Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Wed, 8 Apr 2015 14:27:04 +0200 Subject: [PATCH 43/44] Include composer.phar self-update --- INSTALL/UPDATE.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/INSTALL/UPDATE.txt b/INSTALL/UPDATE.txt index ba4d1b5f8..5965b23f9 100644 --- a/INSTALL/UPDATE.txt +++ b/INSTALL/UPDATE.txt @@ -28,6 +28,7 @@ cd /var/www/MISP/app # "kamisama/cake-resque": ">=4.1.2" vim composer.json +php composer.phar self-update php composer.phar update # To use the scheduler worker for scheduled tasks, do the following: From 116bf8e425e2d1033f1c3191002b9ed94b2ac500 Mon Sep 17 00:00:00 2001 From: Iglocska Date: Wed, 8 Apr 2015 22:47:28 +0200 Subject: [PATCH 44/44] Fix to an issue with the caching - CSV caching was saving to file on each attribute, creating extremely high amounts of I/O - reduced it to saving to file / event - fixed incorrect pathing --- app/Console/Command/EventShell.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/Console/Command/EventShell.php b/app/Console/Command/EventShell.php index e14e8119e..508f2b999 100755 --- a/app/Console/Command/EventShell.php +++ b/app/Console/Command/EventShell.php @@ -45,7 +45,7 @@ class EventShell extends AppShell $eventIds = $this->Event->fetchEventIds($org, $isSiteAdmin); $result = array(); $eventCount = count($eventIds); - $dir = new Folder(APP . DS . '/tmp/cached_exports/xml'); + $dir = new Folder(APP . 'tmp/cached_exports/xml'); if ($isSiteAdmin) { $file = new File($dir->pwd() . DS . 'misp.xml' . '.ADMIN.xml'); } else { @@ -192,7 +192,7 @@ class EventShell extends AppShell $eventIds = $this->Event->fetchEventIds($org, $isSiteAdmin); $eventCount = count($eventIds); $attributes = array(); - $dir = new Folder(APP . DS . '/tmp/cached_exports/' . $extra); + $dir = new Folder(APP . 'tmp/cached_exports/' . $extra); if ($isSiteAdmin) { $file = new File($dir->pwd() . DS . 'misp.' . $extra . '.ADMIN.csv'); } else { @@ -200,11 +200,13 @@ class EventShell extends AppShell } $file->write('uuid,event_id,category,type,value,to_ids,date' . PHP_EOL); foreach ($eventIds as $k => $eventId) { + $chunk = ""; $attributes = $this->Event->csv($org, $isSiteAdmin, $eventId['Event']['id'], $ignore); $attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true); foreach ($attributes as $attribute) { - $file->append($attribute['Attribute']['uuid'] . ',' . $attribute['Attribute']['event_id'] . ',' . $attribute['Attribute']['category'] . ',' . $attribute['Attribute']['type'] . ',' . $attribute['Attribute']['value'] . ',' . intval($attribute['Attribute']['to_ids']) . ',' . $attribute['Attribute']['timestamp'] . PHP_EOL); + $chunk .= $attribute['Attribute']['uuid'] . ',' . $attribute['Attribute']['event_id'] . ',' . $attribute['Attribute']['category'] . ',' . $attribute['Attribute']['type'] . ',' . $attribute['Attribute']['value'] . ',' . intval($attribute['Attribute']['to_ids']) . ',' . $attribute['Attribute']['timestamp'] . PHP_EOL; } + $file->append($chunk); if ($k % 10 == 0) { $this->Job->saveField('progress', $k / $eventCount * 80); }