diff --git a/.gitignore b/.gitignore index 3586ce29c..81ed7980d 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,7 @@ /app/Config/database.php /app/Config/core.php /app/Config/config.php -/cakephp +/app/Lib/cakephp /app/webroot/gpg.asc /app/tmp/logs +*.swp diff --git a/INSTALL/INSTALL.centos6.txt b/INSTALL/INSTALL.centos6.txt index b89243bff..8257f3abe 100644 --- a/INSTALL/INSTALL.centos6.txt +++ b/INSTALL/INSTALL.centos6.txt @@ -26,7 +26,7 @@ rpm -Uvh epel.rpm yum install vim # Install the dependencies: -yum install gcc git zip redis mysql-server php-mysql python-devel python-pip libxslt-devel zlib-devel php-devel +yum install gcc git zip redis mysql-server php-mysql python-devel python-pip libxslt-devel zlib-devel php-devel php-xml yum install php-pear php-pecl-geoip pear channel-update pear.php.net @@ -77,7 +77,7 @@ git submodule update # Once done, install CakeResque along with its dependencies if you intend to use the built in background jobs: cd /var/www/MISP/app curl -s https://getcomposer.org/installer | php -php composer.phar require --no-update kamisama/cake-resque:4.1.0 +php composer.phar require kamisama/cake-resque:4.1.2 php composer.phar config vendor-dir Vendor php composer.phar install @@ -123,8 +123,9 @@ mysql> grant usage on *.* to misp@localhost identified by 'XXXXXXXXX'; mysql> grant all privileges on misp.* to misp@localhost ; mysql> exit -# Import the empty MySQL database from MYSQL.sql cd /var/www/MISP + +# Import the empty MySQL database from MYSQL.sql mysql -u misp -p misp < INSTALL/MYSQL.sql @@ -191,7 +192,8 @@ chmod 700 /var/www/MISP/.gnupg gpg --homedir /var/www/MISP/.gnupg --gen-key chown -R apache:apache /var/www/MISP/.gnupg # Recommended key type: RSA -# The email address should match the one set int he config.php configuration file +# The email address should match the one set in the config.php configuration file +# Make sure that you use the same settings in the MISP Server Settings tool (Described on line 212) # And export the public key to the webroot sudo -u apache gpg --homedir /var/www/MISP/.gnupg --export --armor YOUR-EMAIL > /var/www/MISP/app/webroot/gpg.asc @@ -222,6 +224,11 @@ chown -R apache:apache /var/www/MISP/ # Make sure that the STIX libraries and GnuPG work as intended, if not, refer to INSTALL.txt's paragraphs dealing with these two items +# If anything goes wrong, make sure that you check MISP's logs for errors: +# /var/www/MISP/app/tmp/logs/error.log +# /var/www/MISP/app/tmp/logs/resque-worker-error.log +# /var/www/MISP/app/tmp/logs/resque-scheduler-error.log +# /var/www/MISP/app/tmp/logs/resque-2015-01-01.log //where the actual date is the current date Recommended actions ------------------- diff --git a/INSTALL/INSTALL.centos7.txt b/INSTALL/INSTALL.centos7.txt index 83b5c3df6..faae937ba 100644 --- a/INSTALL/INSTALL.centos7.txt +++ b/INSTALL/INSTALL.centos7.txt @@ -26,7 +26,7 @@ rpm -Uvh epel.rpm yum install vim # Install the dependencies: -yum install git httpd zip php redis mysql-server php-mysql python-devel python-pip libxslt-devel zlib-devel php-devel +yum install git httpd zip php redis mysql-server php-mysql python-devel python-pip libxslt-devel zlib-devel php-devel php-xml yum install php-pear php-pecl-geoip pear channel-update pear.php.net @@ -77,7 +77,7 @@ git submodule update # Once done, install CakeResque along with its dependencies if you intend to use the built in background jobs: cd /var/www/MISP/app curl -s https://getcomposer.org/installer | php -php composer.phar require --no-update kamisama/cake-resque:4.1.0 +php composer.phar require kamisama/cake-resque:4.1.2 php composer.phar config vendor-dir Vendor php composer.phar install @@ -127,8 +127,9 @@ MariaDB [(none)]> grant usage on *.* to misp@localhost identified by 'XXXXXXXXX' MariaDB [(none)]> grant all privileges on misp.* to misp@localhost ; MariaDB [(none)]> exit -# Import the empty MySQL database from MYSQL.sql cd /var/www/MISP + +# Import the empty MySQL database from MYSQL.sql mysql -u misp -p misp < INSTALL/MYSQL.sql @@ -205,7 +206,8 @@ chmod 700 /var/www/MISP/.gnupg gpg --homedir /var/www/MISP/.gnupg --gen-key chown -R apache:apache /var/www/MISP/.gnupg # Recommended key type: RSA -# The email address should match the one set int he config.php configuration file +# The email address should match the one set in the config.php configuration file +# Make sure that you use the same settings in the MISP Server Settings tool (Described on line 226) # And export the public key to the webroot sudo -u apache gpg --homedir /var/www/MISP/.gnupg --export --armor YOUR-EMAIL > /var/www/MISP/app/webroot/gpg.asc @@ -236,6 +238,11 @@ chown -R apache:apache /var/www/MISP/ # Make sure that the STIX libraries and GnuPG work as intended, if not, refer to INSTALL.txt's paragraphs dealing with these two items +# If anything goes wrong, make sure that you check MISP's logs for errors: +# /var/www/MISP/app/tmp/logs/error.log +# /var/www/MISP/app/tmp/logs/resque-worker-error.log +# /var/www/MISP/app/tmp/logs/resque-scheduler-error.log +# /var/www/MISP/app/tmp/logs/resque-2015-01-01.log //where the actual date is the current date Recommended actions ------------------- diff --git a/INSTALL/INSTALL.ubuntu1404.txt b/INSTALL/INSTALL.ubuntu1404.txt index fa0a789fb..4e2e0d61b 100644 --- a/INSTALL/INSTALL.ubuntu1404.txt +++ b/INSTALL/INSTALL.ubuntu1404.txt @@ -14,6 +14,7 @@ Install a minimal ubuntu 14.04-server system with the software: - Mail server You will get some questions, you will probably want to set: +sudo apt-get install postfix - Postfix Configuration: Satellite system Make sure your system is up2date: @@ -64,7 +65,7 @@ git submodule update # Once done, install CakeResque along with its dependencies if you intend to use the built in background jobs: cd /var/www/MISP/app curl -s https://getcomposer.org/installer | php -php composer.phar require --no-update kamisama/cake-resque:4.1.0 +php composer.phar require kamisama/cake-resque:4.1.2 php composer.phar config vendor-dir Vendor php composer.phar install @@ -102,7 +103,8 @@ mysql> exit cd /var/www/MISP # Import the empty MySQL database from MYSQL.sql -mysql -u misp -p misp < INSTALL/MYSQL.sql +mysql -u misp -p misp < INSTALL/MYSQL.sql +#enter the password you set previously 7/ Apache configuration @@ -163,8 +165,8 @@ chown www-data:www-data /var/www/MISP/.gnupg chmod 700 /var/www/MISP/.gnupg sudo -u www-data gpg --homedir /var/www/MISP/.gnupg --gen-key # Recommended key type: RSA -# The email address should match the one set int he bootstrap.php configuration file -# Make sure that you use the same settings in the MISP Server Settings tool tool (Described on line 184) +# The email address should match the one set in the config.php configuration file +# Make sure that you use the same settings in the MISP Server Settings tool (Described on line 184) # And export the public key to the webroot sudo -u www-data gpg --homedir /var/www/MISP/.gnupg --export --armor YOUR-EMAIL > /var/www/MISP/app/webroot/gpg.asc diff --git a/INSTALL/MYSQL.sql b/INSTALL/MYSQL.sql index ab756b9f7..6d414b853 100644 --- a/INSTALL/MYSQL.sql +++ b/INSTALL/MYSQL.sql @@ -221,7 +221,7 @@ CREATE TABLE IF NOT EXISTS `servers` ( `url` varchar(255) COLLATE utf8_bin NOT NULL, `authkey` varchar(40) COLLATE utf8_bin NOT NULL, `org` varchar(255) COLLATE utf8_bin NOT NULL, - `organization` varchar(10) COLLATE utf8_bin NOT NULL, + `organization` varchar(255) COLLATE utf8_bin NOT NULL, `push` tinyint(1) NOT NULL, `pull` tinyint(1) NOT NULL, `lastpulledid` int(11) NOT NULL, diff --git a/INSTALL/UPDATE.txt b/INSTALL/UPDATE.txt new file mode 100644 index 000000000..5965b23f9 --- /dev/null +++ b/INSTALL/UPDATE.txt @@ -0,0 +1,54 @@ +# After installing MISP you can keep it up to date by periodically running the commands below. + +# 1. Update the MISP code to the latest hotfix. If a new major version (2.4.x) has been released, refer to UPGRADE.txt instead. + +cd /var/www/MISP +git pull + +# 2. Update CakePHP to the latest 2.6 code + +cd /var/www/MISP/Lib/cakephp +git fetch origin +git checkout 2.6 + +# 3. Update Mitre's STIX and its dependencies + +cd /var/www/MISP/app/files/scripts/python-cybox +git pull +python setup.py install +cd /var/www/MISP/app/files/scripts/python-stix +git pull +python setup.py install + +# 4. Update CakeResque and it's dependencies + +cd /var/www/MISP/app + +# Edit composer.json so that cake-resque is allowed to be updated +# "kamisama/cake-resque": ">=4.1.2" + +vim composer.json +php composer.phar self-update +php composer.phar update + +# To use the scheduler worker for scheduled tasks, do the following: +cp -fa /var/www/MISP/INSTALL/setup/config.php /var/www/MISP/app/Plugin/CakeResque/Config/config.php + +# 5. Make sure all file permissions are set correctly + +chown -R root:www-data /var/www/MISP +find /var/www/MISP -type d -exec chmod g=rx {} \; +chmod -R g+r,o= /var/www/MISP +chown www-data:www-data /var/www/MISP/app/files +chown www-data:www-data /var/www/MISP/app/files/terms +chown www-data:www-data /var/www/MISP/app/files/scripts/tmp +chown www-data:www-data /var/www/MISP/app/Plugin/CakeResque/tmp +chown -R www-data:www-data /var/www/MISP/app/tmp +chown -R www-data:www-data /var/www/MISP/app/webroot/img/orgs +chown -R www-data:www-data /var/www/MISP/app/webroot/img/custom + +# 6. Restart the CakeResque workers + +su www-data -c 'bash /var/www/MISP/app/Console/worker/start.sh' + +# You can also do this using the MISP application by navigating to the workers tab in the server settings and clicking on the "Restart all workers" button. diff --git a/INSTALL/UPGRADE.txt b/INSTALL/UPGRADE.txt index 291a1669d..b26ffe607 100755 --- a/INSTALL/UPGRADE.txt +++ b/INSTALL/UPGRADE.txt @@ -32,7 +32,7 @@ python setup.py install # install / update CakeResque (using the background workers is optional buy highly recommended) cd /var/www/MISP/app curl -s https://getcomposer.org/installer | php -php composer.phar require --no-update kamisama/cake-resque:4.1.0 +php composer.phar require kamisama/cake-resque:4.1.2 php composer.phar config vendor-dir Vendor php composer.phar install @@ -78,4 +78,4 @@ chown -R www-data:www-data /var/www/MISP/ true, 'full_tags_on_event_index' => true, 'footer_logo' => '', - 'take_ownership_xml_import' => false, + 'take_ownership_xml_import' => false, + 'unpublishedprivate' => false, ), 'GnuPG' => array ( @@ -34,6 +35,14 @@ $config = array ( 'homedir' => '', 'password' => '', ), + 'Proxy' => + array ( + 'host' => '', + 'port' => '', + 'method' => '', + 'user' => '', + 'password' => '', + ), 'SecureAuth' => array ( 'amount' => 5, diff --git a/app/Console/Command/EventShell.php b/app/Console/Command/EventShell.php index ae031324a..508f2b999 100755 --- a/app/Console/Command/EventShell.php +++ b/app/Console/Command/EventShell.php @@ -45,7 +45,7 @@ class EventShell extends AppShell $eventIds = $this->Event->fetchEventIds($org, $isSiteAdmin); $result = array(); $eventCount = count($eventIds); - $dir = new Folder(APP . DS . '/tmp/cached_exports/xml'); + $dir = new Folder(APP . 'tmp/cached_exports/xml'); if ($isSiteAdmin) { $file = new File($dir->pwd() . DS . 'misp.xml' . '.ADMIN.xml'); } else { @@ -192,7 +192,7 @@ class EventShell extends AppShell $eventIds = $this->Event->fetchEventIds($org, $isSiteAdmin); $eventCount = count($eventIds); $attributes = array(); - $dir = new Folder(APP . DS . '/tmp/cached_exports/' . $extra); + $dir = new Folder(APP . 'tmp/cached_exports/' . $extra); if ($isSiteAdmin) { $file = new File($dir->pwd() . DS . 'misp.' . $extra . '.ADMIN.csv'); } else { @@ -200,11 +200,13 @@ class EventShell extends AppShell } $file->write('uuid,event_id,category,type,value,to_ids,date' . PHP_EOL); foreach ($eventIds as $k => $eventId) { + $chunk = ""; $attributes = $this->Event->csv($org, $isSiteAdmin, $eventId['Event']['id'], $ignore); $attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true); foreach ($attributes as $attribute) { - $file->append($attribute['Attribute']['uuid'] . ',' . $attribute['Attribute']['event_id'] . ',' . $attribute['Attribute']['category'] . ',' . $attribute['Attribute']['type'] . ',' . $attribute['Attribute']['value'] . ',' . intval($attribute['Attribute']['to_ids']) . ',' . $attribute['Attribute']['timestamp'] . PHP_EOL); + $chunk .= $attribute['Attribute']['uuid'] . ',' . $attribute['Attribute']['event_id'] . ',' . $attribute['Attribute']['category'] . ',' . $attribute['Attribute']['type'] . ',' . $attribute['Attribute']['value'] . ',' . intval($attribute['Attribute']['to_ids']) . ',' . $attribute['Attribute']['timestamp'] . PHP_EOL; } + $file->append($chunk); if ($k % 10 == 0) { $this->Job->saveField('progress', $k / $eventCount * 80); } @@ -333,7 +335,7 @@ class EventShell extends AppShell // Now that we have figured out when the next execution should happen, it's time to enqueue it. $process_id = CakeResque::enqueueAt( $task['Task']['next_execution_time'], - 'default', + 'cache', 'EventShell', array('enqueueCaching', $task['Task']['next_execution_time']), true diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index f5b494dd3..817c53884 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -301,4 +301,23 @@ class AppController extends Controller { $this->Session->setFlash(__('All done. attribute_count generated from scratch for ' . $k . ' events.')); $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); } + + public function updateDatabase($command) { + if (!$this->_isSiteAdmin()) throw new MethodNotAllowedException(); + $sql = ''; + switch ($command) { + case 'extendServerOrganizationLength': + $sql = 'ALTER TABLE servers MODIFY COLUMN organization varchar(255) NOT NULL'; + $controller = 'Servers'; + break; + default: + $this->Session->setFlash('Invalid command.'); + $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); + break; + } + $this->loadModel($controller); + $this->$controller->query($sql); + $this->Session->setFlash('Done.'); + $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); + } } \ No newline at end of file diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index aa3dda296..e3fbe3ee8 100755 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -58,6 +58,7 @@ class AttributesController extends AppController { 'AND' => array( 'Attribute.distribution >' => 0, 'Event.distribution >' => 0, + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(), ))))); } @@ -1281,9 +1282,17 @@ class AttributesController extends AppController { // merge in private conditions $this->paginate = Set::merge($this->paginate, array( 'conditions' => - array("OR" => array( - array('Event.org =' => $this->Auth->user('org')), - array("AND" => array('Event.org !=' => $this->Auth->user('org')), array('Event.distribution !=' => 0), array('Attribute.distribution !=' => 0)))), + array("OR" => + array( + array('Event.org =' => $this->Auth->user('org')), + array("AND" => + array('Event.org !=' => $this->Auth->user('org')), + array('Event.distribution !=' => 0), + array('Attribute.distribution !=' => 0), + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(), + ) + ) + ) ) ); } @@ -1369,8 +1378,14 @@ class AttributesController extends AppController { public function searchAlternate($data) { $data['AND'][] = array( "OR" => array( - array('Event.org =' => $this->Auth->user('org')), - array("AND" => array('Event.org !=' => $this->Auth->user('org')), array('Event.distribution !=' => 0), array('Attribute.distribution !=' => 0)))); + array('Event.org =' => $this->Auth->user('org')), + array("AND" => array('Event.org !=' => $this->Auth->user('org')), + array('Event.distribution !=' => 0), + array('Attribute.distribution !=' => 0), + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(), + ) + ) + ); $attributes = $this->Attribute->find('all', array( 'conditions' => $data, 'fields' => array( @@ -1555,7 +1570,11 @@ class AttributesController extends AppController { if (!$user['User']['siteAdmin']) { $temp = array(); - $temp['AND'] = array('Event.distribution >' => 0, 'Attribute.distribution >' => 0); + $temp['AND'] = array( + 'Event.distribution >' => 0, + 'Attribute.distribution >' => 0, + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array() + ); $subcondition['OR'][] = $temp; $subcondition['OR'][] = array('Event.org' => $user['User']['org']); array_push($conditions['AND'], $subcondition); diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 1b83b1608..669270ba1 100755 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -76,21 +76,23 @@ class EventsController extends AppController { $this->paginate = Set::merge($this->paginate,array( 'conditions' => array("OR" => array( - array( - 'Event.org_id' => $this->Auth->user('organisation_id') + array( + 'Event.org_id' => $this->Auth->user('organisation_id') + ), + array( + 'AND' => array( + 'Event.distribution >' => 0, + 'Event.distribution <' => 4, + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(), ), - array( - 'AND' => array( - 'Event.distribution >' => 0, - 'Event.distribution <' => 4, - ), - ), - array( - 'AND' => array( - 'Event.distribution' => 4, - 'Event.sharing_group_id' => $sgids - ), - ) + ), + array( + 'AND' => array( + 'Event.distribution' => 4, + 'Event.sharing_group_id' => $sgids, + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(), + ), + ) )))); } } @@ -2474,7 +2476,11 @@ class EventsController extends AppController { if (!$user['User']['siteAdmin']) { $temp = array(); - $temp['AND'] = array('Event.distribution >' => 0, 'Attribute.distribution >' => 0); + $temp['AND'] = array( + 'Event.distribution >' => 0, + 'Attribute.distribution >' => 0, + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array() + ); $subcondition['OR'][] = $temp; $subcondition['OR'][] = array('Event.org' => $user['User']['org']); array_push($conditions['AND'], $subcondition); @@ -3033,7 +3039,7 @@ class EventsController extends AppController { 'fields' => array('Event.uuid', 'Event.timestamp', 'Event.locked'), )); foreach ($events as $k => $v) { - if (!$v['Event']['timestamp'] < $incomingEvents[$v['Event']['uuid']]) { + if ($v['Event']['timestamp'] >= $incomingEvents[$v['Event']['uuid']]) { unset($incomingEvents[$v['Event']['uuid']]); continue; } diff --git a/app/Controller/LogsController.php b/app/Controller/LogsController.php index ab2335d2f..fab481277 100755 --- a/app/Controller/LogsController.php +++ b/app/Controller/LogsController.php @@ -164,19 +164,19 @@ class LogsController extends AppController { // search the db $conditions = array(); - if ($email) { + if (isset($email) && !empty($email)) { $conditions['LOWER(Log.email) LIKE'] = '%' . strtolower($email) . '%'; } - if (isset($org)) { + if (isset($org) && !empty($org)) { $conditions['LOWER(Log.org) LIKE'] = '%' . strtolower($org) . '%'; } if ($action != 'ALL') { $conditions['Log.action ='] = $action; } - if (isset($title)) { + if (isset($title) && !empty($title)) { $conditions['LOWER(Log.title) LIKE'] = '%' . strtolower($title) . '%'; } - if (isset($change)) { + if (isset($change) && !empty($change)) { $conditions['LOWER(Log.change) LIKE'] = '%' . strtolower($change) . '%'; } $this->{$this->defaultModel}->recursive = 0; diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index e745df916..0303c864b 100755 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -289,11 +289,13 @@ class ServersController extends AppController { $tabs = array( 'MISP' => array('count' => 0, 'errors' => 0, 'severity' => 5), 'GnuPG' => array('count' => 0, 'errors' => 0, 'severity' => 5), + 'Proxy' => array('count' => 0, 'errors' => 0, 'severity' => 5), 'Security' => array('count' => 0, 'errors' => 0, 'severity' => 5), 'misc' => array('count' => 0, 'errors' => 0, 'severity' => 5) ); $writeableErrors = array(0 => 'OK', 1 => 'Directory doesn\'t exist', 2 => 'Directory is not writeable'); $gpgErrors = array(0 => 'OK', 1 => 'FAIL: settings not set', 2 => 'FAIL: bad GnuPG.*', 3 => 'FAIL: encrypt failed'); + $proxyErrors = array(0 => 'OK', 1 => 'not configured (so not tested)', 2 => 'Getting URL via proxy failed'); $stixErrors = array(0 => 'ERROR', 1 => 'OK'); $results = $this->Server->serverSettingsRead(); @@ -394,7 +396,29 @@ class ServersController extends AppController { $gpgStatus = 1; } if ($gpgStatus != 0) $diagnostic_errors++; + + // if Proxy is set up in the settings, try to connect to a test URL + $proxyStatus = 0; + $proxy = Configure::read('Proxy'); + if(!empty($proxy['host'])) { + App::uses('SyncTool', 'Tools'); + $syncTool = new SyncTool(); + try { + $HttpSocket = $syncTool->setupHttpSocket(); + $proxyResponse = $HttpSocket->get('http://www.example.com/'); + } catch (Exception $e) { + $proxyStatus = 2; + } + if(empty($proxyResponse) || $proxyResponse->code > 399) { + $proxyStatus = 2; + } + } else { + $proxyStatus = 1; + } + if ($proxyStatus > 1) $diagnostic_errors++; + $this->set('gpgStatus', $gpgStatus); + $this->set('proxyStatus', $proxyStatus); $this->set('diagnostic_errors', $diagnostic_errors); $this->set('tab', $tab); $this->set('tabs', $tabs); @@ -403,6 +427,7 @@ class ServersController extends AppController { $this->set('writeableErrors', $writeableErrors); $this->set('gpgErrors', $gpgErrors); + $this->set('proxyErrors', $proxyErrors); $this->set('stixErrors', $stixErrors); if (Configure::read('MISP.background_jobs')) { @@ -437,7 +462,7 @@ class ServersController extends AppController { foreach ($dumpResults as &$dr) { unset($dr['description']); } - $dump = array('gpgStatus' => $gpgErrors[$gpgStatus], 'stix' => $stixErrors[$stix], 'writeableDirs' => $writeableDirs, 'finalSettings' => $dumpResults); + $dump = array('gpgStatus' => $gpgErrors[$gpgStatus], 'proxyStatus' => $proxyErrors[$proxyStatus], 'stix' => $stixErrors[$stix], 'writeableDirs' => $writeableDirs, 'finalSettings' => $dumpResults); $this->response->body(json_encode($dump, JSON_PRETTY_PRINT)); $this->response->type('json'); $this->response->download('MISP.report.json'); @@ -453,12 +478,16 @@ class ServersController extends AppController { private function __checkVersion() { if (!$this->_isSiteAdmin()) throw new MethodNotAllowedException(); - set_error_handler(function() {}); - $options = array('http' => array('user_agent'=> $_SERVER['HTTP_USER_AGENT'])); - $context = stream_context_create($options); - $tags = file_get_contents('https://api.github.com/repos/MISP/MISP/tags', false, $context); - restore_error_handler(); - if ($tags != false) { + App::uses('SyncTool', 'Tools'); + $syncTool = new SyncTool(); + try { + $HttpSocket = $syncTool->setupHttpSocket(); + $response = $HttpSocket->get('https://api.github.com/repos/MISP/MISP/tags'); + $tags = $response->body; + } catch (Exception $e) { + return false; + } + if ($response->isOK() && !empty($tags)) { $json_decoded_tags = json_decode($tags); // find the latest version tag in the v[major].[minor].[hotfix] format diff --git a/app/Controller/ShadowAttributesController.php b/app/Controller/ShadowAttributesController.php index 09bfcb2a0..bda6fbfcc 100644 --- a/app/Controller/ShadowAttributesController.php +++ b/app/Controller/ShadowAttributesController.php @@ -50,6 +50,7 @@ class ShadowAttributesController extends AppController { 'AND' => array( 'ShadowAttribute.org =' => $this->Auth->user('org'), 'Event.distribution >' => 0, + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(), ), ) ))); @@ -271,7 +272,7 @@ class ShadowAttributesController extends AppController { 'recursive' => -1, 'fields' => array('id', 'orgc', 'distribution', 'org'), )); - if ((($event['Event']['distribution'] == 0 && $event['Event']['org'] != $this->Auth->user('org'))) || ($event['Event']['orgc'] == $this->Auth->user('org'))) { + if (!$this->_isSiteAdmin() && (($event['Event']['distribution'] == 0 && $event['Event']['org'] != $this->Auth->user('org'))) || ($event['Event']['orgc'] == $this->Auth->user('org'))) { $this->Session->setFlash(__('Invalid Event.')); $this->redirect(array('controller' => 'events', 'action' => 'index')); } diff --git a/app/Controller/TagsController.php b/app/Controller/TagsController.php index 527661f6d..7c0a00627 100644 --- a/app/Controller/TagsController.php +++ b/app/Controller/TagsController.php @@ -37,7 +37,15 @@ class TagsController extends AppController { $eventIDs[] = $eventTag['event_id']; } $conditions = array('Event.id' => $eventIDs); - if (!$this->_isSiteAdmin()) $conditions = array_merge($conditions, array('OR' => array(array('Event.distribution >' => 0), array('Event.orgc' => $this->Auth->user('org'))))); + if (!$this->_isSiteAdmin()) $conditions = array_merge( + $conditions, + array('OR' => array( + array('AND' => array( + array('Event.distribution >' => 0), + array('Event.published =' => 1) + )), + array('Event.orgc' => $this->Auth->user('org')) + ))); $events = $this->Event->find('all', array( 'fields' => array('Event.id', 'Event.distribution', 'Event.orgc'), 'conditions' => $conditions @@ -136,4 +144,4 @@ class TagsController extends AppController { $this->set('id', $id); $this->render('ajax/view_tag'); } -} \ No newline at end of file +} diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php index b2f47463e..4eca938a7 100755 --- a/app/Controller/UsersController.php +++ b/app/Controller/UsersController.php @@ -344,8 +344,17 @@ class UsersController extends AppController { if ($this->request->is('post')) { $this->User->create(); // set invited by + $this->loadModel('Role'); + $this->Role->recursive = -1; + $chosenRole = $this->Role->findById($this->request->data['User']['role_id']); $this->request->data['User']['invited_by'] = $this->Auth->user('id'); - $this->request->data['User']['change_pw'] = 1; + if ($chosenRole['Role']['perm_sync']) { + $this->request->data['User']['change_pw'] = 0; + $this->request->data['User']['termsaccepted'] = 1; + } else { + $this->request->data['User']['change_pw'] = 1; + $this->request->data['User']['termsaccepted'] = 0; + } $this->request->data['User']['newsread'] = '2000-01-01'; if (!$this->_isSiteAdmin()) { $this->request->data['User']['organisation_id'] = $this->Auth->User('organisation_id'); diff --git a/app/Lib/Tools/SyncTool.php b/app/Lib/Tools/SyncTool.php index d41ad1938..c55195063 100644 --- a/app/Lib/Tools/SyncTool.php +++ b/app/Lib/Tools/SyncTool.php @@ -2,12 +2,18 @@ class SyncTool { // take a server as parameter and return a HttpSocket object using the ssl options defined in the server settings - public function setupHttpSocket($server) { + public function setupHttpSocket($server = null) { $params = array(); App::uses('HttpSocket', 'Network/Http'); - if ($server['Server']['cert_file']) $params['ssl_cafile'] = APP . "files" . DS . "certs" . DS . $server['Server']['id'] . '.pem'; - if ($server['Server']['self_signed']) $params['ssl_allow_self_signed'] = $server['Server']['self_signed']; + if(!empty($server)) { + if ($server['Server']['cert_file']) $params['ssl_cafile'] = APP . "files" . DS . "certs" . DS . $server['Server']['id'] . '.pem'; + if ($server['Server']['self_signed']) $params['ssl_allow_self_signed'] = $server['Server']['self_signed']; + } $HttpSocket = new HttpSocket($params); + + $proxy = Configure::read('Proxy'); + $HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']); + return $HttpSocket; } } diff --git a/app/Lib/cakephp b/app/Lib/cakephp index a0aac5cfa..4c61b579c 160000 --- a/app/Lib/cakephp +++ b/app/Lib/cakephp @@ -1 +1 @@ -Subproject commit a0aac5cfa9acf2ccbde65c45ad6ee06ecd32fb54 +Subproject commit 4c61b579c40863ada019e1e08c6195ae612c589f diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index c4dbc0b65..3a75c9acd 100755 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -924,7 +924,7 @@ class Attribute extends AppModel { // TODO check if CakePHP has no easy/safe wrapper to execute commands $execRetval = ''; $execOutput = array(); - exec("zip -j -P infected " . $zipfile->path . ' "' . addslashes($fileInZip->path) . '"', $execOutput, $execRetval); + exec("zip -j -P infected " . $zipfile->path . ' \'' . addslashes($fileInZip->path) . '\'', $execOutput, $execRetval); if ($execRetval != 0) { // not EXIT_SUCCESS // do some? }; diff --git a/app/Model/Event.php b/app/Model/Event.php index 463a7e2cc..f92f937bd 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -922,7 +922,10 @@ class Event extends AppModel { $conditions = array(); if (!$isSiteAdmin) { $conditions['OR'] = array( - 'Event.distribution >' => 0, + "AND" => array( + 'Event.distribution >' => 0, + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array() + ), 'Event.org LIKE' => $org ); } @@ -959,19 +962,21 @@ class Event extends AppModel { if (!$user['Role']['perm_site_admin']) { $sgids = $this->SharingGroup->fetchAllAuthorised($user); $conditions['AND']['OR'] = array( - 'Event.org_id' => $user['organisation_id'], - array( - 'AND' => array( - 'Event.distribution >' => 0, - 'Event.distribution <' => 4 - ), + 'Event.org_id' => $user['organisation_id'], + array( + 'AND' => array( + 'Event.distribution >' => 0, + 'Event.distribution <' => 4, + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(), ), - array( - 'AND' => array( - 'Event.sharing_group_id' => $sgids, - 'Event.distribution' => 4, - ) + ), + array( + 'AND' => array( + 'Event.sharing_group_id' => $sgids, + 'Event.distribution' => 4, + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(), ) + ) ); $conditionsAttributes['OR'] = array( 'Attribute.distribution >' => 0, @@ -1078,7 +1083,13 @@ class Event extends AppModel { if ($from) $econditions['AND'][] = array('Event.date >=' => $from); if ($to) $econditions['AND'][] = array('Event.date <=' => $to); // This is for both single event downloads and for full downloads. Org has to be the same as the user's or distribution not org only - if the user is no siteadmin - if(!$isSiteAdmin) $econditions['AND']['OR'] = array('Event.distribution >' => 0, 'Event.org =' => $org); + if(!$isSiteAdmin) $econditions['AND']['OR'] = array( + "AND" => array( + 'Event.distribution >' => 0, + Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(), + ), + 'Event.org =' => $org + ); if ($eventid == 0 && $ignore == 0) $econditions['AND'][] = array('Event.published =' => 1); // If it's a full download (eventid == false) and the user is not a site admin, we need to first find all the events that the user can see and save the IDs @@ -1196,7 +1207,7 @@ class Event extends AppModel { $job = ClassRegistry::init('Job'); $job->create(); $data = array( - 'worker' => 'default', + 'worker' => 'email', 'job_type' => 'publish_alert_email', 'job_input' => 'Event: ' . $id, 'status' => 0, @@ -1207,7 +1218,7 @@ class Event extends AppModel { $job->save($data); $jobId = $job->id; $process_id = CakeResque::enqueue( - 'default', + 'email', 'EventShell', array('alertemail', $user['org'], $jobId, $id) ); @@ -1285,7 +1296,7 @@ class Event extends AppModel { if (Configure::read('MISP.extended_alert_subject')) { $subject = preg_replace( "/\r|\n/", "", $event['Event']['info']); - if (strlen($subject) > 55) { + if (strlen($subject) > 58) { $subject = substr($subject, 0, 55) . '... - '; } else { $subject .= " - "; @@ -1778,7 +1789,7 @@ class Event extends AppModel { $job = ClassRegistry::init('Job'); $job->create(); $data = array( - 'worker' => 'default', + 'worker' => 'email', 'job_type' => 'contact_alert', 'job_input' => 'To entire org: ' . $all, 'status' => 0, @@ -1789,7 +1800,7 @@ class Event extends AppModel { $job->save($data); $jobId = $job->id; $process_id = CakeResque::enqueue( - 'default', + 'email', 'EventShell', array('contactemail', $id, $message, $all, $user['id'], $isSiteAdmin, $jobId) ); @@ -1986,7 +1997,7 @@ class Event extends AppModel { $scriptFile = APP . "files" . DS . "scripts" . DS . "misp2stix.py"; // Execute the python script and point it to the temporary filename - $result = shell_exec('python ' . $scriptFile . ' ' . $randomFileName . ' ' . $returnType); + $result = shell_exec('python ' . $scriptFile . ' ' . $randomFileName . ' ' . $returnType . ' ' . Configure::read('MISP.baseurl') . ' "' . Configure::read('MISP.org') . '"'); // The result of the script will be a returned JSON object with 2 variables: success (boolean) and message // If success = 1 then the temporary output file was successfully written, otherwise an error message is passed along diff --git a/app/Model/Server.php b/app/Model/Server.php index 83c5fd9ea..75aa926d6 100755 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -381,6 +381,14 @@ class Server extends AppModel { 'test' => 'testBool', 'type' => 'boolean' ), + 'unpublishedprivate' => array( + 'level' => 2, + 'description' => 'True will deny access to unpublished events to users outside the organization of the submitter except site admins.', + 'value' => '', + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), ), 'GnuPG' => array( 'branch' => 1, @@ -417,6 +425,49 @@ class Server extends AppModel { 'type' => 'string', ), ), + 'Proxy' => array( + 'branch' => 1, + 'host' => array( + 'level' => 2, + 'description' => 'The hostname of an HTTP proxy for outgoing sync requests. Leave empty to not use a proxy.', + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'port' => array( + 'level' => 2, + 'description' => 'The TCP port for the HTTP proxy.', + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForNumeric', + 'type' => 'numeric', + ), + 'method' => array( + 'level' => 2, + 'description' => 'The authentication method for the HTTP proxy. Currently supported are Basic or Digest. Leave empty for no proxy authentication.', + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'user' => array( + 'level' => 2, + 'description' => 'The authentication username for the HTTP proxy.', + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'password' => array( + 'level' => 2, + 'description' => 'The authentication password for the HTTP proxy.', + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + ), 'Security' => array( 'branch' => 1, 'salt' => array( @@ -1008,7 +1059,7 @@ class Server extends AppModel { public function serverSettingsSaveValue($setting, $value) { Configure::write($setting, $value); - Configure::dump('config.php', 'default', array('MISP', 'GnuPG', 'SecureAuth', 'Security', 'debug')); + Configure::dump('config.php', 'default', array('MISP', 'GnuPG', 'Proxy', 'SecureAuth', 'Security', 'debug')); } public function checkVersion($newest) { diff --git a/app/Model/ShadowAttribute.php b/app/Model/ShadowAttribute.php index 245bec946..237de28cb 100644 --- a/app/Model/ShadowAttribute.php +++ b/app/Model/ShadowAttribute.php @@ -510,7 +510,7 @@ class ShadowAttribute extends AppModel { // TODO check if CakePHP has no easy/safe wrapper to execute commands $execRetval = ''; $execOutput = array(); - exec("zip -j -P infected " . $zipfile->path . ' "' . addslashes($fileInZip->path) . '"', $execOutput, $execRetval); + exec("zip -j -P infected " . $zipfile->path . ' \'' . addslashes($fileInZip->path) . '\'', $execOutput, $execRetval); if ($execRetval != 0) { // not EXIT_SUCCESS // do some? }; diff --git a/app/Plugin/CakeResque b/app/Plugin/CakeResque deleted file mode 160000 index 34f9a7cd3..000000000 --- a/app/Plugin/CakeResque +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 34f9a7cd370bfb2706b9bd6b54fb8ede17e5e5bd diff --git a/app/View/Attributes/alternate_search_result.ctp b/app/View/Attributes/alternate_search_result.ctp index 0a53e37aa..2142b05c9 100644 --- a/app/View/Attributes/alternate_search_result.ctp +++ b/app/View/Attributes/alternate_search_result.ctp @@ -27,7 +27,7 @@ 60) { + if (strlen(h($event['Event']['info'])) > 63) { echo (substr(h($event['Event']['info']), 0, 60)) . '...'; } else echo h($event['Event']['info']); ?> @@ -52,4 +52,4 @@ element('side_menu', array('menuList' => 'event-collection', 'menuItem' => 'searchAttributes')); -?> \ No newline at end of file +?> diff --git a/app/View/Elements/eventattribute.ctp b/app/View/Elements/eventattribute.ctp index 2c6b5cd58..7aa363a4b 100644 --- a/app/View/Elements/eventattribute.ctp +++ b/app/View/Elements/eventattribute.ctp @@ -228,15 +228,16 @@ + + - - ' . $message . ''; ?> - \ No newline at end of file +

+ Proxy +

+

This tool tests whether your HTTP proxy settings are correct.

+
+ 1) { + $colour = 'red'; + } + echo 'Proxy settings....' . $message . ''; + ?> +
+ diff --git a/app/View/Elements/side_menu.ctp b/app/View/Elements/side_menu.ctp index efb022895..1ec17ccc5 100755 --- a/app/View/Elements/side_menu.ctp +++ b/app/View/Elements/side_menu.ctp @@ -16,19 +16,19 @@ ?>
  • >View Event
  • >View Event History
    • +
  • >Edit Event
  • Form->postLink('Delete Event', array('action' => 'delete', $event['Event']['id']), null, __('Are you sure you want to delete # %s?', $event['Event']['id'])); ?>
    • -
  • >Add Attribute
  • >Add Attachment
  • >Populate from OpenIOC
  • >Populate from ThreatConnect
    • - -
  • Populate From Template
    • + +
  • Populate From Template
    • + - -
  • +
  • >Propose Attribute
  • >Propose Attachment
    • diff --git a/app/View/Events/json/view.ctp b/app/View/Events/json/view.ctp index 48fc62226..093151213 100644 --- a/app/View/Events/json/view.ctp +++ b/app/View/Events/json/view.ctp @@ -1,4 +1,4 @@ event2JSON($event)); \ No newline at end of file +echo $converter->event2JSON($event); \ No newline at end of file diff --git a/app/View/Events/view.ctp b/app/View/Events/view.ctp index 29cdacd35..4d9b4bac9 100755 --- a/app/View/Events/view.ctp +++ b/app/View/Events/view.ctp @@ -12,7 +12,7 @@ $mayPublish = ($isAclPublish && $event['Event']['orgc'] == $me['org']); $left = true; } $title = $event['Event']['info']; - if (strlen($title) > 55) $title = substr($title, 0, 55) . '...'; + if (strlen($title) > 58) $title = substr($title, 0, 55) . '...'; ?>
    @@ -252,4 +252,4 @@ $(document).ready(function () { $('#addTagButton').hide(); }); }); - \ No newline at end of file + diff --git a/app/View/Helper/PivotHelper.php b/app/View/Helper/PivotHelper.php index 1a735b8e7..9165628f8 100644 --- a/app/View/Helper/PivotHelper.php +++ b/app/View/Helper/PivotHelper.php @@ -9,7 +9,7 @@ App::uses('AppHelper', 'View/Helper'); $active = ''; $pivot['info'] = h($pivot['info']); // Truncate string if longer than (11 - length of event id) chars to fit the pivot bubble - if (strlen($pivot['info']) > (11 - strlen((string)$pivot['id']))) { + if (strlen($pivot['info']) > (11 - strlen((string)$pivot['id'])) && strlen($pivot['info']) > 9) { $text .= substr($pivot['info'], 0, 6) . '...'; } else { $text .= $pivot['info']; diff --git a/app/View/Pages/administration.ctp b/app/View/Pages/administration.ctp index 69e106b6e..5f5128471 100644 --- a/app/View/Pages/administration.ctp +++ b/app/View/Pages/administration.ctp @@ -16,6 +16,7 @@ if (!$isSiteAdmin) exit();
  • generateLocked (This is for upgrading to hotfix 2.1.8 or later, all events that were created by an organisation that doesn't have users on this instance, or only has a single sync user will have their locked setting set to 1)
  • Verify GPG keys (Check whether every user's GPG key is usable)
  • Upgrade Risk to Threat Level (As of version 2.2 the risk field is replaced by Threat Level. This script will convert all values in risk to threat level.)
    • +
  • Extend Organization length (Hotfix 2.3.57: Increase the max length of the organization field when adding a new server connection.)
    • Overview: General overview of the current state of your MISP installation
  • MISP settings: Basic MISP settings. This includes the way MISP handles the default settings for distribution settings, whether background jobs are enabled, etc
  • GnuPG settings: GPG related settings.
    • +
  • Proxy settings: HTTP proxy related settings.
  • Security settings: Settings controlling the brute-force protection and the application's salt key.
  • Misc settings: You change the debug options here, but make sure that debug is always disabled on a production system.
  • Diagnostics: The diagnostics tool checks if all directories that MISP uses to store data are writeable by the apache user. Also, the tool checks whether the STIX libraries and GPG are working as intended.
    • @@ -240,4 +241,4 @@
  • Message: This field shows when the job was queued by the scheduler for execution.


    • -
    \ No newline at end of file + diff --git a/app/View/Servers/server_settings.ctp b/app/View/Servers/server_settings.ctp index d6f2f54de..cdb97ad23 100644 --- a/app/View/Servers/server_settings.ctp +++ b/app/View/Servers/server_settings.ctp @@ -2,7 +2,7 @@

    Server settings

    element('healthElements/tabs'); - if (in_array($tab, array('MISP', 'Security', 'GnuPG', 'misc'))) { + if (in_array($tab, array('MISP', 'Security', 'GnuPG', 'Proxy', 'misc'))) { echo $this->element('healthElements/settings_tab'); } else if ($tab == 'diagnostics') { echo $this->element('healthElements/diagnostics'); @@ -17,4 +17,4 @@ element('side_menu', array('menuList' => 'admin', 'menuItem' => 'serverSettings')); -?> \ No newline at end of file +?> diff --git a/app/files/scripts/misp2ciq.py b/app/files/scripts/misp2ciq.py index 47998d0e7..98704adc2 100644 --- a/app/files/scripts/misp2ciq.py +++ b/app/files/scripts/misp2ciq.py @@ -1,7 +1,7 @@ from stix.extensions.identity.ciq_identity_3_0 import (CIQIdentity3_0Instance, STIXCIQIdentity3_0, OrganisationInfo, PartyName, Address, ElectronicAddressIdentifier, FreeTextAddress) from stix.common import Identity -def resolveIdentityAttribute(incident, attribute): +def resolveIdentityAttribute(incident, attribute, namespace): ciq_identity = CIQIdentity3_0Instance() identity_spec = STIXCIQIdentity3_0() if attribute["type"] == 'target-user': @@ -20,6 +20,6 @@ def resolveIdentityAttribute(incident, attribute): ciq_identity.id_ = "example:Identity-" + attribute["uuid"] # is this a good idea? - ciq_identity.name = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"] + ciq_identity.name = attribute["type"] + ": " + attribute["value"] + " (MISP Attribute #" + attribute["id"] + ")" incident.add_victim(ciq_identity) return incident diff --git a/app/files/scripts/misp2cybox.py b/app/files/scripts/misp2cybox.py index 4579e3415..5e12e7f17 100644 --- a/app/files/scripts/misp2cybox.py +++ b/app/files/scripts/misp2cybox.py @@ -1,4 +1,4 @@ -from cybox.core import Observable, ObservableComposition +from cybox.core import Object, Observable, ObservableComposition from cybox.objects.file_object import File from cybox.objects.address_object import Address from cybox.objects.hostname_object import Hostname @@ -40,7 +40,11 @@ def generateObservable(indicator, attribute): if (attribute["type"] in simple_type_to_method.keys()): action = getattr(this_module, simple_type_to_method[attribute["type"]], None) if (action != None): - observable = action(attribute) + property = action(attribute) + object = Object(property) + object.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":" + property.__class__.__name__ + "-" + attribute["uuid"] + observable = Observable(object) + observable.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-" + attribute["uuid"] indicator.add_observable(observable) def resolveFileObservable(attribute): @@ -177,30 +181,44 @@ def resolvePatternObservable(attribute): # create an artifact object for the malware-sample type. def createArtifactObject(indicator, attribute): artifact = Artifact(data = attribute["data"]) - indicator.add_observable(artifact) + artifact.parent.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":artifact-" + attribute["uuid"] + observable = Observable(artifact) + observable.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-artifact-" + attribute["uuid"] + indicator.add_observable(observable) # return either a composition if data is set in attribute, or just an observable with a filename if it's not set def returnAttachmentComposition(attribute): file_object = File() file_object.file_name = attribute["value"] + file_object.parent.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":file-" + attribute["uuid"] observable = Observable() if "data" in attribute: artifact = Artifact(data = attribute["data"]) - composition = ObservableComposition(observables = [artifact, file_object]) + artifact.parent.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":artifact-" + attribute["uuid"] + observable_artifact = Observable(artifact) + observable_artifact.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-artifact-" + attribute["uuid"] + observable_file = Observable(file_object) + observable_file.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-file-" + attribute["uuid"] + composition = ObservableComposition(observables = [observable_artifact, observable_file]) observable.observable_composition = composition else: observable = Observable(file_object) + observable.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-" + attribute["uuid"] if attribute["comment"] != "": observable.description = attribute["comment"] return observable # email-attachment are mapped to an email message observable that contains the attachment as a file object -def generateEmailAttachmentObject(indicator, filename): +def generateEmailAttachmentObject(indicator, attribute): file_object = File() - file_object.file_name = filename + file_object.file_name = attribute["value"] email = EmailMessage() email.attachments = Attachments() email.add_related(file_object, "Contains", inline=True) + file_object.parent.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":file-" + attribute["uuid"] email.attachments.append(file_object.parent.id_) - indicator.observable = email + email.parent.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":EmailMessage-" + attribute["uuid"] + observable = Observable(email) + observable.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-" + attribute["uuid"] + indicator.observable = observable diff --git a/app/files/scripts/misp2stix.py b/app/files/scripts/misp2stix.py index 28b0659d1..b192fb1ae 100644 --- a/app/files/scripts/misp2stix.py +++ b/app/files/scripts/misp2stix.py @@ -20,10 +20,6 @@ from cybox.utils import Namespace namespace = ['https://github.com/MISP/MISP', 'MISP'] -cybox.utils.idgen.set_id_namespace(Namespace(namespace[0], namespace[1])) -stix.utils.idgen.set_id_namespace({namespace[0]: namespace[1]}) - - NS_DICT = { "http://cybox.mitre.org/common-2" : 'cyboxCommon', "http://cybox.mitre.org/cybox-2" : 'cybox', @@ -55,7 +51,6 @@ NS_DICT = { "urn:oasis:names:tc:ciq:xal:3" : 'xal', "urn:oasis:names:tc:ciq:xnl:3" : 'xnl', "urn:oasis:names:tc:ciq:xpil:3" : 'xpil', - namespace[0] : namespace[1] } SCHEMALOC_DICT = { @@ -126,7 +121,7 @@ def saveFile(args, pathname, package): def generateMainPackage(events): stix_package = STIXPackage() stix_header = STIXHeader() - stix_header.title="Export from MISP" + stix_header.title="Export from " + namespace[1] + " MISP" stix_header.package_intents="Threat Report" stix_package.stix_header = stix_header return stix_package @@ -134,9 +129,10 @@ def generateMainPackage(events): # generate a package for each event def generateEventPackage(event): package_name = namespace[1] + ':STIXPackage-' + event["Event"]["uuid"] - stix_package = STIXPackage(id_=package_name) + timestamp = getDateFromTimestamp(int(event["Event"]["timestamp"])) + stix_package = STIXPackage(id_=package_name, timestamp=timestamp) stix_header = STIXHeader() - stix_header.title="MISP event #" + event["Event"]["id"] + " uuid: " + event["Event"]["uuid"] + stix_header.title=event["Event"]["info"] + " (MISP Event #" + event["Event"]["id"] + ")" stix_header.package_intents="Threat Report" stix_package.stix_header = stix_header objects = generateSTIXObjects(event) @@ -189,9 +185,8 @@ def resolveAttributes(incident, ttps, attributes): # Create the indicator and pass the attribute further for observable creation - this can be called from resolveattributes directly or from handleNonindicatorAttribute, for some special cases def handleIndicatorAttribute(incident, ttps, attribute): indicator = generateIndicator(attribute) - indicator.title = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"] if attribute["type"] == "email-attachment": - generateEmailAttachmentObject(indicator, attribute["value"]) + generateEmailAttachmentObject(indicator, attribute) else: generateObservable(indicator, attribute) if "data" in attribute: @@ -227,7 +222,7 @@ def handleNonIndicatorAttribute(incident, ttps, attribute): else: addReference(incident, attribute["value"]) elif attribute["type"].startswith('target-'): - resolveIdentityAttribute(incident, attribute) + resolveIdentityAttribute(incident, attribute, namespace[1]) elif attribute["type"] == "attachment": observable = returnAttachmentComposition(attribute) related_observable = RelatedObservable(observable, relationship=attribute["category"]) @@ -236,14 +231,14 @@ def handleNonIndicatorAttribute(incident, ttps, attribute): # TTPs are only used to describe malware names currently (attribute with category Payload Type and type text/comment/other) def generateTTP(incident, attribute): - ttp = TTP() + ttp = TTP(timestamp=getDateFromTimestamp(int(attribute["timestamp"]))) ttp.id_= namespace[1] + ":ttp-" + attribute["uuid"] setTLP(ttp, attribute["distribution"]) - ttp.title = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"] + ttp.title = attribute["category"] + ": " + attribute["value"] + " (MISP Attribute #" + attribute["id"] + ")" if attribute["type"] == "vulnerability": vulnerability = Vulnerability() vulnerability.cve_id = attribute["value"] - et = ExploitTarget() + et = ExploitTarget(timestamp=getDateFromTimestamp(int(attribute["timestamp"]))) et.add_vulnerability(vulnerability) ttp.exploit_targets.append(et) else: @@ -258,9 +253,9 @@ def generateTTP(incident, attribute): # Threat actors are currently only used for the category:attribution / type:(text|comment|other) attributes def generateThreatActor(attribute): - ta = ThreatActor() + ta = ThreatActor(timestamp=getDateFromTimestamp(int(attribute["timestamp"]))) ta.id_= namespace[1] + ":threatactor-" + attribute["uuid"] - ta.title = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"] + ta.title = attribute["category"] + ": " + attribute["value"] + " (MISP Attribute #" + attribute["id"] + ")" if attribute["comment"] != "": ta.description = attribute["value"] + " (" + attribute["comment"] + ")" else: @@ -269,17 +264,17 @@ def generateThreatActor(attribute): # generate the indicator and add the relevant information def generateIndicator(attribute): - indicator = Indicator() + indicator = Indicator(timestamp=getDateFromTimestamp(int(attribute["timestamp"]))) indicator.id_= namespace[1] + ":indicator-" + attribute["uuid"] if attribute["comment"] != "": indicator.description = attribute["comment"] setTLP(indicator, attribute["distribution"]) - indicator.title = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"] + indicator.title = attribute["category"] + ": " + attribute["value"] + " (MISP Attribute #" + attribute["id"] + ")" confidence_description = "Derived from MISP's IDS flag. If an attribute is marked for IDS exports, the confidence will be high, otherwise none" confidence_value = confidence_mapping.get(attribute["to_ids"], None) if confidence_value is None: return indicator - indicator.confidence = Confidence(value=confidence_value, description=confidence_description) + indicator.confidence = Confidence(value=confidence_value, description=confidence_description, timestamp=getDateFromTimestamp(int(attribute["timestamp"]))) return indicator # converts timestamp to the format used by STIX @@ -323,6 +318,13 @@ def addJournalEntry(incident, entry_line): # main def main(args): pathname = os.path.dirname(sys.argv[0]) + if len(sys.argv) > 3: + namespace[0] = sys.argv[3] + if len(sys.argv) > 4: + namespace[1] = sys.argv[4].replace(" ", "_") + NS_DICT[namespace[0]]=namespace[1] + cybox.utils.idgen.set_id_namespace(Namespace(namespace[0], namespace[1])) + stix.utils.idgen.set_id_namespace({namespace[0]: namespace[1]}) events = loadEvent(args, pathname) stix_package = generateMainPackage(events) for event in events: