diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index 15bcecc51..f9a29fe2b 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -182,6 +182,8 @@ class AppController extends Controller if (!empty($this->params['named']['disable_background_processing'])) { Configure::write('MISP.background_jobs', 0); } + Configure::write('CurrentController', $this->params['controller']); + Configure::write('CurrentAction', $this->params['action']); $versionArray = $this->{$this->modelClass}->checkMISPVersion(); $this->mispVersion = implode('.', array_values($versionArray)); $this->Security->blackHoleCallback = 'blackHole'; @@ -298,6 +300,7 @@ class AppController extends Controller } if ($this->Auth->user()) { + Configure::write('CurrentUserId', $this->Auth->user('id')); $this->User->setMonitoring($this->Auth->user()); if (Configure::read('MISP.log_user_ips')) { $redis = $this->{$this->modelClass}->setupRedis(); @@ -606,7 +609,7 @@ class AppController extends Controller ConnectionManager::create('default', $db->config); } $dataSource = $dataSourceConfig['datasource']; - if ($dataSource != 'Database/Mysql' && $dataSource != 'Database/Postgres') { + if (!in_array($dataSource, array('Database/Mysql', 'Database/Postgres', 'Database/MysqlObserver'))) { throw new Exception('datasource not supported: ' . $dataSource); } } diff --git a/app/Controller/Component/RestResponseComponent.php b/app/Controller/Component/RestResponseComponent.php index ae665b9b6..295c166b5 100644 --- a/app/Controller/Component/RestResponseComponent.php +++ b/app/Controller/Component/RestResponseComponent.php @@ -449,7 +449,7 @@ class RestResponseComponent extends Component } if (Configure::read('debug') > 1 && !empty($this->Controller->sql_dump)) { $this->Log = ClassRegistry::init('Log'); - if ($this->Content->sql_dump === 2) { + if ($this->Controller->sql_dump === 2) { $response = array('sql_dump' => $this->Log->getDataSource()->getLog(false, false)); } else { $response['sql_dump'] = $this->Log->getDataSource()->getLog(false, false); diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 4256d7fae..8095d5dd1 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -315,11 +315,11 @@ class EventsController extends AppController break; case 'attribute': $event_id_arrays = $this->__filterOnAttributeValue($v); - foreach ($event_id_arrays[0] as $event_id) { - $this->paginate['conditions']['AND']['OR'][] = array('Event.id' => $event_id); + if (!empty($event_id_arrays[0])) { + $this->paginate['conditions']['AND'][] = array('Event.id' => $event_id_arrays[0]); } - foreach ($event_id_arrays[1] as $event_id) { - $this->paginate['conditions']['AND'][] = array('Event.id !=' => $event_id); + if (!empty($event_id_arrays[1])) { + $this->paginate['conditions']['AND'][] = array('Event.id !=' => $event_id_arrays[1]); } break; case 'published': @@ -342,25 +342,38 @@ class EventsController extends AppController if ($v == "") { continue 2; } - $pieces = explode('|', $v); + if (is_array($v)) { + $pieces = $v; + } else { + $pieces = explode('|', $v); + } $temp = array(); + $eventidConditions = array(); foreach ($pieces as $piece) { $piece = trim($piece); if ($piece[0] == '!') { if (strlen($piece) == 37) { - $this->paginate['conditions']['AND'][] = array('Event.uuid !=' => substr($piece, 1)); + $eventidConditions['NOT']['uuid'][] = substr($piece, 1); } else { - $this->paginate['conditions']['AND'][] = array('Event.id !=' => substr($piece, 1)); + $eventidConditions['NOT']['id'][] = substr($piece, 1); } } else { if (strlen($piece) == 36) { - $temp['OR'][] = array('Event.uuid' => $piece); + $eventidConditions['OR']['uuid'][] = $piece; } else { - $temp['OR'][] = array('Event.id' => $piece); + $eventidConditions['OR']['id'][] = $piece; } } } - $this->paginate['conditions']['AND'][] = $temp; + foreach ($eventidConditions as $operator => $conditionForOperator) { + foreach ($conditionForOperator as $conditionKey => $conditionValue) { + $lookupKey = 'Event.' . $conditionKey; + if ($operator === 'NOT') { + $lookupKey = $lookupKey . ' !='; + } + $this->paginate['conditions']['AND'][] = array($lookupKey => $conditionValue); + } + } break; case 'datefrom': if ($v == "") { @@ -727,8 +740,6 @@ class EventsController extends AppController } else { $rules['order'] = array('Event.' . $passedArgs['sort'] => 'ASC'); } - } else { - $rules['order'] = array('Event.id' => 'DESC'); } $rules['contain'] = $this->paginate['contain']; if (isset($this->paginate['conditions'])) { diff --git a/app/Controller/LogsController.php b/app/Controller/LogsController.php index 5b58a3c93..a72aa3f74 100644 --- a/app/Controller/LogsController.php +++ b/app/Controller/LogsController.php @@ -349,21 +349,25 @@ class LogsController extends AppController } $this->set('list', $list); - // and store into session - $this->Session->write('paginate_conditions_log', $this->paginate); - $this->Session->write('paginate_conditions_log_email', $filters['email']); - $this->Session->write('paginate_conditions_log_org', $filters['org']); - $this->Session->write('paginate_conditions_log_action', $filters['action']); - $this->Session->write('paginate_conditions_log_model', $filters['model']); - $this->Session->write('paginate_conditions_log_model_id', $filters['model_id']); - $this->Session->write('paginate_conditions_log_title', $filters['title']); - $this->Session->write('paginate_conditions_log_change', $filters['change']); - if (Configure::read('MISP.log_client_ip')) { - $this->Session->write('paginate_conditions_log_ip', $filters['ip']); - } + if ($this->_isRest()) { + return $this->RestResponse->viewData($list, $this->response->type()); + } else { + // and store into session + $this->Session->write('paginate_conditions_log', $this->paginate); + $this->Session->write('paginate_conditions_log_email', $filters['email']); + $this->Session->write('paginate_conditions_log_org', $filters['org']); + $this->Session->write('paginate_conditions_log_action', $filters['action']); + $this->Session->write('paginate_conditions_log_model', $filters['model']); + $this->Session->write('paginate_conditions_log_model_id', $filters['model_id']); + $this->Session->write('paginate_conditions_log_title', $filters['title']); + $this->Session->write('paginate_conditions_log_change', $filters['change']); + if (Configure::read('MISP.log_client_ip')) { + $this->Session->write('paginate_conditions_log_ip', $filters['ip']); + } - // set the same view as the index page - $this->render('admin_index'); + // set the same view as the index page + $this->render('admin_index'); + } } else { // get from Session $filters['email'] = $this->Session->read('paginate_conditions_log_email'); diff --git a/app/Lib/Dashboard/CsseCovidTrendsWidget.php b/app/Lib/Dashboard/CsseCovidTrendsWidget.php index 298276206..f9c9ef279 100644 --- a/app/Lib/Dashboard/CsseCovidTrendsWidget.php +++ b/app/Lib/Dashboard/CsseCovidTrendsWidget.php @@ -8,7 +8,7 @@ class CsseCovidTrendsWidget public $height = 5; public $params = array( 'event_info' => 'Substring included in the info field of relevant CSSE COVID-19 events.', - 'type' => 'Type of data used for the widget - confirmed (default), death, recovered, mortality.', + 'type' => 'Type of data used for the widget - confirmed (default), death, recovered, mortality, active.', 'insight' => 'Insight type - raw (default), growth, percent.', 'countries' => 'List of countries to be included (using the names used by the reports, such as Belgium, US, Germany).', 'timeframe' => 'Timeframe for events taken into account in days (going back from now, using the date field, default 10).' @@ -18,11 +18,17 @@ class CsseCovidTrendsWidget '{ "event_info": "%CSSE COVID-19 daily report%", "type": "confirmed", - "insight": "growth", + "insight": "raw", "countries": ["Luxembourg", "Germany", "Belgium", "France"], "timeframe": 20 }'; - //public $cacheLifetime = 600; + + private $__countryAliases = array( + 'Mainland China' => 'China', + 'Korea, South' => 'South Korea' + ); + + public $cacheLifetime = 600; public $autoRefreshDelay = false; private $__countries = array(); @@ -88,7 +94,8 @@ class CsseCovidTrendsWidget 'confirmed' => 'confirmed cases', 'death' => 'mortalities', 'recovered' => 'recoveries', - 'mortality' => 'mortality rate' + 'mortality' => 'mortality rate', + 'active' => 'active cases' ) ); $data['formula'] = sprintf( @@ -136,21 +143,23 @@ class CsseCovidTrendsWidget } if (!empty($options['insight']) && $options['insight'] !== 'raw') { if ($options['insight'] == 'growth') { - foreach ($data as $k => &$countryData) { + foreach ($data as $k => $countryData) { foreach ($countryData as $type => &$value) { - if (empty($previous[$k][$type])) { - $previous[$k][$type] = 0; + if (!isset($previous[$k][$type])) { + $previous[$k][$type] = $data[$k][$type]; } $data[$k]['growth'] = $data[$k][$type] - $previous[$k][$type]; } } } else if ($options['insight'] == 'percent') { - foreach ($data as $k => &$countryData) { + foreach ($data as $k => $countryData) { foreach ($countryData as $type => &$value) { if (empty($previous[$k][$type])) { $previous[$k][$type] = $data[$k][$type]; } - $data[$k]['percent'] = ($data[$k][$type] - $previous[$k][$type]) / $previous[$k][$type]; + if (!empty($previous[$k][$type])) { + $data[$k]['percent'] = 100 * ($data[$k][$type] - $previous[$k][$type]) / $previous[$k][$type]; + } } } } @@ -175,6 +184,15 @@ class CsseCovidTrendsWidget $data[$country][$type] = (empty($data[$country][$type]) ? $temp[$type] : ($data[$country][$type] + $temp[$type])); } } + } else if ($options['type'] === 'active') { + if (empty($data[$country]['active'])) { + $data[$country]['active'] = 0; + } + $data[$country]['active'] = + $data[$country]['active'] + + (empty($temp['confirmed']) ? 0 : $temp['confirmed']) - + (empty($temp['death']) ? 0 : $temp['death']) - + (empty($temp['recovered']) ? 0 : $temp['recovered']); } else { $type = $options['type']; if (!empty($temp[$type])) { @@ -192,6 +210,10 @@ class CsseCovidTrendsWidget if (in_array($attribute['object_relation'], $validFields)) { if ($attribute['object_relation'] !== 'country-region') { $attribute['value'] = intval($attribute['value']); + } else { + if (isset($this->__countryAliases[$attribute['value']])) { + $attribute['value'] = $this->__countryAliases[$attribute['value']]; + } } $temp[$attribute['object_relation']] = $attribute['value']; } diff --git a/app/Lib/Dashboard/CsseCovidWidget.php b/app/Lib/Dashboard/CsseCovidWidget.php index c0a9657d8..76d890f85 100644 --- a/app/Lib/Dashboard/CsseCovidWidget.php +++ b/app/Lib/Dashboard/CsseCovidWidget.php @@ -8,7 +8,7 @@ class CsseCovidWidget public $height = 4; public $params = array( 'event_info' => 'Substring included in the info field of relevant CSSE COVID-19 events.', - 'type' => 'Type of data used for the widget (confirmed, death, recovered, mortality).', + 'type' => 'Type of data used for the widget (confirmed, death, recovered, mortality, active).', 'logarithmic' => 'Use a log10 scale for the graph (set via 0/1).', 'relative' => 'Take the country\'s population size into account (count / 10M)' ); @@ -27,10 +27,10 @@ class CsseCovidWidget 'Holy See' => 'Vatican', 'Congo (Kinshasa)' => 'Democratic Republic of Congo', 'Taiwan*' => 'Taiwan', - 'Korea, South' => 'South Korea' + 'Korea, South' => 'South Korea', + 'Mainland China' => 'China' ); - private $__populationData = array(); public function handler($user, $options = array()) @@ -156,6 +156,15 @@ class CsseCovidWidget $data[$country][$type] = (empty($data[$country][$type]) ? $temp[$type] : ($data[$country][$type] + $temp[$type])); } } + } else if ($options['type'] === 'active') { + if (empty($data[$country]['active'])) { + $data[$country]['active'] = 0; + } + $data[$country]['active'] = + $data[$country]['active'] + + (empty($temp['confirmed']) ? 0 : $temp['confirmed']) - + (empty($temp['death']) ? 0 : $temp['death']) - + (empty($temp['recovered']) ? 0 : $temp['recovered']); } else { $type = $options['type']; if (!empty($temp[$type])) { diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index b7c9ba417..59402ddf2 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -123,7 +123,7 @@ class AppModel extends Model public function isAcceptedDatabaseError($errorMessage, $dataSource) { $isAccepted = false; - if ($dataSource == 'Database/Mysql') { + if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') { $errorDuplicateColumn = 'SQLSTATE[42S21]: Column already exists: 1060 Duplicate column name'; $errorDuplicateIndex = 'SQLSTATE[42000]: Syntax error or access violation: 1061 Duplicate key name'; $errorDropIndex = "/SQLSTATE\[42000\]: Syntax error or access violation: 1091 Can't DROP '[\w]+'; check that column\/key exists/"; @@ -722,7 +722,7 @@ class AppModel extends Model $sqlArray[] = "ALTER TABLE taxonomy_predicates ADD colour varchar(7) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '';"; break; case '2.4.60': - if ($dataSource == 'Database/Mysql') { + if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') { $sqlArray[] = 'CREATE TABLE IF NOT EXISTS `attribute_tags` ( `id` int(11) NOT NULL AUTO_INCREMENT, `attribute_id` int(11) NOT NULL, @@ -1595,7 +1595,7 @@ class AppModel extends Model $dataSource = $dataSourceConfig['datasource']; $this->Log = ClassRegistry::init('Log'); $indexCheckResult = array(); - if ($dataSource == 'Database/Mysql') { + if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') { $indexCheck = "SELECT INDEX_NAME FROM INFORMATION_SCHEMA.STATISTICS WHERE table_schema=DATABASE() AND table_name='" . $table . "' AND index_name LIKE '" . $field . "%';"; $indexCheckResult = $this->query($indexCheck); } elseif ($dataSource == 'Database/Postgres') { @@ -1603,7 +1603,7 @@ class AppModel extends Model $indexCheckResult[] = array('STATISTICS' => array('INDEX_NAME' => $pgIndexName)); } foreach ($indexCheckResult as $icr) { - if ($dataSource == 'Database/Mysql') { + if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') { $dropIndex = 'ALTER TABLE ' . $table . ' DROP INDEX ' . $icr['STATISTICS']['INDEX_NAME'] . ';'; } elseif ($dataSource == 'Database/Postgres') { $dropIndex = 'DROP INDEX IF EXISTS ' . $icr['STATISTICS']['INDEX_NAME'] . ';'; diff --git a/app/Model/Bruteforce.php b/app/Model/Bruteforce.php index ec74a856a..8f214dce2 100644 --- a/app/Model/Bruteforce.php +++ b/app/Model/Bruteforce.php @@ -39,7 +39,7 @@ class Bruteforce extends AppModel $dataSourceConfig = ConnectionManager::getDataSource('default')->config; $dataSource = $dataSourceConfig['datasource']; $expire = date('Y-m-d H:i:s', time()); - if ($dataSource == 'Database/Mysql') { + if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') { $sql = 'DELETE FROM bruteforces WHERE `expire` <= "' . $expire . '";'; } elseif ($dataSource == 'Database/Postgres') { $sql = 'DELETE FROM bruteforces WHERE expire <= \'' . $expire . '\';'; diff --git a/app/Model/Datasource/Database/MysqlObserver.php b/app/Model/Datasource/Database/MysqlObserver.php new file mode 100644 index 000000000..4de61f3ba --- /dev/null +++ b/app/Model/Datasource/Database/MysqlObserver.php @@ -0,0 +1,23 @@ +DecayingModel = ClassRegistry::init('DecayingModel'); } + if (!isset($options['includeEventCorrelations'])) { + $options['includeEventCorrelations'] = true; + } foreach ($possibleOptions as &$opt) { if (!isset($options[$opt])) { $options[$opt] = false; @@ -2161,7 +2164,9 @@ class Event extends AppModel } $event = $this->massageTags($event, 'Event', $options['excludeGalaxy']); // Let's find all the related events and attach it to the event itself - $results[$eventKey]['RelatedEvent'] = $this->getRelatedEvents($user, $event['Event']['id'], $sgids); + if (!empty($options['includeEventCorrelations'])) { + $results[$eventKey]['RelatedEvent'] = $this->getRelatedEvents($user, $event['Event']['id'], $sgids); + } // Let's also find all the relations for the attributes - this won't be in the xml export though if (!empty($options['includeGranularCorrelations'])) { $results[$eventKey]['RelatedAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], $sgids); @@ -2580,7 +2585,7 @@ class Event extends AppModel } return $conditions; } - + public function set_filter_uuid(&$params, $conditions, $options) { if ($options['scope'] === 'Event') { @@ -2696,6 +2701,11 @@ class Event extends AppModel { if (!empty($params[$options['filter']])) { $params[$options['filter']] = $this->convert_filters($params[$options['filter']]); + if (!empty(Configure::read('MISP.attribute_filters_block_only'))) { + if ($options['context'] === 'Event' && !empty($params[$options['filter']]['OR'])) { + unset($params[$options['filter']]['OR']); + } + } $conditions = $this->generic_add_filter($conditions, $params[$options['filter']], 'Attribute.' . $options['filter']); } return $conditions; diff --git a/app/Model/Log.php b/app/Model/Log.php index a1b34f2ae..d1ee39d7b 100644 --- a/app/Model/Log.php +++ b/app/Model/Log.php @@ -154,7 +154,7 @@ class Log extends AppModel $conditions['org'] = $org['Organisation']['name']; } $conditions['AND']['NOT'] = array('action' => array('login', 'logout', 'changepw')); - if ($dataSource == 'Database/Mysql') { + if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') { $validDates = $this->find('all', array( 'fields' => array('DISTINCT UNIX_TIMESTAMP(DATE(created)) AS Date', 'count(id) AS count'), 'conditions' => $conditions, diff --git a/app/Model/Organisation.php b/app/Model/Organisation.php index e309faf08..ef4fdfa87 100644 --- a/app/Model/Organisation.php +++ b/app/Model/Organisation.php @@ -292,7 +292,7 @@ class Organisation extends AppModel $success = true; foreach ($this->organisationAssociations as $model => $data) { foreach ($data['fields'] as $field) { - if ($dataSource == 'Database/Mysql') { + if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') { $sql = 'SELECT `id` FROM `' . $data['table'] . '` WHERE `' . $field . '` = "' . $currentOrg['Organisation']['id'] . '"'; } elseif ($dataSource == 'Database/Postgres') { $sql = 'SELECT "id" FROM "' . $data['table'] . '" WHERE "' . $field . '" = "' . $currentOrg['Organisation']['id'] . '"'; @@ -303,13 +303,13 @@ class Organisation extends AppModel if (!empty($dataMoved['values_changed'][$model][$field])) { $this->Log->create(); try { - if ($dataSource == 'Database/Mysql') { + if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') { $sql = 'UPDATE `' . $data['table'] . '` SET `' . $field . '` = ' . $targetOrg['Organisation']['id'] . ' WHERE `' . $field . '` = ' . $currentOrg['Organisation']['id'] . ';'; } elseif ($dataSource == 'Database/Postgres') { $sql = 'UPDATE "' . $data['table'] . '" SET "' . $field . '" = ' . $targetOrg['Organisation']['id'] . ' WHERE "' . $field . '" = ' . $currentOrg['Organisation']['id'] . ';'; } $result = $this->query($sql); - if ($dataSource == 'Database/Mysql') { + if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') { $sql = 'UPDATE `' . $data['table'] . '` SET `' . $field . '` = ' . $currentOrg['Organisation']['id'] . ' WHERE `id` IN (' . implode(',', $dataMoved['values_changed'][$model][$field]) . ');'; } elseif ($dataSource == 'Database/Postgres') { $sql = 'UPDATE "' . $data['table'] . '" SET "' . $field . '" = ' . $currentOrg['Organisation']['id'] . ' WHERE "id" IN (' . implode(',', $dataMoved['values_changed'][$model][$field]) . ');'; diff --git a/app/Model/Server.php b/app/Model/Server.php index 110051e4d..ea5aef715 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -1072,6 +1072,15 @@ class Server extends AppModel 'test' => 'testForNumeric', 'type' => 'numeric', 'null' => true + ), + 'attribute_filters_block_only' => array( + 'level' => 1, + 'description' => __('This is a performance tweak to change the behaviour of restSearch to use attribute filters solely for blocking. This means that a lookup on the event scope with for example the type field set will be ignored unless it\'s used to strip unwanted attributes from the results. If left disabled, passing [ip-src, ip-dst] for example will return any event with at least one ip-src or ip-dst attribute. This is generally not considered to be too useful and is a heavy burden on the database.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true ) ), 'GnuPG' => array( @@ -4411,7 +4420,7 @@ class Server extends AppModel public function dbSpaceUsage() { $dataSource = $this->getDataSource()->config['datasource']; - if ($dataSource == 'Database/Mysql') { + if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') { $sql = sprintf( 'select TABLE_NAME, sum((DATA_LENGTH+INDEX_LENGTH)/1024/1024) AS used, sum(DATA_FREE)/1024/1024 AS reclaimable from information_schema.tables where table_schema = %s group by TABLE_NAME;', "'" . $this->getDataSource()->config['database'] . "'" @@ -4487,7 +4496,7 @@ class Server extends AppModel 'update_fail_number_reached' => $this->UpdateFailNumberReached(), 'indexes' => array() ); - if ($dataSource == 'Database/Mysql') { + if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') { $dbActualSchema = $this->getActualDBSchema(); $dbExpectedSchema = $this->getExpectedDBSchema(); if ($dbExpectedSchema !== false) { @@ -4645,7 +4654,7 @@ class Server extends AppModel $dbActualSchema = array(); $dbActualIndexes = array(); $dataSource = $this->getDataSource()->config['datasource']; - if ($dataSource == 'Database/Mysql') { + if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') { $sqlGetTable = sprintf('SELECT TABLE_NAME FROM information_schema.tables WHERE table_schema = %s;', "'" . $this->getDataSource()->config['database'] . "'"); $sqlResult = $this->query($sqlGetTable); $tables = HASH::extract($sqlResult, '{n}.tables.TABLE_NAME'); diff --git a/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp b/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp index 23285b24e..58810271b 100644 --- a/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp +++ b/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp @@ -500,13 +500,25 @@ function init() { // variables and functions have their own scope (n return 'translate(' + xpos + ',' + ypos + ')'; }) .on('click', function(d, i) { - d.disabled = !d.disabled; var label_text = d.text; - var label_disabled = d.disabled; - data_nodes.filter(function(d) { return d.name === label_text; }).forEach(function(data) { - data.disabled = label_disabled - }) - _draw() + if (d3.event.ctrlKey) { // hide all others + data_nodes.filter(function(fd) { return fd.name === label_text; }).forEach(function(data) { + data.disabled = false; + }) + data_nodes.filter(function(fd) { return fd.name !== label_text; }).forEach(function(data) { + data.disabled = true; + }) + d.disabled = false; + legend_labels.filter(function(fd) { return fd.text !== label_text}).forEach(function(label_data) { + label_data.disabled = true; + }) + } else { // hide it + d.disabled = !d.disabled; + data_nodes.filter(function(fd) { return fd.name === label_text; }).forEach(function(data) { + data.disabled = d.disabled; + }) + } + _draw(); }); } } diff --git a/app/files/misp-objects b/app/files/misp-objects index 7ef9a2ba5..ef01e6e37 160000 --- a/app/files/misp-objects +++ b/app/files/misp-objects @@ -1 +1 @@ -Subproject commit 7ef9a2ba56efc6553a720d6df27c9ee547e24242 +Subproject commit ef01e6e37b025a71b40515bc0a9d4e11fef20798 diff --git a/app/files/scripts/stix2misp.py b/app/files/scripts/stix2misp.py index dc71dca78..c547a115f 100644 --- a/app/files/scripts/stix2misp.py +++ b/app/files/scripts/stix2misp.py @@ -1088,7 +1088,7 @@ class StixFromMISPParser(StixParser): # Parse STIX object that we know will give MISP objects def parse_misp_object_indicator(self, indicator): item = indicator.item - name = item.title.split(' ')[0] + name = item.title.split(': ')[0] if name not in ('passive-dns'): self.fill_misp_object(item, name, to_ids=True) else: diff --git a/app/files/taxonomies b/app/files/taxonomies index d577ad875..28e7cb79f 160000 --- a/app/files/taxonomies +++ b/app/files/taxonomies @@ -1 +1 @@ -Subproject commit d577ad8758713e4d7c0523bbe2bead64c941ebdb +Subproject commit 28e7cb79f0ec603c232857a3bf7dca519d02cfa1 diff --git a/cti-python-stix2 b/cti-python-stix2 index 65a943d89..e4f08557e 160000 --- a/cti-python-stix2 +++ b/cti-python-stix2 @@ -1 +1 @@ -Subproject commit 65a943d8929c578041f789665b05810ea68986cb +Subproject commit e4f08557ec93c589a71a6e4060134661f1c4b2c0 diff --git a/tools/misp-zmq/slackbot.py b/tools/misp-zmq/slackbot.py new file mode 100644 index 000000000..4679e9248 --- /dev/null +++ b/tools/misp-zmq/slackbot.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +''' +### MISP to Slack #### +ZMQ client to post events, attributes or sighting updates from a MISP instance to a slack channel. + +This tool is part of the MISP core project and released under the GNU Affero +General Public License v3.0 + +Copyright (C) 2020 Christophe Vandeplas + +For instructions on creating your BOT, please read: https://api.slack.com/bot-users +Your bot will need the permissions: +- channels:join +- chat:write +- users:write + +WARNING WARNING - THIS SCRIPT DOES NOT MAGICALLY RESPECT ACLs +MAKE SURE YOU SET THE RIGHT FILTERS IN THE SETTINGS +''' + + +import argparse +import sys +import time +import zmq +import json +try: + import slack +except ImportError: + exit("Missing slackclient dependency. Please 'pip3 install slackclient'") +try: + from slackbot_settings import channel_name, slack_token, misp_url, misp_is_public, allowed_distributions, allowed_sharing_groups, max_value_len, include_attr, include_obj +except ImportError: + exit("Missing slackbot_settings.py. Please create from 'slackbot_settings.py.sample'") + + +def sanitize_value(s): + # very dirty cleanup + s = s.replace('http', 'hxxp') + s = s.replace('.', '[.]') + s = s.replace('@', '[AT]') + s = s.replace('\n', ' ') + # truncate long strings + return (s[:max_value_len] + '..') if len(s) > max_value_len else s + + +def gen_attrs_text(attrs): + attrs_text_lst = [] + type_value_mapping = {} + for a in attrs: + try: + type_value_mapping[a['type']].add(sanitize_value(a['value'])) + except Exception: + type_value_mapping[a['type']] = set() + type_value_mapping[a['type']].add(sanitize_value(a['value'])) + for k, v in type_value_mapping.items(): + attrs_text_lst.append(f"- *{k}*: {','.join(v)}") + attrs_text = '\n'.join(attrs_text_lst) + return attrs_text + + +def publish_event(e): + cnt_attr = len(e.get('Attribute') or '') + cnt_obj = len(e.get('Object') or '') + cnt_tags = len(e.get('Tag') or '') + url = misp_url + '/events/view/' + e['id'] + zmq_message_short = f"New MISP event '{e['info']}' with {cnt_attr} attributes, {cnt_obj} objects and {cnt_tags} tags." + + image_url = 'https://raw.githubusercontent.com/MISP/MISP/2.4/docs/img/misp.png' + if misp_is_public: + image_url = f"{misp_url}/img/orgs/{e['Orgc']['name']}.png" + + zmq_message_blocks = [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": f"*A new MISP <{url}|event> has been published:*\n" + f"Title: {e['info']}\n" + f"Date: {e['date']}\n" + f"Threat Level: {e['threat_level_id']}\n" + f"Contains {cnt_attr} attributes, {cnt_obj} objects and {cnt_tags} tags\n" + f"Full event: <{url}|{url}>" + }, + "accessory": { + "type": "image", + "image_url": image_url, + "alt_text": "MISP or org logo" + } + } + ] + + if 'Tag' in e: + tag_block = { + "type": "actions", + "elements": [ + ] + } + tags = set([t['name'] for t in e['Tag']]) + for a in e['Attribute']: + if 'Tag' in a: + for t in a['Tag']: + tags.add(t['name']) + for o in e['Object']: + for a in o['Attribute']: + if 'Tag' in a: + for t in a['Tag']: + tags.add(t['name']) + + tags = sorted(tags) + for t in tags: + t = t.replace('misp-galaxy:', '').replace('mitre-', '') + tag_block['elements'].append({ + "type": "button", + "text": { + "type": "plain_text", + "text": t + }, + "value": "#" + }) + zmq_message_blocks.append(tag_block) + + # List attributes + if include_attr: + zmq_message_blocks.append({"type": "divider"}) + attrs_text = gen_attrs_text(e['Attribute']) + if attrs_text: + zmq_message_blocks.append( + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": f"*Attributes:*\n{attrs_text}" + } + } + ) + # List Objects + if include_obj: + zmq_message_blocks.append({"type": "divider"}) + for o in e['Object']: + attrs_text = gen_attrs_text(o['Attribute']) + if attrs_text: + # print(json.dumps(o, indent=2)) + zmq_message_blocks.append( + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": f"*{o['name'].capitalize()} object:*\n{attrs_text}" + } + } + ) + + # Send the message + client = slack.WebClient(token=slack_token) + client.users_setPresence(presence='auto') + channel = client.channels_join(name=channel_name) + client.chat_postMessage( + channel=channel['channel']['id'], + text=zmq_message_short, + blocks=zmq_message_blocks + ) + + +parser = argparse.ArgumentParser(description='MISP to Slack bot - ZMQ client to gather events, attributes and sighting updates from a MISP instance') +parser.add_argument("-p", "--port", default="50000", help='set TCP port of the MISP ZMQ (default: 50000)') +parser.add_argument("-r", "--host", default="127.0.0.1", help='set host of the MISP ZMQ (default: 127.0.0.1)') +parser.add_argument("-t", "--sleep", default=0.1, help='sleep time (default: 0.1)', type=int) +args = parser.parse_args() + +port = args.port +host = args.host +context = zmq.Context() +socket = context.socket(zmq.SUB) +socket.connect("tcp://%s:%s" % (host, port)) +socket.setsockopt(zmq.SUBSCRIBE, b'') + +poller = zmq.Poller() +poller.register(socket, zmq.POLLIN) + +while True: + socks = dict(poller.poll(timeout=None)) + if socket in socks and socks[socket] == zmq.POLLIN: + message = socket.recv() + topic, s, m = message.decode('utf-8').partition(" ") + + try: + m_json = json.loads(m) + except Exception: + sys.stderr.write(f'Ignoring non-json message: {m}') + time.sleep(args.sleep) + continue + + if 'status' in m_json: + pass + elif 'Event' in m_json: + # print(m_json) + e = m_json['Event'] + if '*' in allowed_distributions or \ + (e['distribution'] in allowed_distributions and ( + e['distribution'] != '5' or ( + '*' in allowed_sharing_groups or e['sharing_group_id'] in allowed_sharing_groups) + )): + print(f"Publishing event {e['id']} on slack") + publish_event(e) + else: + print(f"Ignoring event {e['id']} as it has a filtered distribution.") + else: + print(f'Non supported message: {m}') + time.sleep(args.sleep) diff --git a/tools/misp-zmq/slackbot_settings.py.sample b/tools/misp-zmq/slackbot_settings.py.sample new file mode 100644 index 000000000..a254e2faa --- /dev/null +++ b/tools/misp-zmq/slackbot_settings.py.sample @@ -0,0 +1,24 @@ +''' +For instructions on creating your BOT, please read: https://api.slack.com/bot-users +Your bot will need the permissions: +- channels:join +- chat:write +- users:write + +WARNING WARNING - THIS SCRIPT DOES NOT MAGICALLY RESPECT ACLs +MAKE SURE YOU SET THE RIGHT FILTERS IN THE SETTINGS +''' + +channel_name = '#name' +slack_token = '' + +misp_url = 'https://192.168.1.1' +misp_is_public = True # set to False if your MISP instance is on a non-internet reachable location. Shows the org icon of the event owner. Otherwise shows the MISP logo. + +# filter for confidentiality +allowed_distributions = ['0', '1', '2', '3', '4'] # * = all, 0/ my org only, 1/ this community, 2/ connected communities, 3/ all communities, 4/ sharing group +allowed_sharing_groups = ['*'] # put here the sharing_group_ids that you allow + +max_value_len = 25 # truncate values longer than X chars +include_attr = True # include attributes in the message +include_obj = True # include objects in the message diff --git a/tools/misp-zmq/sub.py b/tools/misp-zmq/sub.py index 73b8d21f3..17330a89a 100644 --- a/tools/misp-zmq/sub.py +++ b/tools/misp-zmq/sub.py @@ -17,11 +17,11 @@ import pprint pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr) parser = argparse.ArgumentParser(description='Generic ZMQ client to gather events, attributes and sighting updates from a MISP instance') -parser.add_argument("-s","--stats", default=False, action='store_true', help='print regular statistics on stderr') -parser.add_argument("-p","--port", default="50000", help='set TCP port of the MISP ZMQ (default: 50000)') -parser.add_argument("-r","--host", default="127.0.0.1", help='set host of the MISP ZMQ (default: 127.0.0.1)') -parser.add_argument("-o","--only", action="append", default=None, help="set filter (misp_json, misp_json_event, misp_json_attribute or misp_json_sighting) to limit the output a specific type (default: no filter)") -parser.add_argument("-t","--sleep", default=0.1, help='sleep time (default: 0.1)', type=int) +parser.add_argument("-s", "--stats", default=False, action='store_true', help='print regular statistics on stderr') +parser.add_argument("-p", "--port", default="50000", help='set TCP port of the MISP ZMQ (default: 50000)') +parser.add_argument("-r", "--host", default="127.0.0.1", help='set host of the MISP ZMQ (default: 127.0.0.1)') +parser.add_argument("-o", "--only", action="append", default=None, help="set filter (misp_json, misp_json_event, misp_json_attribute or misp_json_sighting) to limit the output a specific type (default: no filter)") +parser.add_argument("-t", "--sleep", default=0.1, help='sleep time (default: 0.1)', type=int) args = parser.parse_args() if args.only is not None: @@ -35,7 +35,7 @@ port = args.port host = args.host context = zmq.Context() socket = context.socket(zmq.SUB) -socket.connect ("tcp://%s:%s" % (host, port)) +socket.connect("tcp://%s:%s" % (host, port)) socket.setsockopt(zmq.SUBSCRIBE, b'') poller = zmq.Poller() @@ -52,9 +52,8 @@ while True: if args.only: if topic not in filters: continue - print (m) + print(m) if args.stats: stats[topic] = stats.get(topic, 0) + 1 pp.pprint(stats) time.sleep(args.sleep) -