MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch '2.4' of https://github.com/MISP/MISP into rework_stix

pull/6022/head
chrisr3d 2020-04-16 17:55:31 +02:00
parent ca7abf45f7 9ca3fafca4
commit 4ffb45eafc
21 changed files with 410 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
app/Controller/AppController.php
View File

@ -182,6 +182,8 @@ class AppController extends Controller
if (!empty($this->params['named']['disable_background_processing'])) {
Configure::write('MISP.background_jobs', 0);
}
Configure::write('CurrentController', $this->params['controller']);
Configure::write('CurrentAction', $this->params['action']);
$versionArray = $this->{$this->modelClass}->checkMISPVersion();
$this->mispVersion = implode('.', array_values($versionArray));
$this->Security->blackHoleCallback = 'blackHole';
@ -298,6 +300,7 @@ class AppController extends Controller
}
if ($this->Auth->user()) {
Configure::write('CurrentUserId', $this->Auth->user('id'));
$this->User->setMonitoring($this->Auth->user());
if (Configure::read('MISP.log_user_ips')) {
$redis = $this->{$this->modelClass}->setupRedis();
@ -606,7 +609,7 @@ class AppController extends Controller
ConnectionManager::create('default', $db->config);
}
$dataSource = $dataSourceConfig['datasource'];
if ($dataSource != 'Database/Mysql' && $dataSource != 'Database/Postgres') {
if (!in_array($dataSource, array('Database/Mysql', 'Database/Postgres', 'Database/MysqlObserver'))) {
throw new Exception('datasource not supported: ' . $dataSource);
}
}

2
app/Controller/Component/RestResponseComponent.php
View File

@ -449,7 +449,7 @@ class RestResponseComponent extends Component
}
if (Configure::read('debug') > 1 && !empty($this->Controller->sql_dump)) {
$this->Log = ClassRegistry::init('Log');
if ($this->Content->sql_dump === 2) {
if ($this->Controller->sql_dump === 2) {
$response = array('sql_dump' => $this->Log->getDataSource()->getLog(false, false));
} else {
$response['sql_dump'] = $this->Log->getDataSource()->getLog(false, false);

35
app/Controller/EventsController.php
View File

@ -315,11 +315,11 @@ class EventsController extends AppController
break;
case 'attribute':
$event_id_arrays = $this->__filterOnAttributeValue($v);
foreach ($event_id_arrays[0] as $event_id) {
$this->paginate['conditions']['AND']['OR'][] = array('Event.id' => $event_id);
if (!empty($event_id_arrays[0])) {
$this->paginate['conditions']['AND'][] = array('Event.id' => $event_id_arrays[0]);
}
foreach ($event_id_arrays[1] as $event_id) {
$this->paginate['conditions']['AND'][] = array('Event.id !=' => $event_id);
if (!empty($event_id_arrays[1])) {
$this->paginate['conditions']['AND'][] = array('Event.id !=' => $event_id_arrays[1]);
}
break;
case 'published':
@ -342,25 +342,38 @@ class EventsController extends AppController
if ($v == "") {
continue 2;
}
$pieces = explode('|', $v);
if (is_array($v)) {
$pieces = $v;
} else {
$pieces = explode('|', $v);
}
$temp = array();
$eventidConditions = array();
foreach ($pieces as $piece) {
$piece = trim($piece);
if ($piece[0] == '!') {
if (strlen($piece) == 37) {
$this->paginate['conditions']['AND'][] = array('Event.uuid !=' => substr($piece, 1));
$eventidConditions['NOT']['uuid'][] = substr($piece, 1);
} else {
$this->paginate['conditions']['AND'][] = array('Event.id !=' => substr($piece, 1));
$eventidConditions['NOT']['id'][] = substr($piece, 1);
}
} else {
if (strlen($piece) == 36) {
$temp['OR'][] = array('Event.uuid' => $piece);
$eventidConditions['OR']['uuid'][] = $piece;
} else {
$temp['OR'][] = array('Event.id' => $piece);
$eventidConditions['OR']['id'][] = $piece;
}
}
}
$this->paginate['conditions']['AND'][] = $temp;
foreach ($eventidConditions as $operator => $conditionForOperator) {
foreach ($conditionForOperator as $conditionKey => $conditionValue) {
$lookupKey = 'Event.' . $conditionKey;
if ($operator === 'NOT') {
$lookupKey = $lookupKey . ' !=';
}
$this->paginate['conditions']['AND'][] = array($lookupKey => $conditionValue);
}
}
break;
case 'datefrom':
if ($v == "") {
@ -727,8 +740,6 @@ class EventsController extends AppController
} else {
$rules['order'] = array('Event.' . $passedArgs['sort'] => 'ASC');
}
} else {
$rules['order'] = array('Event.id' => 'DESC');
}
$rules['contain'] = $this->paginate['contain'];
if (isset($this->paginate['conditions'])) {

32
app/Controller/LogsController.php
View File

@ -349,21 +349,25 @@ class LogsController extends AppController
}
$this->set('list', $list);
// and store into session
$this->Session->write('paginate_conditions_log', $this->paginate);
$this->Session->write('paginate_conditions_log_email', $filters['email']);
$this->Session->write('paginate_conditions_log_org', $filters['org']);
$this->Session->write('paginate_conditions_log_action', $filters['action']);
$this->Session->write('paginate_conditions_log_model', $filters['model']);
$this->Session->write('paginate_conditions_log_model_id', $filters['model_id']);
$this->Session->write('paginate_conditions_log_title', $filters['title']);
$this->Session->write('paginate_conditions_log_change', $filters['change']);
if (Configure::read('MISP.log_client_ip')) {
$this->Session->write('paginate_conditions_log_ip', $filters['ip']);
}
if ($this->_isRest()) {
return $this->RestResponse->viewData($list, $this->response->type());
} else {
// and store into session
$this->Session->write('paginate_conditions_log', $this->paginate);
$this->Session->write('paginate_conditions_log_email', $filters['email']);
$this->Session->write('paginate_conditions_log_org', $filters['org']);
$this->Session->write('paginate_conditions_log_action', $filters['action']);
$this->Session->write('paginate_conditions_log_model', $filters['model']);
$this->Session->write('paginate_conditions_log_model_id', $filters['model_id']);
$this->Session->write('paginate_conditions_log_title', $filters['title']);
$this->Session->write('paginate_conditions_log_change', $filters['change']);
if (Configure::read('MISP.log_client_ip')) {
$this->Session->write('paginate_conditions_log_ip', $filters['ip']);
}
// set the same view as the index page
$this->render('admin_index');
// set the same view as the index page
$this->render('admin_index');
}
} else {
// get from Session
$filters['email'] = $this->Session->read('paginate_conditions_log_email');

40
app/Lib/Dashboard/CsseCovidTrendsWidget.php
View File

@ -8,7 +8,7 @@ class CsseCovidTrendsWidget
public $height = 5;
public $params = array(
'event_info' => 'Substring included in the info field of relevant CSSE COVID-19 events.',
'type' => 'Type of data used for the widget - confirmed (default), death, recovered, mortality.',
'type' => 'Type of data used for the widget - confirmed (default), death, recovered, mortality, active.',
'insight' => 'Insight type - raw (default), growth, percent.',
'countries' => 'List of countries to be included (using the names used by the reports, such as Belgium, US, Germany).',
'timeframe' => 'Timeframe for events taken into account in days (going back from now, using the date field, default 10).'
@ -18,11 +18,17 @@ class CsseCovidTrendsWidget
'{
"event_info": "%CSSE COVID-19 daily report%",
"type": "confirmed",
"insight": "growth",
"insight": "raw",
"countries": ["Luxembourg", "Germany", "Belgium", "France"],
"timeframe": 20
}';
//public $cacheLifetime = 600;
private $__countryAliases = array(
'Mainland China' => 'China',
'Korea, South' => 'South Korea'
);
public $cacheLifetime = 600;
public $autoRefreshDelay = false;
private $__countries = array();
@ -88,7 +94,8 @@ class CsseCovidTrendsWidget
'confirmed' => 'confirmed cases',
'death' => 'mortalities',
'recovered' => 'recoveries',
'mortality' => 'mortality rate'
'mortality' => 'mortality rate',
'active' => 'active cases'
)
);
$data['formula'] = sprintf(
@ -136,21 +143,23 @@ class CsseCovidTrendsWidget
}
if (!empty($options['insight']) && $options['insight'] !== 'raw') {
if ($options['insight'] == 'growth') {
foreach ($data as $k => &$countryData) {
foreach ($data as $k => $countryData) {
foreach ($countryData as $type => &$value) {
if (empty($previous[$k][$type])) {
$previous[$k][$type] = 0;
if (!isset($previous[$k][$type])) {
$previous[$k][$type] = $data[$k][$type];
}
$data[$k]['growth'] = $data[$k][$type] - $previous[$k][$type];
}
}
} else if ($options['insight'] == 'percent') {
foreach ($data as $k => &$countryData) {
foreach ($data as $k => $countryData) {
foreach ($countryData as $type => &$value) {
if (empty($previous[$k][$type])) {
$previous[$k][$type] = $data[$k][$type];
}
$data[$k]['percent'] = ($data[$k][$type] - $previous[$k][$type]) / $previous[$k][$type];
if (!empty($previous[$k][$type])) {
$data[$k]['percent'] = 100 * ($data[$k][$type] - $previous[$k][$type]) / $previous[$k][$type];
}
}
}
}
@ -175,6 +184,15 @@ class CsseCovidTrendsWidget
$data[$country][$type] = (empty($data[$country][$type]) ? $temp[$type] : ($data[$country][$type] + $temp[$type]));
}
}
} else if ($options['type'] === 'active') {
if (empty($data[$country]['active'])) {
$data[$country]['active'] = 0;
}
$data[$country]['active'] =
$data[$country]['active'] +
(empty($temp['confirmed']) ? 0 : $temp['confirmed']) -
(empty($temp['death']) ? 0 : $temp['death']) -
(empty($temp['recovered']) ? 0 : $temp['recovered']);
} else {
$type = $options['type'];
if (!empty($temp[$type])) {
@ -192,6 +210,10 @@ class CsseCovidTrendsWidget
if (in_array($attribute['object_relation'], $validFields)) {
if ($attribute['object_relation'] !== 'country-region') {
$attribute['value'] = intval($attribute['value']);
} else {
if (isset($this->__countryAliases[$attribute['value']])) {
$attribute['value'] = $this->__countryAliases[$attribute['value']];
}
}
$temp[$attribute['object_relation']] = $attribute['value'];
}

15
app/Lib/Dashboard/CsseCovidWidget.php
View File

@ -8,7 +8,7 @@ class CsseCovidWidget
public $height = 4;
public $params = array(
'event_info' => 'Substring included in the info field of relevant CSSE COVID-19 events.',
'type' => 'Type of data used for the widget (confirmed, death, recovered, mortality).',
'type' => 'Type of data used for the widget (confirmed, death, recovered, mortality, active).',
'logarithmic' => 'Use a log10 scale for the graph (set via 0/1).',
'relative' => 'Take the country\'s population size into account (count / 10M)'
);
@ -27,10 +27,10 @@ class CsseCovidWidget
'Holy See' => 'Vatican',
'Congo (Kinshasa)' => 'Democratic Republic of Congo',
'Taiwan*' => 'Taiwan',
'Korea, South' => 'South Korea'
'Korea, South' => 'South Korea',
'Mainland China' => 'China'
);
private $__populationData = array();
public function handler($user, $options = array())
@ -156,6 +156,15 @@ class CsseCovidWidget
$data[$country][$type] = (empty($data[$country][$type]) ? $temp[$type] : ($data[$country][$type] + $temp[$type]));
}
}
} else if ($options['type'] === 'active') {
if (empty($data[$country]['active'])) {
$data[$country]['active'] = 0;
}
$data[$country]['active'] =
$data[$country]['active'] +
(empty($temp['confirmed']) ? 0 : $temp['confirmed']) -
(empty($temp['death']) ? 0 : $temp['death']) -
(empty($temp['recovered']) ? 0 : $temp['recovered']);
} else {
$type = $options['type'];
if (!empty($temp[$type])) {

8
app/Model/AppModel.php
View File

@ -123,7 +123,7 @@ class AppModel extends Model
public function isAcceptedDatabaseError($errorMessage, $dataSource)
{
$isAccepted = false;
if ($dataSource == 'Database/Mysql') {
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
$errorDuplicateColumn = 'SQLSTATE[42S21]: Column already exists: 1060 Duplicate column name';
$errorDuplicateIndex = 'SQLSTATE[42000]: Syntax error or access violation: 1061 Duplicate key name';
$errorDropIndex = "/SQLSTATE\[42000\]: Syntax error or access violation: 1091 Can't DROP '[\w]+'; check that column\/key exists/";
@ -722,7 +722,7 @@ class AppModel extends Model
$sqlArray[] = "ALTER TABLE taxonomy_predicates ADD colour varchar(7) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '';";
break;
case '2.4.60':
if ($dataSource == 'Database/Mysql') {
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
$sqlArray[] = 'CREATE TABLE IF NOT EXISTS `attribute_tags` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`attribute_id` int(11) NOT NULL,
@ -1595,7 +1595,7 @@ class AppModel extends Model
$dataSource = $dataSourceConfig['datasource'];
$this->Log = ClassRegistry::init('Log');
$indexCheckResult = array();
if ($dataSource == 'Database/Mysql') {
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
$indexCheck = "SELECT INDEX_NAME FROM INFORMATION_SCHEMA.STATISTICS WHERE table_schema=DATABASE() AND table_name='" . $table . "' AND index_name LIKE '" . $field . "%';";
$indexCheckResult = $this->query($indexCheck);
} elseif ($dataSource == 'Database/Postgres') {
@ -1603,7 +1603,7 @@ class AppModel extends Model
$indexCheckResult[] = array('STATISTICS' => array('INDEX_NAME' => $pgIndexName));
}
foreach ($indexCheckResult as $icr) {
if ($dataSource == 'Database/Mysql') {
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
$dropIndex = 'ALTER TABLE ' . $table . ' DROP INDEX ' . $icr['STATISTICS']['INDEX_NAME'] . ';';
} elseif ($dataSource == 'Database/Postgres') {
$dropIndex = 'DROP INDEX IF EXISTS ' . $icr['STATISTICS']['INDEX_NAME'] . ';';

2
app/Model/Bruteforce.php
View File

@ -39,7 +39,7 @@ class Bruteforce extends AppModel
$dataSourceConfig = ConnectionManager::getDataSource('default')->config;
$dataSource = $dataSourceConfig['datasource'];
$expire = date('Y-m-d H:i:s', time());
if ($dataSource == 'Database/Mysql') {
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
$sql = 'DELETE FROM bruteforces WHERE `expire` <= "' . $expire . '";';
} elseif ($dataSource == 'Database/Postgres') {
$sql = 'DELETE FROM bruteforces WHERE expire <= \'' . $expire . '\';';

23
app/Model/Datasource/Database/MysqlObserver.php Normal file
View File

@ -0,0 +1,23 @@
<?php
App::uses('Mysql', 'Model/Datasource/Database');
/*
* Overrides the default MySQL database implementation to prepend all queries with debug comments
* - Lightweight and doesn't affect default operations, like a protoss observer it remains cloaked
* whilst trying to help detect potential bugs burrowed in our queries
*/
class MysqlObserver extends Mysql {
public function execute($sql, $options = array(), $params = array()) {
$comment = sprintf(
'%s%s%s',
empty(Configure::read('CurrentUserId')) ? '' : sprintf(
'[User: %s] ',
intval(Configure::read('CurrentUserId'))
),
empty(Configure::read('CurrentController')) ? '' : preg_replace('/[^a-zA-Z0-9_]/', '', Configure::read('CurrentController')) . ' :: ',
empty(Configure::read('CurrentAction')) ? '' : preg_replace('/[^a-zA-Z0-9_]/', '', Configure::read('CurrentAction'))
);
$sql = '/* ' . $comment . ' */ ' . $sql;
return parent::execute($sql, $options, $params);
}
}

14
app/Model/Event.php
View File

@ -1873,6 +1873,9 @@ class Event extends AppModel
if (!empty($options['includeDecayScore'])) {
$this->DecayingModel = ClassRegistry::init('DecayingModel');
}
if (!isset($options['includeEventCorrelations'])) {
$options['includeEventCorrelations'] = true;
}
foreach ($possibleOptions as &$opt) {
if (!isset($options[$opt])) {
$options[$opt] = false;
@ -2161,7 +2164,9 @@ class Event extends AppModel
}
$event = $this->massageTags($event, 'Event', $options['excludeGalaxy']);
// Let's find all the related events and attach it to the event itself
$results[$eventKey]['RelatedEvent'] = $this->getRelatedEvents($user, $event['Event']['id'], $sgids);
if (!empty($options['includeEventCorrelations'])) {
$results[$eventKey]['RelatedEvent'] = $this->getRelatedEvents($user, $event['Event']['id'], $sgids);
}
// Let's also find all the relations for the attributes - this won't be in the xml export though
if (!empty($options['includeGranularCorrelations'])) {
$results[$eventKey]['RelatedAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], $sgids);
@ -2580,7 +2585,7 @@ class Event extends AppModel
}
return $conditions;
}
public function set_filter_uuid(&$params, $conditions, $options)
{
if ($options['scope'] === 'Event') {
@ -2696,6 +2701,11 @@ class Event extends AppModel
{
if (!empty($params[$options['filter']])) {
$params[$options['filter']] = $this->convert_filters($params[$options['filter']]);
if (!empty(Configure::read('MISP.attribute_filters_block_only'))) {
if ($options['context'] === 'Event' && !empty($params[$options['filter']]['OR'])) {
unset($params[$options['filter']]['OR']);
}
}
$conditions = $this->generic_add_filter($conditions, $params[$options['filter']], 'Attribute.' . $options['filter']);
}
return $conditions;

2
app/Model/Log.php
View File

@ -154,7 +154,7 @@ class Log extends AppModel
$conditions['org'] = $org['Organisation']['name'];
}
$conditions['AND']['NOT'] = array('action' => array('login', 'logout', 'changepw'));
if ($dataSource == 'Database/Mysql') {
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
$validDates = $this->find('all', array(
'fields' => array('DISTINCT UNIX_TIMESTAMP(DATE(created)) AS Date', 'count(id) AS count'),
'conditions' => $conditions,

6
app/Model/Organisation.php
View File

@ -292,7 +292,7 @@ class Organisation extends AppModel
$success = true;
foreach ($this->organisationAssociations as $model => $data) {
foreach ($data['fields'] as $field) {
if ($dataSource == 'Database/Mysql') {
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
$sql = 'SELECT `id` FROM `' . $data['table'] . '` WHERE `' . $field . '` = "' . $currentOrg['Organisation']['id'] . '"';
} elseif ($dataSource == 'Database/Postgres') {
$sql = 'SELECT "id" FROM "' . $data['table'] . '" WHERE "' . $field . '" = "' . $currentOrg['Organisation']['id'] . '"';
@ -303,13 +303,13 @@ class Organisation extends AppModel
if (!empty($dataMoved['values_changed'][$model][$field])) {
$this->Log->create();
try {
if ($dataSource == 'Database/Mysql') {
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
$sql = 'UPDATE `' . $data['table'] . '` SET `' . $field . '` = ' . $targetOrg['Organisation']['id'] . ' WHERE `' . $field . '` = ' . $currentOrg['Organisation']['id'] . ';';
} elseif ($dataSource == 'Database/Postgres') {
$sql = 'UPDATE "' . $data['table'] . '" SET "' . $field . '" = ' . $targetOrg['Organisation']['id'] . ' WHERE "' . $field . '" = ' . $currentOrg['Organisation']['id'] . ';';
}
$result = $this->query($sql);
if ($dataSource == 'Database/Mysql') {
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
$sql = 'UPDATE `' . $data['table'] . '` SET `' . $field . '` = ' . $currentOrg['Organisation']['id'] . ' WHERE `id` IN (' . implode(',', $dataMoved['values_changed'][$model][$field]) . ');';
} elseif ($dataSource == 'Database/Postgres') {
$sql = 'UPDATE "' . $data['table'] . '" SET "' . $field . '" = ' . $currentOrg['Organisation']['id'] . ' WHERE "id" IN (' . implode(',', $dataMoved['values_changed'][$model][$field]) . ');';

15
app/Model/Server.php
View File

@ -1072,6 +1072,15 @@ class Server extends AppModel
'test' => 'testForNumeric',
'type' => 'numeric',
'null' => true
),
'attribute_filters_block_only' => array(
'level' => 1,
'description' => __('This is a performance tweak to change the behaviour of restSearch to use attribute filters solely for blocking. This means that a lookup on the event scope with for example the type field set will be ignored unless it\'s used to strip unwanted attributes from the results. If left disabled, passing [ip-src, ip-dst] for example will return any event with at least one ip-src or ip-dst attribute. This is generally not considered to be too useful and is a heavy burden on the database.'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean',
'null' => true
)
),
'GnuPG' => array(
@ -4411,7 +4420,7 @@ class Server extends AppModel
public function dbSpaceUsage()
{
$dataSource = $this->getDataSource()->config['datasource'];
if ($dataSource == 'Database/Mysql') {
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
$sql = sprintf(
'select TABLE_NAME, sum((DATA_LENGTH+INDEX_LENGTH)/1024/1024) AS used, sum(DATA_FREE)/1024/1024 AS reclaimable from information_schema.tables where table_schema = %s group by TABLE_NAME;',
"'" . $this->getDataSource()->config['database'] . "'"
@ -4487,7 +4496,7 @@ class Server extends AppModel
'update_fail_number_reached' => $this->UpdateFailNumberReached(),
'indexes' => array()
);
if ($dataSource == 'Database/Mysql') {
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
$dbActualSchema = $this->getActualDBSchema();
$dbExpectedSchema = $this->getExpectedDBSchema();
if ($dbExpectedSchema !== false) {
@ -4645,7 +4654,7 @@ class Server extends AppModel
$dbActualSchema = array();
$dbActualIndexes = array();
$dataSource = $this->getDataSource()->config['datasource'];
if ($dataSource == 'Database/Mysql') {
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
$sqlGetTable = sprintf('SELECT TABLE_NAME FROM information_schema.tables WHERE table_schema = %s;', "'" . $this->getDataSource()->config['database'] . "'");
$sqlResult = $this->query($sqlGetTable);
$tables = HASH::extract($sqlResult, '{n}.tables.TABLE_NAME');

24
app/View/Elements/dashboard/Widgets/MultiLineChart.ctp
View File

@ -500,13 +500,25 @@ function init<?= $seed ?>() { // variables and functions have their own scope (n
return 'translate(' + xpos + ',' + ypos + ')';
})
.on('click', function(d, i) {
d.disabled = !d.disabled;
var label_text = d.text;
var label_disabled = d.disabled;
data_nodes.filter(function(d) { return d.name === label_text; }).forEach(function(data) {
data.disabled = label_disabled
})
_draw()
if (d3.event.ctrlKey) { // hide all others
data_nodes.filter(function(fd) { return fd.name === label_text; }).forEach(function(data) {
data.disabled = false;
})
data_nodes.filter(function(fd) { return fd.name !== label_text; }).forEach(function(data) {
data.disabled = true;
})
d.disabled = false;
legend_labels.filter(function(fd) { return fd.text !== label_text}).forEach(function(label_data) {
label_data.disabled = true;
})
} else { // hide it
d.disabled = !d.disabled;
data_nodes.filter(function(fd) { return fd.name === label_text; }).forEach(function(data) {
data.disabled = d.disabled;
})
}
_draw();
});
}
}

2
app/files/misp-objects

@ -1 +1 @@
Subproject commit 7ef9a2ba56efc6553a720d6df27c9ee547e24242
Subproject commit ef01e6e37b025a71b40515bc0a9d4e11fef20798

2
app/files/scripts/stix2misp.py
View File

@ -1088,7 +1088,7 @@ class StixFromMISPParser(StixParser):
# Parse STIX object that we know will give MISP objects
def parse_misp_object_indicator(self, indicator):
item = indicator.item
name = item.title.split(' ')[0]
name = item.title.split(': ')[0]
if name not in ('passive-dns'):
self.fill_misp_object(item, name, to_ids=True)
else:

2
app/files/taxonomies

@ -1 +1 @@
Subproject commit d577ad8758713e4d7c0523bbe2bead64c941ebdb
Subproject commit 28e7cb79f0ec603c232857a3bf7dca519d02cfa1

2
cti-python-stix2

@ -1 +1 @@
Subproject commit 65a943d8929c578041f789665b05810ea68986cb
Subproject commit e4f08557ec93c589a71a6e4060134661f1c4b2c0

212
tools/misp-zmq/slackbot.py Normal file
View File

@ -0,0 +1,212 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
'''
### MISP to Slack ####
ZMQ client to post events, attributes or sighting updates from a MISP instance to a slack channel.
This tool is part of the MISP core project and released under the GNU Affero
General Public License v3.0
Copyright (C) 2020 Christophe Vandeplas
For instructions on creating your BOT, please read: https://api.slack.com/bot-users
Your bot will need the permissions:
- channels:join
- chat:write
- users:write
WARNING WARNING - THIS SCRIPT DOES NOT MAGICALLY RESPECT ACLs
MAKE SURE YOU SET THE RIGHT FILTERS IN THE SETTINGS
'''
import argparse
import sys
import time
import zmq
import json
try:
import slack
except ImportError:
exit("Missing slackclient dependency. Please 'pip3 install slackclient'")
try:
from slackbot_settings import channel_name, slack_token, misp_url, misp_is_public, allowed_distributions, allowed_sharing_groups, max_value_len, include_attr, include_obj
except ImportError:
exit("Missing slackbot_settings.py. Please create from 'slackbot_settings.py.sample'")
def sanitize_value(s):
# very dirty cleanup
s = s.replace('http', 'hxxp')
s = s.replace('.', '[.]')
s = s.replace('@', '[AT]')
s = s.replace('\n', ' ')
# truncate long strings
return (s[:max_value_len] + '..') if len(s) > max_value_len else s
def gen_attrs_text(attrs):
attrs_text_lst = []
type_value_mapping = {}
for a in attrs:
try:
type_value_mapping[a['type']].add(sanitize_value(a['value']))
except Exception:
type_value_mapping[a['type']] = set()
type_value_mapping[a['type']].add(sanitize_value(a['value']))
for k, v in type_value_mapping.items():
attrs_text_lst.append(f"- *{k}*: {','.join(v)}")
attrs_text = '\n'.join(attrs_text_lst)
return attrs_text
def publish_event(e):
cnt_attr = len(e.get('Attribute') or '')
cnt_obj = len(e.get('Object') or '')
cnt_tags = len(e.get('Tag') or '')
url = misp_url + '/events/view/' + e['id']
zmq_message_short = f"New MISP event '{e['info']}' with {cnt_attr} attributes, {cnt_obj} objects and {cnt_tags} tags."
image_url = 'https://raw.githubusercontent.com/MISP/MISP/2.4/docs/img/misp.png'
if misp_is_public:
image_url = f"{misp_url}/img/orgs/{e['Orgc']['name']}.png"
zmq_message_blocks = [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": f"*A new MISP <{url}|event> has been published:*\n"
f"Title: {e['info']}\n"
f"Date: {e['date']}\n"
f"Threat Level: {e['threat_level_id']}\n"
f"Contains {cnt_attr} attributes, {cnt_obj} objects and {cnt_tags} tags\n"
f"Full event: <{url}|{url}>"
},
"accessory": {
"type": "image",
"image_url": image_url,
"alt_text": "MISP or org logo"
}
}
]
if 'Tag' in e:
tag_block = {
"type": "actions",
"elements": [
]
}
tags = set([t['name'] for t in e['Tag']])
for a in e['Attribute']:
if 'Tag' in a:
for t in a['Tag']:
tags.add(t['name'])
for o in e['Object']:
for a in o['Attribute']:
if 'Tag' in a:
for t in a['Tag']:
tags.add(t['name'])
tags = sorted(tags)
for t in tags:
t = t.replace('misp-galaxy:', '').replace('mitre-', '')
tag_block['elements'].append({
"type": "button",
"text": {
"type": "plain_text",
"text": t
},
"value": "#"
})
zmq_message_blocks.append(tag_block)
# List attributes
if include_attr:
zmq_message_blocks.append({"type": "divider"})
attrs_text = gen_attrs_text(e['Attribute'])
if attrs_text:
zmq_message_blocks.append(
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": f"*Attributes:*\n{attrs_text}"
}
}
)
# List Objects
if include_obj:
zmq_message_blocks.append({"type": "divider"})
for o in e['Object']:
attrs_text = gen_attrs_text(o['Attribute'])
if attrs_text:
# print(json.dumps(o, indent=2))
zmq_message_blocks.append(
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": f"*{o['name'].capitalize()} object:*\n{attrs_text}"
}
}
)
# Send the message
client = slack.WebClient(token=slack_token)
client.users_setPresence(presence='auto')
channel = client.channels_join(name=channel_name)
client.chat_postMessage(
channel=channel['channel']['id'],
text=zmq_message_short,
blocks=zmq_message_blocks
)
parser = argparse.ArgumentParser(description='MISP to Slack bot - ZMQ client to gather events, attributes and sighting updates from a MISP instance')
parser.add_argument("-p", "--port", default="50000", help='set TCP port of the MISP ZMQ (default: 50000)')
parser.add_argument("-r", "--host", default="127.0.0.1", help='set host of the MISP ZMQ (default: 127.0.0.1)')
parser.add_argument("-t", "--sleep", default=0.1, help='sleep time (default: 0.1)', type=int)
args = parser.parse_args()
port = args.port
host = args.host
context = zmq.Context()
socket = context.socket(zmq.SUB)
socket.connect("tcp://%s:%s" % (host, port))
socket.setsockopt(zmq.SUBSCRIBE, b'')
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
while True:
socks = dict(poller.poll(timeout=None))
if socket in socks and socks[socket] == zmq.POLLIN:
message = socket.recv()
topic, s, m = message.decode('utf-8').partition(" ")
try:
m_json = json.loads(m)
except Exception:
sys.stderr.write(f'Ignoring non-json message: {m}')
time.sleep(args.sleep)
continue
if 'status' in m_json:
pass
elif 'Event' in m_json:
# print(m_json)
e = m_json['Event']
if '*' in allowed_distributions or \
(e['distribution'] in allowed_distributions and (
e['distribution'] != '5' or (
'*' in allowed_sharing_groups or e['sharing_group_id'] in allowed_sharing_groups)
)):
print(f"Publishing event {e['id']} on slack")
publish_event(e)
else:
print(f"Ignoring event {e['id']} as it has a filtered distribution.")
else:
print(f'Non supported message: {m}')
time.sleep(args.sleep)

24
tools/misp-zmq/slackbot_settings.py.sample Normal file
View File

@ -0,0 +1,24 @@
'''
For instructions on creating your BOT, please read: https://api.slack.com/bot-users
Your bot will need the permissions:
- channels:join
- chat:write
- users:write
WARNING WARNING - THIS SCRIPT DOES NOT MAGICALLY RESPECT ACLs
MAKE SURE YOU SET THE RIGHT FILTERS IN THE SETTINGS
'''
channel_name = '#name'
slack_token = ''
misp_url = 'https://192.168.1.1'
misp_is_public = True # set to False if your MISP instance is on a non-internet reachable location. Shows the org icon of the event owner. Otherwise shows the MISP logo.
# filter for confidentiality
allowed_distributions = ['0', '1', '2', '3', '4'] # * = all, 0/ my org only, 1/ this community, 2/ connected communities, 3/ all communities, 4/ sharing group
allowed_sharing_groups = ['*'] # put here the sharing_group_ids that you allow
max_value_len = 25 # truncate values longer than X chars
include_attr = True # include attributes in the message
include_obj = True # include objects in the message

15
tools/misp-zmq/sub.py
View File

@ -17,11 +17,11 @@ import pprint
pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr)
parser = argparse.ArgumentParser(description='Generic ZMQ client to gather events, attributes and sighting updates from a MISP instance')
parser.add_argument("-s","--stats", default=False, action='store_true', help='print regular statistics on stderr')
parser.add_argument("-p","--port", default="50000", help='set TCP port of the MISP ZMQ (default: 50000)')
parser.add_argument("-r","--host", default="127.0.0.1", help='set host of the MISP ZMQ (default: 127.0.0.1)')
parser.add_argument("-o","--only", action="append", default=None, help="set filter (misp_json, misp_json_event, misp_json_attribute or misp_json_sighting) to limit the output a specific type (default: no filter)")
parser.add_argument("-t","--sleep", default=0.1, help='sleep time (default: 0.1)', type=int)
parser.add_argument("-s", "--stats", default=False, action='store_true', help='print regular statistics on stderr')
parser.add_argument("-p", "--port", default="50000", help='set TCP port of the MISP ZMQ (default: 50000)')
parser.add_argument("-r", "--host", default="127.0.0.1", help='set host of the MISP ZMQ (default: 127.0.0.1)')
parser.add_argument("-o", "--only", action="append", default=None, help="set filter (misp_json, misp_json_event, misp_json_attribute or misp_json_sighting) to limit the output a specific type (default: no filter)")
parser.add_argument("-t", "--sleep", default=0.1, help='sleep time (default: 0.1)', type=int)
args = parser.parse_args()
if args.only is not None:
@ -35,7 +35,7 @@ port = args.port
host = args.host
context = zmq.Context()
socket = context.socket(zmq.SUB)
socket.connect ("tcp://%s:%s" % (host, port))
socket.connect("tcp://%s:%s" % (host, port))
socket.setsockopt(zmq.SUBSCRIBE, b'')
poller = zmq.Poller()
@ -52,9 +52,8 @@ while True:
if args.only:
if topic not in filters:
continue
print (m)
print(m)
if args.stats:
stats[topic] = stats.get(topic, 0) + 1
pp.pprint(stats)
time.sleep(args.sleep)