mirror of https://github.com/MISP/MISP
Merge branch '2.4' of https://github.com/MISP/MISP into rework_stix
commit
4ffb45eafc
|
@ -182,6 +182,8 @@ class AppController extends Controller
|
|||
if (!empty($this->params['named']['disable_background_processing'])) {
|
||||
Configure::write('MISP.background_jobs', 0);
|
||||
}
|
||||
Configure::write('CurrentController', $this->params['controller']);
|
||||
Configure::write('CurrentAction', $this->params['action']);
|
||||
$versionArray = $this->{$this->modelClass}->checkMISPVersion();
|
||||
$this->mispVersion = implode('.', array_values($versionArray));
|
||||
$this->Security->blackHoleCallback = 'blackHole';
|
||||
|
@ -298,6 +300,7 @@ class AppController extends Controller
|
|||
}
|
||||
|
||||
if ($this->Auth->user()) {
|
||||
Configure::write('CurrentUserId', $this->Auth->user('id'));
|
||||
$this->User->setMonitoring($this->Auth->user());
|
||||
if (Configure::read('MISP.log_user_ips')) {
|
||||
$redis = $this->{$this->modelClass}->setupRedis();
|
||||
|
@ -606,7 +609,7 @@ class AppController extends Controller
|
|||
ConnectionManager::create('default', $db->config);
|
||||
}
|
||||
$dataSource = $dataSourceConfig['datasource'];
|
||||
if ($dataSource != 'Database/Mysql' && $dataSource != 'Database/Postgres') {
|
||||
if (!in_array($dataSource, array('Database/Mysql', 'Database/Postgres', 'Database/MysqlObserver'))) {
|
||||
throw new Exception('datasource not supported: ' . $dataSource);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -449,7 +449,7 @@ class RestResponseComponent extends Component
|
|||
}
|
||||
if (Configure::read('debug') > 1 && !empty($this->Controller->sql_dump)) {
|
||||
$this->Log = ClassRegistry::init('Log');
|
||||
if ($this->Content->sql_dump === 2) {
|
||||
if ($this->Controller->sql_dump === 2) {
|
||||
$response = array('sql_dump' => $this->Log->getDataSource()->getLog(false, false));
|
||||
} else {
|
||||
$response['sql_dump'] = $this->Log->getDataSource()->getLog(false, false);
|
||||
|
|
|
@ -315,11 +315,11 @@ class EventsController extends AppController
|
|||
break;
|
||||
case 'attribute':
|
||||
$event_id_arrays = $this->__filterOnAttributeValue($v);
|
||||
foreach ($event_id_arrays[0] as $event_id) {
|
||||
$this->paginate['conditions']['AND']['OR'][] = array('Event.id' => $event_id);
|
||||
if (!empty($event_id_arrays[0])) {
|
||||
$this->paginate['conditions']['AND'][] = array('Event.id' => $event_id_arrays[0]);
|
||||
}
|
||||
foreach ($event_id_arrays[1] as $event_id) {
|
||||
$this->paginate['conditions']['AND'][] = array('Event.id !=' => $event_id);
|
||||
if (!empty($event_id_arrays[1])) {
|
||||
$this->paginate['conditions']['AND'][] = array('Event.id !=' => $event_id_arrays[1]);
|
||||
}
|
||||
break;
|
||||
case 'published':
|
||||
|
@ -342,25 +342,38 @@ class EventsController extends AppController
|
|||
if ($v == "") {
|
||||
continue 2;
|
||||
}
|
||||
if (is_array($v)) {
|
||||
$pieces = $v;
|
||||
} else {
|
||||
$pieces = explode('|', $v);
|
||||
}
|
||||
$temp = array();
|
||||
$eventidConditions = array();
|
||||
foreach ($pieces as $piece) {
|
||||
$piece = trim($piece);
|
||||
if ($piece[0] == '!') {
|
||||
if (strlen($piece) == 37) {
|
||||
$this->paginate['conditions']['AND'][] = array('Event.uuid !=' => substr($piece, 1));
|
||||
$eventidConditions['NOT']['uuid'][] = substr($piece, 1);
|
||||
} else {
|
||||
$this->paginate['conditions']['AND'][] = array('Event.id !=' => substr($piece, 1));
|
||||
$eventidConditions['NOT']['id'][] = substr($piece, 1);
|
||||
}
|
||||
} else {
|
||||
if (strlen($piece) == 36) {
|
||||
$temp['OR'][] = array('Event.uuid' => $piece);
|
||||
$eventidConditions['OR']['uuid'][] = $piece;
|
||||
} else {
|
||||
$temp['OR'][] = array('Event.id' => $piece);
|
||||
$eventidConditions['OR']['id'][] = $piece;
|
||||
}
|
||||
}
|
||||
}
|
||||
$this->paginate['conditions']['AND'][] = $temp;
|
||||
foreach ($eventidConditions as $operator => $conditionForOperator) {
|
||||
foreach ($conditionForOperator as $conditionKey => $conditionValue) {
|
||||
$lookupKey = 'Event.' . $conditionKey;
|
||||
if ($operator === 'NOT') {
|
||||
$lookupKey = $lookupKey . ' !=';
|
||||
}
|
||||
$this->paginate['conditions']['AND'][] = array($lookupKey => $conditionValue);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 'datefrom':
|
||||
if ($v == "") {
|
||||
|
@ -727,8 +740,6 @@ class EventsController extends AppController
|
|||
} else {
|
||||
$rules['order'] = array('Event.' . $passedArgs['sort'] => 'ASC');
|
||||
}
|
||||
} else {
|
||||
$rules['order'] = array('Event.id' => 'DESC');
|
||||
}
|
||||
$rules['contain'] = $this->paginate['contain'];
|
||||
if (isset($this->paginate['conditions'])) {
|
||||
|
|
|
@ -349,6 +349,9 @@ class LogsController extends AppController
|
|||
}
|
||||
$this->set('list', $list);
|
||||
|
||||
if ($this->_isRest()) {
|
||||
return $this->RestResponse->viewData($list, $this->response->type());
|
||||
} else {
|
||||
// and store into session
|
||||
$this->Session->write('paginate_conditions_log', $this->paginate);
|
||||
$this->Session->write('paginate_conditions_log_email', $filters['email']);
|
||||
|
@ -364,6 +367,7 @@ class LogsController extends AppController
|
|||
|
||||
// set the same view as the index page
|
||||
$this->render('admin_index');
|
||||
}
|
||||
} else {
|
||||
// get from Session
|
||||
$filters['email'] = $this->Session->read('paginate_conditions_log_email');
|
||||
|
|
|
@ -8,7 +8,7 @@ class CsseCovidTrendsWidget
|
|||
public $height = 5;
|
||||
public $params = array(
|
||||
'event_info' => 'Substring included in the info field of relevant CSSE COVID-19 events.',
|
||||
'type' => 'Type of data used for the widget - confirmed (default), death, recovered, mortality.',
|
||||
'type' => 'Type of data used for the widget - confirmed (default), death, recovered, mortality, active.',
|
||||
'insight' => 'Insight type - raw (default), growth, percent.',
|
||||
'countries' => 'List of countries to be included (using the names used by the reports, such as Belgium, US, Germany).',
|
||||
'timeframe' => 'Timeframe for events taken into account in days (going back from now, using the date field, default 10).'
|
||||
|
@ -18,11 +18,17 @@ class CsseCovidTrendsWidget
|
|||
'{
|
||||
"event_info": "%CSSE COVID-19 daily report%",
|
||||
"type": "confirmed",
|
||||
"insight": "growth",
|
||||
"insight": "raw",
|
||||
"countries": ["Luxembourg", "Germany", "Belgium", "France"],
|
||||
"timeframe": 20
|
||||
}';
|
||||
//public $cacheLifetime = 600;
|
||||
|
||||
private $__countryAliases = array(
|
||||
'Mainland China' => 'China',
|
||||
'Korea, South' => 'South Korea'
|
||||
);
|
||||
|
||||
public $cacheLifetime = 600;
|
||||
public $autoRefreshDelay = false;
|
||||
|
||||
private $__countries = array();
|
||||
|
@ -88,7 +94,8 @@ class CsseCovidTrendsWidget
|
|||
'confirmed' => 'confirmed cases',
|
||||
'death' => 'mortalities',
|
||||
'recovered' => 'recoveries',
|
||||
'mortality' => 'mortality rate'
|
||||
'mortality' => 'mortality rate',
|
||||
'active' => 'active cases'
|
||||
)
|
||||
);
|
||||
$data['formula'] = sprintf(
|
||||
|
@ -136,21 +143,23 @@ class CsseCovidTrendsWidget
|
|||
}
|
||||
if (!empty($options['insight']) && $options['insight'] !== 'raw') {
|
||||
if ($options['insight'] == 'growth') {
|
||||
foreach ($data as $k => &$countryData) {
|
||||
foreach ($data as $k => $countryData) {
|
||||
foreach ($countryData as $type => &$value) {
|
||||
if (empty($previous[$k][$type])) {
|
||||
$previous[$k][$type] = 0;
|
||||
if (!isset($previous[$k][$type])) {
|
||||
$previous[$k][$type] = $data[$k][$type];
|
||||
}
|
||||
$data[$k]['growth'] = $data[$k][$type] - $previous[$k][$type];
|
||||
}
|
||||
}
|
||||
} else if ($options['insight'] == 'percent') {
|
||||
foreach ($data as $k => &$countryData) {
|
||||
foreach ($data as $k => $countryData) {
|
||||
foreach ($countryData as $type => &$value) {
|
||||
if (empty($previous[$k][$type])) {
|
||||
$previous[$k][$type] = $data[$k][$type];
|
||||
}
|
||||
$data[$k]['percent'] = ($data[$k][$type] - $previous[$k][$type]) / $previous[$k][$type];
|
||||
if (!empty($previous[$k][$type])) {
|
||||
$data[$k]['percent'] = 100 * ($data[$k][$type] - $previous[$k][$type]) / $previous[$k][$type];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -175,6 +184,15 @@ class CsseCovidTrendsWidget
|
|||
$data[$country][$type] = (empty($data[$country][$type]) ? $temp[$type] : ($data[$country][$type] + $temp[$type]));
|
||||
}
|
||||
}
|
||||
} else if ($options['type'] === 'active') {
|
||||
if (empty($data[$country]['active'])) {
|
||||
$data[$country]['active'] = 0;
|
||||
}
|
||||
$data[$country]['active'] =
|
||||
$data[$country]['active'] +
|
||||
(empty($temp['confirmed']) ? 0 : $temp['confirmed']) -
|
||||
(empty($temp['death']) ? 0 : $temp['death']) -
|
||||
(empty($temp['recovered']) ? 0 : $temp['recovered']);
|
||||
} else {
|
||||
$type = $options['type'];
|
||||
if (!empty($temp[$type])) {
|
||||
|
@ -192,6 +210,10 @@ class CsseCovidTrendsWidget
|
|||
if (in_array($attribute['object_relation'], $validFields)) {
|
||||
if ($attribute['object_relation'] !== 'country-region') {
|
||||
$attribute['value'] = intval($attribute['value']);
|
||||
} else {
|
||||
if (isset($this->__countryAliases[$attribute['value']])) {
|
||||
$attribute['value'] = $this->__countryAliases[$attribute['value']];
|
||||
}
|
||||
}
|
||||
$temp[$attribute['object_relation']] = $attribute['value'];
|
||||
}
|
||||
|
|
|
@ -8,7 +8,7 @@ class CsseCovidWidget
|
|||
public $height = 4;
|
||||
public $params = array(
|
||||
'event_info' => 'Substring included in the info field of relevant CSSE COVID-19 events.',
|
||||
'type' => 'Type of data used for the widget (confirmed, death, recovered, mortality).',
|
||||
'type' => 'Type of data used for the widget (confirmed, death, recovered, mortality, active).',
|
||||
'logarithmic' => 'Use a log10 scale for the graph (set via 0/1).',
|
||||
'relative' => 'Take the country\'s population size into account (count / 10M)'
|
||||
);
|
||||
|
@ -27,10 +27,10 @@ class CsseCovidWidget
|
|||
'Holy See' => 'Vatican',
|
||||
'Congo (Kinshasa)' => 'Democratic Republic of Congo',
|
||||
'Taiwan*' => 'Taiwan',
|
||||
'Korea, South' => 'South Korea'
|
||||
'Korea, South' => 'South Korea',
|
||||
'Mainland China' => 'China'
|
||||
);
|
||||
|
||||
|
||||
private $__populationData = array();
|
||||
|
||||
public function handler($user, $options = array())
|
||||
|
@ -156,6 +156,15 @@ class CsseCovidWidget
|
|||
$data[$country][$type] = (empty($data[$country][$type]) ? $temp[$type] : ($data[$country][$type] + $temp[$type]));
|
||||
}
|
||||
}
|
||||
} else if ($options['type'] === 'active') {
|
||||
if (empty($data[$country]['active'])) {
|
||||
$data[$country]['active'] = 0;
|
||||
}
|
||||
$data[$country]['active'] =
|
||||
$data[$country]['active'] +
|
||||
(empty($temp['confirmed']) ? 0 : $temp['confirmed']) -
|
||||
(empty($temp['death']) ? 0 : $temp['death']) -
|
||||
(empty($temp['recovered']) ? 0 : $temp['recovered']);
|
||||
} else {
|
||||
$type = $options['type'];
|
||||
if (!empty($temp[$type])) {
|
||||
|
|
|
@ -123,7 +123,7 @@ class AppModel extends Model
|
|||
public function isAcceptedDatabaseError($errorMessage, $dataSource)
|
||||
{
|
||||
$isAccepted = false;
|
||||
if ($dataSource == 'Database/Mysql') {
|
||||
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
|
||||
$errorDuplicateColumn = 'SQLSTATE[42S21]: Column already exists: 1060 Duplicate column name';
|
||||
$errorDuplicateIndex = 'SQLSTATE[42000]: Syntax error or access violation: 1061 Duplicate key name';
|
||||
$errorDropIndex = "/SQLSTATE\[42000\]: Syntax error or access violation: 1091 Can't DROP '[\w]+'; check that column\/key exists/";
|
||||
|
@ -722,7 +722,7 @@ class AppModel extends Model
|
|||
$sqlArray[] = "ALTER TABLE taxonomy_predicates ADD colour varchar(7) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '';";
|
||||
break;
|
||||
case '2.4.60':
|
||||
if ($dataSource == 'Database/Mysql') {
|
||||
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
|
||||
$sqlArray[] = 'CREATE TABLE IF NOT EXISTS `attribute_tags` (
|
||||
`id` int(11) NOT NULL AUTO_INCREMENT,
|
||||
`attribute_id` int(11) NOT NULL,
|
||||
|
@ -1595,7 +1595,7 @@ class AppModel extends Model
|
|||
$dataSource = $dataSourceConfig['datasource'];
|
||||
$this->Log = ClassRegistry::init('Log');
|
||||
$indexCheckResult = array();
|
||||
if ($dataSource == 'Database/Mysql') {
|
||||
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
|
||||
$indexCheck = "SELECT INDEX_NAME FROM INFORMATION_SCHEMA.STATISTICS WHERE table_schema=DATABASE() AND table_name='" . $table . "' AND index_name LIKE '" . $field . "%';";
|
||||
$indexCheckResult = $this->query($indexCheck);
|
||||
} elseif ($dataSource == 'Database/Postgres') {
|
||||
|
@ -1603,7 +1603,7 @@ class AppModel extends Model
|
|||
$indexCheckResult[] = array('STATISTICS' => array('INDEX_NAME' => $pgIndexName));
|
||||
}
|
||||
foreach ($indexCheckResult as $icr) {
|
||||
if ($dataSource == 'Database/Mysql') {
|
||||
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
|
||||
$dropIndex = 'ALTER TABLE ' . $table . ' DROP INDEX ' . $icr['STATISTICS']['INDEX_NAME'] . ';';
|
||||
} elseif ($dataSource == 'Database/Postgres') {
|
||||
$dropIndex = 'DROP INDEX IF EXISTS ' . $icr['STATISTICS']['INDEX_NAME'] . ';';
|
||||
|
|
|
@ -39,7 +39,7 @@ class Bruteforce extends AppModel
|
|||
$dataSourceConfig = ConnectionManager::getDataSource('default')->config;
|
||||
$dataSource = $dataSourceConfig['datasource'];
|
||||
$expire = date('Y-m-d H:i:s', time());
|
||||
if ($dataSource == 'Database/Mysql') {
|
||||
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
|
||||
$sql = 'DELETE FROM bruteforces WHERE `expire` <= "' . $expire . '";';
|
||||
} elseif ($dataSource == 'Database/Postgres') {
|
||||
$sql = 'DELETE FROM bruteforces WHERE expire <= \'' . $expire . '\';';
|
||||
|
|
|
@ -0,0 +1,23 @@
|
|||
<?php
|
||||
App::uses('Mysql', 'Model/Datasource/Database');
|
||||
|
||||
/*
|
||||
* Overrides the default MySQL database implementation to prepend all queries with debug comments
|
||||
* - Lightweight and doesn't affect default operations, like a protoss observer it remains cloaked
|
||||
* whilst trying to help detect potential bugs burrowed in our queries
|
||||
*/
|
||||
class MysqlObserver extends Mysql {
|
||||
public function execute($sql, $options = array(), $params = array()) {
|
||||
$comment = sprintf(
|
||||
'%s%s%s',
|
||||
empty(Configure::read('CurrentUserId')) ? '' : sprintf(
|
||||
'[User: %s] ',
|
||||
intval(Configure::read('CurrentUserId'))
|
||||
),
|
||||
empty(Configure::read('CurrentController')) ? '' : preg_replace('/[^a-zA-Z0-9_]/', '', Configure::read('CurrentController')) . ' :: ',
|
||||
empty(Configure::read('CurrentAction')) ? '' : preg_replace('/[^a-zA-Z0-9_]/', '', Configure::read('CurrentAction'))
|
||||
);
|
||||
$sql = '/* ' . $comment . ' */ ' . $sql;
|
||||
return parent::execute($sql, $options, $params);
|
||||
}
|
||||
}
|
|
@ -1873,6 +1873,9 @@ class Event extends AppModel
|
|||
if (!empty($options['includeDecayScore'])) {
|
||||
$this->DecayingModel = ClassRegistry::init('DecayingModel');
|
||||
}
|
||||
if (!isset($options['includeEventCorrelations'])) {
|
||||
$options['includeEventCorrelations'] = true;
|
||||
}
|
||||
foreach ($possibleOptions as &$opt) {
|
||||
if (!isset($options[$opt])) {
|
||||
$options[$opt] = false;
|
||||
|
@ -2161,7 +2164,9 @@ class Event extends AppModel
|
|||
}
|
||||
$event = $this->massageTags($event, 'Event', $options['excludeGalaxy']);
|
||||
// Let's find all the related events and attach it to the event itself
|
||||
if (!empty($options['includeEventCorrelations'])) {
|
||||
$results[$eventKey]['RelatedEvent'] = $this->getRelatedEvents($user, $event['Event']['id'], $sgids);
|
||||
}
|
||||
// Let's also find all the relations for the attributes - this won't be in the xml export though
|
||||
if (!empty($options['includeGranularCorrelations'])) {
|
||||
$results[$eventKey]['RelatedAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], $sgids);
|
||||
|
@ -2696,6 +2701,11 @@ class Event extends AppModel
|
|||
{
|
||||
if (!empty($params[$options['filter']])) {
|
||||
$params[$options['filter']] = $this->convert_filters($params[$options['filter']]);
|
||||
if (!empty(Configure::read('MISP.attribute_filters_block_only'))) {
|
||||
if ($options['context'] === 'Event' && !empty($params[$options['filter']]['OR'])) {
|
||||
unset($params[$options['filter']]['OR']);
|
||||
}
|
||||
}
|
||||
$conditions = $this->generic_add_filter($conditions, $params[$options['filter']], 'Attribute.' . $options['filter']);
|
||||
}
|
||||
return $conditions;
|
||||
|
|
|
@ -154,7 +154,7 @@ class Log extends AppModel
|
|||
$conditions['org'] = $org['Organisation']['name'];
|
||||
}
|
||||
$conditions['AND']['NOT'] = array('action' => array('login', 'logout', 'changepw'));
|
||||
if ($dataSource == 'Database/Mysql') {
|
||||
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
|
||||
$validDates = $this->find('all', array(
|
||||
'fields' => array('DISTINCT UNIX_TIMESTAMP(DATE(created)) AS Date', 'count(id) AS count'),
|
||||
'conditions' => $conditions,
|
||||
|
|
|
@ -292,7 +292,7 @@ class Organisation extends AppModel
|
|||
$success = true;
|
||||
foreach ($this->organisationAssociations as $model => $data) {
|
||||
foreach ($data['fields'] as $field) {
|
||||
if ($dataSource == 'Database/Mysql') {
|
||||
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
|
||||
$sql = 'SELECT `id` FROM `' . $data['table'] . '` WHERE `' . $field . '` = "' . $currentOrg['Organisation']['id'] . '"';
|
||||
} elseif ($dataSource == 'Database/Postgres') {
|
||||
$sql = 'SELECT "id" FROM "' . $data['table'] . '" WHERE "' . $field . '" = "' . $currentOrg['Organisation']['id'] . '"';
|
||||
|
@ -303,13 +303,13 @@ class Organisation extends AppModel
|
|||
if (!empty($dataMoved['values_changed'][$model][$field])) {
|
||||
$this->Log->create();
|
||||
try {
|
||||
if ($dataSource == 'Database/Mysql') {
|
||||
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
|
||||
$sql = 'UPDATE `' . $data['table'] . '` SET `' . $field . '` = ' . $targetOrg['Organisation']['id'] . ' WHERE `' . $field . '` = ' . $currentOrg['Organisation']['id'] . ';';
|
||||
} elseif ($dataSource == 'Database/Postgres') {
|
||||
$sql = 'UPDATE "' . $data['table'] . '" SET "' . $field . '" = ' . $targetOrg['Organisation']['id'] . ' WHERE "' . $field . '" = ' . $currentOrg['Organisation']['id'] . ';';
|
||||
}
|
||||
$result = $this->query($sql);
|
||||
if ($dataSource == 'Database/Mysql') {
|
||||
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
|
||||
$sql = 'UPDATE `' . $data['table'] . '` SET `' . $field . '` = ' . $currentOrg['Organisation']['id'] . ' WHERE `id` IN (' . implode(',', $dataMoved['values_changed'][$model][$field]) . ');';
|
||||
} elseif ($dataSource == 'Database/Postgres') {
|
||||
$sql = 'UPDATE "' . $data['table'] . '" SET "' . $field . '" = ' . $currentOrg['Organisation']['id'] . ' WHERE "id" IN (' . implode(',', $dataMoved['values_changed'][$model][$field]) . ');';
|
||||
|
|
|
@ -1072,6 +1072,15 @@ class Server extends AppModel
|
|||
'test' => 'testForNumeric',
|
||||
'type' => 'numeric',
|
||||
'null' => true
|
||||
),
|
||||
'attribute_filters_block_only' => array(
|
||||
'level' => 1,
|
||||
'description' => __('This is a performance tweak to change the behaviour of restSearch to use attribute filters solely for blocking. This means that a lookup on the event scope with for example the type field set will be ignored unless it\'s used to strip unwanted attributes from the results. If left disabled, passing [ip-src, ip-dst] for example will return any event with at least one ip-src or ip-dst attribute. This is generally not considered to be too useful and is a heavy burden on the database.'),
|
||||
'value' => false,
|
||||
'errorMessage' => '',
|
||||
'test' => 'testBool',
|
||||
'type' => 'boolean',
|
||||
'null' => true
|
||||
)
|
||||
),
|
||||
'GnuPG' => array(
|
||||
|
@ -4411,7 +4420,7 @@ class Server extends AppModel
|
|||
public function dbSpaceUsage()
|
||||
{
|
||||
$dataSource = $this->getDataSource()->config['datasource'];
|
||||
if ($dataSource == 'Database/Mysql') {
|
||||
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
|
||||
$sql = sprintf(
|
||||
'select TABLE_NAME, sum((DATA_LENGTH+INDEX_LENGTH)/1024/1024) AS used, sum(DATA_FREE)/1024/1024 AS reclaimable from information_schema.tables where table_schema = %s group by TABLE_NAME;',
|
||||
"'" . $this->getDataSource()->config['database'] . "'"
|
||||
|
@ -4487,7 +4496,7 @@ class Server extends AppModel
|
|||
'update_fail_number_reached' => $this->UpdateFailNumberReached(),
|
||||
'indexes' => array()
|
||||
);
|
||||
if ($dataSource == 'Database/Mysql') {
|
||||
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
|
||||
$dbActualSchema = $this->getActualDBSchema();
|
||||
$dbExpectedSchema = $this->getExpectedDBSchema();
|
||||
if ($dbExpectedSchema !== false) {
|
||||
|
@ -4645,7 +4654,7 @@ class Server extends AppModel
|
|||
$dbActualSchema = array();
|
||||
$dbActualIndexes = array();
|
||||
$dataSource = $this->getDataSource()->config['datasource'];
|
||||
if ($dataSource == 'Database/Mysql') {
|
||||
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
|
||||
$sqlGetTable = sprintf('SELECT TABLE_NAME FROM information_schema.tables WHERE table_schema = %s;', "'" . $this->getDataSource()->config['database'] . "'");
|
||||
$sqlResult = $this->query($sqlGetTable);
|
||||
$tables = HASH::extract($sqlResult, '{n}.tables.TABLE_NAME');
|
||||
|
|
|
@ -500,13 +500,25 @@ function init<?= $seed ?>() { // variables and functions have their own scope (n
|
|||
return 'translate(' + xpos + ',' + ypos + ')';
|
||||
})
|
||||
.on('click', function(d, i) {
|
||||
d.disabled = !d.disabled;
|
||||
var label_text = d.text;
|
||||
var label_disabled = d.disabled;
|
||||
data_nodes.filter(function(d) { return d.name === label_text; }).forEach(function(data) {
|
||||
data.disabled = label_disabled
|
||||
if (d3.event.ctrlKey) { // hide all others
|
||||
data_nodes.filter(function(fd) { return fd.name === label_text; }).forEach(function(data) {
|
||||
data.disabled = false;
|
||||
})
|
||||
_draw()
|
||||
data_nodes.filter(function(fd) { return fd.name !== label_text; }).forEach(function(data) {
|
||||
data.disabled = true;
|
||||
})
|
||||
d.disabled = false;
|
||||
legend_labels.filter(function(fd) { return fd.text !== label_text}).forEach(function(label_data) {
|
||||
label_data.disabled = true;
|
||||
})
|
||||
} else { // hide it
|
||||
d.disabled = !d.disabled;
|
||||
data_nodes.filter(function(fd) { return fd.name === label_text; }).forEach(function(data) {
|
||||
data.disabled = d.disabled;
|
||||
})
|
||||
}
|
||||
_draw();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1 +1 @@
|
|||
Subproject commit 7ef9a2ba56efc6553a720d6df27c9ee547e24242
|
||||
Subproject commit ef01e6e37b025a71b40515bc0a9d4e11fef20798
|
|
@ -1088,7 +1088,7 @@ class StixFromMISPParser(StixParser):
|
|||
# Parse STIX object that we know will give MISP objects
|
||||
def parse_misp_object_indicator(self, indicator):
|
||||
item = indicator.item
|
||||
name = item.title.split(' ')[0]
|
||||
name = item.title.split(': ')[0]
|
||||
if name not in ('passive-dns'):
|
||||
self.fill_misp_object(item, name, to_ids=True)
|
||||
else:
|
||||
|
|
|
@ -1 +1 @@
|
|||
Subproject commit d577ad8758713e4d7c0523bbe2bead64c941ebdb
|
||||
Subproject commit 28e7cb79f0ec603c232857a3bf7dca519d02cfa1
|
|
@ -1 +1 @@
|
|||
Subproject commit 65a943d8929c578041f789665b05810ea68986cb
|
||||
Subproject commit e4f08557ec93c589a71a6e4060134661f1c4b2c0
|
|
@ -0,0 +1,212 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
'''
|
||||
### MISP to Slack ####
|
||||
ZMQ client to post events, attributes or sighting updates from a MISP instance to a slack channel.
|
||||
|
||||
This tool is part of the MISP core project and released under the GNU Affero
|
||||
General Public License v3.0
|
||||
|
||||
Copyright (C) 2020 Christophe Vandeplas
|
||||
|
||||
For instructions on creating your BOT, please read: https://api.slack.com/bot-users
|
||||
Your bot will need the permissions:
|
||||
- channels:join
|
||||
- chat:write
|
||||
- users:write
|
||||
|
||||
WARNING WARNING - THIS SCRIPT DOES NOT MAGICALLY RESPECT ACLs
|
||||
MAKE SURE YOU SET THE RIGHT FILTERS IN THE SETTINGS
|
||||
'''
|
||||
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import time
|
||||
import zmq
|
||||
import json
|
||||
try:
|
||||
import slack
|
||||
except ImportError:
|
||||
exit("Missing slackclient dependency. Please 'pip3 install slackclient'")
|
||||
try:
|
||||
from slackbot_settings import channel_name, slack_token, misp_url, misp_is_public, allowed_distributions, allowed_sharing_groups, max_value_len, include_attr, include_obj
|
||||
except ImportError:
|
||||
exit("Missing slackbot_settings.py. Please create from 'slackbot_settings.py.sample'")
|
||||
|
||||
|
||||
def sanitize_value(s):
|
||||
# very dirty cleanup
|
||||
s = s.replace('http', 'hxxp')
|
||||
s = s.replace('.', '[.]')
|
||||
s = s.replace('@', '[AT]')
|
||||
s = s.replace('\n', ' ')
|
||||
# truncate long strings
|
||||
return (s[:max_value_len] + '..') if len(s) > max_value_len else s
|
||||
|
||||
|
||||
def gen_attrs_text(attrs):
|
||||
attrs_text_lst = []
|
||||
type_value_mapping = {}
|
||||
for a in attrs:
|
||||
try:
|
||||
type_value_mapping[a['type']].add(sanitize_value(a['value']))
|
||||
except Exception:
|
||||
type_value_mapping[a['type']] = set()
|
||||
type_value_mapping[a['type']].add(sanitize_value(a['value']))
|
||||
for k, v in type_value_mapping.items():
|
||||
attrs_text_lst.append(f"- *{k}*: {','.join(v)}")
|
||||
attrs_text = '\n'.join(attrs_text_lst)
|
||||
return attrs_text
|
||||
|
||||
|
||||
def publish_event(e):
|
||||
cnt_attr = len(e.get('Attribute') or '')
|
||||
cnt_obj = len(e.get('Object') or '')
|
||||
cnt_tags = len(e.get('Tag') or '')
|
||||
url = misp_url + '/events/view/' + e['id']
|
||||
zmq_message_short = f"New MISP event '{e['info']}' with {cnt_attr} attributes, {cnt_obj} objects and {cnt_tags} tags."
|
||||
|
||||
image_url = 'https://raw.githubusercontent.com/MISP/MISP/2.4/docs/img/misp.png'
|
||||
if misp_is_public:
|
||||
image_url = f"{misp_url}/img/orgs/{e['Orgc']['name']}.png"
|
||||
|
||||
zmq_message_blocks = [
|
||||
{
|
||||
"type": "section",
|
||||
"text": {
|
||||
"type": "mrkdwn",
|
||||
"text": f"*A new MISP <{url}|event> has been published:*\n"
|
||||
f"Title: {e['info']}\n"
|
||||
f"Date: {e['date']}\n"
|
||||
f"Threat Level: {e['threat_level_id']}\n"
|
||||
f"Contains {cnt_attr} attributes, {cnt_obj} objects and {cnt_tags} tags\n"
|
||||
f"Full event: <{url}|{url}>"
|
||||
},
|
||||
"accessory": {
|
||||
"type": "image",
|
||||
"image_url": image_url,
|
||||
"alt_text": "MISP or org logo"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
if 'Tag' in e:
|
||||
tag_block = {
|
||||
"type": "actions",
|
||||
"elements": [
|
||||
]
|
||||
}
|
||||
tags = set([t['name'] for t in e['Tag']])
|
||||
for a in e['Attribute']:
|
||||
if 'Tag' in a:
|
||||
for t in a['Tag']:
|
||||
tags.add(t['name'])
|
||||
for o in e['Object']:
|
||||
for a in o['Attribute']:
|
||||
if 'Tag' in a:
|
||||
for t in a['Tag']:
|
||||
tags.add(t['name'])
|
||||
|
||||
tags = sorted(tags)
|
||||
for t in tags:
|
||||
t = t.replace('misp-galaxy:', '').replace('mitre-', '')
|
||||
tag_block['elements'].append({
|
||||
"type": "button",
|
||||
"text": {
|
||||
"type": "plain_text",
|
||||
"text": t
|
||||
},
|
||||
"value": "#"
|
||||
})
|
||||
zmq_message_blocks.append(tag_block)
|
||||
|
||||
# List attributes
|
||||
if include_attr:
|
||||
zmq_message_blocks.append({"type": "divider"})
|
||||
attrs_text = gen_attrs_text(e['Attribute'])
|
||||
if attrs_text:
|
||||
zmq_message_blocks.append(
|
||||
{
|
||||
"type": "section",
|
||||
"text": {
|
||||
"type": "mrkdwn",
|
||||
"text": f"*Attributes:*\n{attrs_text}"
|
||||
}
|
||||
}
|
||||
)
|
||||
# List Objects
|
||||
if include_obj:
|
||||
zmq_message_blocks.append({"type": "divider"})
|
||||
for o in e['Object']:
|
||||
attrs_text = gen_attrs_text(o['Attribute'])
|
||||
if attrs_text:
|
||||
# print(json.dumps(o, indent=2))
|
||||
zmq_message_blocks.append(
|
||||
{
|
||||
"type": "section",
|
||||
"text": {
|
||||
"type": "mrkdwn",
|
||||
"text": f"*{o['name'].capitalize()} object:*\n{attrs_text}"
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
# Send the message
|
||||
client = slack.WebClient(token=slack_token)
|
||||
client.users_setPresence(presence='auto')
|
||||
channel = client.channels_join(name=channel_name)
|
||||
client.chat_postMessage(
|
||||
channel=channel['channel']['id'],
|
||||
text=zmq_message_short,
|
||||
blocks=zmq_message_blocks
|
||||
)
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser(description='MISP to Slack bot - ZMQ client to gather events, attributes and sighting updates from a MISP instance')
|
||||
parser.add_argument("-p", "--port", default="50000", help='set TCP port of the MISP ZMQ (default: 50000)')
|
||||
parser.add_argument("-r", "--host", default="127.0.0.1", help='set host of the MISP ZMQ (default: 127.0.0.1)')
|
||||
parser.add_argument("-t", "--sleep", default=0.1, help='sleep time (default: 0.1)', type=int)
|
||||
args = parser.parse_args()
|
||||
|
||||
port = args.port
|
||||
host = args.host
|
||||
context = zmq.Context()
|
||||
socket = context.socket(zmq.SUB)
|
||||
socket.connect("tcp://%s:%s" % (host, port))
|
||||
socket.setsockopt(zmq.SUBSCRIBE, b'')
|
||||
|
||||
poller = zmq.Poller()
|
||||
poller.register(socket, zmq.POLLIN)
|
||||
|
||||
while True:
|
||||
socks = dict(poller.poll(timeout=None))
|
||||
if socket in socks and socks[socket] == zmq.POLLIN:
|
||||
message = socket.recv()
|
||||
topic, s, m = message.decode('utf-8').partition(" ")
|
||||
|
||||
try:
|
||||
m_json = json.loads(m)
|
||||
except Exception:
|
||||
sys.stderr.write(f'Ignoring non-json message: {m}')
|
||||
time.sleep(args.sleep)
|
||||
continue
|
||||
|
||||
if 'status' in m_json:
|
||||
pass
|
||||
elif 'Event' in m_json:
|
||||
# print(m_json)
|
||||
e = m_json['Event']
|
||||
if '*' in allowed_distributions or \
|
||||
(e['distribution'] in allowed_distributions and (
|
||||
e['distribution'] != '5' or (
|
||||
'*' in allowed_sharing_groups or e['sharing_group_id'] in allowed_sharing_groups)
|
||||
)):
|
||||
print(f"Publishing event {e['id']} on slack")
|
||||
publish_event(e)
|
||||
else:
|
||||
print(f"Ignoring event {e['id']} as it has a filtered distribution.")
|
||||
else:
|
||||
print(f'Non supported message: {m}')
|
||||
time.sleep(args.sleep)
|
|
@ -0,0 +1,24 @@
|
|||
'''
|
||||
For instructions on creating your BOT, please read: https://api.slack.com/bot-users
|
||||
Your bot will need the permissions:
|
||||
- channels:join
|
||||
- chat:write
|
||||
- users:write
|
||||
|
||||
WARNING WARNING - THIS SCRIPT DOES NOT MAGICALLY RESPECT ACLs
|
||||
MAKE SURE YOU SET THE RIGHT FILTERS IN THE SETTINGS
|
||||
'''
|
||||
|
||||
channel_name = '#name'
|
||||
slack_token = ''
|
||||
|
||||
misp_url = 'https://192.168.1.1'
|
||||
misp_is_public = True # set to False if your MISP instance is on a non-internet reachable location. Shows the org icon of the event owner. Otherwise shows the MISP logo.
|
||||
|
||||
# filter for confidentiality
|
||||
allowed_distributions = ['0', '1', '2', '3', '4'] # * = all, 0/ my org only, 1/ this community, 2/ connected communities, 3/ all communities, 4/ sharing group
|
||||
allowed_sharing_groups = ['*'] # put here the sharing_group_ids that you allow
|
||||
|
||||
max_value_len = 25 # truncate values longer than X chars
|
||||
include_attr = True # include attributes in the message
|
||||
include_obj = True # include objects in the message
|
|
@ -17,11 +17,11 @@ import pprint
|
|||
pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr)
|
||||
|
||||
parser = argparse.ArgumentParser(description='Generic ZMQ client to gather events, attributes and sighting updates from a MISP instance')
|
||||
parser.add_argument("-s","--stats", default=False, action='store_true', help='print regular statistics on stderr')
|
||||
parser.add_argument("-p","--port", default="50000", help='set TCP port of the MISP ZMQ (default: 50000)')
|
||||
parser.add_argument("-r","--host", default="127.0.0.1", help='set host of the MISP ZMQ (default: 127.0.0.1)')
|
||||
parser.add_argument("-o","--only", action="append", default=None, help="set filter (misp_json, misp_json_event, misp_json_attribute or misp_json_sighting) to limit the output a specific type (default: no filter)")
|
||||
parser.add_argument("-t","--sleep", default=0.1, help='sleep time (default: 0.1)', type=int)
|
||||
parser.add_argument("-s", "--stats", default=False, action='store_true', help='print regular statistics on stderr')
|
||||
parser.add_argument("-p", "--port", default="50000", help='set TCP port of the MISP ZMQ (default: 50000)')
|
||||
parser.add_argument("-r", "--host", default="127.0.0.1", help='set host of the MISP ZMQ (default: 127.0.0.1)')
|
||||
parser.add_argument("-o", "--only", action="append", default=None, help="set filter (misp_json, misp_json_event, misp_json_attribute or misp_json_sighting) to limit the output a specific type (default: no filter)")
|
||||
parser.add_argument("-t", "--sleep", default=0.1, help='sleep time (default: 0.1)', type=int)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.only is not None:
|
||||
|
@ -35,7 +35,7 @@ port = args.port
|
|||
host = args.host
|
||||
context = zmq.Context()
|
||||
socket = context.socket(zmq.SUB)
|
||||
socket.connect ("tcp://%s:%s" % (host, port))
|
||||
socket.connect("tcp://%s:%s" % (host, port))
|
||||
socket.setsockopt(zmq.SUBSCRIBE, b'')
|
||||
|
||||
poller = zmq.Poller()
|
||||
|
@ -52,9 +52,8 @@ while True:
|
|||
if args.only:
|
||||
if topic not in filters:
|
||||
continue
|
||||
print (m)
|
||||
print(m)
|
||||
if args.stats:
|
||||
stats[topic] = stats.get(topic, 0) + 1
|
||||
pp.pprint(stats)
|
||||
time.sleep(args.sleep)
|
||||
|
||||
|
|
Loading…
Reference in New Issue