MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'develop' of github.com:MISP/MISP into develop

pull/8580/head
Alexandre Dulaunoy 2022-09-09 14:06:47 +02:00
parent 892fa9f741 25e9684297
commit c5688914ba
No known key found for this signature in database
GPG Key ID: 09E2CD4944E6CBCD
9 changed files with 288 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
PyMISP

@ -1 +1 @@
Subproject commit 3ca8717e6c7780718cd20328040799261a39a281
Subproject commit a897b8d62c0c193672b6a529d01c01de2afb6b4d

2
app/Controller/AppController.php
View File

@ -36,7 +36,7 @@ class AppController extends Controller
public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
private $__queryVersion = '144';
public $pyMispVersion = '2.4.160';
public $pyMispVersion = '2.4.162';
public $phpmin = '7.2';
public $phprec = '7.4';
public $phptoonew = '8.0';

4
app/Controller/AttributesController.php
View File

@ -758,6 +758,8 @@ class AttributesController extends AppController
if (count($existingAttribute) && !$existingAttribute['Attribute']['deleted']) {
$this->request->data['Attribute']['id'] = $existingAttribute['Attribute']['id'];
$this->request->data['Attribute']['event_id'] = $existingAttribute['Attribute']['event_id'];
$this->request->data['Attribute']['object_id'] = $existingAttribute['Attribute']['object_id'];
$this->request->data['Attribute']['uuid'] = $existingAttribute['Attribute']['uuid'];
$skipTimeCheck = false;
if (!isset($this->request->data['Attribute']['timestamp'])) {
$this->request->data['Attribute']['timestamp'] = $dateObj->getTimestamp();
@ -790,7 +792,7 @@ class AttributesController extends AppController
}
$this->Attribute->Object->updateTimestamp($existingAttribute['Attribute']['object_id']);
} else {
$result = $this->Attribute->save($this->request->data);
$result = $this->Attribute->save($this->request->data, array('fieldList' => Attribute::EDITABLE_FIELDS));
if ($result) {
$this->Attribute->AttributeTag->handleAttributeTags($this->Auth->user(), $this->request->data['Attribute'], $attribute['Event']['id'], $capture=true);
}

25
app/Lib/Tools/QueryTool.php
View File

@ -1,25 +0,0 @@
<?php
class QueryTool
{
const PDO_MAP = array(
'integer' => PDO::PARAM_INT,
'float' => PDO::PARAM_STR,
'boolean' => PDO::PARAM_BOOL,
'string' => PDO::PARAM_STR,
'text' => PDO::PARAM_STR,
);
public function quickDelete($table, $field, $value, $model)
{
$db = $model->getDataSource();
$connection = $db->getConnection();
if (in_array($db->config['datasource'], ['Database/Mysql', 'Database/MysqlObserver', 'Database/MysqlExtended'])) {
$query = $connection->prepare('DELETE FROM ' . $table . ' WHERE ' . $field . ' = :value');
} elseif ($db->config['datasource'] == 'Database/Postgres' ) {
$query = $connection->prepare('DELETE FROM "' . $table . '" WHERE "' . $field . '" = :value');
}
$query->bindValue(':value', $value, self::PDO_MAP[$db->introspectType($value)]);
$query->execute();
}
}

164
app/Model/AppModel.php
View File

@ -83,7 +83,7 @@ class AppModel extends Model
75 => false, 76 => true, 77 => false, 78 => false, 79 => false, 80 => false,
81 => false, 82 => false, 83 => false, 84 => false, 85 => false, 86 => false,
87 => false, 88 => false, 89 => false, 90 => false, 91 => false, 92 => false,
93 => false, 94 => false, 95 => false,
93 => false, 94 => false, 95 => false, 96 => false,
);
const ADVANCED_UPDATES_DESCRIPTION = array(
@ -265,6 +265,10 @@ class AppModel extends Model
}
$dbUpdateSuccess = true;
break;
case 96:
$this->removeDuplicatedUUIDs();
$dbUpdateSuccess = $this->updateDatabase('createUUIDsConstraints');
break;
default:
$dbUpdateSuccess = $this->updateDatabase($command);
break;
@ -1930,6 +1934,15 @@ class AppModel extends Model
$indexArray[] = array('shadow_attributes', 'first_seen');
$indexArray[] = array('shadow_attributes', 'last_seen');
break;
case 'createUUIDsConstraints':
$tables_to_check = ['events', 'attributes', 'objects', 'sightings', 'dashboards', 'inbox', 'organisations', 'tag_collections'];
foreach ($tables_to_check as $table) {
if (!$this->__checkIndexExists($table, 'uuid', true)) {
$this->__dropIndex($table, 'uuid');
$this->__addIndex($table, 'uuid', null, true);
}
}
break;
default:
return false;
}
@ -2180,6 +2193,18 @@ class AppModel extends Model
return $additionResult;
}
private function __checkIndexExists($table, $column_name, $is_unique = false): bool
{
$query = sprintf(
'SHOW INDEX FROM %s WHERE Column_name = \'%s\' and Non_unique = %s;',
$table,
$column_name,
!empty($is_unique) ? '0' : '1'
);
$existing_index = $this->query($query);
return !empty($existing_index);
}
public function cleanCacheFiles()
{
Cache::clear();
@ -2704,6 +2729,143 @@ class AppModel extends Model
return true;
}
public function removeDuplicatedUUIDs()
{
$removedResults = array(
'Event' => $this->removeDuplicateEventUUIDs(),
'Attribute' => $this->removeDuplicateAttributeUUIDs(),
'Object' => $this->__removeDuplicateUUIDsGeneric(ClassRegistry::init('MispObject'), 'timestamp'),
'Sighting' => $this->__removeDuplicateUUIDsGeneric(ClassRegistry::init('Sighting'), 'date_sighting'),
'Dashboard' => $this->__removeDuplicateUUIDsGeneric(ClassRegistry::init('Dashboard'), 'timestamp'),
'Inbox' => $this->__removeDuplicateUUIDsGeneric(ClassRegistry::init('Inbox'), 'timestamp'),
'TagCollection' => $this->__removeDuplicateUUIDsGeneric(ClassRegistry::init('TagCollection')),
// 'GalaxyCluster' => $this->__removeDuplicateUUIDsGeneric(ClassRegistry::init('GalaxyCluster')),
);
$this->Log->createLogEntry('SYSTEM', 'update_database', 'Server', 0, __('Removed duplicated UUIDs'), __('Event: %s, Attribute: %s, Object: %s, Sighting: %s, Dashboard: %s, Inbox: %s, TagCollection: %s', h($removedResults['Event']), h($removedResults['Attribute']), h($removedResults['Object']), h($removedResults['Sighting']), h($removedResults['Dashboard']), h($removedResults['Inbox']), h($removedResults['TagCollection'])));
}
private function __removeDuplicateUUIDsGeneric($model, $sort_by=false): int
{
$className = get_class($model);
$alias = $model->alias;
$this->Log = ClassRegistry::init('Log');
$duplicates = $model->find('all', array(
'fields' => array('uuid', 'count(uuid) as occurrence'),
'recursive' => -1,
'group' => array('uuid HAVING occurrence > 1'),
));
$counter = 0;
foreach ($duplicates as $duplicate) {
$options = [
'recursive' => -1,
'conditions' => array('uuid' => $duplicate[$alias]['uuid']),
];
if (!empty($sort_by)) {
$options['order'] = "$sort_by DESC";
}
$fetched_duplicates = $model->find('all', $options);
unset($fetched_duplicates[0]);
foreach ($fetched_duplicates as $fetched_duplicate) {
$model->delete($fetched_duplicate[$alias]['id']);
$this->Log->createLogEntry('SYSTEM', 'delete', $className, $fetched_duplicate[$alias]['id'], __('Removed %s (%s)', $className, $fetched_duplicate[$alias]['id']), __('%s\'s UUID duplicated (%s)', $className, $fetched_duplicate[$alias]['uuid']));
$counter++;
}
}
return $counter;
}
public function removeDuplicateAttributeUUIDs()
{
$this->Attribute = ClassRegistry::init('Attribute');
$this->Log = ClassRegistry::init('Log');
$duplicates = $this->Attribute->find('all', array(
'fields' => array('Attribute.uuid', 'count(Attribute.uuid) as occurrence'),
'recursive' => -1,
'group' => array('Attribute.uuid HAVING occurrence > 1'),
'order' => false,
));
$counter = 0;
foreach ($duplicates as $duplicate) {
$attributes = $this->Attribute->find('all', array(
'recursive' => -1,
'conditions' => array('uuid' => $duplicate['Attribute']['uuid']),
'contain' => array(
'AttributeTag' => array(
'fields' => array('tag_id')
)
),
'order' => 'timestamp DESC',
));
$tagIDsOfFirstAttribute = Hash::extract($attributes[0]['AttributeTag'], '{n}.tag_id');
$eventIDOfFirstAttribute = $attributes[0]['Attribute']['event_id'];
unset($attributes[0]);
foreach ($attributes as $attribute) {
$tagIDs = Hash::extract($attribute['AttributeTag'], '{n}.tag_id');
$logTag = false;
$logEventID = false;
if (empty(array_diff($tagIDs, $tagIDsOfFirstAttribute))) {
$logTag = true;
}
if ($eventIDOfFirstAttribute != $attribute['Attribute']['event_id']) {
$logEventID = true;
}
$success = $this->Attribute->delete($attribute['Attribute']['id']);
if (empty($success)) {
$this->Log->createLogEntry('SYSTEM', 'delete', 'Attribute', $attribute['Attribute']['id'], __('Could not remove attribute (%s)', $attribute['Attribute']['id']), __('Deletion was rejected.'));
continue;
}
$logMessage = __('Attribute\'s UUID duplicated (%s).', $attribute['Attribute']['uuid']);
if ($logEventID) {
$logMessage .= __(' Was part of another event_id (%s) than the one that was kept (%s).', $attribute['Attribute']['event_id'], $eventIDOfFirstAttribute);
} else if ($logTag) {
$logMessage .= __(' Tag IDs attached [%s]', implode($tagIDs));
} else {
}
$this->Log->createLogEntry('SYSTEM', 'delete', 'Attribute', $attribute['Attribute']['id'], __('Removed attribute (%s)', $attribute['Attribute']['id']), $logMessage);
$counter++;
}
}
return $counter;
}
public function removeDuplicateEventUUIDs()
{
$this->Event = ClassRegistry::init('Event');
$this->Log = ClassRegistry::init('Log');
$duplicates = $this->Event->find('all', array(
'fields' => array('Event.uuid', 'count(Event.uuid) as occurrence'),
'recursive' => -1,
'group' => array('Event.uuid HAVING occurrence > 1'),
));
$counter = 0;
// load this so we can remove the blocklist item that will be created, this is the one case when we do not want it.
if (Configure::read('MISP.enableEventBlocklisting') !== false) {
$this->EventBlocklist = ClassRegistry::init('EventBlocklist');
}
foreach ($duplicates as $duplicate) {
$events = $this->Event->find('all', array(
'recursive' => -1,
'conditions' => array('uuid' => $duplicate['Event']['uuid']),
'order' => 'timestamp DESC',
));
unset($events[0]);
foreach ($events as $event) {
$uuid = $event['Event']['uuid'];
$this->Event->delete($event['Event']['id']);
$this->Log->createLogEntry('SYSTEM', 'delete', 'Event', $event['Event']['id'], __('Removed event (%s)', $event['Event']['id']), __('Event\'s UUID duplicated (%s)', $event['Event']['uuid']));
$counter++;
// remove the blocklist entry that we just created with the event deletion, if the feature is enabled
// We do not want to block the UUID, since we just deleted a copy
if (Configure::read('MISP.enableEventBlocklisting') !== false) {
$this->EventBlocklist->deleteAll(array('EventBlocklist.event_uuid' => $uuid));
}
}
}
return $counter;
}
public function checkFilename($filename)
{
return preg_match('@^([a-z0-9_.]+[a-z0-9_.\- ]*[a-z0-9_.\-]|[a-z0-9_.])+$@i', $filename);

110
app/Model/Datasource/Database/MysqlExtended.php
View File

@ -7,6 +7,16 @@ App::uses('Mysql', 'Model/Datasource/Database');
*/
class MysqlExtended extends Mysql
{
/**
* Output MD5 as binary, that is faster and uses less memory
* @param string $value
* @return string
*/
public function cacheMethodHasher($value)
{
return md5($value, true);
}
/**
* Builds and generates an SQL statement from an array. Handles final clean-up before conversion.
*
@ -87,16 +97,15 @@ class MysqlExtended extends Mysql
* @return string|null Rendered SQL expression to be run, otherwise null.\
* @see DboSource::renderStatement()
*/
public function renderStatement($type, $data)
{
if ($type === 'select' && $data['indexHint'] != null) {
if ($type === 'select') {
extract($data);
$having = !empty($having) ? " $having" : '';
return trim("SELECT {$fields} FROM {$table} {$alias} {$indexHint} {$joins} {$conditions} {$group}{$having} {$order} {$limit}{$lock}");
} else {
return parent::renderStatement($type, $data);
$lock = !empty($lock) ? " $lock" : '';
return rtrim("SELECT {$fields} FROM {$table} {$alias} {$indexHint} {$joins} {$conditions} {$group}{$having} {$order} {$limit}{$lock}");
}
return parent::renderStatement($type, $data);
}
/**
@ -105,12 +114,93 @@ class MysqlExtended extends Mysql
* @param string|null $useIndexHint USE INDEX hint
* @return string
*/
private function __buildIndexHint($useIndexHint = null): string
private function __buildIndexHint($useIndexHint = null): ?string
{
$index = '';
if (isset($useIndexHint)) {
$index = 'USE INDEX ' . $useIndexHint;
return isset($useIndexHint) ? ('USE INDEX ' . $useIndexHint) : null;
}
/**
* Reduce memory usage for insertMulti
*
* @param string $table
* @param array $fields
* @param array $values
* @return bool
*/
public function insertMulti($table, $fields, $values)
{
$table = $this->fullTableName($table);
$holder = implode(',', array_fill(0, count($fields), '?'));
$fields = implode(',', array_map([$this, 'name'], $fields));
$pdoMap = [
'integer' => PDO::PARAM_INT,
'float' => PDO::PARAM_STR,
'boolean' => PDO::PARAM_BOOL,
'string' => PDO::PARAM_STR,
'text' => PDO::PARAM_STR
];
$columnMap = [];
foreach ($values[key($values)] as $key => $val) {
if (is_int($val)) {
$columnMap[$key] = PDO::PARAM_INT;
} elseif (is_bool($val)) {
$columnMap[$key] = PDO::PARAM_BOOL;
} else {
$type = $this->introspectType($val);
$columnMap[$key] = $pdoMap[$type];
}
}
return $index;
$sql = "INSERT INTO $table ($fields) VALUES ";
$sql .= implode(',', array_fill(0, count($values), "($holder)"));
$statement = $this->_connection->prepare($sql);
$valuesList = array();
$i = 1;
foreach ($values as $value) {
foreach ($value as $col => $val) {
if ($this->fullDebug) {
$valuesList[] = $val;
}
$statement->bindValue($i, $val, $columnMap[$col]);
$i++;
}
}
$result = $statement->execute();
$statement->closeCursor();
if ($this->fullDebug) {
$this->logQuery($sql, $valuesList);
}
return $result;
}
/**
* {@inheritDoc}
*/
public function value($data, $column = null, $null = true)
{
// Fast check if data is int, then return value
if (is_int($data)) {
return $data;
}
// No need to quote bool values
if (is_bool($data)) {
return $data ? '1' : '0';
}
// No need to call expensive array_map
if (is_array($data) && !empty($data)) {
$output = [];
foreach ($data as $d) {
if (is_int($d)) {
$output[] = $d;
} else {
$output[] = parent::value($d, $column);
}
}
return $output;
}
return parent::value($data, $column, $null);
}
}

19
app/Model/Event.php
View File

@ -1147,6 +1147,10 @@ class Event extends AppModel
return $data[0];
}
/**
* @param array $event
* @return bool
*/
public function quickDelete(array $event)
{
$id = (int)$event['Event']['id'];
@ -1156,7 +1160,7 @@ class Event extends AppModel
'fields' => array('Thread.id'),
'recursive' => -1
));
$thread_id = !empty($thread) ? $thread['Thread']['id'] : false;
$thread_id = !empty($thread) ? (int)$thread['Thread']['id'] : false;
$relations = array(
array(
'table' => 'attributes',
@ -1231,10 +1235,17 @@ class Event extends AppModel
)
);
}
App::uses('QueryTool', 'Tools');
$queryTool = new QueryTool();
$db = $this->getDataSource();
$db->begin();
$connection = $db->getConnection();
foreach ($relations as $relation) {
$queryTool->quickDelete($relation['table'], $relation['foreign_key'], $relation['value'], $this);
$query = $connection->prepare('DELETE FROM ' . $db->name($relation['table']) . ' WHERE ' . $db->name($relation['foreign_key']) . ' = :value');
$query->bindValue(':value', $relation['value'], PDO::PARAM_INT);
$query->execute();
}
if (!$db->commit()) {
return false;
}
$this->set($event);
return $this->delete(null, false);

2
app/files/taxonomies

@ -1 +1 @@
Subproject commit fc12a106f5481e466275799de71e29dd7cf764f7
Subproject commit aa251b6a4006372e66f3bf5e946111f6d5f6a2d6

8
db_schema.json
View File

@ -8698,7 +8698,7 @@
"restrict_to_org_id": false,
"restrict_to_permission_flag": false,
"user_id": false,
"uuid": false
"uuid": true
},
"decaying_models": {
"all_orgs": false,
@ -8836,7 +8836,7 @@
"title": false,
"type": false,
"user_agent_sha256": false,
"uuid": false
"uuid": true
},
"jobs": {
"id": true
@ -9029,7 +9029,7 @@
"id": true,
"org_id": false,
"user_id": false,
"uuid": false
"uuid": true
},
"tag_collection_tags": {
"id": true,
@ -9116,5 +9116,5 @@
"uuid": false
}
},
"db_version": "95"
"db_version": "96"
}