MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

new: [CLI] Command for recompressing data stored in audit logs table

pull/8749/head
Jakub Onderka 2022-11-13 08:31:19 +01:00
parent 13e6c10739
commit 483104bf80
2 changed files with 63 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
app/Console/Command/LogShell.php
View File

@ -30,6 +30,9 @@ class LogShell extends AppShell
),
),
]);
$parser->addSubcommand('recompress', [
'help' => __('Recompress compressed data in logs.'),
]);
return $parser;
}
@ -148,6 +151,9 @@ class LogShell extends AppShell
$this->out('Change field:');
$this->out('-------------');
$this->out(str_pad(__('Compressed items:'), 20) . $this->AuditLog->compressionStats['compressed']);
$this->out(str_pad(__('ZSTD compressed:'), 20) . $this->AuditLog->compressionStats['zstd_compressed']);
$this->out(str_pad(__('Brotli compressed:'), 20) . $this->AuditLog->compressionStats['brotli_compressed']);
$this->out(str_pad(__('Total size:'), 20) . CakeNumber::toReadableSize($this->AuditLog->compressionStats['bytes_total']));
$this->out(str_pad(__('Uncompressed size:'), 20) . CakeNumber::toReadableSize($this->AuditLog->compressionStats['bytes_uncompressed']));
$this->out(str_pad(__('Compressed size:'), 20) . CakeNumber::toReadableSize($this->AuditLog->compressionStats['bytes_compressed']));
}
@ -175,4 +181,9 @@ class LogShell extends AppShell
$this->out(str_pad(__('Index size:'), 20) . CakeNumber::toReadableSize($usage['index_in_bytes']));
$this->out(str_pad(__('Reclaimable size:'), 20) . CakeNumber::toReadableSize($usage['reclaimable_in_bytes']), 2);
}
public function recompress()
{
$this->AuditLog->recompress();
}
}

67
app/Model/AuditLog.php
View File

@ -10,7 +10,7 @@ class AuditLog extends AppModel
{
const BROTLI_HEADER = "\xce\xb2\xcf\x81",
ZSTD_HEADER = "\x28\xb5\x2f\xfd";
const COMPRESS_MIN_LENGTH = 200;
const COMPRESS_MIN_LENGTH = 256;
const ACTION_ADD = 'add',
ACTION_EDIT = 'edit',
@ -57,8 +57,11 @@ class AuditLog extends AppModel
public $compressionStats = [
'compressed' => 0,
'bytes_total' => 0,
'bytes_compressed' => 0,
'bytes_uncompressed' => 0,
'brotli_compressed' => 0,
'zstd_compressed' => 0,
];
public $belongsTo = [
@ -143,6 +146,24 @@ class AuditLog extends AppModel
return '';
}
/**
* @param mixed $change
* @return string
* @throws JsonException
*/
private function encodeChange($change)
{
$change = JsonTool::encode($change);
if ($this->compressionEnabled && strlen($change) >= self::COMPRESS_MIN_LENGTH) {
if (function_exists('zstd_compress')) {
return zstd_compress($change, 4);
} else {
return self::BROTLI_HEADER . brotli_compress($change, 4, BROTLI_TEXT);
}
}
return $change;
}
/**
* @param string $change
* @return array|string
@ -150,11 +171,14 @@ class AuditLog extends AppModel
*/
private function decodeChange($change)
{
$len = strlen($change);
$this->compressionStats['bytes_total'] += $len;
$header = substr($change, 0, 4);
if ($header === self::ZSTD_HEADER) {
$this->compressionStats['compressed']++;
$this->compressionStats['zstd_compressed']++;
if (function_exists('zstd_uncompress')) {
$this->compressionStats['bytes_compressed'] += strlen($change);
$this->compressionStats['bytes_compressed'] += $len;
$change = zstd_uncompress($change);
$this->compressionStats['bytes_uncompressed'] += strlen($change);
if ($change === false) {
@ -165,8 +189,9 @@ class AuditLog extends AppModel
}
} else if ($header === self::BROTLI_HEADER) {
$this->compressionStats['compressed']++;
$this->compressionStats['brotli_compressed']++;
if (function_exists('brotli_uncompress')) {
$this->compressionStats['bytes_compressed'] += strlen($change);
$this->compressionStats['bytes_compressed'] += $len;
$change = brotli_uncompress(substr($change, 4));
$this->compressionStats['bytes_uncompressed'] += strlen($change);
if ($change === false) {
@ -233,15 +258,7 @@ class AuditLog extends AppModel
}
if (isset($auditLog['change'])) {
$change = JsonTool::encode($auditLog['change']);
if ($this->compressionEnabled && strlen($change) >= self::COMPRESS_MIN_LENGTH) {
if (function_exists('zstd_compress')) {
$change = zstd_compress($change, 4);
} else {
$change = self::BROTLI_HEADER . brotli_compress($change, 4, BROTLI_TEXT);
}
}
$auditLog['change'] = $change;
$auditLog['change'] = $this->encodeChange($auditLog['change']);
}
}
@ -344,15 +361,35 @@ class AuditLog extends AppModel
}
}
/**
* @throws JsonException
* @throws Exception
*/
public function recompress()
{
$changes = $this->find('all', [
'fields' => ['AuditLog.id', 'AuditLog.change'],
'recursive' => -1,
'conditions' => ['length(AuditLog.change) >=' => self::BROTLI_MIN_LENGTH],
'conditions' => ['OR' => [
['length(AuditLog.change) >=' => self::COMPRESS_MIN_LENGTH],
['AuditLog.change LIKE' => self::ZSTD_HEADER . '%'],
['AuditLog.change LIKE' => self::BROTLI_HEADER . '%'],
]],
]);
foreach ($changes as $change) {
$this->save($change, true, ['id', 'change']);
$options = [
'validate' => false,
'callbacks' => false,
'fieldList' => ['change'],
];
foreach (array_chunk($changes, 100) as $chunk) {
$toSave = [];
foreach ($chunk as $change) {
$change['AuditLog']['change'] = $this->encodeChange($change['AuditLog']['change']);
$toSave[] = $change;
}
$this->saveMany($toSave, $options);
}
}