MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [internal] Optimise AuditLog

pull/7912/head
Jakub Onderka 2021-10-31 10:46:28 +01:00
parent 6f75336754
commit f1283996ad
2 changed files with 44 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
app/Model/AppModel.php
View File

@ -2537,22 +2537,17 @@ class AppModel extends Model
return self::$loadedPubSubTool; return self::$loadedPubSubTool;
} }
public function getElasticSearchTool() protected function getElasticSearchTool()
{ {
if (!$this->elasticSearchClient) { if (!$this->elasticSearchClient) {
$this->loadElasticSearchTool(); App::uses('ElasticSearchClient', 'Tools');
$client = new ElasticSearchClient();
$client->initTool();
$this->elasticSearchClient = $client;
} }
return $this->elasticSearchClient; return $this->elasticSearchClient;
} }
public function loadElasticSearchTool()
{
App::uses('ElasticSearchClient', 'Tools');
$client = new ElasticSearchClient();
$client->initTool();
$this->elasticSearchClient = $client;
}
// generate a generic subquery - options needs to include conditions // generate a generic subquery - options needs to include conditions
protected function subQueryGenerator(AppModel $model, array $options, $lookupKey, $negation = false) protected function subQueryGenerator(AppModel $model, array $options, $lookupKey, $negation = false)
{ {

64
app/Model/AuditLog.php
View File

@ -41,6 +41,15 @@ class AuditLog extends AppModel
/** @var bool */ /** @var bool */
private $compressionEnabled; private $compressionEnabled;
/** @var bool */
private $pubToZmq;
/** @var bool */
private $elasticLogging;
/** @var bool */
private $logClientIp;
/** /**
* Null when not defined, false when not enabled * Null when not defined, false when not enabled
* @var Syslog|null|false * @var Syslog|null|false
@ -72,19 +81,22 @@ class AuditLog extends AppModel
{ {
parent::__construct($id, $table, $ds); parent::__construct($id, $table, $ds);
$this->compressionEnabled = Configure::read('MISP.log_new_audit_compress') && function_exists('brotli_compress'); $this->compressionEnabled = Configure::read('MISP.log_new_audit_compress') && function_exists('brotli_compress');
$this->pubToZmq = $this->pubToZmq('audit');
$this->elasticLogging = Configure::read('Plugin.ElasticSearch_logging_enable');
$this->logClientIp = Configure::read('MISP.log_client_ip');
} }
public function afterFind($results, $primary = false) public function afterFind($results, $primary = false)
{ {
foreach ($results as $key => $result) { foreach ($results as &$result) {
if (isset($result['AuditLog']['ip'])) { if (isset($result['AuditLog']['ip'])) {
$results[$key]['AuditLog']['ip'] = inet_ntop($result['AuditLog']['ip']); $result['AuditLog']['ip'] = inet_ntop($result['AuditLog']['ip']);
} }
if (isset($result['AuditLog']['change']) && $result['AuditLog']['change']) { if (isset($result['AuditLog']['change']) && $result['AuditLog']['change']) {
$results[$key]['AuditLog']['change'] = $this->decodeChange($result['AuditLog']['change']); $result['AuditLog']['change'] = $this->decodeChange($result['AuditLog']['change']);
} }
if (isset($result['AuditLog']['action']) && isset($result['AuditLog']['model']) && isset($result['AuditLog']['model_id'])) { if (isset($result['AuditLog']['action']) && isset($result['AuditLog']['model']) && isset($result['AuditLog']['model_id'])) {
$results[$key]['AuditLog']['title'] = $this->generateUserFriendlyTitle($result['AuditLog']); $result['AuditLog']['title'] = $this->generateUserFriendlyTitle($result['AuditLog']);
} }
} }
return $results; return $results;
@ -160,51 +172,52 @@ class AuditLog extends AppModel
public function beforeSave($options = array()) public function beforeSave($options = array())
{ {
if (!isset($this->data['AuditLog']['ip']) && Configure::read('MISP.log_client_ip')) { $auditLog = &$this->data['AuditLog'];
if (!isset($auditLog['ip']) && $this->logClientIp) {
$ipHeader = Configure::read('MISP.log_client_ip_header') ?: 'REMOTE_ADDR'; $ipHeader = Configure::read('MISP.log_client_ip_header') ?: 'REMOTE_ADDR';
if (isset($_SERVER[$ipHeader])) { if (isset($_SERVER[$ipHeader])) {
$this->data['AuditLog']['ip'] = inet_pton($_SERVER[$ipHeader]); $auditLog['ip'] = inet_pton($_SERVER[$ipHeader]); // convert to binary form
} }
} }
if (!isset($this->data['AuditLog']['user_id'])) { if (!isset($auditLog['user_id'])) {
$this->data['AuditLog']['user_id'] = $this->userInfo()['id']; $auditLog['user_id'] = $this->userInfo()['id'];
} }
if (!isset($this->data['AuditLog']['org_id'])) { if (!isset($auditLog['org_id'])) {
$this->data['AuditLog']['org_id'] = $this->userInfo()['org_id']; $auditLog['org_id'] = $this->userInfo()['org_id'];
} }
if (!isset($this->data['AuditLog']['request_type'])) { if (!isset($auditLog['request_type'])) {
$this->data['AuditLog']['request_type'] = $this->userInfo()['request_type']; $auditLog['request_type'] = $this->userInfo()['request_type'];
} }
if (!isset($this->data['AuditLog']['authkey_id'])) { if (!isset($auditLog['authkey_id'])) {
$this->data['AuditLog']['authkey_id'] = $this->userInfo()['authkey_id']; $auditLog['authkey_id'] = $this->userInfo()['authkey_id'];
} }
if (!isset($this->data['AuditLog']['request_id'] ) && isset($_SERVER['HTTP_X_REQUEST_ID'])) { if (!isset($auditLog['request_id'] ) && isset($_SERVER['HTTP_X_REQUEST_ID'])) {
$this->data['AuditLog']['request_id'] = $_SERVER['HTTP_X_REQUEST_ID']; $auditLog['request_id'] = $_SERVER['HTTP_X_REQUEST_ID'];
} }
// Truncate request_id // Truncate request_id
if (isset($this->data['AuditLog']['request_id']) && strlen($this->data['AuditLog']['request_id']) > 255) { if (isset($auditLog['request_id']) && strlen($auditLog['request_id']) > 255) {
$this->data['AuditLog']['request_id'] = substr($this->data['AuditLog']['request_id'], 0, 255); $auditLog['request_id'] = substr($auditLog['request_id'], 0, 255);
} }
// Truncate model title // Truncate model title
if (isset($this->data['AuditLog']['model_title']) && mb_strlen($this->data['AuditLog']['model_title']) > 255) { if (isset($auditLog['model_title']) && mb_strlen($auditLog['model_title']) > 255) {
$this->data['AuditLog']['model_title'] = mb_substr($this->data['AuditLog']['model_title'], 0, 252) . '...'; $auditLog['model_title'] = mb_substr($auditLog['model_title'], 0, 252) . '...';
} }
$this->logData($this->data); $this->logData($this->data);
if (isset($this->data['AuditLog']['change'])) { if (isset($auditLog['change'])) {
$change = json_encode($this->data['AuditLog']['change'], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); $change = json_encode($auditLog['change'], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
if ($this->compressionEnabled && strlen($change) >= self::BROTLI_MIN_LENGTH) { if ($this->compressionEnabled && strlen($change) >= self::BROTLI_MIN_LENGTH) {
$change = self::BROTLI_HEADER . brotli_compress($change, 4, BROTLI_TEXT); $change = self::BROTLI_HEADER . brotli_compress($change, 4, BROTLI_TEXT);
} }
$this->data['AuditLog']['change'] = $change; $auditLog['change'] = $change;
} }
} }
@ -214,14 +227,14 @@ class AuditLog extends AppModel
*/ */
private function logData(array $data) private function logData(array $data)
{ {
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_audit_notifications_enable')) { if ($this->pubToZmq) {
$pubSubTool = $this->getPubSubTool(); $pubSubTool = $this->getPubSubTool();
$pubSubTool->publish($data, 'audit', 'log'); $pubSubTool->publish($data, 'audit', 'log');
} }
$this->publishKafkaNotification('audit', $data, 'log'); $this->publishKafkaNotification('audit', $data, 'log');
if (Configure::read('Plugin.ElasticSearch_logging_enable')) { if ($this->elasticLogging) {
// send off our logs to distributed /dev/null // send off our logs to distributed /dev/null
$logIndex = Configure::read("Plugin.ElasticSearch_log_index"); $logIndex = Configure::read("Plugin.ElasticSearch_log_index");
$elasticSearchClient = $this->getElasticSearchTool(); $elasticSearchClient = $this->getElasticSearchTool();
@ -344,6 +357,7 @@ class AuditLog extends AppModel
'conditions' => $conditions, 'conditions' => $conditions,
'group' => ['Date'], 'group' => ['Date'],
'order' => ['Date'], 'order' => ['Date'],
'callbacks' => false,
]); ]);
} elseif ($dataSource === 'Database/Postgres') { } elseif ($dataSource === 'Database/Postgres') {
if (!empty($conditions['org_id'])) { if (!empty($conditions['org_id'])) {