MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [internal] Optimise AuditLog

pull/7912/head
Jakub Onderka 2021-10-31 10:46:28 +01:00
parent 6f75336754
commit f1283996ad
2 changed files with 44 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
app/Model/AppModel.php
View File

@ -2537,22 +2537,17 @@ class AppModel extends Model
return self::$loadedPubSubTool;
}
public function getElasticSearchTool()
protected function getElasticSearchTool()
{
if (!$this->elasticSearchClient) {
$this->loadElasticSearchTool();
App::uses('ElasticSearchClient', 'Tools');
$client = new ElasticSearchClient();
$client->initTool();
$this->elasticSearchClient = $client;
}
return $this->elasticSearchClient;
}
public function loadElasticSearchTool()
{
App::uses('ElasticSearchClient', 'Tools');
$client = new ElasticSearchClient();
$client->initTool();
$this->elasticSearchClient = $client;
}
// generate a generic subquery - options needs to include conditions
protected function subQueryGenerator(AppModel $model, array $options, $lookupKey, $negation = false)
{

64
app/Model/AuditLog.php
View File

@ -41,6 +41,15 @@ class AuditLog extends AppModel
/** @var bool */
private $compressionEnabled;
/** @var bool */
private $pubToZmq;
/** @var bool */
private $elasticLogging;
/** @var bool */
private $logClientIp;
/**
* Null when not defined, false when not enabled
* @var Syslog|null|false
@ -72,19 +81,22 @@ class AuditLog extends AppModel
{
parent::__construct($id, $table, $ds);
$this->compressionEnabled = Configure::read('MISP.log_new_audit_compress') && function_exists('brotli_compress');
$this->pubToZmq = $this->pubToZmq('audit');
$this->elasticLogging = Configure::read('Plugin.ElasticSearch_logging_enable');
$this->logClientIp = Configure::read('MISP.log_client_ip');
}
public function afterFind($results, $primary = false)
{
foreach ($results as $key => $result) {
foreach ($results as &$result) {
if (isset($result['AuditLog']['ip'])) {
$results[$key]['AuditLog']['ip'] = inet_ntop($result['AuditLog']['ip']);
$result['AuditLog']['ip'] = inet_ntop($result['AuditLog']['ip']);
}
if (isset($result['AuditLog']['change']) && $result['AuditLog']['change']) {
$results[$key]['AuditLog']['change'] = $this->decodeChange($result['AuditLog']['change']);
$result['AuditLog']['change'] = $this->decodeChange($result['AuditLog']['change']);
}
if (isset($result['AuditLog']['action']) && isset($result['AuditLog']['model']) && isset($result['AuditLog']['model_id'])) {
$results[$key]['AuditLog']['title'] = $this->generateUserFriendlyTitle($result['AuditLog']);
$result['AuditLog']['title'] = $this->generateUserFriendlyTitle($result['AuditLog']);
}
}
return $results;
@ -160,51 +172,52 @@ class AuditLog extends AppModel
public function beforeSave($options = array())
{
if (!isset($this->data['AuditLog']['ip']) && Configure::read('MISP.log_client_ip')) {
$auditLog = &$this->data['AuditLog'];
if (!isset($auditLog['ip']) && $this->logClientIp) {
$ipHeader = Configure::read('MISP.log_client_ip_header') ?: 'REMOTE_ADDR';
if (isset($_SERVER[$ipHeader])) {
$this->data['AuditLog']['ip'] = inet_pton($_SERVER[$ipHeader]);
$auditLog['ip'] = inet_pton($_SERVER[$ipHeader]); // convert to binary form
}
}
if (!isset($this->data['AuditLog']['user_id'])) {
$this->data['AuditLog']['user_id'] = $this->userInfo()['id'];
if (!isset($auditLog['user_id'])) {
$auditLog['user_id'] = $this->userInfo()['id'];
}
if (!isset($this->data['AuditLog']['org_id'])) {
$this->data['AuditLog']['org_id'] = $this->userInfo()['org_id'];
if (!isset($auditLog['org_id'])) {
$auditLog['org_id'] = $this->userInfo()['org_id'];
}
if (!isset($this->data['AuditLog']['request_type'])) {
$this->data['AuditLog']['request_type'] = $this->userInfo()['request_type'];
if (!isset($auditLog['request_type'])) {
$auditLog['request_type'] = $this->userInfo()['request_type'];
}
if (!isset($this->data['AuditLog']['authkey_id'])) {
$this->data['AuditLog']['authkey_id'] = $this->userInfo()['authkey_id'];
if (!isset($auditLog['authkey_id'])) {
$auditLog['authkey_id'] = $this->userInfo()['authkey_id'];
}
if (!isset($this->data['AuditLog']['request_id'] ) && isset($_SERVER['HTTP_X_REQUEST_ID'])) {
$this->data['AuditLog']['request_id'] = $_SERVER['HTTP_X_REQUEST_ID'];
if (!isset($auditLog['request_id'] ) && isset($_SERVER['HTTP_X_REQUEST_ID'])) {
$auditLog['request_id'] = $_SERVER['HTTP_X_REQUEST_ID'];
}
// Truncate request_id
if (isset($this->data['AuditLog']['request_id']) && strlen($this->data['AuditLog']['request_id']) > 255) {
$this->data['AuditLog']['request_id'] = substr($this->data['AuditLog']['request_id'], 0, 255);
if (isset($auditLog['request_id']) && strlen($auditLog['request_id']) > 255) {
$auditLog['request_id'] = substr($auditLog['request_id'], 0, 255);
}
// Truncate model title
if (isset($this->data['AuditLog']['model_title']) && mb_strlen($this->data['AuditLog']['model_title']) > 255) {
$this->data['AuditLog']['model_title'] = mb_substr($this->data['AuditLog']['model_title'], 0, 252) . '...';
if (isset($auditLog['model_title']) && mb_strlen($auditLog['model_title']) > 255) {
$auditLog['model_title'] = mb_substr($auditLog['model_title'], 0, 252) . '...';
}
$this->logData($this->data);
if (isset($this->data['AuditLog']['change'])) {
$change = json_encode($this->data['AuditLog']['change'], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
if (isset($auditLog['change'])) {
$change = json_encode($auditLog['change'], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
if ($this->compressionEnabled && strlen($change) >= self::BROTLI_MIN_LENGTH) {
$change = self::BROTLI_HEADER . brotli_compress($change, 4, BROTLI_TEXT);
}
$this->data['AuditLog']['change'] = $change;
$auditLog['change'] = $change;
}
}
@ -214,14 +227,14 @@ class AuditLog extends AppModel
*/
private function logData(array $data)
{
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_audit_notifications_enable')) {
if ($this->pubToZmq) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->publish($data, 'audit', 'log');
}
$this->publishKafkaNotification('audit', $data, 'log');
if (Configure::read('Plugin.ElasticSearch_logging_enable')) {
if ($this->elasticLogging) {
// send off our logs to distributed /dev/null
$logIndex = Configure::read("Plugin.ElasticSearch_log_index");
$elasticSearchClient = $this->getElasticSearchTool();
@ -344,6 +357,7 @@ class AuditLog extends AppModel
'conditions' => $conditions,
'group' => ['Date'],
'order' => ['Date'],
'callbacks' => false,
]);
} elseif ($dataSource === 'Database/Postgres') {
if (!empty($conditions['org_id'])) {