From 23dc460359b980becc23be813619a200ade7a822 Mon Sep 17 00:00:00 2001 From: iglocska Date: Wed, 17 Nov 2021 14:44:07 +0100 Subject: [PATCH] new: [auditlog system] added - port of Jakub Onderka's implementation from MISP - Still not fully realised, lacking search functionalities --- src/Controller/AuditLogsController.php | 36 ++++ src/Model/Behavior/AuditLogBehavior.php | 214 ++++++++++++++++++++ src/Model/Entity/AuditLog.php | 68 +++++++ src/Model/Table/AuditLogsTable.php | 250 ++++++++++++++++++++++++ 4 files changed, 568 insertions(+) create mode 100644 src/Controller/AuditLogsController.php create mode 100644 src/Model/Behavior/AuditLogBehavior.php create mode 100644 src/Model/Entity/AuditLog.php create mode 100644 src/Model/Table/AuditLogsTable.php diff --git a/src/Controller/AuditLogsController.php b/src/Controller/AuditLogsController.php new file mode 100644 index 0000000..58a2133 --- /dev/null +++ b/src/Controller/AuditLogsController.php @@ -0,0 +1,36 @@ +CRUD->index([ + 'contain' => $this->containFields, + 'filters' => $this->filterFields, + 'quickFilters' => $this->quickFilterFields, + 'afterFind' => function($data) { + $data['request_ip'] = inet_ntop(stream_get_contents($data['request_ip'])); + $data['change'] = stream_get_contents($data['change']); + return $data; + } + ]); + $responsePayload = $this->CRUD->getResponsePayload(); + if (!empty($responsePayload)) { + return $responsePayload; + } + $this->set('metaGroup', 'Administration'); + } +} diff --git a/src/Model/Behavior/AuditLogBehavior.php b/src/Model/Behavior/AuditLogBehavior.php new file mode 100644 index 0000000..3eacfe2 --- /dev/null +++ b/src/Model/Behavior/AuditLogBehavior.php @@ -0,0 +1,214 @@ + true, + 'lastpushedid' => true, + 'timestamp' => true, + 'revision' => true, + 'modified' => true, + 'date_modified' => true, // User + 'current_login' => true, // User + 'last_login' => true, // User + 'newsread' => true, // User + 'proposal_email_lock' => true, // Event + ]; + + public function initialize(array $config): void + { + $this->config = $config; + } + + public function beforeSave(EventInterface $event, EntityInterface $entity, ArrayObject $options) + { + $fields = $entity->extract($entity->getVisible(), true); + $skipFields = $this->skipFields; + $fieldsToFetch = array_filter($fields, function($key) use ($skipFields) { + return strpos($key, '_') !== 0 && !isset($skipFields[$key]); + }, ARRAY_FILTER_USE_KEY); + // Do not fetch old version when just few fields will be fetched + $fieldToFetch = []; + if (!empty($options['fieldList'])) { + foreach ($options['fieldList'] as $field) { + if (!isset($this->skipFields[$field])) { + $fieldToFetch[] = $field; + } + } + if (empty($fieldToFetch)) { + $this->old = null; + return true; + } + } + if ($entity->id) { + $this->old = $this->_table->find()->where(['id' => $entity->id])->contain($fieldToFetch)->first(); + } else { + $this->old = null; + } + return true; + } + + public function afterSave(EventInterface $event, EntityInterface $entity, ArrayObject $options) + { + if ($entity->id) { + $id = $entity->id; + } else { + $id = null; + } + + if ($entity->isNew()) { + $action = $entity->getConstant('ACTION_ADD'); + } else { + $action = $entity->getConstant('ACTION_EDIT'); + if (isset($entity['deleted'])) { + if ($entity['deleted']) { + $action = $entity->getConstant('ACTION_SOFT_DELETE'); + } else if (!$entity['deleted'] && $this->old['deleted']) { + $action = $entity->getConstant('ACTION_UNDELETE'); + } + } + } + $changedFields = $this->changedFields($entity, isset($options['fieldList']) ? $options['fieldList'] : null); + if (empty($changedFields)) { + return; + } + + $modelTitleField = $this->_table->getDisplayField(); + if (is_callable($modelTitleField)) { + $modelTitle = $modelTitleField($entity, isset($this->old) ? $this->old : []); + } else if (isset($entity[$modelTitleField])) { + $modelTitle = $entity[$modelTitleField]; + } else if ($this->old[$modelTitleField]) { + $modelTitle = $this->old[$modelTitleField]; + } + $this->auditLogs()->insert([ + 'action' => $action, + 'model' => $entity->getSource(), + 'model_id' => $id, + 'model_title' => $modelTitle, + 'change' => $changedFields + ]); + } + + public function beforeDelete(EventInterface $event, EntityInterface $entity, ArrayObject $options) + { + $this->old = $this->_table->find()->where(['id' => $entity->id])->first(); + return true; + } + + public function afterDelete(EventInterface $event, EntityInterface $entity, ArrayObject $options) + { + $modelTitleField = $this->_table->getDisplayField(); + if (is_callable($modelTitleField)) { + $modelTitle = $modelTitleField($entity, []); + } else if (isset($entity[$modelTitleField])) { + $modelTitle = $entity[$modelTitleField]; + } + + $logEntity = $this->auditLogs()->newEntity([ + 'action' => $entity->getConstant('ACTION_DELETE'), + 'model' => $entity->getSource(), + 'model_id' => $this->old->id, + 'model_title' => $modelTitle, + 'change' => $this->changedFields($entity) + ]); + $logEntity->save(); + } + + /** + * @param Model $model + * @param array|null $fieldsToSave + * @return array + */ + private function changedFields(EntityInterface $entity, $fieldsToSave = null) + { + $dbFields = $this->_table->getSchema()->typeMap(); + $changedFields = []; + foreach ($entity->extract($entity->getVisible()) as $key => $value) { + if (isset($this->skipFields[$key])) { + continue; + } + if (!isset($dbFields[$key])) { + continue; + } + if ($fieldsToSave && !in_array($key, $fieldsToSave, true)) { + continue; + } + if (isset($entity[$key]) && isset($this->old[$key])) { + $old = $this->old[$key]; + } else { + $old = null; + } + // Normalize + if (is_bool($old)) { + $old = $old ? 1 : 0; + } + if (is_bool($value)) { + $value = $value ? 1 : 0; + } + $dbType = $dbFields[$key]; + if ($dbType === 'integer' || $dbType === 'tinyinteger' || $dbType === 'biginteger' || $dbType === 'boolean') { + $value = (int)$value; + if ($old !== null) { + $old = (int)$old; + } + } + if ($value == $old) { + continue; + } + if ($key === 'password' || $key === 'authkey') { + $value = '*****'; + if ($old !== null) { + $old = $value; + } + } + + if ($old === null) { + $changedFields[$key] = $value; + } else { + $changedFields[$key] = [$old, $value]; + } + } + return $changedFields; + } + + /** + * @return AuditLogs + */ + public function auditLogs() + { + if ($this->AuditLogs === null) { + $this->AuditLogs = TableRegistry::getTableLocator()->get('AuditLogs'); + } + return $this->AuditLogs; + } + + public function log() + { + + } +} diff --git a/src/Model/Entity/AuditLog.php b/src/Model/Entity/AuditLog.php new file mode 100644 index 0000000..beabeb8 --- /dev/null +++ b/src/Model/Entity/AuditLog.php @@ -0,0 +1,68 @@ +compressionEnabled = Configure::read('Cerebrate.log_compress') && function_exists('brotli_compress'); + parent::__construct($properties, $options); + } + + protected function _getTitle(): String + { + return $this->generateUserFriendlyTitle($this); + } + + /** + * @param string $change + * @return array|string + * @throws JsonException + */ + private function decodeChange($change) + { + if (substr($change, 0, 4) === self::BROTLI_HEADER) { + if (function_exists('brotli_uncompress')) { + $change = brotli_uncompress(substr($change, 4)); + if ($change === false) { + return 'Compressed'; + } + } else { + return 'Compressed'; + } + } + return json_decode($change, true); + } + + /** + * @param array $auditLog + * @return string + */ + public function generateUserFriendlyTitle($auditLog) + { + if (in_array($auditLog['action'], [self::ACTION_TAG, self::ACTION_TAG_LOCAL, self::ACTION_REMOVE_TAG, self::ACTION_REMOVE_TAG_LOCAL], true)) { + $attached = ($auditLog['action'] === self::ACTION_TAG || $auditLog['action'] === self::ACTION_TAG_LOCAL); + $local = ($auditLog['action'] === self::ACTION_TAG_LOCAL || $auditLog['action'] === self::ACTION_REMOVE_TAG_LOCAL) ? __('local') : __('global'); + if ($attached) { + return __('Attached %s tag "%s" to %s #%s', $local, $auditLog['model_title'], strtolower($auditLog['model']), $auditLog['model_id']); + } else { + return __('Detached %s tag "%s" from %s #%s', $local, $auditLog['model_title'], strtolower($auditLog['model']), $auditLog['model_id']); + } + } + + + $title = "{$auditLog['model']} #{$auditLog['model_id']}"; + + if (isset($auditLog['model_title']) && $auditLog['model_title']) { + $title .= ": {$auditLog['model_title']}"; + } + return $title; + } +} diff --git a/src/Model/Table/AuditLogsTable.php b/src/Model/Table/AuditLogsTable.php new file mode 100644 index 0000000..374a156 --- /dev/null +++ b/src/Model/Table/AuditLogsTable.php @@ -0,0 +1,250 @@ +addBehavior('UUID'); + $this->addBehavior('Timestamp', [ + 'Model.beoreSave' => [ + 'created_at' => 'new' + ] + ]); + $this->belongsTo('Users'); + $this->setDisplayField('info'); + $this->compressionEnabled = Configure::read('Cerebrate.log_new_audit_compress') && function_exists('brotli_compress'); + } + + public function beforeMarshal(EventInterface $event, ArrayObject $data, ArrayObject $options) + { + if (!isset($data['request_ip'])) { + $ipHeader = 'REMOTE_ADDR'; + if (isset($_SERVER[$ipHeader])) { + $data['request_ip'] = $_SERVER[$ipHeader]; + } else { + $data['request_ip'] = '127.0.0.1'; + } + } + foreach (['user_id', 'request_type', 'authkey_id'] as $field) { + if (!isset($data[$field])) { + if (!isset($userInfo)) { + $userInfo = $this->userInfo(); + } + if (!empty($userInfo[$field])) { + $data[$field] = $userInfo[$field]; + } else { + $data[$field] = 0; + } + } + } + + if (!isset($data['request_id'] ) && isset($_SERVER['HTTP_X_REQUEST_ID'])) { + $data['request_id'] = $_SERVER['HTTP_X_REQUEST_ID']; + } + + // Truncate request_id + if (isset($data['request_id']) && strlen($data['request_id']) > 255) { + $data['request_id'] = substr($data['request_id'], 0, 255); + } + + // Truncate model title + if (isset($data['model_title']) && mb_strlen($data['model_title']) > 255) { + $data['model_title'] = mb_substr($data['model_title'], 0, 252) . '...'; + } + + if (isset($data['change'])) { + $change = json_encode($data['change'], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); + if ($this->compressionEnabled && strlen($change) >= self::BROTLI_MIN_LENGTH) { + $change = self::BROTLI_HEADER . brotli_compress($change, 4, BROTLI_TEXT); + } + $data['change'] = $change; + } + } + + public function beforeSave(EventInterface $event, EntityInterface $entity, ArrayObject $options) + { + $entity->request_ip = inet_pton($entity->request_ip); + $this->logData($entity); + return true; + } + + /** + * @param array $data + * @return bool + */ + private function logData(EntityInterface $entity) + { + if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_audit_notifications_enable')) { + $pubSubTool = $this->getPubSubTool(); + $pubSubTool->publish($data, 'audit', 'log'); + } + + //$this->publishKafkaNotification('audit', $data, 'log'); + + if (Configure::read('Plugin.ElasticSearch_logging_enable')) { + // send off our logs to distributed /dev/null + $logIndex = Configure::read("Plugin.ElasticSearch_log_index"); + $elasticSearchClient = $this->getElasticSearchTool(); + $elasticSearchClient->pushDocument($logIndex, "log", $data); + } + + // write to syslogd as well if enabled + if ($this->syslog === null) { + if (Configure::read('Security.syslog')) { + $options = []; + $syslogToStdErr = Configure::read('Security.syslog_to_stderr'); + if ($syslogToStdErr !== null) { + $options['to_stderr'] = $syslogToStdErr; + } + $syslogIdent = Configure::read('Security.syslog_ident'); + if ($syslogIdent) { + $options['ident'] = $syslogIdent; + } + $this->syslog = new SysLog($options); + } else { + $this->syslog = false; + } + } + if ($this->syslog) { + $entry = $data['action']; + $title = $entity->generateUserFriendlyTitle(); + if ($title) { + $entry .= " -- $title"; + } + $this->syslog->write('info', $entry); + } + return true; + } + + /** + * @return array + */ + public function userInfo() + { + if ($this->user !== null) { + return $this->user; + } + + $this->user = ['id' => 0, /*'org_id' => 0, */'authkey_id' => 0, 'request_type' => self::REQUEST_TYPE_DEFAULT]; + + $isShell = (php_sapi_name() === 'cli'); + if ($isShell) { + // do not start session for shell commands and fetch user info from configuration + $this->user['request_type'] = self::REQUEST_TYPE_CLI; + $currentUserId = Configure::read('CurrentUserId'); + if (!empty($currentUserId)) { + $this->user['id'] = $currentUserId; + $userFromDb = $this->Users->find()->where(['id' => $currentUserId])->first(); + $this->user['name'] = $userFromDb['name']; + $this->user['org_id'] = $userFromDb['org_id']; + } + } else { + $authUser = Router::getRequest()->getSession()->read('authUser'); + if (!empty($authUser)) { + $this->user['id'] = $authUser['id']; + $this->user['user_id'] = $authUser['id']; + $this->user['name'] = $authUser['name']; + //$this->user['org_id'] = $authUser['org_id']; + if (isset($authUser['logged_by_authkey']) && $authUser['logged_by_authkey']) { + $this->user['request_type'] = self::REQUEST_TYPE_API; + } + if (isset($authUser['authkey_id'])) { + $this->user['authkey_id'] = $authUser['authkey_id']; + } + } + } + return $this->user; + } + + public function insert(array $data) + { + $logEntity = $this->newEntity($data); + if ($logEntity->getErrors()) { + throw new Exception($logEntity->getErrors()); + } else { + $this->save($logEntity); + } + } + + /** + * @param string|int $org + * @return array + */ + public function returnDates($org = 'all') + { + $conditions = []; + if ($org !== 'all') { + $org = $this->Organisation->fetchOrg($org); + if (empty($org)) { + throw new NotFoundException('Invalid organisation.'); + } + $conditions['org_id'] = $org['id']; + } + + $dataSource = ConnectionManager::getDataSource('default')->config['datasource']; + if ($dataSource === 'Database/Mysql' || $dataSource === 'Database/MysqlObserver') { + $validDates = $this->find('all', [ + 'recursive' => -1, + 'fields' => ['DISTINCT UNIX_TIMESTAMP(DATE(created)) AS Date', 'count(id) AS count'], + 'conditions' => $conditions, + 'group' => ['Date'], + 'order' => ['Date'], + ]); + } elseif ($dataSource === 'Database/Postgres') { + if (!empty($conditions['org_id'])) { + $condOrg = sprintf('WHERE org_id = %s', intval($conditions['org_id'])); + } else { + $condOrg = ''; + } + $sql = 'SELECT DISTINCT EXTRACT(EPOCH FROM CAST(created AS DATE)) AS "Date", COUNT(id) AS count + FROM audit_logs + ' . $condOrg . ' + GROUP BY "Date" ORDER BY "Date"'; + $validDates = $this->query($sql); + } + $data = []; + foreach ($validDates as $date) { + $data[(int)$date[0]['Date']] = (int)$date[0]['count']; + } + return $data; + } +}