diff --git a/app/Config/routes.php b/app/Config/routes.php index 673c57177..032267db2 100644 --- a/app/Config/routes.php +++ b/app/Config/routes.php @@ -33,6 +33,7 @@ Router::connect('/roles/admin_index/*', array('controller' => 'roles', 'action' => 'index', 'admin' => true)); Router::connect('/logs/admin_search/*', array('controller' => 'logs', 'action' => 'search', 'admin' => true)); Router::connect('/audit_logs/admin_index/*', array('controller' => 'audit_logs', 'action' => 'index', 'admin' => true)); + Router::connect('/access_logs/admin_index/*', array('controller' => 'access_logs', 'action' => 'index', 'admin' => true)); Router::connect('/logs/admin_index/*', array('controller' => 'logs', 'action' => 'index', 'admin' => true)); Router::connect('/regexp/admin_index/*', array('controller' => 'regexp', 'action' => 'index', 'admin' => true)); diff --git a/app/Controller/AccessLogsController.php b/app/Controller/AccessLogsController.php new file mode 100644 index 000000000..e49ecc7c1 --- /dev/null +++ b/app/Controller/AccessLogsController.php @@ -0,0 +1,169 @@ + -1, + 'limit' => 60, + 'fields' => ['id', 'created', 'user_id', 'org_id', 'authkey_id', 'ip', 'request_method', 'request_id', 'controller', 'action', 'url', 'response_code', 'memory_usage', 'duration'], + 'contain' => [ + 'User' => ['fields' => ['id', 'email', 'org_id']], + 'Organisation' => ['fields' => ['id', 'name', 'uuid']], + ], + 'order' => [ + 'AccessLog.id' => 'DESC' + ], + ]; + + public function admin_index() + { + $params = $this->IndexFilter->harvestParameters([ + 'created', + 'ip', + 'user', + 'org', + 'request_id', + 'authkey_id', + 'api_request', + 'request_method', + 'controller', + 'action', + 'url', + 'response_code', + ]); + + $conditions = $this->__searchConditions($params); + + if ($this->_isRest()) { + $list = $this->AccessLog->find('all', [ + 'conditions' => $conditions, + 'contain' => $this->paginate['contain'], + ]); + return $this->RestResponse->viewData($list, 'json'); + } + + $this->paginate['conditions'] = $conditions; + $list = $this->paginate(); + + $this->set('list', $list); + $this->set('title_for_layout', __('Access logs')); + } + + public function admin_request($id) + { + $request = $this->AccessLog->find('first', [ + 'conditions' => ['AccessLog.id' => $id], + 'fields' => ['AccessLog.request'], + ]); + if (empty($request)) { + throw new NotFoundException(__('Access log not found')); + } + + list($contentType, $encoding, $data) = explode("\n", $request['AccessLog']['request'], 3); + $contentType = explode(';', $contentType, 2)[0]; + + if ($contentType === 'application/x-www-form-urlencoded') { + parse_str($data, $output); + $data = var_export($output, true); + } + + $this->set('request', $data); + } + + /** + * @param array $params + * @return array + */ + private function __searchConditions(array $params) + { + $qbRules = []; + foreach ($params as $key => $value) { + if ($key === 'created') { + $qbRules[] = [ + 'id' => $key, + 'operator' => is_array($value) ? 'between' : 'greater_or_equal', + 'value' => $value, + ]; + } else { + if (is_array($value)) { + $value = implode('||', $value); + } + $qbRules[] = [ + 'id' => $key, + 'value' => $value, + ]; + } + } + $this->set('qbRules', $qbRules); + + $conditions = []; + if (isset($params['user'])) { + if (is_numeric($params['user'])) { + $conditions['AccessLog.user_id'] = $params['user']; + } else { + $user = $this->User->find('first', [ + 'conditions' => ['User.email' => $params['user']], + 'fields' => ['id'], + ]); + if (!empty($user)) { + $conditions['AccessLog.user_id'] = $user['User']['id']; + } else { + $conditions['AccessLog.user_id'] = -1; + } + } + } + if (isset($params['ip'])) { + $conditions['AccessLog.ip'] = inet_pton($params['ip']); + } + foreach (['authkey_id', 'request_id', 'controller', 'action'] as $field) { + if (isset($params[$field])) { + $conditions['AccessLog.' . $field] = $params[$field]; + } + } + if (isset($params['url'])) { + $conditions['AccessLog.url LIKE'] = "%{$params['url']}%"; + } + if (isset($params['request_method'])) { + $methodId = array_flip(AccessLog::REQUEST_TYPES)[$params['request_method']] ?? -1; + $conditions['AccessLog.request_method'] = $methodId; + } + if (isset($params['org'])) { + if (is_numeric($params['org'])) { + $conditions['AccessLog.org_id'] = $params['org']; + } else { + $org = $this->AccessLog->Organisation->fetchOrg($params['org']); + if ($org) { + $conditions['AccessLog.org_id'] = $org['id']; + } else { + $conditions['AccessLog.org_id'] = -1; + } + } + } + if (isset($params['created'])) { + $tempData = is_array($params['created']) ? $params['created'] : [$params['created']]; + foreach ($tempData as $k => $v) { + $tempData[$k] = $this->AccessLog->resolveTimeDelta($v); + } + if (count($tempData) === 1) { + $conditions['AccessLog.created >='] = date("Y-m-d H:i:s", $tempData[0]); + } else { + if ($tempData[0] < $tempData[1]) { + $temp = $tempData[1]; + $tempData[1] = $tempData[0]; + $tempData[0] = $temp; + } + $conditions['AND'][] = ['AccessLog.created <=' => date("Y-m-d H:i:s", $tempData[0])]; + $conditions['AND'][] = ['AccessLog.created >=' => date("Y-m-d H:i:s", $tempData[1])]; + } + } + return $conditions; + } +} \ No newline at end of file diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index e92adf25a..fe4db0024 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -665,27 +665,22 @@ class AppController extends Controller { $userMonitoringEnabled = Configure::read('Security.user_monitoring_enabled'); if ($userMonitoringEnabled) { - $redis = $this->User->setupRedis(); - $userMonitoringEnabled = $redis && $redis->sismember('misp:monitored_users', $user['id']); + try { + $userMonitoringEnabled = RedisTool::init()->sismember('misp:monitored_users', $user['id']); + } catch (Exception $e) { + $userMonitoringEnabled = false; + } } - if (Configure::read('MISP.log_paranoid') || $userMonitoringEnabled) { - $change = 'HTTP method: ' . $_SERVER['REQUEST_METHOD'] . PHP_EOL . 'Target: ' . $this->request->here; - if ( - ( - $this->request->is('post') || - $this->request->is('put') - ) && - ( - !empty(Configure::read('MISP.log_paranoid_include_post_body')) || - $userMonitoringEnabled - ) - ) { - $payload = $this->request->input(); - $change .= PHP_EOL . 'Request body: ' . $payload; - } - $this->Log = ClassRegistry::init('Log'); - $this->Log->createLogEntry($user, 'request', 'User', $user['id'], 'Paranoid log entry', $change); + $shouldBeLogged = $userMonitoringEnabled || + Configure::read('MISP.log_paranoid') || + (Configure::read('MISP.log_paranoid_api') && $user['logged_by_authkey']); + + if ($shouldBeLogged) { + $includeRequestBody = !empty(Configure::read('MISP.log_paranoid_include_post_body')) || $userMonitoringEnabled; + /** @var AccessLog $accessLog */ + $accessLog = ClassRegistry::init('AccessLog'); + $accessLog->logRequest($user, $this->_remoteIp(), $this->request, $includeRequestBody); } } diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php index 6f0a02598..2de04e085 100644 --- a/app/Controller/Component/ACLComponent.php +++ b/app/Controller/Component/ACLComponent.php @@ -386,16 +386,20 @@ class ACLComponent extends Component 'testForStolenAttributes' => array(), 'pruneUpdateLogs' => array() ), - 'auditLogs' => [ - 'admin_index' => ['perm_audit'], - 'fullChange' => ['perm_audit'], - 'eventIndex' => ['*'], - 'returnDates' => ['*'], - ], - 'modules' => array( - 'index' => array('perm_auth'), - 'queryEnrichment' => array('perm_auth'), - ), + 'auditLogs' => [ + 'admin_index' => ['perm_audit'], + 'fullChange' => ['perm_audit'], + 'eventIndex' => ['*'], + 'returnDates' => ['*'], + ], + 'accessLogs' => [ + 'admin_index' => [], + 'admin_request' => [], + ], + 'modules' => array( + 'index' => array('perm_auth'), + 'queryEnrichment' => array('perm_auth'), + ), 'news' => array( 'add' => array(), 'edit' => array(), diff --git a/app/Model/AccessLog.php b/app/Model/AccessLog.php new file mode 100644 index 000000000..ec9b82b05 --- /dev/null +++ b/app/Model/AccessLog.php @@ -0,0 +1,230 @@ + 'Unknown', + 1 => 'GET', + 2 => 'HEAD', + 3 => 'POST', + 4 => 'PUT', + 5 => 'DELETE', + 6 => 'OPTIONS', + 7 => 'TRACE', + 8 => 'PATCH', + ]; + + public $actsAs = [ + 'Containable', + ]; + + public $compressionStats = [ + 'compressed' => 0, + 'bytes_compressed' => 0, + 'bytes_uncompressed' => 0, + ]; + + public $belongsTo = [ + 'User' => [ + 'className' => 'User', + 'foreignKey' => 'user_id', + ], + 'Organisation' => [ + 'className' => 'Organisation', + 'foreignKey' => 'org_id', + ], + ]; + + public function afterFind($results, $primary = false) + { + foreach ($results as &$result) { + if (isset($result['AccessLog']['ip'])) { + $result['AccessLog']['ip'] = inet_ntop($result['AccessLog']['ip']); + } + if (isset($result['AccessLog']['request_method'])) { + $result['AccessLog']['request_method'] = self::REQUEST_TYPES[$result['AccessLog']['request_method']]; + } + if (!empty($result['AccessLog']['request'])) { + $result['AccessLog']['request'] = $this->decodeRequest($result['AccessLog']['request']); + } + } + return $results; + } + + public function beforeSave($options = []) + { + $accessLog = &$this->data['AccessLog']; + + $this->externalLog($accessLog); + + if (Configure::read('MISP.log_paranoid_skip_db')) { + return; + } + + // Truncate + foreach (['request_id', 'user_agent', 'url'] as $field) { + if (isset($accessLog[$field]) && strlen($accessLog[$field]) > 255) { + $accessLog[$field] = substr($accessLog[$field], 0, 255); + } + } + + if (isset($accessLog['ip'])) { + $accessLog['ip'] = inet_pton($accessLog['ip']); + } + + if (isset($accessLog['request_method'])) { + $requestMethodIds = array_flip(self::REQUEST_TYPES); + $accessLog['request_method'] = $requestMethodIds[$accessLog['request_method']] ?? 0; + } + + if (isset($accessLog['request'])) { + $accessLog['request'] = $this->encodeRequest($accessLog['request']); + } + } + + /** + * @param array $user + * @param string $remoteIp + * @param CakeRequest $request + * @param bool $includeRequestBody + * @return bool + * @throws Exception + */ + public function logRequest(array $user, $remoteIp, CakeRequest $request, $includeRequestBody = true) + { + $requestTime = $_SERVER['REQUEST_TIME_FLOAT'] ?? microtime(true); + $now = DateTime::createFromFormat('U.u', $requestTime); + $logClientIp = Configure::read('MISP.log_client_ip'); + + $dataToSave = [ + 'created' => $now->format('Y-m-d H:i:s.u'), + 'request_id' => $_SERVER['HTTP_X_REQUEST_ID'] ?? null, + 'user_id' => (int)$user['id'], + 'org_id' => (int)$user['org_id'], + 'authkey_id' => isset($user['authkey_id']) ? (int)$user['authkey_id'] : null, + 'ip' => $logClientIp ? $remoteIp : null, + 'user_agent' => $_SERVER['HTTP_USER_AGENT'] ?? null, + 'request_method' => $_SERVER['REQUEST_METHOD'], + 'controller' => $request->params['controller'], + 'action' => $request->params['action'], + 'url' => $request->here, + ]; + + if ($includeRequestBody && $request->is(['post', 'put', 'delete'])) { + $requestContentType = $_SERVER['CONTENT_TYPE'] ?? null; + $requestEncoding = $_SERVER['HTTP_CONTENT_ENCODING'] ?? null; + $dataToSave['request'] = "$requestContentType\n$requestEncoding\n{$request->input()}"; + } + + // Save data on shutdown + register_shutdown_function(function () use ($dataToSave, $requestTime) { + session_write_close(); // close session to allow concurrent requests + $this->saveOnShutdown($dataToSave, $requestTime); + }); + + return true; + } + + /** + * @param array $data + * @param float $requestTime + * @return bool + * @throws Exception + */ + private function saveOnShutdown(array $data, $requestTime) + { + $data['response_code'] = http_response_code(); + $data['memory_usage'] = memory_get_peak_usage(); + $data['duration'] = (int)((microtime(true) - $requestTime) * 1000); + + try { + return $this->save($data, ['atomic' => false]); + } catch (Exception $e) { + $this->logException("Could not insert access log to database", $e, LOG_WARNING); + return false; + } + } + + /** + * @param array $data + * @return void + */ + public function externalLog(array $data) + { + if ($this->pubToZmq('audit')) { + $this->getPubSubTool()->publish($data, 'audit', 'log'); + } + + $this->publishKafkaNotification('audit', $data, 'log'); + + if (Configure::read('Plugin.ElasticSearch_logging_enable')) { + // send off our logs to distributed /dev/null + $logIndex = Configure::read("Plugin.ElasticSearch_log_index"); + $elasticSearchClient = $this->getElasticSearchTool(); + $elasticSearchClient->pushDocument($logIndex, "log", $data); + } + } + + /** + * @param string $request + * @return string + */ + private function decodeRequest($request) + { + $header = substr($request, 0, 4); + if ($header === self::BROTLI_HEADER) { + $this->compressionStats['compressed']++; + if (function_exists('brotli_uncompress')) { + $this->compressionStats['bytes_compressed'] += strlen($request); + $request = brotli_uncompress(substr($request, 4)); + $this->compressionStats['bytes_uncompressed'] += strlen($request); + if ($request === false) { + return 'Compressed'; + } + } else { + return 'Compressed'; + } + } elseif ($header === self::ZSTD_HEADER) { + $this->compressionStats['compressed']++; + if (function_exists('zstd_uncompress')) { + $this->compressionStats['bytes_compressed'] += strlen($request); + $request = zstd_uncompress($request); + $this->compressionStats['bytes_uncompressed'] += strlen($request); + if ($request === false) { + return 'Compressed'; + } + } else { + return 'Compressed'; + } + } + return $request; + } + + /** + * @param string $request + * @return string + */ + private function encodeRequest($request) + { + $compressionEnabled = Configure::read('MISP.log_new_audit_compress') && + (function_exists('brotli_compress') || function_exists('zstd_compress')); + + if ($compressionEnabled && strlen($request) >= self::COMPRESS_MIN_LENGTH) { + if (function_exists('zstd_compress')) { + return zstd_compress($request, 4); + } else { + return self::BROTLI_HEADER . brotli_compress($request, 4, BROTLI_TEXT); + } + } + return $request; + } +} \ No newline at end of file diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 8465e5794..27b4268bc 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -85,7 +85,7 @@ class AppModel extends Model 81 => false, 82 => false, 83 => false, 84 => false, 85 => false, 86 => false, 87 => false, 88 => false, 89 => false, 90 => false, 91 => false, 92 => false, 93 => false, 94 => false, 95 => true, 96 => false, 97 => true, 98 => false, - 99 => false + 99 => false, 100 => false, ); const ADVANCED_UPDATES_DESCRIPTION = array( @@ -1882,6 +1882,28 @@ class AppModel extends Model $sqlArray[] = "ALTER TABLE `event_tags` ADD `relationship_type` varchar(191) NULL DEFAULT '';"; $sqlArray[] = "ALTER TABLE `attribute_tags` ADD `relationship_type` varchar(191) NULL DEFAULT '';"; break; + case 100: + $sqlArray[] = "CREATE TABLE IF NOT EXISTS `access_logs` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `created` datetime(4) NOT NULL, + `user_id` int(11) NOT NULL, + `org_id` int(11) NOT NULL, + `authkey_id` int(11) DEFAULT NULL, + `ip` varbinary(16) DEFAULT NULL, + `request_method` tinyint NOT NULL, + `user_agent` varchar(255) DEFAULT NULL, + `request_id` varchar(255) DEFAULT NULL, + `controller` varchar(20) NOT NULL, + `action` varchar(20) NOT NULL, + `url` varchar(255) NOT NULL, + `request` blob, + `response_code` smallint NOT NULL, + `memory_usage` int(11) NOT NULL, + `duration` int(11) NOT NULL, + PRIMARY KEY (`id`), + INDEX `user_id` (`user_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"; + break; case 'fixNonEmptySharingGroupID': $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; diff --git a/app/Model/Log.php b/app/Model/Log.php index 2d6b983f2..65d882eeb 100644 --- a/app/Model/Log.php +++ b/app/Model/Log.php @@ -145,9 +145,6 @@ class Log extends AppModel } } $this->logData($this->data); - if ($this->data['Log']['action'] === 'request' && !empty(Configure::read('MISP.log_paranoid_skip_db'))) { - return false; - } return true; } @@ -243,9 +240,6 @@ class Log extends AppModel ]]); if (!$result) { - if ($action === 'request' && !empty(Configure::read('MISP.log_paranoid_skip_db'))) { - return null; - } if (!empty(Configure::read('MISP.log_skip_db_logs_completely'))) { return null; } @@ -349,9 +343,8 @@ class Log extends AppModel public function logData($data) { - if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_audit_notifications_enable')) { - $pubSubTool = $this->getPubSubTool(); - $pubSubTool->publish($data, 'audit', 'log'); + if ($this->pubToZmq('audit')) { + $this->getPubSubTool()->publish($data, 'audit', 'log'); } $this->publishKafkaNotification('audit', $data, 'log'); @@ -363,11 +356,6 @@ class Log extends AppModel $elasticSearchClient->pushDocument($logIndex, "log", $data); } - // Do not save request action logs to syslog, because they contain no information - if ($data['Log']['action'] === 'request') { - return true; - } - // write to syslogd as well if enabled if ($this->syslog === null) { if (Configure::read('Security.syslog')) { diff --git a/app/Model/Server.php b/app/Model/Server.php index d1825e2b8..fd99b7573 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -5576,6 +5576,14 @@ class Server extends AppModel 'type' => 'boolean', 'null' => true ), + 'log_paranoid_api' => array( + 'level' => 0, + 'description' => __('If this functionality is enabled all API requests will be logged.'), + 'value' => false, + 'test' => 'testBoolFalse', + 'type' => 'boolean', + 'null' => true + ), 'log_paranoid_skip_db' => array( 'level' => 0, 'description' => __('You can decide to skip the logging of the paranoid logs to the database.'), diff --git a/app/View/AccessLogs/admin_index.ctp b/app/View/AccessLogs/admin_index.ctp new file mode 100644 index 000000000..4add1c8c7 --- /dev/null +++ b/app/View/AccessLogs/admin_index.ctp @@ -0,0 +1,336 @@ +
+

+
+
+
+ + +
+
+ Html->script('moment.min'); + echo $this->Html->script('doT'); + echo $this->Html->script('extendext'); + echo $this->Html->css('query-builder.default'); + echo $this->Html->script('query-builder'); + ?> + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LightPaginator->sort('created') ?>LightPaginator->sort('user_id', __('User')) ?>LightPaginator->sort('ip', __('IP')) ?>LightPaginator->sort('org_id', __('Org')) ?>LightPaginator->sort('request_method', __('Method')) ?>LightPaginator->sort('url', __('URL')) ?>LightPaginator->sort('response_code', __('Code')) ?>LightPaginator->sort('memory_usage', __('Memory')) ?>LightPaginator->sort('duration', __('Duration')) ?>
Time->time($item['AccessLog']['created']); ?>' . h($item['User']['email']) . ''; + } else { + echo __('Deleted user #%s', h($item['AccessLog']['user_id'])); + } + + if (!empty($item['AccessLog']['authkey_id'])) { + echo ' '; + } + ?> + OrgImg->getOrgLogo($item, 24); + } else if ($item['AccessLog']['org_id'] != 0) { + echo __('Deleted org #%s', h($item['AccessLog']['org_id'])); + } + ?> + + + ' : '' ?> + ms
+

+

+ +
+ +element('/genericElements/SideMenu/side_menu', ['menuList' => 'logs', 'menuItem' => 'listAccessLogs']); + diff --git a/app/View/AccessLogs/admin_request.ctp b/app/View/AccessLogs/admin_request.ctp new file mode 100644 index 000000000..79ad3c115 --- /dev/null +++ b/app/View/AccessLogs/admin_request.ctp @@ -0,0 +1 @@ +
diff --git a/app/View/AuditLogs/admin_index.ctp b/app/View/AuditLogs/admin_index.ctp index 81c556d48..eb6af731f 100644 --- a/app/View/AuditLogs/admin_index.ctp +++ b/app/View/AuditLogs/admin_index.ctp @@ -14,7 +14,7 @@ echo $this->Html->css('query-builder.default'); echo $this->Html->script('query-builder'); ?> -