mirror of https://github.com/MISP/MISP
add: [3.x] migrate access logs controller
Luciano Righetti
parent
2fb1716f4a
commit
dd6c6aa164
|
|
@ -1,4 +1,5 @@
|
|||
<?php
|
||||
|
||||
/*
|
||||
* Local configuration file to provide any overrides to your app.php configuration.
|
||||
* Copy and save this file as app_local.php and make changes as required.
|
||||
|
|
@ -50,7 +51,11 @@ return [
|
|||
'MISP' => [
|
||||
'dark' => 0,
|
||||
'email' => 'email@example.com',
|
||||
'default_event_distribution' => '1'
|
||||
'default_event_distribution' => '1',
|
||||
'log_paranoid' => true,
|
||||
'log_paranoid_include_sql_queries' => true,
|
||||
'log_new_audit_compress' => true,
|
||||
'log_paranoid_include_post_body' => true,
|
||||
],
|
||||
'BackgroundJobs' => [
|
||||
'enabled' => true,
|
||||
|
|
|
|||
28
phpcs.xml
28
phpcs.xml
|
|
@ -14,7 +14,33 @@
|
|||
<arg name="parallel" value="75" />
|
||||
<arg value="np" />
|
||||
|
||||
<rule ref="PSR12" />
|
||||
<!-- PSR12 Standard -->
|
||||
<rule ref="PSR12">
|
||||
<exclude name="PSR12.Files.FileHeader.SpacingAfterBlock" />
|
||||
<exclude name="PSR12.Files.FileHeader.IncorrectOrder" />
|
||||
<!--
|
||||
Property and method names with underscore prefix are allowed in CakePHP.
|
||||
Not using underscore prefix is a recommendation of PSR2, not a requirement.
|
||||
-->
|
||||
<exclude name="PSR2.Classes.PropertyDeclaration.Underscore" />
|
||||
<exclude name="PSR2.Methods.MethodDeclaration.Underscore" />
|
||||
</rule>
|
||||
|
||||
<!-- Relax rules from PSR1 -->
|
||||
<rule ref="PSR1.Classes.ClassDeclaration.MissingNamespace">
|
||||
<exclude-pattern>*/config/Migrations/*</exclude-pattern>
|
||||
<exclude-pattern>*/config/Seeds/*</exclude-pattern>
|
||||
</rule>
|
||||
<rule ref="PSR1.Files.SideEffects">
|
||||
<exclude-pattern>*/config/*</exclude-pattern>
|
||||
<exclude-pattern>*/tests/*</exclude-pattern>
|
||||
</rule>
|
||||
<rule ref="PSR1.Methods.CamelCapsMethodName">
|
||||
<exclude-pattern>*/src/Controller/*</exclude-pattern>
|
||||
<exclude-pattern>*/src/Command/*</exclude-pattern>
|
||||
<exclude-pattern>*/src/Shell/*</exclude-pattern>
|
||||
<exclude-pattern>*/tests/*</exclude-pattern>
|
||||
</rule>
|
||||
|
||||
<!-- Arrays -->
|
||||
<rule ref="NormalizedArrays.Arrays.ArrayBraceSpacing" />
|
||||
|
|
|
|||
|
|
@ -0,0 +1,241 @@
|
|||
<?php
|
||||
|
||||
namespace App\Controller\Admin;
|
||||
|
||||
use App\Controller\AppController;
|
||||
use App\Model\Entity\AccessLog;
|
||||
use Cake\Core\Configure;
|
||||
use Cake\Http\Exception\NotFoundException;
|
||||
|
||||
class AccessLogsController extends AppController
|
||||
{
|
||||
public $paginate = [
|
||||
'recursive' => -1,
|
||||
'limit' => 60,
|
||||
'fields' => ['id', 'created', 'user_id', 'org_id', 'authkey_id', 'ip', 'request_method', 'user_agent', 'request_id', 'controller', 'action', 'url', 'response_code', 'memory_usage', 'duration', 'query_count'],
|
||||
'contain' => [
|
||||
'Users' => ['fields' => ['id', 'email', 'org_id']],
|
||||
'Organisations' => ['fields' => ['id', 'name', 'uuid']],
|
||||
],
|
||||
'order' => [
|
||||
'AccessLogs.id' => 'DESC'
|
||||
],
|
||||
];
|
||||
|
||||
public function index()
|
||||
{
|
||||
$params = $this->harvestParameters(
|
||||
[
|
||||
'created',
|
||||
'ip',
|
||||
'user',
|
||||
'org',
|
||||
'request_id',
|
||||
'authkey_id',
|
||||
'api_request',
|
||||
'request_method',
|
||||
'controller',
|
||||
'action',
|
||||
'url',
|
||||
'user_agent',
|
||||
'memory_usage',
|
||||
'duration',
|
||||
'query_count',
|
||||
'response_code',
|
||||
]
|
||||
);
|
||||
|
||||
$conditions = $this->__searchConditions($params);
|
||||
|
||||
if ($this->ParamHandler->isRest()) {
|
||||
$list = $this->AccessLogs->find(
|
||||
'all',
|
||||
[
|
||||
'conditions' => $conditions,
|
||||
'contain' => $this->paginate['contain'],
|
||||
]
|
||||
);
|
||||
foreach ($list as $item) {
|
||||
if (!empty($item['request'])) {
|
||||
$item['request'] = base64_encode($item['request']);
|
||||
}
|
||||
}
|
||||
return $this->RestResponse->viewData($list->toArray(), 'json');
|
||||
}
|
||||
if (empty(Configure::read('MISP.log_skip_access_logs_in_application_logs'))) {
|
||||
$this->Flash->warning(__('Access logs are logged in both application logs and access logs. Make sure you reconfigure your log monitoring tools and update MISP.log_skip_access_logs_in_application_logs.'));
|
||||
}
|
||||
|
||||
$this->AccessLog->virtualFields['has_query_log'] = 'query_log IS NOT NULL';
|
||||
$this->paginate['fields'][] = 'has_query_log';
|
||||
$this->paginate['conditions'] = $conditions;
|
||||
$list = $this->paginate();
|
||||
|
||||
$this->set('list', $list);
|
||||
$this->set('title_for_layout', __('Access logs'));
|
||||
}
|
||||
|
||||
public function request($id)
|
||||
{
|
||||
$request = $this->AccessLogs->find(
|
||||
'all',
|
||||
[
|
||||
'conditions' => ['AccessLogs.id' => $id],
|
||||
'fields' => ['AccessLogs.request'],
|
||||
]
|
||||
)->first();
|
||||
if (empty($request)) {
|
||||
throw new NotFoundException(__('Access log not found'));
|
||||
}
|
||||
|
||||
if (empty($request['request'])) {
|
||||
throw new NotFoundException(__('Request body is empty'));
|
||||
}
|
||||
|
||||
$contentType = explode(';', $request['request_content_type'], 2)[0];
|
||||
if ($contentType === 'application/x-www-form-urlencoded' || $contentType === 'multipart/form-data') {
|
||||
parse_str($request['request'], $output);
|
||||
// highlight PHP array
|
||||
$highlighted = highlight_string("<?php " . var_export($output, true), true);
|
||||
$highlighted = trim($highlighted);
|
||||
$highlighted = preg_replace("|^\\<code\\>\\<span style\\=\"color\\: #[a-fA-F0-9]{0,6}\"\\>|", "", $highlighted, 1); // remove prefix
|
||||
$highlighted = preg_replace("|\\</code\\>\$|", "", $highlighted, 1); // remove suffix 1
|
||||
$highlighted = trim($highlighted); // remove line breaks
|
||||
$highlighted = preg_replace("|\\</span\\>\$|", "", $highlighted, 1); // remove suffix 2
|
||||
$highlighted = trim($highlighted); // remove line breaks
|
||||
$highlighted = preg_replace("|^(\\<span style\\=\"color\\: #[a-fA-F0-9]{0,6}\"\\>)(<\\?php )(.*?)(\\</span\\>)|", "\$1\$3\$4", $highlighted); // remove custom added "<?php "
|
||||
$data = $highlighted;
|
||||
} else {
|
||||
$data = h($request['request']);
|
||||
}
|
||||
|
||||
$this->set('request', $data);
|
||||
}
|
||||
|
||||
public function queryLog($id)
|
||||
{
|
||||
$request = $this->AccessLog->find(
|
||||
'first',
|
||||
[
|
||||
'conditions' => ['AccessLogs.id' => $id],
|
||||
'fields' => ['AccessLogs.query_log'],
|
||||
]
|
||||
);
|
||||
if (empty($request)) {
|
||||
throw new NotFoundException(__('Access log not found'));
|
||||
}
|
||||
|
||||
if (empty($request['query_log'])) {
|
||||
throw new NotFoundException(__('Query log is empty'));
|
||||
}
|
||||
|
||||
$this->set('queryLog', $request['query_log']);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $params
|
||||
* @return array
|
||||
*/
|
||||
private function __searchConditions(array $params)
|
||||
{
|
||||
$qbRules = [];
|
||||
foreach ($params as $key => $value) {
|
||||
if ($key === 'created') {
|
||||
$qbRules[] = [
|
||||
'id' => $key,
|
||||
'operator' => is_array($value) ? 'between' : 'greater_or_equal',
|
||||
'value' => $value,
|
||||
];
|
||||
} else {
|
||||
if (is_array($value)) {
|
||||
$value = implode('||', $value);
|
||||
}
|
||||
$qbRules[] = [
|
||||
'id' => $key,
|
||||
'value' => $value,
|
||||
];
|
||||
}
|
||||
}
|
||||
$this->set('qbRules', $qbRules);
|
||||
|
||||
$conditions = [];
|
||||
if (isset($params['user'])) {
|
||||
if (is_numeric($params['user'])) {
|
||||
$conditions['AccessLogs.user_id'] = $params['user'];
|
||||
} else {
|
||||
$user = $this->Users->find(
|
||||
'first',
|
||||
[
|
||||
'conditions' => ['Users.email' => $params['user']],
|
||||
'fields' => ['id'],
|
||||
]
|
||||
);
|
||||
if (!empty($user)) {
|
||||
$conditions['AccessLogs.user_id'] = $user['id'];
|
||||
} else {
|
||||
$conditions['AccessLogs.user_id'] = -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (isset($params['ip'])) {
|
||||
$conditions['AccessLogs.ip'] = inet_pton($params['ip']);
|
||||
}
|
||||
foreach (['authkey_id', 'request_id', 'controller', 'action'] as $field) {
|
||||
if (isset($params[$field])) {
|
||||
$conditions['AccessLogs.' . $field] = $params[$field];
|
||||
}
|
||||
}
|
||||
if (isset($params['url'])) {
|
||||
$conditions['AccessLogs.url LIKE'] = "%{$params['url']}%";
|
||||
}
|
||||
if (isset($params['user_agent'])) {
|
||||
$conditions['AccessLogs.user_agent LIKE'] = "%{$params['user_agent']}%";
|
||||
}
|
||||
if (isset($params['memory_usage'])) {
|
||||
$conditions['AccessLogs.memory_usage >='] = ($params['memory_usage'] * 1024);
|
||||
}
|
||||
if (isset($params['memory_usage'])) {
|
||||
$conditions['AccessLogs.memory_usage >='] = ($params['memory_usage'] * 1024);
|
||||
}
|
||||
if (isset($params['duration'])) {
|
||||
$conditions['AccessLogs.duration >='] = $params['duration'];
|
||||
}
|
||||
if (isset($params['query_count'])) {
|
||||
$conditions['AccessLogs.query_count >='] = $params['query_count'];
|
||||
}
|
||||
if (isset($params['request_method'])) {
|
||||
$methodId = array_flip(AccessLog::REQUEST_TYPES)[$params['request_method']] ?? -1;
|
||||
$conditions['AccessLogs.request_method'] = $methodId;
|
||||
}
|
||||
if (isset($params['org'])) {
|
||||
if (is_numeric($params['org'])) {
|
||||
$conditions['AccessLogs.org_id'] = $params['org'];
|
||||
} else {
|
||||
$org = $this->AccessLog->Organisation->fetchOrg($params['org']);
|
||||
if ($org) {
|
||||
$conditions['AccessLogs.org_id'] = $org['id'];
|
||||
} else {
|
||||
$conditions['AccessLogs.org_id'] = -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (isset($params['created'])) {
|
||||
$tempData = is_array($params['created']) ? $params['created'] : [$params['created']];
|
||||
foreach ($tempData as $k => $v) {
|
||||
$tempData[$k] = $this->AccessLog->resolveTimeDelta($v);
|
||||
}
|
||||
if (count($tempData) === 1) {
|
||||
$conditions['AccessLogs.created >='] = date("Y-m-d H:i:s", $tempData[0]);
|
||||
} else {
|
||||
if ($tempData[0] < $tempData[1]) {
|
||||
$temp = $tempData[1];
|
||||
$tempData[1] = $tempData[0];
|
||||
$tempData[0] = $temp;
|
||||
}
|
||||
$conditions['AND'][] = ['AccessLogs.created <=' => date("Y-m-d H:i:s", $tempData[0])];
|
||||
$conditions['AND'][] = ['AccessLogs.created >=' => date("Y-m-d H:i:s", $tempData[1])];
|
||||
}
|
||||
}
|
||||
return $conditions;
|
||||
}
|
||||
}
|
||||
|
|
@ -19,6 +19,7 @@ declare(strict_types=1);
|
|||
namespace App\Controller;
|
||||
|
||||
use App\Lib\Tools\JsonTool;
|
||||
use App\Lib\Tools\RedisTool;
|
||||
use Cake\Controller\Controller;
|
||||
use Cake\Core\Configure;
|
||||
use Cake\Event\EventInterface;
|
||||
|
|
@ -134,6 +135,7 @@ class AppController extends Controller
|
|||
'contain' => ['Roles', /*'UserSettings',*/ 'Organisations']
|
||||
]
|
||||
);
|
||||
$this->__accessMonitor($user->toArray());
|
||||
if (!empty($user['disabled'])) {
|
||||
$this->Authentication->logout();
|
||||
$this->Flash->error(__('The user account is disabled.'));
|
||||
|
|
@ -412,4 +414,63 @@ class AppController extends Controller
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return string|null
|
||||
*/
|
||||
protected function _remoteIp()
|
||||
{
|
||||
$ipHeader = Configure::read('MISP.log_client_ip_header') ?: 'REMOTE_ADDR';
|
||||
return isset($_SERVER[$ipHeader]) ? trim($_SERVER[$ipHeader]) : $_SERVER['REMOTE_ADDR'];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $user
|
||||
* @throws Exception
|
||||
*/
|
||||
private function __accessMonitor(array $user)
|
||||
{
|
||||
$userMonitoringEnabled = Configure::read('Security.user_monitoring_enabled');
|
||||
if ($userMonitoringEnabled) {
|
||||
try {
|
||||
$userMonitoringEnabled = RedisTool::init()->sismember('misp:monitored_users', $user['id']);
|
||||
} catch (Exception $e) {
|
||||
$userMonitoringEnabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
$shouldBeLogged = $userMonitoringEnabled ||
|
||||
Configure::read('MISP.log_paranoid') ||
|
||||
(Configure::read('MISP.log_paranoid_api') && isset($user['logged_by_authkey']) && $user['logged_by_authkey']);
|
||||
|
||||
if ($shouldBeLogged) {
|
||||
$includeRequestBody = !empty(Configure::read('MISP.log_paranoid_include_post_body')) || $userMonitoringEnabled;
|
||||
/** @var AccessLog $accessLog */
|
||||
$accessLogsTable = $this->fetchTable('AccessLogs');
|
||||
$accessLogsTable->logRequest($user, $this->_remoteIp(), $this->request, $includeRequestBody);
|
||||
}
|
||||
|
||||
if (
|
||||
empty(Configure::read('MISP.log_skip_access_logs_in_application_logs')) &&
|
||||
$shouldBeLogged
|
||||
) {
|
||||
$change = 'HTTP method: ' . $_SERVER['REQUEST_METHOD'] . PHP_EOL . 'Target: ' . $this->request->getAttribute('here');
|
||||
;
|
||||
if (
|
||||
(
|
||||
$this->request->is('post') ||
|
||||
$this->request->is('put')
|
||||
) &&
|
||||
(
|
||||
!empty(Configure::read('MISP.log_paranoid_include_post_body')) ||
|
||||
$userMonitoringEnabled
|
||||
)
|
||||
) {
|
||||
$payload = $this->request->getBody();
|
||||
$change .= PHP_EOL . 'Request body: ' . $payload;
|
||||
}
|
||||
$logsTable = $this->fetchTable('Logs');
|
||||
$logsTable->createLogEntry($user, 'request', 'User', $user['id'], 'Paranoid log entry', $change);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,23 @@
|
|||
<?php
|
||||
|
||||
namespace App\Model\Entity;
|
||||
|
||||
use App\Model\Entity\AppModel;
|
||||
|
||||
class AccessLog extends AppModel
|
||||
{
|
||||
const BROTLI_HEADER = "\xce\xb2\xcf\x81";
|
||||
const COMPRESS_MIN_LENGTH = 256;
|
||||
|
||||
const REQUEST_TYPES = [
|
||||
0 => 'Unknown',
|
||||
1 => 'GET',
|
||||
2 => 'HEAD',
|
||||
3 => 'POST',
|
||||
4 => 'PUT',
|
||||
5 => 'DELETE',
|
||||
6 => 'OPTIONS',
|
||||
7 => 'TRACE',
|
||||
8 => 'PATCH',
|
||||
];
|
||||
}
|
||||
|
|
@ -0,0 +1,371 @@
|
|||
<?php
|
||||
|
||||
namespace App\Model\Table;
|
||||
|
||||
use App\Model\Entity\AccessLog;
|
||||
use App\Model\Table\AppTable;
|
||||
use Cake\Datasource\EntityInterface;
|
||||
use Cake\Event\EventInterface;
|
||||
use ArrayObject;
|
||||
use App\Lib\Tools\JsonTool;
|
||||
use Cake\Collection\CollectionInterface;
|
||||
use Cake\ORM\Query;
|
||||
use Cake\Chronos\Chronos;
|
||||
use Cake\Http\ServerRequest;
|
||||
use Cake\Core\Configure;
|
||||
use Exception;
|
||||
use Cake\Datasource\ConnectionManager;
|
||||
use DebugKit\Database\Log\DebugLog;
|
||||
use App\Lib\Tools\LogExtendedTrait;
|
||||
|
||||
class AccessLogsTable extends AppTable
|
||||
{
|
||||
use LogExtendedTrait;
|
||||
|
||||
public function initialize(array $config): void
|
||||
{
|
||||
parent::initialize($config);
|
||||
$this->belongsTo(
|
||||
'Users',
|
||||
[
|
||||
'className' => 'User',
|
||||
'foreignKey' => 'user_id',
|
||||
'propertyName' => 'User',
|
||||
]
|
||||
);
|
||||
$this->belongsTo(
|
||||
'Organisations',
|
||||
[
|
||||
'className' => 'Organisation',
|
||||
'foreignKey' => 'org_id',
|
||||
'propertyName' => 'Organisation',
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
public function beforeMarshal(EventInterface $event, ArrayObject $data, ArrayObject $options)
|
||||
{
|
||||
// Truncate
|
||||
foreach (['request_id', 'user_agent', 'url'] as $field) {
|
||||
if (isset($entity[$field]) && strlen($entity[$field]) > 255) {
|
||||
$entity[$field] = substr($entity[$field], 0, 255);
|
||||
}
|
||||
}
|
||||
|
||||
if (isset($data['ip'])) {
|
||||
$data['ip'] = inet_pton($data['ip']);
|
||||
}
|
||||
|
||||
if (isset($data['request_method'])) {
|
||||
$requestMethodIds = array_flip(AccessLog::REQUEST_TYPES);
|
||||
$data['request_method'] = $requestMethodIds[$data['request_method']] ?? 0;
|
||||
}
|
||||
|
||||
if (!empty($data['request'])) {
|
||||
$data['request'] = $this->compress($data['request']);
|
||||
}
|
||||
|
||||
if (!empty($data['query_log'])) {
|
||||
$data['query_log'] = $this->compress(JsonTool::encode($data['query_log']));
|
||||
}
|
||||
|
||||
// In database save size in kb to avoid overflow signed int type
|
||||
if (isset($data['memory_usage'])) {
|
||||
$data['memory_usage'] = $data['memory_usage'] >> 10; // same as /= 1024
|
||||
}
|
||||
}
|
||||
|
||||
public function beforeFind(EventInterface $event, Query $query, ArrayObject $options)
|
||||
{
|
||||
$query->formatResults(
|
||||
function (CollectionInterface $results) {
|
||||
return $results->map(
|
||||
function ($row) {
|
||||
if (isset($row['ip'])) {
|
||||
$row['ip'] = inet_ntop($row['ip']);
|
||||
}
|
||||
if (isset($row['request_method'])) {
|
||||
$row['request_method'] = AccessLog::REQUEST_TYPES[$row['request_method']];
|
||||
}
|
||||
if (!empty($row['request'])) {
|
||||
$decoded = $this->decodeRequest($row['request']);
|
||||
if ($decoded) {
|
||||
list($contentType, $encoding, $data) = $decoded;
|
||||
$row['request'] = $data;
|
||||
$row['request_content_type'] = $contentType;
|
||||
$row['request_content_encoding'] = $encoding;
|
||||
} else {
|
||||
$row['request'] = false;
|
||||
}
|
||||
}
|
||||
if (!empty($row['query_log'])) {
|
||||
$row['query_log'] = JsonTool::decode($this->decompress(stream_get_contents($row['query_log'])));
|
||||
}
|
||||
if (!empty($row['memory_usage'])) {
|
||||
$row['memory_usage'] = $row['memory_usage'] * 1024;
|
||||
}
|
||||
|
||||
return $row;
|
||||
}
|
||||
);
|
||||
},
|
||||
$query::APPEND
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $user
|
||||
* @param string $remoteIp
|
||||
* @param ServerRequest $request
|
||||
* @param bool $includeRequestBody
|
||||
* @return bool
|
||||
* @throws Exception
|
||||
*/
|
||||
public function logRequest(array $user, $remoteIp, ServerRequest $request, $includeRequestBody = true)
|
||||
{
|
||||
$requestTime = $this->requestTime();
|
||||
$logClientIp = Configure::read('MISP.log_client_ip');
|
||||
$includeSqlQueries = Configure::read('MISP.log_paranoid_include_sql_queries');
|
||||
|
||||
if ($includeSqlQueries) {
|
||||
ConnectionManager::get('default')->enableQueryLogging(); // Enable SQL logging
|
||||
}
|
||||
|
||||
$dataToSave = [
|
||||
'created' => $requestTime,
|
||||
'request_id' => $_SERVER['HTTP_X_REQUEST_ID'] ?? null,
|
||||
'user_id' => (int)$user['id'],
|
||||
'org_id' => (int)$user['org_id'],
|
||||
'authkey_id' => isset($user['authkey_id']) ? (int)$user['authkey_id'] : null,
|
||||
'ip' => $logClientIp ? $remoteIp : null,
|
||||
'user_agent' => $_SERVER['HTTP_USER_AGENT'] ?? null,
|
||||
'request_method' => $_SERVER['REQUEST_METHOD'],
|
||||
'controller' => $request->getParam('controller'),
|
||||
'action' => $request->getParam('action'),
|
||||
'url' => $request->getAttribute('here'),
|
||||
];
|
||||
|
||||
if ($includeRequestBody && $request->is(['post', 'put', 'delete'])) {
|
||||
$dataToSave['request'] = $this->requestBody($request);
|
||||
} else {
|
||||
$dataToSave['request'] = null;
|
||||
}
|
||||
|
||||
// Save data on shutdown
|
||||
register_shutdown_function(function () use ($dataToSave, $requestTime, $includeSqlQueries) {
|
||||
session_write_close(); // close session to allow concurrent requests
|
||||
$this->saveOnShutdown($dataToSave, $requestTime, $includeSqlQueries);
|
||||
});
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Chronos $duration
|
||||
* @return int Number of deleted entries
|
||||
*/
|
||||
public function deleteOldLogs(Chronos $duration)
|
||||
{
|
||||
$this->deleteAll([
|
||||
['created <' => $duration->format('Y-m-d H:i:s.u')],
|
||||
], false);
|
||||
|
||||
$deleted = $this->getAffectedRows();
|
||||
if ($deleted > 100) {
|
||||
$dataSource = $this->getDataSource();
|
||||
$dataSource->query('OPTIMIZE TABLE ' . $dataSource->name($this->table));
|
||||
}
|
||||
return $deleted;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param CakeRequest $request
|
||||
* @return string
|
||||
*/
|
||||
private function requestBody(ServerRequest $request)
|
||||
{
|
||||
$requestContentType = $_SERVER['CONTENT_TYPE'] ?? null;
|
||||
$requestEncoding = $_SERVER['HTTP_CONTENT_ENCODING'] ?? null;
|
||||
|
||||
if (substr($requestContentType, 0, 19) === 'multipart/form-data') {
|
||||
$input = http_build_query($request->getBody(), '', '&');
|
||||
} else {
|
||||
$input = $request->getData();
|
||||
if (is_array($input)) {
|
||||
$input = JsonTool::encode($input);
|
||||
}
|
||||
}
|
||||
|
||||
return "$requestContentType\n$requestEncoding\n$input";
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $data
|
||||
* @param Chronos $requestTime
|
||||
* @param bool $includeSqlQueries
|
||||
* @return bool
|
||||
* @throws Exception
|
||||
*/
|
||||
private function saveOnShutdown(array $data, Chronos $requestTime, $includeSqlQueries)
|
||||
{
|
||||
$sqlLog = ConnectionManager::get('default')->getLogger();
|
||||
|
||||
if ($sqlLog === null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$queries = [];
|
||||
$queryCount = 0;
|
||||
$queryTotalTime = 0;
|
||||
|
||||
if ($sqlLog instanceof DebugLog) {
|
||||
$queries = $sqlLog->queries();
|
||||
$queryCount = $sqlLog->totalRows();
|
||||
$queryTotalTime = $sqlLog->totalTime();
|
||||
}
|
||||
|
||||
if ($includeSqlQueries && !empty($queries)) {
|
||||
foreach ($queries as &$log) {
|
||||
$log['query'] = $this->escapeNonUnicode($log['query']);
|
||||
unset($log['affected']); // affected is the same as numRows
|
||||
unset($log['params']); // no need to save for your use case
|
||||
}
|
||||
$data['query_log'] = ['time' => $queryTotalTime, 'log' => $queries];
|
||||
}
|
||||
|
||||
$data['response_code'] = http_response_code();
|
||||
$data['memory_usage'] = memory_get_peak_usage();
|
||||
$data['query_count'] = $queryCount;
|
||||
$data['duration'] = (int)((microtime(true) - $requestTime->format('U.u')) * 1000); // in milliseconds
|
||||
|
||||
$accessLogEntity = $this->newEntity($data);
|
||||
|
||||
try {
|
||||
return $this->save($accessLogEntity, ['atomic' => false]);
|
||||
} catch (Exception $e) {
|
||||
$this->logException("Could not insert access log to database", $e, LOG_WARNING);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $data
|
||||
* @return void
|
||||
*/
|
||||
public function externalLog(array $data)
|
||||
{
|
||||
if ($this->pubToZmq('audit')) {
|
||||
$this->getPubSubTool()->publish($data, 'audit', 'log');
|
||||
}
|
||||
|
||||
$this->publishKafkaNotification('audit', $data, 'log');
|
||||
// In future add support for sending logs to elastic
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Chronos
|
||||
*/
|
||||
private function requestTime()
|
||||
{
|
||||
$requestTime = $_SERVER['REQUEST_TIME_FLOAT'] ?? microtime(true);
|
||||
$requestTime = (string) $requestTime;
|
||||
// Fix string if float value doesnt contain decimal part
|
||||
if (strpos($requestTime, '.') === false) {
|
||||
$requestTime .= '.0';
|
||||
}
|
||||
return Chronos::createFromFormat('U.u', $requestTime);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $request
|
||||
* @return array|false
|
||||
*/
|
||||
private function decodeRequest($request)
|
||||
{
|
||||
$request = $this->decompress(stream_get_contents($request));
|
||||
if ($request === false) {
|
||||
return false;
|
||||
}
|
||||
|
||||
list($contentType, $encoding, $data) = explode("\n", $request, 3);
|
||||
|
||||
if ($encoding === 'gzip') {
|
||||
$data = gzdecode($data);
|
||||
} elseif ($encoding === 'br') {
|
||||
if (function_exists('brotli_uncompress')) {
|
||||
$data = brotli_uncompress($data);
|
||||
} else {
|
||||
$data = false;
|
||||
}
|
||||
}
|
||||
|
||||
return [$contentType, $encoding, $data];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $data
|
||||
* @return false|string
|
||||
*/
|
||||
private function decompress($data)
|
||||
{
|
||||
$header = substr($data, 0, 4);
|
||||
if ($header === AccessLog::BROTLI_HEADER) {
|
||||
if (function_exists('brotli_uncompress')) {
|
||||
$data = brotli_uncompress(substr($data, 4));
|
||||
if ($data === false) {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return $data;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $data
|
||||
* @return string
|
||||
*/
|
||||
private function compress($data)
|
||||
{
|
||||
$compressionEnabled = Configure::read('MISP.log_new_audit_compress') &&
|
||||
function_exists('brotli_compress');
|
||||
|
||||
if ($compressionEnabled && strlen($data) >= AccessLog::COMPRESS_MIN_LENGTH) {
|
||||
return AccessLog::BROTLI_HEADER . brotli_compress($data, 4, BROTLI_TEXT);
|
||||
}
|
||||
return $data;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param $string
|
||||
* @return string
|
||||
*/
|
||||
private function escapeNonUnicode($string)
|
||||
{
|
||||
if (json_encode($string, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_LINE_TERMINATORS) !== false) {
|
||||
return $string; // string is valid unicode
|
||||
}
|
||||
|
||||
if (function_exists('mb_str_split')) {
|
||||
$result = mb_str_split($string);
|
||||
} else {
|
||||
$result = [];
|
||||
$length = mb_strlen($string);
|
||||
for ($i = 0; $i < $length; $i++) {
|
||||
$result[] = mb_substr($string, $i, 1);
|
||||
}
|
||||
}
|
||||
|
||||
$string = '';
|
||||
foreach ($result as $char) {
|
||||
if (strlen($char) === 1 && !preg_match('/[[:print:]]/', $char)) {
|
||||
$string .= '\x' . bin2hex($char);
|
||||
} else {
|
||||
$string .= $char;
|
||||
}
|
||||
}
|
||||
|
||||
return $string;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Test\Fixture;
|
||||
|
||||
use Cake\TestSuite\Fixture\TestFixture;
|
||||
|
||||
class AccessLogsFixture extends TestFixture
|
||||
{
|
||||
public $connection = 'test';
|
||||
|
||||
public const ACCESS_LOG_1_ID = 1000;
|
||||
|
||||
public function init(): void
|
||||
{
|
||||
$faker = \Faker\Factory::create();
|
||||
|
||||
$this->records = [
|
||||
[
|
||||
'id' => self::ACCESS_LOG_1_ID,
|
||||
'created' => '#000000',
|
||||
'user_id' => UsersFixture::USER_ADMIN_ID,
|
||||
'org_id' => OrganisationsFixture::ORGANISATION_A_ID,
|
||||
'request_method' => 0,
|
||||
'controller' => 'UsersController',
|
||||
'action' => 'index',
|
||||
'url' => 'http://localhost',
|
||||
'request' => null,
|
||||
'response_code' => 200,
|
||||
'memory_usage' => 0,
|
||||
'duration' => 1,
|
||||
'query_count' => 0,
|
||||
'query_log' => null,
|
||||
]
|
||||
];
|
||||
parent::init();
|
||||
}
|
||||
}
|
||||
|
|
@ -4,15 +4,12 @@ declare(strict_types=1);
|
|||
|
||||
namespace App\Test\Helper;
|
||||
|
||||
use Cake\TestSuite\IntegrationTestTrait;
|
||||
use Cake\Core\Configure;
|
||||
use Cake\Http\ServerRequestFactory;
|
||||
use Cake\Http\ServerRequest;
|
||||
use Cake\Http\Exception\NotImplementedException;
|
||||
use \League\OpenAPIValidation\PSR7\ValidatorBuilder;
|
||||
use \League\OpenAPIValidation\PSR7\RequestValidator;
|
||||
use \League\OpenAPIValidation\PSR7\ResponseValidator;
|
||||
use \League\OpenAPIValidation\PSR7\OperationAddress;
|
||||
use Cake\Http\ServerRequest;
|
||||
use Cake\Http\ServerRequestFactory;
|
||||
use Cake\TestSuite\IntegrationTestTrait;
|
||||
use League\OpenAPIValidation\PSR7\OperationAddress;
|
||||
|
||||
/**
|
||||
* Trait ApiTestTrait
|
||||
|
|
@ -56,15 +53,16 @@ trait ApiTestTrait
|
|||
|
||||
// somehow this is not set automatically in test environment
|
||||
$_SERVER['HTTP_AUTHORIZATION'] = $authToken;
|
||||
$_SERVER['REMOTE_ADDR'] = '127.0.0.1';
|
||||
|
||||
$this->configRequest([
|
||||
'headers' => [
|
||||
'Accept' => 'application/json',
|
||||
'Authorization' => $this->_authToken,
|
||||
'Content-Type' => 'application/json'
|
||||
$this->configRequest(
|
||||
[
|
||||
'headers' => [
|
||||
'Accept' => 'application/json',
|
||||
'Authorization' => $this->_authToken,
|
||||
'Content-Type' => 'application/json'
|
||||
]
|
||||
]
|
||||
]);
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -85,7 +83,7 @@ trait ApiTestTrait
|
|||
|
||||
/**
|
||||
* Load OpenAPI specification validator
|
||||
*
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function initializeOpenApiValidator(): void
|
||||
|
|
@ -100,7 +98,7 @@ trait ApiTestTrait
|
|||
|
||||
/**
|
||||
* Validates the API request against the OpenAPI spec
|
||||
*
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function assertRequestMatchesOpenApiSpec(): void
|
||||
|
|
@ -110,8 +108,8 @@ trait ApiTestTrait
|
|||
|
||||
/**
|
||||
* Validates the API response against the OpenAPI spec
|
||||
*
|
||||
* @param string $path The path to the API endpoint
|
||||
*
|
||||
* @param string $path The path to the API endpoint
|
||||
* @param string $method The HTTP method used to call the endpoint
|
||||
* @return void
|
||||
*/
|
||||
|
|
@ -121,15 +119,15 @@ trait ApiTestTrait
|
|||
$this->_validator->getResponseValidator()->validate($address, $this->_response);
|
||||
}
|
||||
|
||||
/**
|
||||
/**
|
||||
* Validates a record exists in the database
|
||||
*
|
||||
*
|
||||
* @param string $table The table name
|
||||
* @param array $conditions The conditions to check
|
||||
* @return void
|
||||
* @throws \Exception
|
||||
* @throws \Cake\Datasource\Exception\RecordNotFoundException
|
||||
*
|
||||
*
|
||||
* @see https://book.cakephp.org/4/en/orm-query-builder.html
|
||||
*/
|
||||
public function assertDbRecordExists(string $table, array $conditions): void
|
||||
|
|
@ -141,15 +139,15 @@ trait ApiTestTrait
|
|||
$this->assertNotEmpty($record);
|
||||
}
|
||||
|
||||
/**
|
||||
/**
|
||||
* Validates a record do not exists in the database
|
||||
*
|
||||
*
|
||||
* @param string $table The table name
|
||||
* @param array $conditions The conditions to check
|
||||
* @return void
|
||||
* @throws \Exception
|
||||
* @throws \Cake\Datasource\Exception\RecordNotFoundException
|
||||
*
|
||||
*
|
||||
* @see https://book.cakephp.org/4/en/orm-query-builder.html
|
||||
*/
|
||||
public function assertDbRecordNotExists(string $table, array $conditions): void
|
||||
|
|
@ -161,9 +159,9 @@ trait ApiTestTrait
|
|||
$this->assertEmpty($record);
|
||||
}
|
||||
|
||||
/**
|
||||
/**
|
||||
* Parses the response body and returns the decoded JSON
|
||||
*
|
||||
*
|
||||
* @return array
|
||||
* @throws \Exception
|
||||
*/
|
||||
|
|
@ -176,9 +174,9 @@ trait ApiTestTrait
|
|||
return json_decode((string)$this->_response->getBody(), true);
|
||||
}
|
||||
|
||||
/**
|
||||
/**
|
||||
* Gets a database records as an array
|
||||
*
|
||||
*
|
||||
* @param string $table The table name
|
||||
* @param array $conditions The conditions to check
|
||||
* @return array
|
||||
|
|
@ -191,10 +189,10 @@ trait ApiTestTrait
|
|||
|
||||
/**
|
||||
* This method intercepts IntegrationTestTrait::_buildRequest()
|
||||
* in the quest to get a PSR-7 request object and saves it for
|
||||
* in the quest to get a PSR-7 request object and saves it for
|
||||
* later inspection, also validates it against the OpenAPI spec.
|
||||
* @see \Cake\TestSuite\IntegrationTestTrait::_buildRequest()
|
||||
*
|
||||
*
|
||||
* @param string $url The URL
|
||||
* @param string $method The HTTP method
|
||||
* @param array|string $data The request data.
|
||||
|
|
@ -248,6 +246,14 @@ trait ApiTestTrait
|
|||
$data = json_encode($data);
|
||||
}
|
||||
|
||||
// somehow this is not set automatically in test environment
|
||||
$_SERVER['REMOTE_ADDR'] = '127.0.0.1';
|
||||
$_SERVER['HTTP_USER_AGENT'] = 'CakePHP TestSuite';
|
||||
$_SERVER['REQUEST_METHOD'] = $method;
|
||||
$_SERVER['CONTENT_TYPE'] = $this->_request['headers']['Content-Type'];
|
||||
$_SERVER['HTTP_CONTENT_ENCODING'] = $this->_request['headers']['Content-Type'];
|
||||
|
||||
|
||||
$this->_sendRequestOriginal($url, $method, $data);
|
||||
|
||||
// Validate response against OpenAPI spec
|
||||
|
|
@ -270,7 +276,7 @@ trait ApiTestTrait
|
|||
/**
|
||||
* Create a PSR-7 request from the request spec.
|
||||
* @see \Cake\TestSuite\MiddlewareDispatcher::_createRequest()
|
||||
*
|
||||
*
|
||||
* @param array<string, mixed> $spec The request spec.
|
||||
* @return \Cake\Http\ServerRequest
|
||||
*/
|
||||
|
|
|
|||
Loading…
Reference in New Issue