MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #9465 from JakubOnderka/logging 

ECS logging
pull/9106/head
Jakub Onderka 2023-12-28 12:58:28 +01:00 committed by GitHub
parent 268eb1a1a4 2b159eb025
commit 3e4edf46f7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 415 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/Controller/UsersController.php
View File

@ -3179,7 +3179,7 @@ class UsersController extends AppController
// We didn't filter the data at SQL query too much, nor by age, as we want to show "enough" data, even if old
$rows = 0;
// group authentications by type of loginprofile, to make the list shorter
foreach($logs as $logEntry) {
foreach ($logs as $logEntry) {
$loginProfile = $this->UserLoginProfile->_fromLog($logEntry['Log']);
if (!$loginProfile) continue; // skip if empty log
$loginProfile['ip'] = $logEntry['Log']['ip'] ?? null; // transitional workaround

14
app/Lib/Tools/SendEmail.php
View File

@ -555,14 +555,18 @@ class SendEmail
}
try {
return [
'contents' => $email->send(),
'encrypted' => $encrypted,
'subject' => $subject,
];
$content = $email->send();
} catch (Exception $e) {
throw new SendEmailException('The message could not be sent.', 0, $e);
}
return [
'to' => $user['User']['email'],
'message_id' => $email->messageId(),
'contents' => $content,
'encrypted' => $encrypted,
'subject' => $subject,
];
}
/**

6
app/Model/AuditLog.php
View File

@ -25,11 +25,7 @@ class AuditLog extends AppModel
ACTION_REMOVE_GALAXY = 'remove_galaxy',
ACTION_REMOVE_GALAXY_LOCAL = 'remove_local_galaxy',
ACTION_PUBLISH = 'publish',
ACTION_PUBLISH_SIGHTINGS = 'publish_sightings',
ACTION_LOGIN = 'login',
ACTION_PASSWDCHANGE = 'password_change',
ACTION_LOGOUT = 'logout',
ACTION_LOGIN_FAILED = 'login_failed';
ACTION_PUBLISH_SIGHTINGS = 'publish_sightings';
const REQUEST_TYPE_DEFAULT = 0,
REQUEST_TYPE_API = 1,

65
app/Model/Log.php
View File

@ -159,11 +159,6 @@ class Log extends AppModel
if (!in_array($this->data['Log']['model'], ['Log', 'Workflow'])) {
$trigger_id = 'log-after-save';
$workflowErrors = [];
$logging = [
'model' => 'Log',
'action' => 'execute_workflow',
'id' => $this->data['Log']['user_id']
];
$this->executeTrigger($trigger_id, $this->data, $workflowErrors);
}
return true;
@ -206,7 +201,7 @@ class Log extends AppModel
$validDates = $this->query($sql);
}
$data = array();
foreach ($validDates as $k => $date) {
foreach ($validDates as $date) {
$data[$date[0]['Date']] = intval($date[0]['count']);
}
return $data;
@ -286,7 +281,7 @@ class Log extends AppModel
public function validationError($user, $action, $model, $title, array $validationErrors, array $fullObject)
{
$this->log($title, LOG_WARNING);
$change = 'Validation errors: ' . json_encode($validationErrors) . ' Full ' . $model . ': ' . json_encode($fullObject);
$change = 'Validation errors: ' . JsonTool::encode($validationErrors) . ' Full ' . $model . ': ' . JsonTool::encode($fullObject);
$this->createLogEntry($user, $action, $model, 0, $title, $change);
}
@ -365,7 +360,7 @@ class Log extends AppModel
}
}
public function logData($data)
private function logData(array $data)
{
if ($this->pubToZmq('audit')) {
$this->getPubSubTool()->publish($data, 'audit', 'log');
@ -386,6 +381,14 @@ class Log extends AppModel
}
// write to syslogd as well if enabled
$this->sendToSyslog($data);
$this->sendToEcs($data);
return true;
}
private function sendToSyslog(array $data)
{
if ($this->syslog === null) {
if (Configure::read('Security.syslog')) {
$options = [];
@ -407,8 +410,7 @@ class Log extends AppModel
if (isset($data['Log']['action'])) {
if (in_array($data['Log']['action'], self::ERROR_ACTIONS, true)) {
$action = LOG_ERR;
}
if (in_array($data['Log']['action'], self::WARNING_ACTIONS, true)) {
} else if (in_array($data['Log']['action'], self::WARNING_ACTIONS, true)) {
$action = LOG_WARNING;
}
}
@ -420,11 +422,44 @@ class Log extends AppModel
if (!empty($data['Log']['description'])) {
$entry .= " -- {$data['Log']['description']}";
} else if (!empty($data['Log']['change'])) {
$entry .= " -- " . json_encode($data['Log']['change']);
$entry .= " -- " . JsonTool::encode($data['Log']['change']);
}
$this->syslog->write($action, $entry);
}
return true;
}
/**
* @param array $data
* @return void
* @throws JsonException
*/
private function sendToEcs(array $data)
{
if (!Configure::read('Security.ecs_log')) {
return;
}
$action = $data['Log']['action'];
$type = 'info';
if (isset($action)) {
if (in_array($action, self::ERROR_ACTIONS, true)) {
$type = 'error';
} else if (in_array($action, self::WARNING_ACTIONS, true)) {
$type = 'warning';
}
}
$message = $action;
if (!empty($data['Log']['title'])) {
$message .= " -- {$data['Log']['title']}";
}
if (!empty($data['Log']['description'])) {
$message .= " -- {$data['Log']['description']}";
} else if (!empty($data['Log']['change'])) {
$message .= " -- " . JsonTool::encode($data['Log']['change']);
}
EcsLog::writeApplicationLog($type, $action, $message);
}
public function filterSiteAdminSensitiveLogs($list)
@ -1176,11 +1211,17 @@ class Log extends AppModel
return $this->elasticSearchClient;
}
/**
* @param $data
* @param $options
* @return array|bool|mixed
*/
public function saveOrFailSilently($data, $options = null)
{
try {
return $this->save($data, $options);
} catch (Exception $e) {
$this->logException('Could not save log to database', $e);
return false;
}
}

36
app/Model/User.php
View File

@ -12,6 +12,7 @@ App::uses('BlowfishConstantPasswordHasher', 'Controller/Component/Auth');
* @property Organisation $Organisation
* @property Role $Role
* @property UserSetting $UserSetting
* @property UserLoginProfile $UserLoginProfile
* @property Event $Event
* @property AuthKey $AuthKey
* @property Server $Server
@ -889,7 +890,11 @@ class User extends AppModel
$logTitle = $result['encrypted'] ? 'Encrypted email' : 'Email';
// Intentional two spaces to pass test :)
$logTitle .= $replyToLog . ' to ' . $user['User']['email'] . ' sent, titled "' . $result['subject'] . '".';
$logTitle .= $replyToLog . ' to ' . $result['to'] . ' sent, titled "' . $result['subject'] . '".';
if (Configure::read('Security.ecs_log')) {
EcsLog::writeEmailLog($logTitle, $result, $replyToUser ? $replyToUser['User']['email'] : null);
}
$log->create();
$log->saveOrFailSilently(array(
@ -1264,37 +1269,44 @@ class User extends AppModel
return $newkey;
}
public function extralog($user, $action = null, $description = null, $fieldsResult = null, $modifiedUser = null)
/**
* @param string|array $user
* @param string $action
* @param string $description
* @param string $fieldsResult
* @param array|null $modifiedUser
* @return void
* @throws JsonException
*/
public function extralog($user, $action, $description = null, $fieldsResult = null, $modifiedUser = null)
{
if (!is_array($user) && $user === 'SYSTEM') {
if ($user === 'SYSTEM') {
$user = [
'id' => 0,
'email' => 'SYSTEM',
'Organisation' => [
'name' => 'SYSTEM'
]
],
];
}
// new data
$model = 'User';
$modelId = $user['id'];
if (!empty($modifiedUser)) {
$modelId = $modifiedUser['User']['id'];
}
if ($action == 'login') {
if ($action === 'login') {
$description = "User (" . $user['id'] . "): " . $user['email'];
$fieldsResult = json_encode($this->UserLoginProfile->_getUserProfile());
} elseif ($action == 'logout') {
$fieldsResult = JsonTool::encode($this->UserLoginProfile->_getUserProfile());
} else if ($action === 'logout') {
$description = "User (" . $user['id'] . "): " . $user['email'];
} elseif ($action == 'edit') {
} else if ($action === 'edit') {
$description = "User (" . $modifiedUser['User']['id'] . "): " . $modifiedUser['User']['email'];
} elseif ($action == 'change_pw') {
} else if ($action === 'change_pw') {
$description = "User (" . $modifiedUser['User']['id'] . "): " . $modifiedUser['User']['email'];
$fieldsResult = "Password changed.";
}
// query
$result = $this->loadLog()->createLogEntry($user, $action, $model, $modelId, $description, $fieldsResult);
$result = $this->loadLog()->createLogEntry($user, $action, 'User', $modelId, $description, $fieldsResult);
// write to syslogd as well
if ($result) {
App::import('Lib', 'SysLog.SysLog');

92
app/Model/UserLoginProfile.php
View File

@ -1,7 +1,9 @@
<?php
App::uses('AppModel', 'Model');
/**
* @property User $User
*/
class UserLoginProfile extends AppModel
{
public $actsAs = array(
@ -20,34 +22,37 @@ class UserLoginProfile extends AppModel
'rule' => '/^(trusted|malicious)$/',
'message' => 'Must be one of: trusted, malicious'
],
];
public $order = array("UserLoginProfile.id" => "DESC");
public $belongsTo = [
'User' => [
'className' => 'User',
'foreignKey' => 'user_id',
'conditions' => '',
'fields' => '',
'order' => ''
]];
'User' => [
'className' => 'User',
'foreignKey' => 'user_id',
'conditions' => '',
'fields' => '',
'order' => ''
]
];
protected $browscapCacheDir = APP . DS . 'tmp' . DS . 'browscap';
protected $browscapIniFile = APP . DS . 'files' . DS . 'browscap'. DS . 'browscap.ini'; // Browscap file managed by MISP - https://browscap.org/stream?q=Lite_PHP_BrowsCapINI
protected $geoIpDbFile = APP . DS . 'files' . DS . 'geo-open' . DS . 'GeoOpen-Country.mmdb'; // GeoIP file managed by MISP - https://data.public.lu/en/datasets/geo-open-ip-address-geolocation-per-country-in-mmdb-format/
const BROWSER_CACHE_DIR = APP . DS . 'tmp' . DS . 'browscap';
const BROWSER_INI_FILE = APP . DS . 'files' . DS . 'browscap'. DS . 'browscap.ini'; // Browscap file managed by MISP - https://browscap.org/stream?q=Lite_PHP_BrowsCapINI
const GEOIP_DB_FILE = APP . DS . 'files' . DS . 'geo-open' . DS . 'GeoOpen-Country.mmdb'; // GeoIP file managed by MISP - https://data.public.lu/en/datasets/geo-open-ip-address-geolocation-per-country-in-mmdb-format/
private $userProfile;
private $knownUserProfiles = [];
public function _buildBrowscapCache() {
public function _buildBrowscapCache()
{
$this->log("Browscap - building new cache from browscap.ini file.", "info");
$fileCache = new \Doctrine\Common\Cache\FilesystemCache($this->browscapCacheDir);
$fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR);
$cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache);
$logger = new \Monolog\Logger('name');
$bc = new \BrowscapPHP\BrowscapUpdater($cache, $logger);
$bc->convertFile($this->browscapIniFile);
$bc->convertFile(UserLoginProfile::BROWSER_INI_FILE);
}
public function beforeSave($options = [])
@ -56,7 +61,8 @@ class UserLoginProfile extends AppModel
return true;
}
public function hash($data) {
public function hash($data)
{
unset($data['hash']);
unset($data['created_at']);
return md5(serialize($data));
@ -66,12 +72,13 @@ class UserLoginProfile extends AppModel
* slow function - don't call it too often
* @return array
*/
public function _getUserProfile() {
public function _getUserProfile()
{
if (!$this->userProfile) {
// below uses https://github.com/browscap/browscap-php
if (class_exists('\BrowscapPHP\Browscap')) {
try {
$fileCache = new \Doctrine\Common\Cache\FilesystemCache($this->browscapCacheDir);
$fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR);
$cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache);
$logger = new \Monolog\Logger('name');
$bc = new \BrowscapPHP\Browscap($cache, $logger);
@ -82,7 +89,7 @@ class UserLoginProfile extends AppModel
}
} else {
// a primitive OS & browser extraction capability
$ua = env('HTTP_USER_AGENT');
$ua = $_SERVER['HTTP_USER_AGENT'] ?? null;
$browser = new stdClass();
$browser->browser_name_pattern = $ua;
if (mb_strpos($ua, 'Linux') !== false) $browser->platform = "Linux";
@ -95,16 +102,16 @@ class UserLoginProfile extends AppModel
}
$ip = $this->_remoteIp();
if (class_exists('GeoIp2\Database\Reader')) {
$geoDbReader = new GeoIp2\Database\Reader($this->geoIpDbFile);
$geoDbReader = new GeoIp2\Database\Reader(UserLoginProfile::GEOIP_DB_FILE);
$record = $geoDbReader->country($ip);
$country = $record->country->isoCode;
} else {
$country = 'None';
}
$this->userProfile = [
'user_agent' => env('HTTP_USER_AGENT'),
'user_agent' => $_SERVER['HTTP_USER_AGENT'] ?? null,
'ip' => $ip,
'accept_lang' => env('HTTP_ACCEPT_LANGUAGE'),
'accept_lang' => $_SERVER['HTTP_ACCEPT_LANGUAGE'] ?? null,
'geoip' => $country,
'ua_pattern' => $browser->browser_name_pattern,
'ua_platform' => $browser->platform,
@ -114,16 +121,20 @@ class UserLoginProfile extends AppModel
return $this->userProfile;
}
public function _fromLog($logEntry) {
$data = json_decode('{"user_agent": "", "ip": "", "accept_lang":"", "geoip":"", "ua_pattern":"", "ua_platform":"", "ua_browser":""}', true);
$data = array_merge($data, json_decode($logEntry['change'], true) ?? []);
public function _fromLog($logEntry)
{
$data = ["user_agent" => "", "ip" => "", "accept_lang" => "", "geoip" => "", "ua_pattern" => "", "ua_platform" => "", "ua_browser" => ""];
$data = array_merge($data, JsonTool::decode($logEntry['change']) ?? []);
$data['ip'] = $logEntry['ip'];
$data['timestamp'] = $logEntry['created'];
if ($data['user_agent'] == "") return false;
if ($data['user_agent'] == "") {
return false;
}
return $data;
}
public function _isSimilar($a, $b) {
public function _isSimilar($a, $b)
{
// if one is not initialized
if (!$a || !$b) return false;
// transition for old logs where UA was not known
@ -146,7 +157,8 @@ class UserLoginProfile extends AppModel
return false;
}
public function _isIdentical($a, $b) {
public function _isIdentical($a, $b)
{
if ($a['ip'] == $b['ip'] &&
$a['ua_browser'] == $b['ua_browser'] &&
$a['ua_platform'] == $b['ua_platform'] &&
@ -157,7 +169,8 @@ class UserLoginProfile extends AppModel
return false;
}
public function _getTrustStatus($userProfileToCheck, $user_id = null) {
public function _getTrustStatus($userProfileToCheck, $user_id = null)
{
if (!$user_id) {
$user_id = AuthComponent::user('id');
}
@ -183,17 +196,19 @@ class UserLoginProfile extends AppModel
return 'unknown';
}
public function _isTrusted() {
public function _isTrusted()
{
if (strpos($this->_getTrustStatus($this->_getUserProfile()), 'trusted') !== false) {
return true;
}
return false;
}
public function _isSuspicious() {
public function _isSuspicious()
{
// previously marked loginuserprofile as malicious by the user
if (strpos($this->_getTrustStatus($this->_getUserProfile()), 'malicious') !== false) {
return _('A user reported a similar login profile as malicious.');
return __('A user reported a similar login profile as malicious.');
}
// same IP as previous malicious user
$maliciousWithSameIP = $this->find('first', [
@ -205,7 +220,7 @@ class UserLoginProfile extends AppModel
'fields' => array('UserLoginProfile.*')]
);
if ($maliciousWithSameIP) {
return _('The source IP was reported as as malicious by a user.');
return __('The source IP was reported as as malicious by a user.');
}
// LATER - use other data to identify suspicious logins, such as:
// - what with use-case where a user marks something as legitimate, but is marked by someone else as suspicious?
@ -214,7 +229,8 @@ class UserLoginProfile extends AppModel
return false;
}
public function email_newlogin($user) {
public function email_newlogin($user)
{
if (!Configure::read('MISP.disable_emailing')) {
$date_time = date('c');
@ -233,7 +249,8 @@ class UserLoginProfile extends AppModel
}
}
public function email_report_malicious($user, $userLoginProfile) {
public function email_report_malicious($user, $userLoginProfile)
{
// inform the org admin
$date_time = $userLoginProfile['timestamp']; // LATER not ideal as timestamp is string without timezone info
$body = new SendEmailTemplate('userloginprofile_report_malicious');
@ -259,7 +276,8 @@ class UserLoginProfile extends AppModel
}
}
public function email_suspicious($user, $suspiciousness_reason) {
public function email_suspicious($user, $suspiciousness_reason)
{
if (!Configure::read('MISP.disable_emailing')) {
$date_time = date('c');
// inform the user
@ -300,6 +318,4 @@ class UserLoginProfile extends AppModel
}
}
}
}

273
app/Plugin/EcsLog/Lib/Log/Engine/EcsLog.php Normal file
View File

@ -0,0 +1,273 @@
<?php
App::uses('JsonTool', 'Tools');
/**
* Logging class that sends logs in JSON format to UNIX socket in Elastic Common Schema (ECS) format
* Logs are separated by new line characters, so basically it is send as JSONL
*/
class EcsLog implements CakeLogInterface
{
const ECS_VERSION = '8.11';
const SOCKET_PATH = '/run/vector';
/** @var false|resource */
private static $socket;
/** @var array[] */
private static $meta;
/**
* @param string $type The type of log you are making.
* @param string $message The message you want to log.
* @return void
* @throws JsonException
*/
public function write($type, $message)
{
$message = [
'@timestamp' => self::timestamp(),
'ecs' => [
'version' => self::ECS_VERSION,
],
'event' => [
'kind' => 'event',
'provider' => 'misp',
'module' => 'system',
'dataset' => 'system.logs',
],
'log' => [
'level' => $type,
],
'message' => $message,
];
static::writeMessage($message);
}
/**
* @param string $type
* @param string $action
* @param string $message
* @return void
* @throws JsonException
*/
public static function writeApplicationLog($type, $action, $message)
{
if ($action === 'email') {
return; // do not log email actions as it is logged with more details by `writeEmailLog` function
}
$message = [
'@timestamp' => self::timestamp(),
'ecs' => [
'version' => self::ECS_VERSION,
],
'event' => [
'kind' => 'event',
'provider' => 'misp',
'module' => 'application',
'dataset' => 'application.logs',
'action' => $action,
],
'log' => [
'level' => $type,
],
'message' => $message,
];
if (in_array($action, ['auth', 'auth_fail', 'auth_alert', 'change_pw', 'login', 'login_fail', 'logout', 'password_reset'], true)) {
$message['event']['category'] = 'authentication';
if (in_array($action, ['auth_fail', 'login_fail'], true)) {
$message['event']['outcome'] = 'failure';
}
}
static::writeMessage($message);
}
/**
* Include more meta information about email than would provide default `writeApplicationLog` log
* @param string $logTitle
* @param array $emailResult
* @param string|null $replyTo
* @return void
* @throws JsonException
*/
public static function writeEmailLog($logTitle, array $emailResult, $replyTo = null)
{
$message = [
'@timestamp' => self::timestamp(),
'ecs' => [
'version' => self::ECS_VERSION,
],
'event' => [
'kind' => 'event',
'provider' => 'misp',
'module' => 'application',
'dataset' => 'application.logs',
'category' => 'email',
'action' => 'email',
'type' => 'info',
],
'email' => [
'message_id' => $emailResult['message_id'],
'subject' => $emailResult['subject'],
'to' => [
'address' => $emailResult['to'],
],
],
'message' => $logTitle,
];
if ($replyTo) {
$message['email']['reply_to'] = ['address' => $replyTo];
}
static::writeMessage($message);
}
/**
* @return string|null
*/
private static function clientIp()
{
$ipHeader = Configure::read('MISP.log_client_ip_header') ?: 'REMOTE_ADDR';
return isset($_SERVER[$ipHeader]) ? trim($_SERVER[$ipHeader]) : $_SERVER['REMOTE_ADDR'];
}
/**
* @return array[]
*/
private static function createLogMeta()
{
if (self::$meta) {
return self::$meta;
}
$meta = ['process' => ['pid' => getmypid()]];
// Add metadata if log was generated because of HTTP request
if (PHP_SAPI !== 'cli') {
if (isset($_SERVER['HTTP_X_REQUEST_ID'])) {
$meta['http'] = ['request' => ['id' => $_SERVER['HTTP_X_REQUEST_ID']]];
}
$clientIp = static::clientIp();
$client = [
'ip' => $_SERVER['REMOTE_ADDR'],
'port' => $_SERVER['REMOTE_PORT'],
];
if ($clientIp === $_SERVER['REMOTE_ADDR']) {
$meta['client'] = $client;
} else {
$meta['client'] = [
'ip' => $clientIp,
'nat' => $client,
];
}
if (strpos($_SERVER['HTTP_HOST'], ':') !== 0) {
list($domain, $port) = explode(':', $_SERVER['HTTP_HOST'], 2);
$meta['url'] = [
'domain' => $domain,
'port' => (int) $port,
'path' => $_SERVER['REQUEST_URI'],
];
} else {
$meta['url'] = [
'domain' => $_SERVER['HTTP_HOST'],
'path' => $_SERVER['REQUEST_URI'],
];
}
} else {
$meta['process']['argv'] = $_SERVER['argv'];
}
$userMeta = self::createUserMeta();
if ($userMeta) {
$meta['user'] = $userMeta;
}
return self::$meta = $meta;
}
/**
* Get user metadata (use unique id and email address)
* @return array|null
*/
private static function createUserMeta()
{
if (PHP_SAPI === 'cli') {
$currentUserId = Configure::read('CurrentUserId');
if (!empty($currentUserId)) {
/** @var User $userModel */
$userModel = ClassRegistry::init('User');
$user = $userModel->find('first', [
'recursive' => -1,
'conditions' => ['id' => $currentUserId],
'fields' => ['sub', 'email'],
]);
if (!empty($user)) {
return [
'id' => $user['User']['sub'] ?? $currentUserId,
'email' => $user['User']['email'],
];
}
}
} else {
App::uses('AuthComponent', 'Controller/Component');
$authUser = AuthComponent::user();
if (!empty($authUser)) {
return [
'id' => $authUser['sub'] ?? $authUser['id'],
'email' => $authUser['email'],
];
}
}
return null;
}
/**
* @return string
*/
public static function timestamp()
{
return date('Y-m-d\TH:i:s.uP');
}
/**
* @param array $message
* @return void
* @throws JsonException
*/
private static function writeMessage(array $message)
{
if (static::$socket === null) {
static::connect();
}
if (static::$socket) {
$message = array_merge($message, self::createLogMeta());
$data = JsonTool::encode($message) . "\n";
$bytesWritten = fwrite(static::$socket, $data);
// In case of failure, try reconnect and send log again
if ($bytesWritten === false) {
static::connect();
if (static::$socket) {
fwrite(static::$socket, $data);
}
}
}
}
private static function connect()
{
static::$socket = stream_socket_client('unix://' . static::SOCKET_PATH, $errorCode, $errorMessage);
}
}