Merge pull request #9470 from JakubOnderka/logging

fix: [internal] ECS: Reliable logging
pull/9481/head
Jakub Onderka 2024-01-02 09:25:03 +01:00 committed by GitHub
commit c213088785
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 315 additions and 116 deletions

View File

@ -11,7 +11,7 @@ class Stix1Export extends StixExport
{
return [
ProcessTool::pythonBin(),
$this->__framing_script,
self::FRAMING_SCRIPT,
'stix1',
'-s', $this->__scope,
'-v', $this->__version,
@ -25,7 +25,7 @@ class Stix1Export extends StixExport
{
$command = [
ProcessTool::pythonBin(),
$this->__scripts_dir . 'misp2stix.py',
self::SCRIPTS_DIR . 'misp2stix.py',
'-s', $this->__scope,
'-v', $this->__version,
'-f', $this->__return_format,
@ -33,6 +33,10 @@ class Stix1Export extends StixExport
'-i',
];
$command = array_merge($command, $this->__filenames);
return ProcessTool::execute($command, null, true);
try {
return ProcessTool::execute($command, null, true);
} catch (ProcessException $e) {
return $e->stdout();
}
}
}

View File

@ -11,16 +11,20 @@ class Stix2Export extends StixExport
{
return [
ProcessTool::pythonBin(),
$this->__framing_script,
self::FRAMING_SCRIPT,
'stix2',
'-v', $this->__version,
'--uuid', CakeText::uuid(),
];
}
/**
* @return string
* @throws Exception
*/
protected function __parse_misp_data()
{
$scriptFile = $this->__scripts_dir . 'stix2/misp2stix2.py';
$scriptFile = self::SCRIPTS_DIR . 'stix2/misp2stix2.py';
$command = [
ProcessTool::pythonBin(),
$scriptFile,
@ -28,7 +32,11 @@ class Stix2Export extends StixExport
'-i',
];
$command = array_merge($command, $this->__filenames);
$result = ProcessTool::execute($command, null, true);
try {
$result = ProcessTool::execute($command, null, true);
} catch (ProcessException $e) {
$result = $e->stdout();
}
$result = preg_split("/\r\n|\n|\r/", trim($result));
return end($result);
}

View File

@ -6,13 +6,14 @@ App::uses('ProcessTool', 'Tools');
abstract class StixExport
{
const SCRIPTS_DIR = APP . 'files/scripts/',
FRAMING_SCRIPT = APP . 'files/scripts/misp_framing.py';
public $additional_params = array(
'includeEventTags' => 1,
'includeGalaxy' => 1
);
protected $__return_format = 'json';
protected $__scripts_dir = APP . 'files/scripts/';
protected $__framing_script = APP . 'files/scripts/misp_framing.py';
protected $__return_type = null;
/** @var array Full paths to files to convert */

View File

@ -1,7 +1,7 @@
<?php
class ProcessException extends Exception
{
/** @var string|null */
/** @var string */
private $stderr;
/** @var string */
@ -10,14 +10,13 @@ class ProcessException extends Exception
/**
* @param string|array $command
* @param int $returnCode
* @param string|null $stderr
* @param string $stderr
* @param string $stdout
*/
public function __construct($command, $returnCode, $stderr, $stdout)
{
$commandForException = is_array($command) ? implode(' ', $command) : $command;
$stderrToMessage = $stderr === null ? 'Logged to tmp/logs/exec-errors.log' : "'$stderr'";
$message = "Command '$commandForException' finished with error code $returnCode.\nSTDERR: $stderrToMessage\nSTDOUT: '$stdout'";
$message = "Command '$commandForException' finished with error code $returnCode.\nSTDERR: '$stderr'\nSTDOUT: '$stdout'";
$this->stderr = $stderr;
$this->stdout = $stdout;
parent::__construct($message, $returnCode);
@ -41,21 +40,20 @@ class ProcessTool
/**
* @param array $command If command is array, it is not necessary to escape arguments
* @param string|null $cwd
* @param bool $stderrToFile IF true, log stderrr output to LOG_FILE
* @param bool $logToFile If true, log stderr output to LOG_FILE
* @return string Stdout
* @throws ProcessException
* @throws Exception
*/
public static function execute(array $command, $cwd = null, $stderrToFile = false)
public static function execute(array $command, $cwd = null, $logToFile = false)
{
$descriptorSpec = [
1 => ['pipe', 'w'], // stdout
2 => ['pipe', 'w'], // stderr
];
if ($stderrToFile) {
if ($logToFile) {
self::logMessage('Running command ' . implode(' ', $command));
$descriptorSpec[2] = ['file', self::LOG_FILE, 'a'];
}
// PHP older than 7.4 do not support proc_open with array, so we need to convert values to string manually
@ -75,20 +73,24 @@ class ProcessTool
throw new Exception("Could not get STDOUT of command '$commandForException'.");
}
if ($stderrToFile) {
$stderr = null;
} else {
$stderr = stream_get_contents($pipes[2]);
$stderr = stream_get_contents($pipes[2]);
if ($stderr === false) {
$commandForException = self::commandFormat($command);
throw new Exception("Could not get STDERR of command '$commandForException'.");
}
$returnCode = proc_close($process);
if ($stderrToFile) {
self::logMessage("Process finished with return code $returnCode");
if ($logToFile) {
self::logMessage("Process finished with return code $returnCode", $stderr);
}
if ($returnCode !== 0) {
throw new ProcessException($command, $returnCode, $stderr, $stdout);
$exception = new ProcessException($command, $returnCode, $stderr, $stdout);
if ($logToFile && Configure::read('Security.ecs_log')) {
EcsLog::handleException($exception);
}
throw $exception;
}
return $stdout;
@ -116,9 +118,17 @@ class ProcessTool
return Configure::read('MISP.python_bin') ?: 'python3';
}
private static function logMessage($message)
/**
* @param string $message
* @param string|null $stderr
* @return void
*/
private static function logMessage($message, $stderr = null)
{
$logMessage = '[' . date("Y-m-d H:i:s") . ' ' . getmypid() . "] $message\n";
if ($stderr) {
$logMessage = rtrim($stderr) . "\n" . $logMessage;
}
file_put_contents(self::LOG_FILE, $logMessage, FILE_APPEND | LOCK_EX);
}

View File

@ -5922,61 +5922,24 @@ class Event extends AppModel
/**
* @param array $user
* @param string $file Path
* @param string $stix_version
* @param string $original_file
* @param string $stixVersion
* @param string $originalFile
* @param bool $publish
* @param int $distribution
* @param int $sharingGroupId
* @param bool $galaxiesAsTags
* @param bool $debug
* @return int|string|array
* @throws JsonException
* @throws InvalidArgumentException
* @throws Exception
*/
public function upload_stix(array $user, $file, $stix_version, $original_file, $publish, $distribution, $sharingGroupId, $galaxiesAsTags, $debug = false)
public function upload_stix(array $user, $file, $stixVersion, $originalFile, $publish, $distribution, $sharingGroupId, $galaxiesAsTags, $debug = false)
{
$scriptDir = APP . 'files' . DS . 'scripts';
if ($stix_version == '2' || $stix_version == '2.0' || $stix_version == '2.1') {
$scriptFile = $scriptDir . DS . 'stix2' . DS . 'stix2misp.py';
$output_path = $file . '.out';
$shell_command = [
ProcessTool::pythonBin(),
$scriptFile,
'-i', $file,
'--distribution', $distribution
];
if ($distribution == 4) {
array_push($shell_command, '--sharing_group_id', $sharingGroupId);
}
if ($galaxiesAsTags) {
$shell_command[] = '--galaxies_as_tags';
}
if ($debug) {
$shell_command[] = '--debug';
}
$stix_version = "STIX 2.1";
} elseif ($stix_version == '1' || $stix_version == '1.1' || $stix_version == '1.2') {
$scriptFile = $scriptDir . DS . 'stix2misp.py';
$output_path = $file . '.json';
$shell_command = [
ProcessTool::pythonBin(),
$scriptFile,
$file,
Configure::read('MISP.default_event_distribution'),
Configure::read('MISP.default_attribute_distribution'),
$this->__getTagNamesFromSynonyms($scriptDir)
];
$stix_version = "STIX 1.1";
} else {
throw new InvalidArgumentException('Invalid STIX version');
}
$decoded = $this->convertStixToMisp($stixVersion, $file, $distribution, $sharingGroupId, $galaxiesAsTags, $debug);
$result = ProcessTool::execute($shell_command, null, true);
$result = preg_split("/\r\n|\n|\r/", trim($result));
$result = trim(end($result));
$tempFile = file_get_contents($file);
unlink($file);
$decoded = JsonTool::decode($result);
if (!empty($decoded['success'])) {
$data = FileAccessTool::readAndDelete($output_path);
$data = $this->jsonDecode($data);
$data = JsonTool::decodeArray($decoded['converted']);
if (empty($data['Event'])) {
$data = array('Event' => $data);
}
@ -6000,15 +5963,13 @@ class Event extends AppModel
}
}
}
if (!empty($decoded['stix_version'])) {
$stix_version = 'STIX ' . $decoded['stix_version'];
}
$stixVersion = $decoded['stix_version'];
$created_id = false;
$validationIssues = false;
$result = $this->_add($data, true, $user, '', null, false, null, $created_id, $validationIssues);
if ($result === true) {
if ($original_file) {
$this->add_original_file($tempFile, $original_file, $created_id, $stix_version);
if ($originalFile) {
$this->add_original_file($decoded['original'], $originalFile, $created_id, $stixVersion);
}
if ($publish && $user['Role']['perm_publish']) {
$this->publish($created_id);
@ -6031,6 +5992,76 @@ class Event extends AppModel
return $response;
}
/**
* @param string $stixVersion
* @param string $file
* @param int $distribution
* @param int $sharingGroupId
* @param bool $galaxiesAsTags
* @param bool $debug
* @return array
* @throws Exception
*/
private function convertStixToMisp($stixVersion, $file, $distribution, $sharingGroupId, $galaxiesAsTags, $debug)
{
$scriptDir = APP . 'files' . DS . 'scripts';
if ($stixVersion === '2' || $stixVersion === '2.0' || $stixVersion === '2.1') {
$scriptFile = $scriptDir . DS . 'stix2' . DS . 'stix2misp.py';
$outputPath = $file . '.out';
$shellCommand = [
ProcessTool::pythonBin(),
$scriptFile,
'-i', $file,
'--distribution', $distribution,
];
if ($distribution == 4) {
array_push($shellCommand, '--sharing_group_id', $sharingGroupId);
}
if ($galaxiesAsTags) {
$shellCommand[] = '--galaxies_as_tags';
}
if ($debug) {
$shellCommand[] = '--debug';
}
$stixVersion = "STIX 2.1";
} else if ($stixVersion === '1' || $stixVersion === '1.1' || $stixVersion === '1.2') {
$scriptFile = $scriptDir . DS . 'stix2misp.py';
$outputPath = $file . '.json';
$shellCommand = [
ProcessTool::pythonBin(),
$scriptFile,
$file,
Configure::read('MISP.default_event_distribution'),
Configure::read('MISP.default_attribute_distribution'),
$this->__getTagNamesFromSynonyms($scriptDir)
];
$stixVersion = "STIX 1.1";
} else {
throw new InvalidArgumentException('Invalid STIX version');
}
try {
$stdout = ProcessTool::execute($shellCommand, null, true);
} catch (ProcessException $e) {
$stdout = $e->stdout();
}
$stdout = preg_split("/\r\n|\n|\r/", trim($stdout));
$stdout = trim(end($stdout));
$decoded = JsonTool::decode($stdout);
if (empty($decoded['stix_version'])) {
$decoded['stix_version'] = $stixVersion;
}
$decoded['original'] = FileAccessTool::readAndDelete($file);
if (!empty($decoded['success'])) {
$decoded['converted'] = FileAccessTool::readAndDelete($outputPath);
}
return $decoded;
}
private function __handleGalaxiesAndClusters($user, &$data)
{
if (!empty($data['Galaxy'])) {

View File

@ -442,6 +442,10 @@ class Log extends AppModel
$log = $data['Log'];
$action = $log['action'];
if ($action === 'email') {
return; // do not log email actions as it is logged with more details by `writeEmailLog` function
}
if (in_array($action, self::ERROR_ACTIONS, true)) {
$type = 'error';
} else if (in_array($action, self::WARNING_ACTIONS, true)) {

View File

@ -9,19 +9,33 @@ class EcsLog implements CakeLogInterface
{
const ECS_VERSION = '8.11';
/** @var string Unix socket path where logs will be send in JSONL format */
const SOCKET_PATH = '/run/vector';
/** @var false|resource */
private static $socket;
/** @var string[] */
private static $messageBuffer = [];
/** @var array[] */
private static $meta;
const LOG_LEVEL_STRING = [
LOG_EMERG => 'emergency',
LOG_ALERT => 'alert',
LOG_CRIT => 'critical',
LOG_ERR => 'error',
LOG_WARNING => 'warning',
LOG_NOTICE => 'notice',
LOG_INFO => 'info',
LOG_DEBUG => 'debug',
];
/**
* @param string $type The type of log you are making.
* @param string $message The message you want to log.
* @return void
* @throws JsonException
*/
public function write($type, $message)
{
@ -50,14 +64,9 @@ class EcsLog implements CakeLogInterface
* @param string $action
* @param string $message
* @return void
* @throws JsonException
*/
public static function writeApplicationLog($type, $action, $message)
{
if ($action === 'email') {
return; // do not log email actions as it is logged with more details by `writeEmailLog` function
}
$message = [
'@timestamp' => self::now(),
'ecs' => [
@ -93,7 +102,6 @@ class EcsLog implements CakeLogInterface
* @param array $emailResult
* @param string|null $replyTo
* @return void
* @throws JsonException
*/
public static function writeEmailLog($logTitle, array $emailResult, $replyTo = null)
{
@ -128,6 +136,87 @@ class EcsLog implements CakeLogInterface
static::writeMessage($message);
}
/**
* @param int $code
* @param string $description
* @param string|null $file
* @param int|null $line
* @return void
*/
public static function handleError($code, $description, $file = null, $line = null)
{
list($name, $log) = ErrorHandler::mapErrorCode($code);
$level = self::LOG_LEVEL_STRING[$log];
$message = [
'@timestamp' => self::now(),
'ecs' => [
'version' => self::ECS_VERSION,
],
'event' => [
'kind' => 'event',
'provider' => 'misp',
'module' => 'system',
'dataset' => 'system.logs',
'type' => 'error',
],
'error' => [
'code' => $code,
'message' => $description,
],
'log' => [
'level' => $level,
'origin' => [
'file' => [
'name' => $file,
'line' => $line,
],
],
],
];
static::writeMessage($message);
}
/**
* @param Exception $exception
* @return void
*/
public static function handleException(Exception $exception)
{
$code = $exception->getCode();
$code = ($code && is_int($code)) ? $code : 1;
$message = [
'@timestamp' => self::now(),
'ecs' => [
'version' => self::ECS_VERSION,
],
'event' => [
'kind' => 'event',
'provider' => 'misp',
'module' => 'system',
'dataset' => 'system.logs',
'type' => 'error',
],
'error' => [
'code' => $code,
'type' => get_class($exception),
'message' => $exception->getMessage(),
'stack_trace' => $exception->getTraceAsString(),
],
'log' => [
'level' => 'error',
'origin' => [
'file' => [
'name' => $exception->getFile(),
'line' => $exception->getLine(),
],
],
],
];
static::writeMessage($message);
}
/**
* @return string|null
*/
@ -250,39 +339,67 @@ class EcsLog implements CakeLogInterface
* ISO 8601 timestamp with microsecond precision
* @return string
*/
public static function now()
private static function now()
{
return (new DateTime())->format('Y-m-d\TH:i:s.uP');
}
/**
* @param array $message
* @return void
* @throws JsonException
* @return bool True when message was successfully send to socket, false if message was saved to buffer
*/
private static function writeMessage(array $message)
{
$message = array_merge($message, self::createLogMeta());
try {
$data = JsonTool::encode($message) . "\n";
} catch (JsonException $e) {
return null;
}
if (static::$socket === null) {
static::connect();
}
if (static::$socket) {
$message = array_merge($message, self::createLogMeta());
$data = JsonTool::encode($message) . "\n";
$bytesWritten = fwrite(static::$socket, $data);
if ($bytesWritten !== false) {
return true;
}
// In case of failure, try reconnect and send log again
if ($bytesWritten === false) {
static::connect();
if (static::$socket) {
fwrite(static::$socket, $data);
static::connect();
if (static::$socket) {
$bytesWritten = fwrite(static::$socket, $data);
if ($bytesWritten !== false) {
return true;
}
}
}
// If sending message was not successful, save to buffer
self::$messageBuffer[] = $data;
if (count(self::$messageBuffer) > 100) {
array_shift(self::$messageBuffer); // remove oldest log
}
return false;
}
private static function connect()
{
static::$socket = null;
if (!file_exists(static::SOCKET_PATH)) {
return;
}
static::$socket = stream_socket_client('unix://' . static::SOCKET_PATH, $errorCode, $errorMessage);
if (static::$socket) {
foreach (self::$messageBuffer as $message) {
fwrite(static::$socket, $message);
}
self::$messageBuffer = [];
}
}
}

View File

@ -78,9 +78,13 @@ class StixExport:
if self._parser.errors:
self._handle_errors()
print(json.dumps(results))
except Exception as e:
print(json.dumps({'error': e.__str__()}))
error = type(e).__name__ + ': ' + e.__str__()
print(json.dumps({'error': error}))
traceback.print_tb(e.__traceback__)
print(error, file=sys.stderr)
sys.exit(1)
class StixAttributesExport(StixExport):
@ -157,19 +161,23 @@ class StixEventsExport(StixExport):
if __name__ == "__main__":
argparser = argparse.ArgumentParser(description='Export MISP into STIX1.')
argparser.add_argument('-s', '--scope', default='Event', choices=['Attribute', 'Event'], help='Scope: which kind of data is exported.')
argparser.add_argument('-v', '--version', default='1.1.1', choices=['1.1.1', '1.2'], help='STIX version (1.1.1 or 1.2).')
argparser.add_argument('-f', '--format', default='xml', choices=['json', 'xml'], help='Output format (xml or json).')
argparser.add_argument('-s', '--scope', default='Event', choices=('Attribute', 'Event'), help='Scope: which kind of data is exported.')
argparser.add_argument('-v', '--version', default='1.1.1', choices=('1.1.1', '1.2'), help='STIX version (1.1.1 or 1.2).')
argparser.add_argument('-f', '--format', default='xml', choices=('json', 'xml'), help='Output format (xml or json).')
argparser.add_argument('-i', '--input', nargs='+', help='Input file(s) containing MISP standard format.')
argparser.add_argument('-o', '--orgname', default='MISP', help='Default Org name to use if no Orgc value is provided.')
argparser.add_argument('-d', '--debug', action='store_true', help='Allow debug mode with warnings.')
try:
args = argparser.parse_args()
if args.input is None:
print(json.dumps({'error': 'No input file provided.'}))
else:
arguments = (args.orgname, args.format, args.version, args.debug)
exporter = globals()[f'Stix{args.scope}sExport'](*arguments)
exporter.parse_misp_files(args.input)
except SystemExit:
print(json.dumps({'error': 'Arguments error, please check you entered a valid version and provided input file names.'}))
sys.exit(1)
if args.input is None:
print(json.dumps({'error': 'No input file provided.'}))
sys.exit(1)
arguments = (args.orgname, args.format, args.version, args.debug)
exporter = globals()[f'Stix{args.scope}sExport'](*arguments)
exporter.parse_misp_files(args.input)
sys.exit(0)

View File

@ -49,14 +49,14 @@ def _process_misp_files(
version: str, input_names: Union[list, None], debug: bool):
if input_names is None:
print(json.dumps({'error': 'No input file provided.'}))
return
sys.exit(1)
try:
parser = MISPtoSTIX20Parser() if version == '2.0' else MISPtoSTIX21Parser()
for name in input_names:
parser.parse_json_content(name)
with open(f'{name}.out', 'wt', encoding='utf-8') as f:
f.write(
f'{json.dumps(parser.stix_objects, cls=STIXJSONEncoder)}'
json.dumps(parser.stix_objects, cls=STIXJSONEncoder)
)
if parser.errors:
_handle_messages('Errors', parser.errors)
@ -64,8 +64,11 @@ def _process_misp_files(
_handle_messages('Warnings', parser.warnings)
print(json.dumps({'success': 1}))
except Exception as e:
print(json.dumps({'error': e.__str__()}))
error = type(e).__name__ + ': ' + e.__str__()
print(json.dumps({'error': error}))
traceback.print_tb(e.__traceback__)
print(error, file=sys.stderr)
sys.exit(1)
if __name__ == "__main__":
@ -84,7 +87,6 @@ if __name__ == "__main__":
)
try:
args = argparser.parse_args()
_process_misp_files(args.version, args.input, args.debug)
except SystemExit:
print(
json.dumps(
@ -94,3 +96,7 @@ if __name__ == "__main__":
}
)
)
sys.exit(1)
_process_misp_files(args.version, args.input, args.debug)
sys.exit(0)

View File

@ -44,7 +44,7 @@ def _handle_return_message(traceback):
return '\n - '.join(traceback)
def _process_stix_file(args: argparse.ArgumentParser):
def _process_stix_file(args: argparse.Namespace):
try:
with open(args.input, 'rt', encoding='utf-8') as f:
bundle = stix2_parser(
@ -81,8 +81,11 @@ def _process_stix_file(args: argparse.ArgumentParser):
file=sys.stderr
)
except Exception as e:
print(json.dumps({'error': e.__str__()}))
error = type(e).__name__ + ': ' + e.__str__()
print(json.dumps({'error': error}))
traceback.print_tb(e.__traceback__)
print(error, file=sys.stderr)
sys.exit(1)
if __name__ == '__main__':
@ -109,8 +112,7 @@ if __name__ == '__main__':
)
try:
args = argparser.parse_args()
_process_stix_file(args)
except SystemExit:
except SystemExit as e:
print(
json.dumps(
{
@ -118,4 +120,6 @@ if __name__ == '__main__':
}
)
)
sys.exit(1)
_process_stix_file(args)

View File

@ -96,7 +96,7 @@ class StixParser():
# Convert the MISP event we create from the STIX document into json format
# and write it in the output file
def saveFile(self):
def save_to_file(self):
for attribute in self.misp_event.attributes:
attribute_uuid = uuid.UUID(attribute.uuid) if isinstance(attribute.uuid, str) else attribute.uuid
if attribute_uuid.version not in _RFC_UUID_VERSIONS:
@ -1547,19 +1547,20 @@ def generate_event(filename, tries=0):
return STIXPackage.from_xml(filename)
except NamespaceNotFoundError:
if tries == 1:
print(json.dump({'error': 'Cannot handle STIX namespace'}))
sys.exit()
print(json.dumps({'error': 'Cannot handle STIX namespace'}))
sys.exit(1)
_update_namespaces()
return generate_event(filename, 1)
except NotImplementedError:
print(json.dumps({'error': 'Missing python library: stix_edh'}))
sys.exit(1)
except Exception as e:
try:
import maec
print(json.dumps({'error': f'Error while loading the STIX file: {e.__str__()}'}))
except ImportError:
print(json.dumps({'error': 'Missing python library: maec'}))
sys.exit(0)
sys.exit(1)
def is_from_misp(event):
@ -1567,7 +1568,7 @@ def is_from_misp(event):
title = event.stix_header.title
except AttributeError:
return False
return ('Export from ' in title and 'MISP' in title)
return 'Export from ' in title and 'MISP' in title
def main(args):
@ -1578,11 +1579,16 @@ def main(args):
stix_parser = StixFromMISPParser() if from_misp else ExternalStixParser()
stix_parser.load_event(args[2:], filename, from_misp, event.version)
stix_parser.build_misp_event(event)
stix_parser.saveFile()
stix_parser.save_to_file()
print(json.dumps({'success': 1}))
sys.exit(0)
except Exception as e:
print(json.dumps({'error': e.__str__()}))
error = type(e).__name__ + ': ' + e.__str__()
print(json.dumps({'error': error}))
traceback.print_tb(e.__traceback__)
print(error, file=sys.stderr)
sys.exit(1)
if __name__ == "__main__":
main(sys.argv)