MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'develop' into 2.4

pull/8534/head v2.4.160
iglocska 2022-08-05 15:57:53 +02:00
parent 488dbcd07f 3dd18f0ed4
commit 71d4e2cd10
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
178 changed files with 20343 additions and 7159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -50,6 +50,8 @@ app/Lib/EventWarning/Custom/*
!/app/files/misp-objects/*
!/app/files/misp-decaying-models
!/app/files/misp-decaying-models/*
!/app/files/misp-workflow-blueprints
!/app/files/misp-workflow-blueprints/*
/app/files/scripts/*.pyc
/app/files/scripts/*.py~
/app/files/scripts/__pycache__

3
.gitmodules vendored
View File

@ -48,3 +48,6 @@
[submodule "app/files/scripts/python-maec"]
path = app/files/scripts/python-maec
url = https://github.com/MAECProject/python-maec
[submodule "app/files/misp-workflow-blueprints"]
path = app/files/misp-workflow-blueprints
url = https://github.com/MISP/misp-workflow-blueprints

2
PyMISP

@ -1 +1 @@
Subproject commit cd4b5d533b68de19b714be7f83f231640404f0bf
Subproject commit 962b296f0c2de3366deaf69ce3484187f255f5e0

2
VERSION.json
View File

@ -1 +1 @@
{"major":2, "minor":4, "hotfix":159}
{"major":2, "minor":4, "hotfix":160}

40
app/Console/Command/AdminShell.php
View File

@ -16,8 +16,9 @@ App::uses('JsonTool', 'Tools');
*/
class AdminShell extends AppShell
{
public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Allowedlist', 'Server', 'Organisation', 'AdminSetting', 'Galaxy', 'Taxonomy', 'Warninglist', 'Noticelist', 'ObjectTemplate', 'Bruteforce', 'Role', 'Feed', 'SharingGroupBlueprint');
public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Allowedlist', 'Server', 'Organisation', 'AdminSetting', 'Galaxy', 'Taxonomy', 'Warninglist', 'Noticelist', 'ObjectTemplate', 'Bruteforce', 'Role', 'Feed', 'SharingGroupBlueprint', 'Correlation');
public $tasks = array('ConfigLoad');
public function getOptionParser()
{
$parser = parent::getOptionParser();
@ -99,6 +100,7 @@ class AdminShell extends AppShell
public function jobGenerateCorrelation()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Generate correlation'] . PHP_EOL);
}
@ -1186,4 +1188,38 @@ class AdminShell extends AppShell
);
$this->out($message);
}
public function truncateTable()
{
$this->ConfigLoad->execute();
if (!isset($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Truncate table correlation'] . PHP_EOL);
}
$userId = $this->args[0];
if ($userId) {
$user = $this->User->getAuthUser($userId);
} else {
$user = [
'id' => 0,
'email' => 'SYSTEM',
'Organisation' => [
'name' => 'SYSTEM'
]
];
}
if (empty($this->args[1])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Truncate table correlation'] . PHP_EOL);
}
if (!empty($this->args[2])) {
$jobId = $this->args[2];
}
$table = trim($this->args[1]);
$this->Correlation->truncate($user, $table);
if ($jobId) {
$this->Job->id = $jobId;
$this->Job->saveField('progress', 100);
$this->Job->saveField('date_modified', date("Y-m-d H:i:s"));
$this->Job->saveField('message', __('Database truncated: ' . $table));
}
}
}

14
app/Console/Command/EventShell.php
View File

@ -40,6 +40,9 @@ class EventShell extends AppShell
'event_id' => ['help' => __('Event ID'), 'required' => true],
'user_id' => ['help' => __('User ID'), 'required' => true],
],
'options' => [
'send' => ['help' => __('Send email to given user'), 'boolean' => true],
],
],
]);
$parser->addSubcommand('duplicateTags', [
@ -607,6 +610,7 @@ class EventShell extends AppShell
public function testEventNotificationEmail()
{
list($eventId, $userId) = $this->args;
$send = $this->param('send');
$user = $this->getUser($userId);
$eventForUser = $this->Event->fetchEvent($user, [
@ -626,10 +630,16 @@ class EventShell extends AppShell
App::uses('SendEmail', 'Tools');
App::uses('GpgTool', 'Tools');
$sendEmail = new SendEmail(GpgTool::initializeGpg());
$sendEmail->setTransport('Debug');
if (!$send) {
$sendEmail->setTransport('Debug');
}
$result = $sendEmail->sendToUser(['User' => $user], null, $emailTemplate);
echo $result['contents']['headers'] . "\n\n" . $result['contents']['message'] . "\n";
if ($send) {
var_dump($result);
} else {
echo $result['contents']['headers'] . "\n\n" . $result['contents']['message'] . "\n";
}
}
/**

262
app/Console/Command/Ls22Shell.php
View File

@ -52,6 +52,55 @@ class Ls22Shell extends AppShell
),
),
]);
$parser->addSubcommand('checkSyncConnections', [
'help' => __('Check the given sync connection(s) for the given server(s).'),
'parser' => array(
'options' => array(
'instances' => [
'help' => 'Path to the instance file, by default "instances.csv" from the local directory',
'short' => 'i',
'required' => true
],
'misp_url_filter' => [
'help' => 'The url of the instance to execute changes on. If not set, all are updated.',
'short' => 'm',
'required' => false
],
'synced_misp_url_filter' => [
'help' => 'The sync connection to modify on each valid instance (as selected by the misp_url_filter). If not set, all sync connections on the selected instances will be updated.',
'short' => 's',
'required' => false
]
),
),
]);
$parser->addSubcommand('modifySyncConnection', [
'help' => __('Modify sync connection(s).'),
'parser' => array(
'options' => array(
'instances' => [
'help' => 'Path to the instance file, by default "instances.csv" from the local directory',
'short' => 'i',
'required' => true
],
'misp_url_filter' => [
'help' => 'The url of the instance to execute changes on. If not set, all are updated.',
'short' => 'm',
'required' => false
],
'synced_misp_url_filter' => [
'help' => 'The sync connection to modify on each valid instance (as selected by the misp_url_filter). If not set, all sync connections on the selected instances will be updated.',
'short' => 's',
'required' => false
],
'json' => [
'help' => 'JSON delta to push (such as \'{"push": 1}\').',
'short' => 'j',
'required' => true
]
),
),
]);
$parser->addSubcommand('addWarninglist', [
'help' => __('Inject warninglist'),
'parser' => array(
@ -104,6 +153,11 @@ class Ls22Shell extends AppShell
'help' => 'Upper bound of the date. Accepts timestamp or date distance (such as 1d or 5h). Defaults to unbounded.',
'short' => 't',
'required' => false
],
'org' => [
'help' => 'Name the org that should be evaluated. If not set, all will be included.',
'short' => 'o',
'required' => false
]
),
),
@ -111,6 +165,103 @@ class Ls22Shell extends AppShell
return $parser;
}
public function checkSyncConnections()
{
$this->__getInstances($this->param('instances'));
$results = [];
$instanceFilter = $this->param('misp_url_filter');
$syncedInstanceFilter = $this->param('synced_misp_url_filter');
foreach ($this->__servers as $server) {
if (!empty($instanceFilter) && strtolower(trim($server['Server']['url'])) !== strtolower(trim($instanceFilter))) {
continue;
}
$HttpSocket = $this->Server->setupHttpSocket($server, null);
$request = $this->Server->setupSyncRequest($server, 'Server');
$start_time = microtime(true);
$response = $HttpSocket->get($server['Server']['url'] . '/servers/index', false, $request);
$baseline = round((microtime(true) - $start_time) * 1000);
if (!$response->isOk()) {
$this->out($server['Server']['url'] . ': ' . '<error>Connection or auth failed</error>', 1, Shell::NORMAL);
continue;
}
$synced_servers = json_decode($response->body, true);
foreach ($synced_servers as $synced_server) {
$success = false;
if (empty($syncedInstanceFilter) || strtolower($synced_server['Server']['url']) === strtolower($syncedInstanceFilter)) {
$start_time = microtime(true);
$response = $HttpSocket->get($server['Server']['url'] . '/servers/testConnection/' . $synced_server['Server']['id'], '{}', $request);
$execution_time = round((microtime(true) - $start_time) * 1000) - $baseline;
if ($response->isOk()) {
$success = true;
}
$this->out(
sprintf(
'%s connection to %s: %s (%sms)',
$server['Server']['url'],
$synced_server['Server']['url'],
sprintf(
'<%s>%s</%s>',
$success ? 'info' : 'error',
$success ? 'Success' : 'Failed',
$success ? 'info' : 'error'
),
$execution_time
),
1,
Shell::NORMAL
);
}
}
}
}
public function modifySyncConnection()
{
$this->__getInstances($this->param('instances'));
$results = [];
$instanceFilter = $this->param('misp_url_filter');
$syncedInstanceFilter = $this->param('synced_misp_url_filter');
$json = $this->param('json');
foreach ($this->__servers as $server) {
if (!empty($instanceFilter) && strtolower(trim($server['Server']['url'])) !== strtolower(trim($instanceFilter))) {
continue;
}
$HttpSocket = $this->Server->setupHttpSocket($server, null);
$request = $this->Server->setupSyncRequest($server, 'Server');
$response = $HttpSocket->get($server['Server']['url'] . '/servers/index', false, $request);
if (!$response->isOk()) {
$this->out($server['Server']['url'] . ': ' . '<error>Connection or auth failed</error>', 1, Shell::NORMAL);
}
$synced_servers = json_decode($response->body, true);
$success = false;
foreach ($synced_servers as $synced_server) {
if (empty($syncedInstanceFilter) || strtolower($synced_server['Server']['url']) === strtolower($syncedInstanceFilter)) {
debug($json);
$response = $HttpSocket->post($server['Server']['url'] . '/servers/edit/' . $synced_server['Server']['id'], $json, $request);
debug($response->body);
if ($response->isOk()) {
$success = true;
}
$this->out(
sprintf(
'%s connection to %s: %s',
$server['Server']['url'],
$synced_server['Server']['url'],
sprintf(
'<%s>%s</%s>',
$success ? 'info' : 'error',
$success ? 'Success' : 'Failed',
$success ? 'info' : 'error'
)
),
1,
Shell::NORMAL
);
}
}
}
}
public function enableTaxonomy()
{
$taxonomyToEnable = $this->param('taxonomy');
@ -162,16 +313,29 @@ class Ls22Shell extends AppShell
$HttpSocket = $this->Server->setupHttpSocket($server, null);
$request = $this->Server->setupSyncRequest($server, 'Server');
$start_time = microtime(true);
$response = $HttpSocket->get($server['Server']['url'] . '/users/view/me', false, $request);
$execution_time = round((microtime(true) - $start_time) * 1000);
$statusWrapped = sprintf(
'<%s>%s</%s>',
$response->isOk() ? 'info' : 'error',
$response->isOk() ? 'OK (' . $execution_time . 'ms)' : 'Failed. (' . $response->code . ')',
$response->isOk() ? 'info' : 'error'
);
$fatal_error = false;
try {
$response = $HttpSocket->get($server['Server']['url'] . '/users/view/me', false, $request);
} catch (Exception $e) {
$fatal_error = true;
echo "\x07";
$statusWrapped = sprintf(
'<error>%s %s: %s</error>',
'Something went wrong while trying to reach',
$server['Server']['url'],
$e->getMessage()
);
}
if (!$fatal_error) {
$execution_time = round((microtime(true) - $start_time) * 1000);
$statusWrapped = sprintf(
'<%s>%s</%s>',
$response->isOk() ? 'info' : 'error',
$response->isOk() ? 'OK (' . $execution_time . 'ms)' : 'Failed. (' . $response->code . ')',
$response->isOk() ? 'info' : 'error'
);
}
$this->out($server['Server']['url'] . ': ' . $statusWrapped, 1, Shell::NORMAL);
$results[$server['Server']['url']] = $response->isOk() ? $execution_time : false;
}
}
@ -215,28 +379,30 @@ class Ls22Shell extends AppShell
}
$HttpSocket = $this->Server->setupHttpSocket($server, null);
$request = $this->Server->setupSyncRequest($server);
$response = $HttpSocket->get($server['Server']['url'] . '/organisations/index', false, $request);
$response = $HttpSocket->get($server['Server']['url'] . '/organisations/index/scope:all', false, $request);
$orgs = json_decode($response->body(), true);
$this->out(__('Organisations fetched. %d found.', count($orgs)), 1, Shell::VERBOSE);
$org_mapping = [];
foreach ($orgs as $org) {
$name = explode(' ', $org['Organisation']['name']);
if ($name[0] !== 'BT') {
if (!empty($this->param('org')) && $org['Organisation']['name'] !== $this->param('org')) {
continue;
}
if ($org['Organisation']['name'] === 'YT') {
continue;
}
$org_mapping[$org['Organisation']['name']] = $org['Organisation']['id'];
}
if (!empty($this->param['from'])) {
$time_range[] = $this->param['from'];
}
if (!empty($this->param['to'])) {
if (empty($time_range)) {
$time_range[] = '365d';
}
$time_range[] = $this->param['to'];
}
foreach ($org_mapping as $org_name => $org_id) {
$time_range = [];
if (!empty($this->param['from'])) {
$time_range[] = $this->param['from'];
}
if (!empty($this->param['to'])) {
if (empty($time_range)) {
$time_range[] = '365d';
}
$time_range[] = $this->param['to'];
}
$params = [
'org' => $org_id
];
@ -256,7 +422,8 @@ class Ls22Shell extends AppShell
'other' => 0,
'attribute_attack' => 0,
'attribute_other' => 0,
'score' => 0
'score' => 0,
'warnings' => 0
];
foreach ($events['response'] as $event) {
if (!empty($event['Event']['Tag'])) {
@ -290,6 +457,9 @@ class Ls22Shell extends AppShell
}
}
}
if (!empty($attribute['warnings'])) {
$result[$org_name]['warnings'] += 1;
}
}
$results[$org_name]['attribute_count'] += count($event['Event']['Attribute']);
if (!empty($event['Event']['Object'])) {
@ -319,11 +489,18 @@ class Ls22Shell extends AppShell
foreach ($results as $k => $result) {
$totalCount = $result['attribute_count'] + $result['object_count'];
if ($totalCount) {
if (empty($result['warnings'])) {
$results[$k]['metrics']['warnings'] = 100;
} else if (100 * $result['warnings'] < $result['attribute_count']) {
$results[$k]['metrics']['warnings'] = 50;
} else {
$results[$k]['metrics']['warnings'] = 0;
}
$results[$k]['metrics']['connectedness'] = 100 * ($result['connected_elements'] / ($result['attribute_count'] + $result['object_count']));
$results[$k]['metrics']['attack_weight'] = 100 * (2*($result['attack']) + $result['attribute_attack']) / ($result['attribute_count'] + $result['object_count']);
$results[$k]['metrics']['other_weight'] = 100 * (2*($result['other']) + $result['attribute_other']) / ($result['attribute_count'] + $result['object_count']);
}
foreach (['connectedness', 'attack_weight', 'other_weight'] as $metric) {
foreach (['connectedness', 'attack_weight', 'other_weight', 'warnings'] as $metric) {
if (empty($results[$k]['metrics'][$metric])) {
$results[$k]['metrics'][$metric] = 0;
}
@ -331,8 +508,17 @@ class Ls22Shell extends AppShell
$results[$k]['metrics'][$metric] = 100;
}
}
$results[$k]['score'] = round(40 * $results[$k]['metrics']['connectedness'] + 40 * $results[$k]['metrics']['attack_weight'] + 20 * $results[$k]['metrics']['other_weight']) / 100;
$scores[$k] = $results[$k]['score'];
$results[$k]['score'] = round(
20 * $results[$k]['metrics']['warnings'] +
20 * $results[$k]['metrics']['connectedness'] +
40 * $results[$k]['metrics']['attack_weight'] +
20 * $results[$k]['metrics']['other_weight']
) / 100;
$scores[$k]['total'] = $results[$k]['score'];
$scores[$k]['warnings'] = round(20 * $results[$k]['metrics']['warnings']);
$scores[$k]['connectedness'] = round(20 * $results[$k]['metrics']['connectedness']);
$scores[$k]['attack_weight'] = round(40 * $results[$k]['metrics']['attack_weight']);
$scores[$k]['other_weight'] = round(20 * $results[$k]['metrics']['other_weight']);
}
arsort($scores, SORT_DESC);
$this->out(str_repeat('=', 128), 1, Shell::NORMAL);
@ -344,18 +530,34 @@ class Ls22Shell extends AppShell
), 1, Shell::NORMAL);
$this->out(str_repeat('=', 128), 1, Shell::NORMAL);
foreach ($scores as $org => $score) {
$score_string = str_repeat('█', round($score));
$score_string[0] = str_repeat('█', round($score['warnings']/100));
$score_string[1] = str_repeat('█', round($score['connectedness']/100));
$score_string[2] = str_repeat('█', round($score['attack_weight']/100));
$score_string[3] = str_repeat('█', round($score['other_weight']/100));
$this->out(sprintf(
'| %s | %s | %s |',
str_pad($org, 10, ' ', STR_PAD_RIGHT),
sprintf(
'<info>%s</info>%s',
$score_string,
str_repeat(' ', 100 - mb_strlen($score_string))
'<error>%s</error><warning>%s</warning><question>%s</question><info>%s</info>%s',
$score_string[0],
$score_string[1],
$score_string[2],
$score_string[3],
str_repeat(' ', 100 - mb_strlen(implode('', $score_string)))
),
str_pad($score . '%', 8, ' ', STR_PAD_RIGHT)
str_pad($score['total'] . '%', 8, ' ', STR_PAD_RIGHT)
), 1, Shell::NORMAL);
}
$this->out(str_repeat('=', 128), 1, Shell::NORMAL);
$this->out(sprintf(
'| Legend: %s %s %s %s %s |',
'<error>█: Warnings</error>',
'<warning>█: Connectedness</warning>',
'<question>█: ATT&CK context</question>',
'<info>█: Other Context</info>',
str_repeat(' ', 52)
), 1, Shell::NORMAL);
$this->out(str_repeat('=', 128), 1, Shell::NORMAL);
file_put_contents(APP . 'tmp/report.json', json_encode($results, JSON_PRETTY_PRINT));
}
}

77
app/Console/Command/WorkflowShell.php Normal file
View File

@ -0,0 +1,77 @@
<?php
declare(strict_types=1);
require_once 'AppShell.php';
class WorkflowShell extends AppShell {
public $uses = ['Job', 'Workflow'];
public $tasks = ['ConfigLoad'];
public function executeWorkflowForTrigger()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || empty($this->args[3])) {
die(__('Invalid number of arguments.'));
}
$trigger_id = $this->args[0];
$data = JsonTool::decode($this->args[1]);
$logging = JsonTool::decode($this->args[2]);
$jobId = $this->args[3];
$blockingErrors = [];
$executionSuccess = $this->Workflow->executeWorkflowForTrigger($trigger_id, $data, $blockingErrors);
$job = $this->Job->read(null, $jobId);
$job['Job']['progress'] = 100;
$job['Job']['status'] = Job::STATUS_COMPLETED;
$job['Job']['date_modified'] = date("Y-m-d H:i:s");
if ($executionSuccess) {
$job['Job']['message'] = __('Workflow for trigger `%s` completed execution', $trigger_id);
} else {
$errorMessage = implode(', ', $blockingErrors);
$message = __('Error while executing workflow for trigger `%s`: %s. %s%s', $trigger_id, $logging['message'], PHP_EOL . __('Returned message: %s', $errorMessage));
$job['Job']['message'] = $message;
}
$this->Job->save($job);
}
public function walkGraph()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || empty($this->args[3])) {
die(__('Invalid number of arguments.'));
}
$workflow_id = (int)$this->args[0];
$workflow = $this->Workflow->fetchWorkflow($workflow_id);
$node_id_to_exec = (int)$this->args[1];
$roamingData = JsonTool::decode($this->args[2]);
$for_path = $this->args[3];
$jobId = $this->args[4];
$concurrentErrors = [];
$walkResult = [];
$executionSuccess = $this->Workflow->walkGraph(
$workflow,
$node_id_to_exec,
$for_path,
$roamingData,
$concurrentErrors,
$walkResult
);
$job = $this->Job->read(null, $jobId);
$job['Job']['progress'] = 100;
$job['Job']['status'] = Job::STATUS_COMPLETED;
$job['Job']['date_modified'] = date("Y-m-d H:i:s");
if ($executionSuccess) {
$job['Job']['message'] = __('Workflow concurrent task executed %s nodes starting from node %s.', count($walkResult['executed_nodes']), $node_id_to_exec);
} else {
$message = __('Error while executing workflow concurrent task. %s', PHP_EOL . implode(', ', $concurrentErrors));
$this->Workflow->logExecutionError($workflow, $message);
$job['Job']['message'] = $message;
}
$this->Job->save($job);
}
}

61
app/Controller/AppController.php
View File

@ -4,6 +4,7 @@ App::uses('Controller', 'Controller');
App::uses('File', 'Utility');
App::uses('RequestRearrangeTool', 'Tools');
App::uses('BlowfishConstantPasswordHasher', 'Controller/Component/Auth');
App::uses('BetterCakeEventManager', 'Tools');
/**
* Application Controller
@ -34,8 +35,8 @@ class AppController extends Controller
public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
private $__queryVersion = '141';
public $pyMispVersion = '2.4.159';
private $__queryVersion = '143';
public $pyMispVersion = '2.4.160';
public $phpmin = '7.2';
public $phprec = '7.4';
public $phptoonew = '8.0';
@ -1259,17 +1260,17 @@ class AppController extends Controller
]);
}
}
/** @var TmpFileTool $final */
$final = $model->restSearch($user, $returnFormat, $filters, false, false, $elementCounter, $renderView);
if (!empty($renderView) && !empty($final)) {
if ($renderView) {
$this->layout = false;
$final = json_decode($final->intoString(), true);
foreach ($final as $key => $data) {
$this->set($key, $data);
}
$this->set($final);
$this->render('/Events/module_views/' . $renderView);
} else {
$filename = $this->RestSearch->getFilename($filters, $scope, $responseType);
return $this->RestResponse->viewData($final, $responseType, false, true, $filename, array('X-Result-Count' => $elementCounter, 'X-Export-Module-Used' => $returnFormat, 'X-Response-Format' => $responseType));
$headers = ['X-Result-Count' => $elementCounter, 'X-Export-Module-Used' => $returnFormat, 'X-Response-Format' => $responseType];
return $this->RestResponse->viewData($final, $responseType, false, true, $filename, $headers);
}
}
@ -1443,6 +1444,16 @@ class AppController extends Controller
return parent::_getViewObject();
}
public function getEventManager()
{
if (empty($this->_eventManager)) {
$this->_eventManager = new BetterCakeEventManager();
$this->_eventManager->attach($this->Components);
$this->_eventManager->attach($this);
}
return $this->_eventManager;
}
/**
* Close session without writing changes to them and return current user.
* @return array
@ -1462,18 +1473,34 @@ class AppController extends Controller
protected function _jsonDecode($dataToDecode)
{
try {
if (defined('JSON_THROW_ON_ERROR')) {
// JSON_THROW_ON_ERROR is supported since PHP 7.3
return json_decode($dataToDecode, true, 512, JSON_THROW_ON_ERROR);
} else {
$decoded = json_decode($dataToDecode, true);
if ($decoded === null) {
throw new UnexpectedValueException('Could not parse JSON: ' . json_last_error_msg(), json_last_error());
}
return $decoded;
}
return JsonTool::decode($dataToDecode);
} catch (Exception $e) {
throw new HttpException('Invalid JSON input. Make sure that the JSON input is a correctly formatted JSON string. This request has been blocked to avoid an unfiltered request.', 405, $e);
}
}
/**
* Mimics what PaginateComponent::paginate() would do, when Model::paginate() is not called
*
* @param integer $page
* @param integer $limit
* @param integer $current
* @param string $type
* @return void
*/
protected function __setPagingParams(int $page, int $limit, int $current, string $type = 'named')
{
$this->request->params['paging'] = [
'Correlation' => [
'page' => $page,
'limit' => $limit,
'current' => $current,
'pageCount' => 0,
'prevPage' => $page > 1,
'nextPage' => $current >= $limit,
'options' => [],
'paramType' => $type
]
];
}
}

240
app/Controller/AttributesController.php
View File

@ -84,10 +84,24 @@ class AttributesController extends AppController
'fields' => ['Orgc.id', 'Orgc.name', 'Orgc.uuid'],
]);
$orgTable = Hash::combine($orgTable, '{n}.Orgc.id', '{n}.Orgc');
$sgids = $this->Attribute->SharingGroup->authorizedIds($this->Auth->user());
foreach ($attributes as &$attribute) {
if (isset($orgTable[$attribute['Event']['orgc_id']])) {
$attribute['Event']['Orgc'] = $orgTable[$attribute['Event']['orgc_id']];
}
$temp = $this->Attribute->Correlation->getRelatedAttributes(
$this->Auth->user(),
$sgids,
$attribute['Attribute'],
[],
true
);
foreach ($temp as &$t) {
$t['info'] = $t['Event']['info'];
$t['org_id'] = $t['Event']['org_id'];
$t['date'] = $t['Event']['date'];
}
$attribute['Event']['RelatedAttribute'][$attribute['Attribute']['id']] = $temp;
}
list($attributes, $sightingsData) = $this->__searchUI($attributes);
@ -732,7 +746,13 @@ class AttributesController extends AppController
}
}
$dateObj = new DateTime();
$existingAttribute = $this->Attribute->findByUuid($this->Attribute->data['Attribute']['uuid']);
$existingAttribute = $this->Attribute->find('first', [
'conditions' => [
'Attribute.uuid' => $this->Attribute->data['Attribute']['uuid']
],
'recursive' => -1
]
);
// check if the attribute has a timestamp already set (from a previous instance that is trying to edit via synchronisation)
// check which attribute is newer
if (count($existingAttribute) && !$existingAttribute['Attribute']['deleted']) {
@ -828,7 +848,10 @@ class AttributesController extends AppController
}
}
} else {
$this->request->data = $this->Attribute->read(null, $id);
$this->request->data = $this->Attribute->find('first', [
'recursive' => -1,
'conditions' => ['Attribute.id' => $id]
]);
}
$this->set('attribute', $this->request->data);
if (!empty($this->request->data['Attribute']['object_id'])) {
@ -877,7 +900,9 @@ class AttributesController extends AppController
// ajax edit - post a single edited field and this method will attempt to save it and return a json with the validation errors if they occur.
public function editField($id)
{
$attribute = $this->__fetchAttribute($id);
$attribute = $this->Attribute->fetchAttributeSimple($this->Auth->user(), [
'conditions' => ['Attribute.id' => $id],
]);
if (empty($attribute)) {
return new CakeResponse(array('body'=> json_encode(array('fail' => false, 'errors' => 'Invalid attribute')), 'status' => 200, 'type' => 'json'));
}
@ -1459,57 +1484,63 @@ class AttributesController extends AppController
return new CakeResponse(array('body'=> json_encode(array('saved' => true)), 'status' => 200, 'type' => 'json'));
}
private function __getSearchFilters(&$exception)
{
if (isset($this->request->data['Attribute'])) {
$this->request->data = $this->request->data['Attribute'];
}
$checkForEmpty = array('value', 'tags', 'uuid', 'org', 'type', 'category', 'first_seen', 'last_seen');
foreach ($checkForEmpty as $field) {
if (empty($this->request->data[$field]) || $this->request->data[$field] === 'ALL') {
unset($this->request->data[$field]);
}
}
if (empty($this->request->data['to_ids'])) {
unset($this->request->data['to_ids']);
$this->request->data['ignore'] = 1;
}
$paramArray = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags', 'first_seen', 'last_seen');
$filterData = array(
'request' => $this->request,
'named_params' => $this->request->params['named'],
'paramArray' => $paramArray,
'additional_delimiters' => PHP_EOL
);
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
if (!empty($filters['uuid'])) {
if (!is_array($filters['uuid'])) {
$filters['uuid'] = array($filters['uuid']);
}
$uuid = array();
$ids = array();
foreach ($filters['uuid'] as $k => $filter) {
if ($filter[0] === '!') {
$filter = substr($filter, 1);
}
if (Validation::uuid($filter)) {
$uuid[] = $filters['uuid'][$k];
} else {
$ids[] = $filters['uuid'][$k];
}
}
if (empty($uuid)) {
unset($filters['uuid']);
} else {
$filters['uuid'] = $uuid;
}
if (!empty($ids)) {
$filters['eventid'] = $ids;
}
}
return $filters;
}
public function search($continue = false)
{
$exception = null;
$filters = $this->__getSearchFilters($exception);
if ($this->request->is('post') || !empty($this->request->params['named']['tags'])) {
if (isset($this->request->data['Attribute'])) {
$this->request->data = $this->request->data['Attribute'];
}
$checkForEmpty = array('value', 'tags', 'uuid', 'org', 'type', 'category', 'first_seen', 'last_seen');
foreach ($checkForEmpty as $field) {
if (empty($this->request->data[$field]) || $this->request->data[$field] === 'ALL') {
unset($this->request->data[$field]);
}
}
if (empty($this->request->data['to_ids'])) {
unset($this->request->data['to_ids']);
$this->request->data['ignore'] = 1;
}
$paramArray = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags', 'first_seen', 'last_seen');
$filterData = array(
'request' => $this->request,
'named_params' => $this->request->params['named'],
'paramArray' => $paramArray,
'additional_delimiters' => PHP_EOL
);
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
if (!empty($filters['uuid'])) {
if (!is_array($filters['uuid'])) {
$filters['uuid'] = array($filters['uuid']);
}
$uuid = array();
$ids = array();
foreach ($filters['uuid'] as $k => $filter) {
if ($filter[0] === '!') {
$filter = substr($filter, 1);
}
if (Validation::uuid($filter)) {
$uuid[] = $filters['uuid'][$k];
} else {
$ids[] = $filters['uuid'][$k];
}
}
if (empty($uuid)) {
unset($filters['uuid']);
} else {
$filters['uuid'] = $uuid;
}
if (!empty($ids)) {
$filters['eventid'] = $ids;
}
}
unset($filterData);
if ($filters === false) {
return $exception;
}
@ -1536,7 +1567,8 @@ class AttributesController extends AppController
$this->Session->write('search_attributes_filters', null);
}
if (isset($filters)) {
if (!empty($filters)) {
$filters['includeCorrelations'] = 1;
$params = $this->Attribute->restSearch($this->Auth->user(), 'json', $filters, true);
if (!isset($params['conditions']['Attribute.deleted'])) {
$params['conditions']['Attribute.deleted'] = 0;
@ -1555,6 +1587,7 @@ class AttributesController extends AppController
'fields' => ['Orgc.id', 'Orgc.name', 'Orgc.uuid'],
]);
$orgTable = array_column(array_column($orgTable, 'Orgc'), null, 'id');
$sgids = $this->Attribute->SharingGroup->authorizedIds($this->Auth->user());
foreach ($attributes as &$attribute) {
if (isset($orgTable[$attribute['Event']['orgc_id']])) {
$attribute['Event']['Orgc'] = $orgTable[$attribute['Event']['orgc_id']];
@ -1562,6 +1595,21 @@ class AttributesController extends AppController
if (isset($orgTable[$attribute['Event']['org_id']])) {
$attribute['Event']['Org'] = $orgTable[$attribute['Event']['org_id']];
}
if (isset($filters['includeCorrelations'])) {
$temp = $this->Attribute->Correlation->getRelatedAttributes(
$this->Auth->user(),
$sgids,
$attribute['Attribute'],
[],
true
);
foreach ($temp as &$t) {
$t['info'] = $t['Event']['info'];
$t['org_id'] = $t['Event']['org_id'];
$t['date'] = $t['Event']['date'];
}
$attribute['Event']['RelatedAttribute'][$attribute['Attribute']['id']] = $temp;
}
}
if ($this->_isRest()) {
return $this->RestResponse->viewData($attributes, $this->response->type());
@ -1877,36 +1925,38 @@ class AttributesController extends AppController
public function generateCorrelation()
{
$this->request->allowMethod(['post']);
if ($this->request->is('post')) {
if (!Configure::read('MISP.background_jobs')) {
$k = $this->Attribute->generateCorrelation();
$this->Flash->success(__('All done. %s attributes processed.', $k));
$this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
} else {
/** @var Job $job */
$job = ClassRegistry::init('Job');
$jobId = $job->createJob(
'SYSTEM',
Job::WORKER_DEFAULT,
'generate correlation',
'All attributes',
'Job created.'
);
if (!Configure::read('MISP.background_jobs')) {
$k = $this->Attribute->generateCorrelation();
$this->Flash->success(__('All done. %s attributes processed.', $k));
$this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
} else {
/** @var Job $job */
$job = ClassRegistry::init('Job');
$jobId = $job->createJob(
'SYSTEM',
Job::WORKER_DEFAULT,
'generate correlation',
'All attributes',
'Job created.'
);
$this->Attribute->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[
'jobGenerateCorrelation',
$this->Attribute->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[
'jobGenerateCorrelation',
$jobId
],
true,
$jobId
],
true,
$jobId
);
);
$this->Flash->success(__('Job queued. You can view the progress if you navigate to the active jobs view (Administration -> Jobs).'));
$this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
$this->Flash->success(__('Job queued. You can view the progress if you navigate to the active jobs view (Administration -> Jobs).'));
$this->redirect(Router::url($this->referer(), true));
}
} else {
$this->render('ajax/recorrelationConfirmation');
}
}
@ -2417,7 +2467,7 @@ class AttributesController extends AppController
} else {
$data[$attribute[0]['Attribute']['type']] = $attribute[0]['Attribute']['value'];
}
$result = $this->Module->queryModuleServer($data, true);
$result = $this->Module->queryModuleServer($data, true, 'Enrichment', false, $attribute[0]);
if ($result) {
if (!is_array($result)) {
$resultArray[$type] = ['error' => $result];
@ -2770,14 +2820,18 @@ class AttributesController extends AppController
$tag_id = $this->request->data['tag'];
}
$this->Attribute->id = $id;
if (!$this->Attribute->exists()) {
$attribute = $this->__fetchAttribute($id);
$attribute = $this->Attribute->find('first', [
'recursive' => -1,
'conditions' => ['Attribute.id' => $id],
'fields' => ['Attribute.deleted', 'Attribute.event_id', 'Attribute.id', 'Attribute.object_id']
]);
if (empty($attribute)) {
throw new NotFoundException(__('Invalid attribute'));
}
$this->Attribute->read();
if ($this->Attribute->data['Attribute']['deleted']) {
if ($attribute['Attribute']['deleted']) {
throw new NotFoundException(__('Invalid attribute'));
}
$eventId = $this->Attribute->data['Attribute']['event_id'];
if (empty($tag_id)) {
return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Invalid Tag.')), 'status' => 200, 'type' => 'json'));
}
@ -2792,12 +2846,13 @@ class AttributesController extends AppController
$id = $this->request->data['Attribute']['id'];
}
$this->Attribute->Event->recursive = -1;
$event = $this->Attribute->Event->read(array(), $eventId);
$event = $this->Attribute->Event->find('first', [
'recursive' => -1,
'conditons' => ['Event.id' => $attribute['Attribute']['event_id']]
]);
if (!$this->_isRest()) {
$this->Attribute->Event->insertLock($this->Auth->user(), $eventId);
$this->Attribute->Event->insertLock($this->Auth->user(), $attribute['Attribute']['event_id']);
}
$this->Attribute->recursive = -1;
$attributeTag = $this->Attribute->AttributeTag->find('first', array(
'conditions' => array(
'attribute_id' => $id,
@ -2825,11 +2880,11 @@ class AttributesController extends AppController
$date = new DateTime();
$event['Event']['timestamp'] = $date->getTimestamp();
$this->Attribute->Event->save($event);
if ($this->Attribute->data['Attribute']['object_id'] != 0) {
$this->Attribute->Object->updateTimestamp($this->Attribute->data['Attribute']['object_id'], $date->getTimestamp());
if ($attribute['Attribute']['object_id'] != 0) {
$this->Attribute->Object->updateTimestamp($attribute['Attribute']['object_id'], $date->getTimestamp());
}
$this->Attribute->data['Attribute']['timestamp'] = $date->getTimestamp();
$this->Attribute->save($this->Attribute->data);
$attribute['Attribute']['timestamp'] = $date->getTimestamp();
$this->Attribute->save($attribute);
}
$log = ClassRegistry::init('Log');
$log->createLogEntry($this->Auth->user(), 'tag', 'Attribute', $id, 'Removed tag (' . $tag_id . ') "' . $tag['Tag']['name'] . '" from attribute (' . $id . ')', 'Attribute (' . $id . ') untagged of Tag (' . $tag_id . ')');
@ -2945,6 +3000,7 @@ class AttributesController extends AppController
/**
* @param int|string $id Attribute ID or UUID
* @return array
* @throws Exception
*/
private function __fetchAttribute($id)
{

7
app/Controller/AuditLogsController.php
View File

@ -52,6 +52,7 @@ class AuditLogsController extends AppController
'GalaxyClusterRelation',
'News',
'Warninglist',
'Workflow',
];
public $paginate = [
@ -67,9 +68,9 @@ class AuditLogsController extends AppController
],
];
public function __construct($id = false, $table = null, $ds = null)
public function __construct($request = null, $response = null)
{
parent::__construct($id, $table, $ds);
parent::__construct($request, $response);
$this->actions = [
AuditLog::ACTION_ADD => __('Add'),
AuditLog::ACTION_EDIT => __('Edit'),
@ -344,7 +345,7 @@ class AuditLogsController extends AppController
return ['event_id' => $event['Event']['id']];
}
$event = $this->Event->fetchEvent($this->Auth->user(), [
$event = $this->AuditLog->Event->fetchEvent($this->Auth->user(), [
'eventid' => $event['Event']['id'],
'sgReferenceOnly' => 1,
'deleted' => [0, 1],

43
app/Controller/Component/ACLComponent.php
View File

@ -98,7 +98,10 @@ class ACLComponent extends Component
],
'correlations' => [
'generateTopCorrelations' => [],
'top' => []
'overCorrelations' => [],
'switchEngine' => [],
'top' => [],
'truncate' => []
],
'cryptographicKeys' => [
'add' => ['perm_add'],
@ -499,6 +502,7 @@ class ACLComponent extends Component
'servers' => array(
'add' => array(),
'dbSchemaDiagnostic' => array(),
'dbConfiguration' => array(),
'cache' => array(),
'changePriority' => array(),
'checkout' => array(),
@ -669,6 +673,7 @@ class ACLComponent extends Component
'view' => array('*'),
'unhideTag' => array('perm_tagger'),
'hideTag' => array('perm_tagger'),
'normalizeCustomTagsToTaxonomyFormat' => [],
),
'templateElements' => array(
'add' => array('perm_template'),
@ -738,6 +743,7 @@ class ACLComponent extends Component
'verifyGPG' => array(),
'view' => array('*'),
'getGpgPublicKey' => array('*'),
'unsubscribe' => ['*'],
),
'userSettings' => array(
'index' => array('*'),
@ -762,6 +768,32 @@ class ACLComponent extends Component
'export' => ['*'],
'import' => ['perm_warninglist'],
),
'workflows' => [
'index'=> [],
'rebuildRedis'=> [],
'edit'=> [],
'delete'=> [],
'view'=> [],
'editor'=> [],
'triggers'=> [],
'moduleIndex'=> [],
'moduleView'=> [],
'toggleModule'=> [],
'checkGraph'=> [],
'executeWorkflow'=> [],
'debugToggleField'=> [],
'massToggleField'=> [],
],
'workflowBlueprints' => [
'add' => [],
'delete' => [],
'edit' => [],
'export' => [],
'import' => [],
'index' => [],
'update' => [],
'view' => [],
],
'allowedlists' => array(
'admin_add' => array('perm_regexp_access'),
'admin_delete' => array('perm_regexp_access'),
@ -979,6 +1011,8 @@ class ACLComponent extends Component
private function __findAllFunctions()
{
$functionsToIgnore = ['beforeFilter', 'afterFilter', 'beforeRender', 'getEventManager'];
$functionFinder = '/function[\s\n]+(\S+)[\s\n]*\(/';
$dir = new Folder(APP . 'Controller');
$files = $dir->find('.*\.php');
@ -989,11 +1023,11 @@ class ACLComponent extends Component
$controllerName = '*';
}
$functionArray = array();
$fileContents = file_get_contents(APP . 'Controller' . DS . $file);
$fileContents = FileAccessTool::readFromFile(APP . 'Controller' . DS . $file);
$fileContents = preg_replace('/\/\*[^\*]+?\*\//', '', $fileContents);
preg_match_all($functionFinder, $fileContents, $functionArray);
foreach ($functionArray[1] as $function) {
if ($function[0] !== '_' && $function !== 'beforeFilter' && $function !== 'afterFilter' && $function !== 'beforeRender') {
if ($function[0] !== '_' && !in_array($function, $functionsToIgnore, true)) {
$results[$controllerName][] = $function;
}
}
@ -1014,8 +1048,7 @@ class ACLComponent extends Component
$missing = array();
foreach ($results as $controller => $functions) {
foreach ($functions as $function) {
if (!isset(self::ACL_LIST[$controller])
|| !in_array($function, array_keys(self::ACL_LIST[$controller]))) {
if (!isset(self::ACL_LIST[$controller]) || !in_array($function, array_keys(self::ACL_LIST[$controller]))) {
$missing[$controller][] = $function;
}
}

19
app/Controller/Component/Auth/BlowfishConstantPasswordHasher.php
View File

@ -1,8 +1,21 @@
<?php
App::uses('BlowfishPasswordHasher', 'Controller/Component/Auth');
App::uses('AbstractPasswordHasher', 'Controller/Component/Auth');
class BlowfishConstantPasswordHasher extends BlowfishPasswordHasher
class BlowfishConstantPasswordHasher extends AbstractPasswordHasher
{
/**
* @param string $password
* @return string
*/
public function hash($password)
{
$hash = password_hash($password, PASSWORD_BCRYPT);
if ($hash === false) {
throw new RuntimeException('Could not generate hashed password');
}
return $hash;
}
/**
* @param string $password
* @param string $hashedPassword
@ -10,6 +23,6 @@ class BlowfishConstantPasswordHasher extends BlowfishPasswordHasher
*/
public function check($password, $hashedPassword)
{
return hash_equals($hashedPassword, Security::hash($password, 'blowfish', $hashedPassword));
return password_verify($password, $hashedPassword);
}
}

24
app/Controller/Component/RestResponseComponent.php
View File

@ -636,9 +636,8 @@ class RestResponseComponent extends Component
$headers['ETag'] = $etag;
}
if ($this->signContents) {
$this->CryptographicKey = ClassRegistry::init('CryptographicKey');
$data = $response->intoString();
$headers['x-pgp-signature'] = base64_encode($this->CryptographicKey->signWithInstanceKey($data));
$headers['x-pgp-signature'] = $this->sign($data);
$cakeResponse = new CakeResponse(['body' => $data, 'status' => $code, 'type' => $type]);
} else {
App::uses('CakeResponseFile', 'Tools');
@ -657,7 +656,7 @@ class RestResponseComponent extends Component
}
$cakeResponse = new CakeResponse(['body' => $response, 'status' => $code, 'type' => $type]);
if ($this->signContents) {
$headers['x-pgp-signature'] = base64_encode($this->CryptographicKey->signWithInstanceKey($response));
$headers['x-pgp-signature'] = $this->sign($response);
}
}
@ -682,6 +681,25 @@ class RestResponseComponent extends Component
return $cakeResponse;
}
/**
* @param string $response
* @return string Signature as base64 encoded string
* @throws Crypt_GPG_BadPassphraseException
* @throws Crypt_GPG_Exception
* @throws Crypt_GPG_KeyNotFoundException
* @throws Exception
*/
private function sign($response)
{
/** @var CryptographicKey $cryptographicKey */
$cryptographicKey = ClassRegistry::init('CryptographicKey');
$signature = $cryptographicKey->signWithInstanceKey($response);
if (!$signature) {
throw new Exception('Could not sign data.');
}
return base64_encode($signature);
}
/**
* Detect if request comes from automatic tool (like other MISP instance or PyMISP) or AJAX
* @return bool

128
app/Controller/CorrelationsController.php
View File

@ -44,6 +44,9 @@ class CorrelationsController extends AppController
}
}
}
$this->__setPagingParams($query['page'], $query['limit'], count($data), 'named');
$this->set('age', $age);
$this->set('age_unit', $unit);
$this->set('data', $data);
@ -72,4 +75,129 @@ class CorrelationsController extends AppController
$this->redirect(['controller' => 'correlations', 'action' => 'top']);
}
}
public function overCorrelations()
{
$query = [
'limit' => 50,
'page' => 1,
'order' => 'occurrence desc'
];
foreach (array_keys($query) as $custom_param) {
if (isset($this->params['named'][$custom_param])) {
$query[$custom_param] = $this->params['named'][$custom_param];
}
}
if (isset($this->params['named']['scope'])) {
$limit = $this->Correlation->OverCorrelatingValue->getLimit();
if ($this->params['named']['scope'] === 'over_correlating') {
$query['conditions'][] = ['occurrence >=' => $limit];
} else if ($this->params['named']['scope'] === 'not_over_correlating') {
$query['conditions'][] = ['occurrence <' => $limit];
}
}
$data = $this->Correlation->OverCorrelatingValue->getOverCorrelations($query);
$data = $this->Correlation->attachExclusionsToOverCorrelations($data);
if ($this->_isRest()) {
return $this->RestResponse->viewData($data, 'json');
} else {
$this->__setPagingParams($query['page'], $query['limit'], count($data), 'named');
$this->set('data', $data);
$this->set('title_for_layout', __('Index of over correlating values'));
$this->set('menuData', [
'menuList' => 'correlationExclusions',
'menuItem' => 'over'
]);
}
}
public function switchEngine(string $engine)
{
$this->loadModel('Server');
if (!isset($this->Correlation->validEngines[$engine])) {
throw new MethodNotAllowedException(__('Not a valid engine choice. Please make sure you pass one of the following: ', implode(', ', array_keys($this->Correlation->validEngines))));
}
if ($this->request->is('post')) {
$setting = $this->Server->getSettingData('MISP.correlation_engine');
$result = $this->Server->serverSettingsEditValue($this->Auth->user(), $setting, $engine);
if ($result === true) {
$message = __('Engine switched.');
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('Correlations', 'switchEngine', false, $this->response->type(), $message);
} else {
$this->Flash->success($message);
$this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'correlations']);
}
} else {
$message = __('Couldn\'t switch to the requested engine.');
if ($this->_isRest()) {
return $this->RestResponse->saveFailResponse('Correlations', 'switchEngine', false, $message, $this->response->type());
} else {
$this->Flash->error($message);
$this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'correlations']);
}
}
} else {
$this->set('engine', $engine);
$this->render('ajax/switch_engine_confirmation');
}
}
public function truncate(string $engine)
{
if (!isset($this->Correlation->validEngines[$engine])) {
throw new MethodNotAllowedException(__('Not a valid engine choice. Please make sure you pass one of the following: ', implode(', ', array_keys($this->Correlation->validEngines))));
}
if ($this->request->is('post')) {
if (!Configure::read('MISP.background_jobs')) {
$result = $this->Correlation->truncate($this->Auth->user(), $engine);
$message = $result ? __('Table truncated.') : __('Could not truncate table');
if ($this->_isRest()) {
if ($result) {
$this->RestResponse->saveSuccessResponse('Correlations', 'truncate', false, $this->response->type(), $message);
} else {
$this->RestResponse->saveFailResponse('Correlations', 'truncate', false, $message, $this->response->type());
}
} else {
$this->Flash->{$result ? 'success' : 'error'}($message);
$this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'correlations']);
}
} else {
$job = ClassRegistry::init('Job');
$jobId = $job->createJob(
'SYSTEM',
Job::WORKER_DEFAULT,
'truncate table',
$this->Correlation->validEngines[$engine],
'Job created.'
);
$this->Correlation->Attribute->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[
'truncateTable',
$this->Auth->user('id'),
$engine,
$jobId
],
true,
$jobId
);
$message = __('Job queued. You can view the progress if you navigate to the active jobs view (Administration -> Jobs).');
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('Correlations', 'truncate', false, $this->response->type(), $message);
} else {
$this->Flash->success($message);
$this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'correlations']);
}
}
} else {
$this->set('engine', $engine);
$this->set('table_name', $this->Correlation->validEngines[$engine]);
$this->render('ajax/truncate_confirmation');
}
}
}

138
app/Controller/EventsController.php
View File

@ -619,7 +619,7 @@ class EventsController extends AppController
if (empty($usersToMatch)) {
$nothing = true;
} else {
$this->paginate['conditions']['AND'][] = ['Event.user_id' => array_unique($usersToMatch)];
$this->paginate['conditions']['AND'][] = ['Event.user_id' => array_unique($usersToMatch, SORT_REGULAR)];
}
}
break;
@ -2960,8 +2960,12 @@ class EventsController extends AppController
$result = $this->Event->publishRouter($event['Event']['id'], null, $this->Auth->user());
if (!Configure::read('MISP.background_jobs')) {
if (!is_array($result)) {
// redirect to the view event page
$message = __('Event published without alerts');
if ($result === true) {
$message = __('Event published without alerts');
} else {
$message = __('Event publishing failed due to a blocking module failing. The reason for the failure: %s', $result);
$errors['Module'] = 'Module failure.';
}
} else {
$lastResult = array_pop($result);
$resultString = (count($result) > 0) ? implode(', ', $result) . ' and ' . $lastResult : $lastResult;
@ -2969,11 +2973,6 @@ class EventsController extends AppController
$message = __('Event published but not pushed to %s, re-try later. If the issue persists, make sure that the correct sync user credentials are used for the server link and that the sync user on the remote server has authentication privileges.', $resultString);
}
} else {
// update the DB to set the published flag
// for background jobs, this should be done already
$event['Event']['published'] = 1;
$event['Event']['publish_timestamp'] = time();
$this->Event->save($event, true, ['id', 'published', 'publish_timestamp', 'info']); // info field is required because of SysLogLogableBehavior
$message = 'Job queued';
}
if ($this->_isRest()) {
@ -2983,7 +2982,11 @@ class EventsController extends AppController
return $this->RestResponse->saveSuccessResponse('Events', 'publish', $event['Event']['id'], false, $message);
}
} else {
$this->Flash->success($message);
if (!empty($errors)) {
$this->Flash->error($message);
} else {
$this->Flash->success($message);
}
$this->redirect(array('action' => 'view', $event['Event']['id']));
}
} else {
@ -3020,13 +3023,19 @@ class EventsController extends AppController
$errors['failed_servers'] = $result;
$message = __('Not published given no connection to %s but email sent to all participants.', $resultString);
}
} elseif (!is_bool($emailResult)) {
// Performs all the actions required to publish an event
$result = $this->Event->publishRouter($event['Event']['id'], null, $this->Auth->user());
if (!is_array($result)) {
if ($result === true) {
$message = __('Published but no email sent given GnuPG is not configured.');
$errors['GnuPG'] = 'GnuPG not set up.';
} else {
$message = $result;
$errors['Module'] = 'Module failure.';
}
// redirect to the view event page
$message = __('Published but no email sent given GnuPG is not configured.');
$errors['GnuPG'] = 'GnuPG not set up.';
} else {
$lastResult = array_pop($result);
$resultString = (count($result) > 0) ? implode(', ', $result) . ' and ' . $lastResult : $lastResult;
@ -3315,20 +3324,40 @@ class EventsController extends AppController
public function restSearchExport($id = null, $returnFormat = null)
{
if (is_null($returnFormat)) {
if (is_numeric($id)) {
$idList = [$id];
} else {
$idList = $this->_jsonDecode($id);
}
if ($returnFormat === null) {
$exportFormats = [
'attack' => __('Attack matrix'),
'attack-sightings' => __('Attack matrix by sightings'),
'context' => __('Aggregated context data'),
'context-markdown' => __('Aggregated context data as Markdown'),
'csv' => __('CSV'),
'hashes' => __('Hashes'),
'hosts' => __('Hosts file'),
'json' => __('MISP JSON'),
'netfilter' => __('Netfilter'),
'opendata' => __('Open data'),
'openioc' => __('OpenIOC'),
'rpz' => __('RPZ'),
'snort' => __('Snort rules'),
'stix' => __('STIX 1 XML'),
'stix-json' => __('STIX 1 JSON'),
'stix2' => __('STIX 2'),
'suricata' => __('Suricata rules'),
'text' => __('Text file'),
'xml' => __('MISP XML'),
'yara' => __('YARA rules'),
'yara-json' => __('YARA rules (JSON)'),
];
$idList = is_numeric($id) ? [$id] : $this->_jsonDecode($id);
if (empty($idList)) {
throw new NotFoundException(__('Invalid input.'));
}
$this->set('idList', $idList);
$this->set('exportFormats', array_keys($this->Event->validFormats));
$this->set('exportFormats', $exportFormats);
$this->render('ajax/eventRestSearchExportConfirmationForm');
} else {
$returnFormat = empty($this->Event->validFormats[$returnFormat]) ? 'json' : $returnFormat;
$returnFormat = !isset($this->Event->validFormats[$returnFormat]) ? 'json' : $returnFormat;
$idList = $id;
if (!is_array($idList)) {
if (is_numeric($idList) || Validation::uuid($idList)) {
@ -3341,19 +3370,17 @@ class EventsController extends AppController
throw new NotFoundException(__('Invalid input.'));
}
$filters = [
'eventid' => $idList
'eventid' => $idList,
'published' => [true, false], // fetch published and unpublished events
];
$elementCounter = 0;
$renderView = false;
$validFormat = $this->Event->validFormats[$returnFormat];
$responseType = $validFormat[0];
$responseType = $this->Event->validFormats[$returnFormat][0];
$final = $this->Event->restSearch($this->Auth->user(), $returnFormat, $filters, false, false, $elementCounter, $renderView);
if (!empty($renderView) && !empty($final)) {
if ($renderView) {
$final = json_decode($final->intoString(), true);
foreach ($final as $key => $data) {
$this->set($key, $data);
}
$this->set($final);
$this->set('responseType', $responseType);
$this->set('returnFormat', $returnFormat);
$this->set('renderView', $renderView);
@ -4097,29 +4124,31 @@ class EventsController extends AppController
public function filterEventIdsForPush()
{
if ($this->request->is('post')) {
$incomingIDs = array();
$incomingEvents = array();
foreach ($this->request->data as $event) {
$incomingIDs[] = $event['Event']['uuid'];
$incomingEvents[$event['Event']['uuid']] = $event['Event']['timestamp'];
}
$events = $this->Event->find('all', array(
'conditions' => array('Event.uuid' => $incomingIDs),
'recursive' => -1,
'fields' => array('Event.uuid', 'Event.timestamp', 'Event.locked'),
));
foreach ($events as $event) {
if ($event['Event']['timestamp'] >= $incomingEvents[$event['Event']['uuid']]) {
unset($incomingEvents[$event['Event']['uuid']]);
continue;
}
if ($event['Event']['locked'] == 0) {
unset($incomingEvents[$event['Event']['uuid']]);
}
}
return $this->RestResponse->viewData(array_keys($incomingEvents), $this->response->type());
if (!$this->request->is('post')) {
throw new MethodNotAllowedException(__('This endpoint requires a POST request.'));
}
$incomingUuids = [];
$incomingEvents = [];
foreach ($this->request->data as $event) {
$incomingUuids[] = $event['Event']['uuid'];
$incomingEvents[$event['Event']['uuid']] = $event['Event']['timestamp'];
}
$events = $this->Event->find('all', [
'conditions' => ['Event.uuid' => $incomingUuids],
'recursive' => -1,
'fields' => ['Event.uuid', 'Event.timestamp', 'Event.locked'],
]);
foreach ($events as $event) {
if ($event['Event']['timestamp'] >= $incomingEvents[$event['Event']['uuid']]) {
unset($incomingEvents[$event['Event']['uuid']]);
continue;
}
if ($event['Event']['locked'] == 0) {
unset($incomingEvents[$event['Event']['uuid']]);
}
}
return $this->RestResponse->viewData(array_keys($incomingEvents), $this->response->type());
}
public function checkuuid($uuid)
@ -5105,7 +5134,14 @@ class EventsController extends AppController
if (!Configure::read('Plugin.' . $type . '_services_enable')) {
throw new MethodNotAllowedException(__('%s services are not enabled.', $type));
}
$attribute = $this->Event->Attribute->fetchAttributes($this->Auth->user(), array('conditions' => array('Attribute.id' => $attribute_id), 'flatten' => 1));
$attribute = $this->Event->Attribute->fetchAttributes($this->Auth->user(), [
'conditions' => [
'Attribute.id' => $attribute_id
],
'flatten' => 1,
'includeEventTags' => 1,
'contain' => ['Event' => ['fields' => ['distribution', 'sharing_group_id']]],
]);
if (empty($attribute)) {
throw new MethodNotAllowedException(__('Attribute not found or you are not authorised to see it.'));
}
@ -5166,7 +5202,7 @@ class EventsController extends AppController
if (!empty($options)) {
$data['config'] = $options;
}
$result = $this->Module->queryModuleServer($data, false, $type);
$result = $this->Module->queryModuleServer($data, false, $type, false, $attribute[0]);
if (!$result) {
throw new InternalErrorException(__('%s service not reachable.', $type));
}
@ -5212,7 +5248,7 @@ class EventsController extends AppController
if (!empty($options)) {
$data['config'] = $options;
}
$result = $this->Module->queryModuleServer($data, false, $type);
$result = $this->Module->queryModuleServer($data, false, $type, false, $attribute[0]);
if (!$result) {
throw new InternalErrorException(__('%s service not reachable.', $type));
}

24
app/Controller/LogsController.php
View File

@ -30,17 +30,17 @@ class LogsController extends AppController
public function admin_index()
{
$paramArray = array('id', 'title', 'created', 'model', 'model_id', 'action', 'user_id', 'change', 'email', 'org', 'description', 'ip');
$filterData = array(
'request' => $this->request,
'named_params' => $this->params['named'],
'paramArray' => $paramArray,
'ordered_url_params' => func_get_args()
);
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
unset($filterData);
if ($this->_isRest()) {
$paramArray = array('id', 'title', 'created', 'model', 'model_id', 'action', 'user_id', 'change', 'email', 'org', 'description', 'ip');
$filterData = array(
'request' => $this->request,
'named_params' => $this->params['named'],
'paramArray' => $paramArray,
'ordered_url_params' => func_get_args()
);
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
unset($filterData);
if ($filters === false) {
return $exception;
}
@ -100,6 +100,9 @@ class LogsController extends AppController
if (isset($this->params['named']['filter']) && in_array($this->params['named']['filter'], array_keys($validFilters))) {
$this->paginate['conditions']['Log.action'] = $validFilters[$this->params['named']['filter']]['values'];
}
foreach ($filters as $key => $value) {
$this->paginate['conditions']["Log.$key"] = $value;
}
$this->set('validFilters', $validFilters);
$this->set('filter', isset($this->params['named']['filter']) ? $this->params['named']['filter'] : false);
$this->set('list', $this->paginate());
@ -389,6 +392,7 @@ class LogsController extends AppController
'Galaxy',
'GalaxyCluster',
'GalaxyClusterRelation',
'Workflow',
];
sort($models);
$models = array('' => 'ALL') + $this->_arrayToValuesIndexArray($models);

32
app/Controller/ServersController.php
View File

@ -948,7 +948,14 @@ class ServersController extends AppController
public function serverSettingsReloadSetting($setting, $id)
{
$pathToSetting = explode('.', $setting);
if (strpos($setting, 'Plugin.Enrichment') !== false || strpos($setting, 'Plugin.Import') !== false || strpos($setting, 'Plugin.Export') !== false || strpos($setting, 'Plugin.Cortex') !== false) {
if (
strpos($setting, 'Plugin.Enrichment') !== false ||
strpos($setting, 'Plugin.Import') !== false ||
strpos($setting, 'Plugin.Export') !== false ||
strpos($setting, 'Plugin.Cortex') !== false ||
strpos($setting, 'Plugin.Action') !== false ||
strpos($setting, 'Plugin.Workflow') !== false
) {
$settingObject = $this->Server->getCurrentServerSettings();
} else {
$settingObject = $this->Server->serverSettings;
@ -1065,6 +1072,11 @@ class ServersController extends AppController
$diagnostic_errors = 0;
App::uses('File', 'Utility');
App::uses('Folder', 'Utility');
if ($tab === 'correlations') {
$this->loadModel('Correlation');
$correlation_metrics = $this->Correlation->collectMetrics();
$this->set('correlation_metrics', $correlation_metrics);
}
if ($tab === 'files') {
$files = $this->Server->grabFiles();
$this->set('files', $files);
@ -1146,6 +1158,7 @@ class ServersController extends AppController
// get the DB diagnostics
$dbDiagnostics = $this->Server->dbSpaceUsage();
$dbSchemaDiagnostics = $this->Server->dbSchemaDiagnostic();
$dbConfiguration = $this->Server->dbConfiguration();
$redisInfo = $this->Server->redisInfo();
@ -1166,7 +1179,7 @@ class ServersController extends AppController
$securityAudit = (new SecurityAudit())->run($this->Server);
$view = compact('gpgStatus', 'sessionErrors', 'proxyStatus', 'sessionStatus', 'zmqStatus', 'moduleStatus', 'yaraStatus', 'gpgErrors', 'proxyErrors', 'zmqErrors', 'stix', 'moduleErrors', 'moduleTypes', 'dbDiagnostics', 'dbSchemaDiagnostics', 'redisInfo', 'attachmentScan', 'securityAudit');
$view = compact('gpgStatus', 'sessionErrors', 'proxyStatus', 'sessionStatus', 'zmqStatus', 'moduleStatus', 'yaraStatus', 'gpgErrors', 'proxyErrors', 'zmqErrors', 'stix', 'moduleErrors', 'moduleTypes', 'dbDiagnostics', 'dbSchemaDiagnostics', 'dbConfiguration', 'redisInfo', 'attachmentScan', 'securityAudit');
} else {
$view = [];
}
@ -1207,6 +1220,7 @@ class ServersController extends AppController
'readableFiles' => $readableFiles,
'dbDiagnostics' => $dbDiagnostics,
'dbSchemaDiagnostics' => $dbSchemaDiagnostics,
'dbConfiguration' => $dbConfiguration,
'redisInfo' => $redisInfo,
'finalSettings' => $dumpResults,
'extensions' => $extensions,
@ -1444,7 +1458,6 @@ class ServersController extends AppController
}
$this->set('id', $id);
}
$setting = $this->Server->getSettingData($settingName);
if ($setting === false) {
throw new NotFoundException(__('Setting %s is invalid.', $settingName));
@ -1916,7 +1929,7 @@ class ServersController extends AppController
$dbVersion = $this->AdminSetting->getSetting('db_version');
$updateProgress = $this->Server->getUpdateProgress();
$updateProgress['db_version'] = $dbVersion;
$maxUpdateNumber = max(array_keys($this->Server->db_changes));
$maxUpdateNumber = max(array_keys(Server::DB_CHANGES));
$updateProgress['complete_update_remaining'] = max($maxUpdateNumber - $dbVersion, 0);
$updateProgress['update_locked'] = $this->Server->isUpdateLocked();
$updateProgress['lock_remaining_time'] = $this->Server->getLockRemainingTime();
@ -2205,6 +2218,17 @@ class ServersController extends AppController
}
}
public function dbConfiguration()
{
$dbConfiguration = $this->Server->dbConfiguration();
if ($this->_isRest()) {
return $this->RestResponse->viewData($dbConfiguration, $this->response->type());
} else {
$this->set('dbConfiguration', $dbConfiguration);
$this->render('/Elements/healthElements/db_config_diagnostic');
}
}
public function cspReport()
{
if (!$this->request->is('post')) {

27
app/Controller/ShadowAttributesController.php
View File

@ -518,19 +518,19 @@ class ShadowAttributesController extends AppController
}
} else {
$shadowAttribute = array(
'ShadowAttribute' => array(
'value' => $filename,
'category' => $this->request->data['ShadowAttribute']['category'],
'type' => 'attachment',
'event_id' => $this->request->data['ShadowAttribute']['event_id'],
'comment' => $this->request->data['ShadowAttribute']['comment'],
'data' => base64_encode($tmpfile->read()),
'to_ids' => 0,
'email' => $this->Auth->user('email'),
'org_id' => $this->Auth->user('org_id'),
'event_uuid' => $event['Event']['uuid'],
'event_org_id' => $event['Event']['orgc_id'],
)
'ShadowAttribute' => array(
'value' => $filename,
'category' => $this->request->data['ShadowAttribute']['category'],
'type' => 'attachment',
'event_id' => $this->request->data['ShadowAttribute']['event_id'],
'comment' => $this->request->data['ShadowAttribute']['comment'],
'data' => base64_encode($tmpfile->read()),
'to_ids' => 0,
'email' => $this->Auth->user('email'),
'org_id' => $this->Auth->user('org_id'),
'event_uuid' => $event['Event']['uuid'],
'event_org_id' => $event['Event']['orgc_id'],
)
);
$this->ShadowAttribute->create();
$r = $this->ShadowAttribute->save($shadowAttribute);
@ -585,6 +585,7 @@ class ShadowAttributesController extends AppController
$this->set('typeDefinitions', $this->ShadowAttribute->typeDefinitions);
$this->set('categoryDefinitions', $this->ShadowAttribute->categoryDefinitions);
$this->set('isMalwareSampleCategory', $isMalwareSampleCategory);
$this->set('mayModify', $this->__canModifyEvent($event));
$this->set('event', $event);
$this->set('title_for_layout', __('Propose attachment'));
}

10
app/Controller/TaxonomiesController.php
View File

@ -594,4 +594,14 @@ class TaxonomiesController extends AppController
return $taxonomyIds;
}
public function normalizeCustomTagsToTaxonomyFormat()
{
$this->request->allowMethod(['post', 'put']);
$conversionResult = $this->Taxonomy->normalizeCustomTagsToTaxonomyFormat();
$this->Flash->success(__('%s tags successfully converted. %s row updated.', $conversionResult['tag_converted'], $conversionResult['row_updated']));
$this->redirect(array('controller' => 'taxonomies', 'action' => 'index'));
}
}

18
app/Controller/UsersController.php
View File

@ -118,6 +118,24 @@ class UsersController extends AppController
return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Something went wrong, please try again later.')), 'status'=>200, 'type' => 'json'));
}
public function unsubscribe($code)
{
$user = $this->Auth->user();
if (!hash_equals($this->User->unsubscribeCode($user), rtrim($code, '.'))) {
$this->Flash->error(__('Invalid unsubscribe code.'));
$this->redirect(['action' => 'view', 'me']);
}
if ($user['autoalert']) {
$this->User->updateField($this->Auth->user(), 'autoalert', false);
$this->Flash->success(__('Successfully unsubscribed from event alert.'));
} else {
$this->Flash->info(__('Already unsubscribed from event alert.'));
}
$this->redirect(['action' => 'view', 'me']);
}
public function edit()
{
$currentUser = $this->User->find('first', array(

127
app/Controller/WorkflowBlueprintsController.php Normal file
View File

@ -0,0 +1,127 @@
<?php
App::uses('AppController', 'Controller');
class WorkflowBlueprintsController extends AppController
{
public $components = array(
'RequestHandler'
);
public function update($force = false)
{
$this->request->allowMethod(['post', 'put']);
$this->WorkflowBlueprint->update($force);
$message = __('Default workflow blueprints updated');
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('WorkflowBlueprint', 'update', false, $this->response->type(), $message);
} else {
$this->Flash->success($message);
$this->redirect(array('controller' => 'workflowBlueprints', 'action' => 'index'));
}
}
public function index()
{
$params = [
'filters' => ['name', 'uuid', 'timestamp'],
'quickFilters' => ['name', 'uuid'],
];
$this->CRUD->index($params);
if ($this->IndexFilter->isRest()) {
return $this->restResponsePayload;
}
$this->set('menuData', ['menuList' => 'workflowBlueprints', 'menuItem' => 'index']);
}
public function add($fromEditor = false)
{
$params = [
'beforeSave' => function(array $blueprint) {
$blueprint['WorkflowBlueprint']['default'] = false;
return $blueprint;
},
];
$this->CRUD->add($params);
if ($this->IndexFilter->isRest()) {
return $this->restResponsePayload;
}
$this->set('fromEditor', !empty($fromEditor));
$this->set('menuData', ['menuList' => 'workflowBlueprints', 'menuItem' => 'add']);
}
public function edit($id)
{
$params = [
'beforeSave' => function (array $blueprint) {
$blueprint['WorkflowBlueprint']['default'] = false;
return $blueprint;
},
];
$this->CRUD->edit($id, $params);
if ($this->IndexFilter->isRest()) {
return $this->restResponsePayload;
}
$this->request->data['WorkflowBlueprint']['data'] = JsonTool::encode($this->data['WorkflowBlueprint']['data']);
$this->set('menuData', ['menuList' => 'workflowBlueprints', 'menuItem' => 'edit']);
$this->set('id', $id);
$this->render('add');
}
public function delete($id)
{
$params = [
];
$this->CRUD->delete($id, $params);
if ($this->IndexFilter->isRest()) {
return $this->restResponsePayload;
}
$this->set('menuData', ['menuList' => 'workflowBlueprints', 'menuItem' => 'delete']);
}
public function view($id)
{
$filters = $this->IndexFilter->harvestParameters(['format']);
if (!empty($filters['format'])) {
if ($filters['format'] == 'dot') {
$dot = $this->WorkflowBlueprint->getDotNotation($id);
return $this->RestResponse->viewData($dot, $this->response->type());
} else if ($filters['format'] == 'mermaid') {
$mermaid = $this->WorkflowBlueprint->getMermaid($id);
return $this->RestResponse->viewData($mermaid, $this->response->type());
}
}
$this->CRUD->view($id, [
]);
if ($this->IndexFilter->isRest()) {
return $this->restResponsePayload;
}
$this->set('id', $id);
$this->set('menuData', ['menuList' => 'workflowBlueprints', 'menuItem' => 'view']);
}
public function import()
{
if ($this->request->is('post') || $this->request->is('put')) {
$workflowBlueprintData = JsonTool::decode($this->request->data['WorkflowBlueprint']['data']);
if ($workflowBlueprintData === null) {
throw new MethodNotAllowedException(__('Error while decoding JSON'));
}
$this->request->data['WorkflowBlueprint']['data'] = JsonTool::encode($workflowBlueprintData);
$this->add();
}
}
public function export($id)
{
$workflowBlueprint = $this->WorkflowBlueprint->find('first', [
'conditions' => [
'id' => $id,
]
]);
$content = JsonTool::encode($workflowBlueprint, JSON_PRETTY_PRINT);
$this->response->body($content);
$this->response->type('json');
$this->response->download(sprintf('workflowblueprint_%s_%s.json', $workflowBlueprint['WorkflowBlueprint']['name'], time()));
return $this->response;
}
}

425
app/Controller/WorkflowsController.php Normal file
View File

@ -0,0 +1,425 @@
<?php
App::uses('AppController', 'Controller');
class WorkflowsController extends AppController
{
public $components = array(
'RequestHandler'
);
private $toggleableFields = ['enabled'];
public function beforeFilter()
{
parent::beforeFilter();
$this->Security->unlockedActions[] = 'checkGraph';
$requirementErrors = [];
if (empty(Configure::read('MISP.background_jobs'))) {
$requirementErrors[] = __('Background workers must be enabled to use workflows');
$this->render('error');
}
if (empty(Configure::read('Plugin.Workflow_enable'))) {
$requirementErrors[] = __('The workflow plugin must be enabled to use workflows. Go to `/servers/serverSettings/Plugin` the enable the `Plugin.Workflow` setting');
$this->render('error');
}
try {
$this->Workflow->setupRedisWithException();
} catch (Exception $e) {
$requirementErrors[] = $e->getMessage();
}
if (!empty($requirementErrors)) {
$this->set('requirementErrors', $requirementErrors);
$this->render('error');
}
}
public function index()
{
$params = [
'filters' => ['name', 'uuid'],
'quickFilters' => ['name', 'uuid'],
];
$this->CRUD->index($params);
if ($this->IndexFilter->isRest()) {
return $this->restResponsePayload;
}
$this->set('menuData', array('menuList' => 'workflows', 'menuItem' => 'index'));
}
public function rebuildRedis()
{
$this->Workflow->rebuildRedis();
}
public function edit($id)
{
$this->set('id', $id);
$savedWorkflow = $this->Workflow->fetchWorkflow($id);
if ($this->request->is('post') || $this->request->is('put')) {
$newWorkflow = $this->request->data;
$newWorkflow['Workflow']['data'] = JsonTool::decode($newWorkflow['Workflow']['data']);
$newWorkflow = $this->__applyDataFromSavedWorkflow($newWorkflow, $savedWorkflow);
$result = $this->Workflow->editWorkflow($newWorkflow);
$redirectTarget = ['action' => 'view', $id];
if (!empty($result['errors'])) {
return $this->__getFailResponseBasedOnContext($result['errors'], null, 'edit', $this->Workflow->id, $redirectTarget);
} else {
$successMessage = __('Workflow saved.');
$savedWorkflow = $result['saved'];
return $this->__getSuccessResponseBasedOnContext($successMessage, $savedWorkflow, 'edit', false, $redirectTarget);
}
} else {
$savedWorkflow['Workflow']['data'] = JsonTool::encode($savedWorkflow['Workflow']['data']);
$this->request->data = $savedWorkflow;
}
$this->set('menuData', array('menuList' => 'workflows', 'menuItem' => 'edit'));
$this->render('add');
}
public function delete($id)
{
$params = [
];
$this->CRUD->delete($id, $params);
if ($this->IndexFilter->isRest()) {
return $this->restResponsePayload;
}
}
public function view($id)
{
$filters = $this->IndexFilter->harvestParameters(['format']);
if (!empty($filters['format'])) {
if ($filters['format'] == 'dot') {
$dot = $this->Workflow->getDotNotation($id);
return $this->RestResponse->viewData($dot, $this->response->type());
} else if ($filters['format'] == 'mermaid') {
$mermaid = $this->Workflow->getMermaid($id);
return $this->RestResponse->viewData($mermaid, $this->response->type());
}
}
$this->CRUD->view($id, [
]);
if ($this->IndexFilter->isRest()) {
return $this->restResponsePayload;
}
$this->set('id', $id);
$this->set('menuData', array('menuList' => 'workflows', 'menuItem' => 'view'));
}
public function editor($id)
{
$trigger_id = false;
$workflow = false;
if (is_numeric($id)) {
$workflow_id = $id;
} else {
$trigger_id = $id;
}
$modules = $this->Workflow->getModulesByType();
if (!empty($trigger_id)) {
$trigger_ids = Hash::extract($modules['modules_trigger'], '{n}.id');
if (!in_array($trigger_id, $trigger_ids)) {
return $this->__getFailResponseBasedOnContext(
[__('Unkown trigger %s', $trigger_id)],
null,
'add',
$trigger_id,
['controller' => 'workflows', 'action' => 'triggers']
);
}
$workflow = $this->Workflow->fetchWorkflowByTrigger($trigger_id, false);
if (empty($workflow)) { // Workflow do not exists yet. Create it.
$result = $this->Workflow->addWorkflow([
'name' => sprintf('Workflow for trigger %s', $trigger_id),
'data' => $this->Workflow->genGraphDataForTrigger($trigger_id),
'trigger_id' => $trigger_id,
]);
if (!empty($result['errors'])) {
return $this->__getFailResponseBasedOnContext(
[__('Could not create workflow for trigger %s', $trigger_id), $result['errors']],
null,
'add',
$trigger_id,
['controller' => 'workflows', 'action' => 'editor']
);
}
$workflow = $this->Workflow->fetchWorkflowByTrigger($trigger_id, false);
}
} else {
$workflow = $this->Workflow->fetchWorkflow($workflow_id);
}
$modules = $this->Workflow->attachNotificationToModules($modules, $workflow);
$this->loadModel('WorkflowBlueprint');
$workflowBlueprints = $this->WorkflowBlueprint->find('all');
$this->set('selectedWorkflow', $workflow);
$this->set('workflowTriggerId', $trigger_id);
$this->set('modules', $modules);
$this->set('workflowBlueprints', $workflowBlueprints);
}
public function executeWorkflow($workflow_id)
{
if ($this->request->is('post') || $this->request->is('put')) {
$blockingErrors = [];
$data = JsonTool::decode($this->request->data['Workflow']['data']);
$result = $this->Workflow->executeWorkflow($workflow_id, $data, $blockingErrors);
if (!empty($logging) && empty($result['success'])) {
$logging['message'] = !empty($logging['message']) ? $logging['message'] : __('Error while executing workflow.');
$errorMessage = implode(', ', $blockingErrors);
$this->Workflow->loadLog()->createLogEntry('SYSTEM', $logging['action'], $logging['model'], $logging['id'], $logging['message'], __('Returned message: %s', $errorMessage));
}
return $this->RestResponse->viewData([
'success' => $result['success'],
'outcome' => $result['outcomeText'],
], $this->response->type());
}
$this->render('ajax/executeWorkflow');
}
public function triggers()
{
$triggers = $this->Workflow->getModulesByType('trigger');
$triggers = $this->Workflow->attachWorkflowToTriggers($triggers);
$data = $triggers;
App::uses('CustomPaginationTool', 'Tools');
$customPagination = new CustomPaginationTool();
$customPagination->truncateAndPaginate($data, $this->params, 'Workflow', true);
if ($this->_isRest()) {
return $this->RestResponse->viewData($data, $this->response->type());
}
$this->set('data', $data);
$this->set('menuData', ['menuList' => 'workflows', 'menuItem' => 'index_trigger']);
}
public function moduleIndex()
{
$modules = $this->Workflow->getModulesByType();
$errorWhileLoading = $this->Workflow->getModuleLoadingError();
$this->Module = ClassRegistry::init('Module');
$mispModules = $this->Module->getModules('Action');
$this->set('module_service_error', !is_array($mispModules));
$filters = $this->IndexFilter->harvestParameters(['type', 'actiontype', 'enabled']);
$moduleType = $filters['type'] ?? 'action';
$actionType = $filters['actiontype'] ?? 'all';
$enabledState = $filters['enabled'] ?? false;
if ($moduleType == 'all' || $moduleType == 'custom') {
$data = array_merge(
$modules["modules_action"],
$modules["modules_logic"]
);
} else {
$data = $modules["modules_{$moduleType}"];
}
if ($actionType == 'mispmodule') {
$data = array_filter($data, function($module) {
return !empty($module['is_misp_module']);
});
} else if ($actionType == 'blocking') {
$data = array_filter($data, function ($module) {
return !empty($module['blocking']);
});
} else if ($moduleType == 'custom') {
$data = array_filter($data, function ($module) {
return !empty($module['is_custom']);
});
}
if ($enabledState !== false) {
$moduleType = !empty($enabledState) ? 'enabled' : 'disabled';
$data = array_filter($data, function ($module) use ($enabledState) {
return !empty($enabledState) ? empty($module['disabled']) : !empty($module['disabled']);
});
}
if ($this->_isRest()) {
return $this->RestResponse->viewData($data, $this->response->type());
}
App::uses('CustomPaginationTool', 'Tools');
$customPagination = new CustomPaginationTool();
$params = $customPagination->createPaginationRules($data, $this->passedArgs, 'Workflow');
$params = $customPagination->applyRulesOnArray($data, $params, 'Workflow');
$params['options'] = array_merge($params['options'], $filters);
$this->params['paging'] = [$this->modelClass => $params];
$this->set('data', $data);
$this->set('indexType', $moduleType);
$this->set('actionType', $actionType);
$this->set('errorWhileLoading', $errorWhileLoading);
$this->set('menuData', ['menuList' => 'workflows', 'menuItem' => 'index_module']);
}
public function moduleView($module_id)
{
$module = $this->Workflow->getModuleByID($module_id);
if (empty($module)) {
throw new NotFoundException(__('Invalid trigger ID'));
}
$is_trigger = $module['module_type'] == 'trigger';
if ($is_trigger) {
$module = $this->Workflow->attachWorkflowToTriggers([$module])[0];
$module['listening_workflows'] = $this->Workflow->getListeningWorkflowForTrigger($module);
}
if ($this->_isRest()) {
return $this->RestResponse->viewData($module, $this->response->type());
}
$this->set('data', $module);
$this->set('menuData', ['menuList' => 'workflows', 'menuItem' => 'view_module']);
}
public function toggleModule($module_id, $enabled, $is_trigger=false)
{
$this->request->allowMethod(['post', 'put']);
$saved = $this->Workflow->toggleModule($module_id, $enabled, $is_trigger);
if ($saved) {
return $this->__getSuccessResponseBasedOnContext(
__('%s module %s', ($enabled ? 'Enabled' : 'Disabled'), $module_id),
null,
'toggle_module',
$module_id,
['action' => (!empty($is_trigger) ? 'triggers' : 'moduleIndex')]
);
} else {
return $this->__getFailResponseBasedOnContext(
__('Could not %s module %s', ($enabled ? 'Enabled' : 'Disabled'), $module_id),
null,
'toggle_module',
$module_id,
['action' => (!empty($is_trigger) ? 'triggers' : 'moduleIndex')]
);
}
}
public function debugToggleField($workflow_id, $enabled)
{
if (!$this->request->is('ajax')) {
throw new MethodNotAllowedException(__('This action is available via AJAX only.'));
}
$this->layout = false;
$this->render('ajax/getDebugToggleField');
if ($this->request->is('post') || $this->request->is('put')) {
$success = $this->Workflow->toggleDebug($workflow_id, $enabled);
if (!empty($success)) {
return $this->__getSuccessResponseBasedOnContext(
__('%s debug mode', ($enabled ? __('Enabled') : __('Disabled'))),
null,
'toggle_debug',
$workflow_id,
['action' => 'triggers']
);
} else {
return $this->__getFailResponseBasedOnContext(
__('Could not %s debug mode', ($enabled ? __('enable') : __('disable'))),
null,
'toggle_debug',
$workflow_id,
['action' => 'triggers']
);
}
}
}
public function massToggleField($fieldName, $enabled, $is_trigger=false)
{
if (!in_array($fieldName, $this->toggleableFields)) {
throw new MethodNotAllowedException(__('The field `%s` cannot be toggled', $fieldName));
}
if ($this->request->is('post') || $this->request->is('put')) {
$module_ids = JsonTool::decode($this->request->data['Workflow']['module_ids']);
$enabled_count = $this->Workflow->toggleModules($module_ids, $enabled, $is_trigger);
if (!empty($enabled_count)) {
return $this->__getSuccessResponseBasedOnContext(
__('%s %s modules', ($enabled ? 'Enabled' : 'Disabled'), $enabled_count),
null,
'toggle_module',
$module_ids,
['action' => (!empty($is_trigger) ? 'triggers' : 'moduleIndex')]
);
} else {
return $this->__getFailResponseBasedOnContext(
__('Could not %s modules', ($enabled ? 'enable' : 'disable')),
null,
'toggle_module',
$module_ids,
['action' => (!empty($is_trigger) ? 'triggers' : 'moduleIndex')]
);
}
}
}
private function __getSuccessResponseBasedOnContext($message, $data = null, $action = '', $id = false, $redirect = array())
{
if ($this->_isRest()) {
if (!is_null($data)) {
return $this->RestResponse->viewData($data, $this->response->type());
} else {
return $this->RestResponse->saveSuccessResponse('Workflow', $action, $id, false, $message);
}
} elseif ($this->request->is('ajax')) {
return $this->RestResponse->saveSuccessResponse('Workflow', $action, $id, false, $message, $data);
} else {
$this->Flash->success($message);
$this->redirect($redirect);
}
return;
}
private function __getFailResponseBasedOnContext($message, $data = null, $action = '', $id = false, $redirect = array())
{
if (is_array($message)) {
$message = implode(', ', $message);
}
if ($this->_isRest()) {
if ($data !== null) {
return $this->RestResponse->viewData($data, $this->response->type());
} else {
return $this->RestResponse->saveFailResponse('Workflow', $action, $id, $message);
}
} elseif ($this->request->is('ajax')) {
return $this->RestResponse->saveFailResponse('Workflow', $action, $id, $message, false, $data);
} else {
$this->Flash->error($message);
$this->redirect($redirect);
}
}
private function __applyDataFromSavedWorkflow($newWorkflow, $savedWorkflow)
{
if (!isset($newWorkflow['Workflow'])) {
$newWorkflow = ['Workflow' => $newWorkflow];
}
$ignoreFieldList = ['id', 'uuid'];
foreach (Workflow::CAPTURE_FIELDS_EDIT as $field) {
if (!in_array($field, $ignoreFieldList) && isset($newWorkflow['Workflow'][$field])) {
$savedWorkflow['Workflow'][$field] = $newWorkflow['Workflow'][$field];
}
}
return $savedWorkflow;
}
public function checkGraph()
{
$this->request->allowMethod(['post']);
$graphData = JsonTool::decode($this->request->data['graph']);
$cycles = [];
$isAcyclic = $this->Workflow->workflowGraphTool->isAcyclic($graphData, $cycles);
$edgesMultipleOutput = [];
$hasMultipleOutputConnection = $this->Workflow->workflowGraphTool->hasMultipleOutputConnection($graphData, $edgesMultipleOutput);
$edgesWarnings = [];
$hasPathWarnings = $this->Workflow->hasPathWarnings($graphData, $edgesWarnings);
$data = [
'is_acyclic' => [
'is_acyclic' => $isAcyclic,
'cycles' => $cycles,
],
'multiple_output_connection' => [
'has_multiple_output_connection' => $hasMultipleOutputConnection,
'edges' => $edgesMultipleOutput,
],
'path_warnings' => [
'has_path_warnings' => $hasPathWarnings,
'edges' => $edgesWarnings,
],
];
return $this->RestResponse->viewData($data, 'json');
}
}

23
app/Lib/Export/HashesExport.php
View File

@ -6,16 +6,16 @@ class HashesExport
'flatten' => 1
);
public $validTypes = array(
const VALID_TYPES = array(
'simple' => array(
'md5', 'sha1', 'sha256', 'sha224', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'tlsh',
'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'pehash', 'authentihash',
'impfuzzy'
'md5', 'sha1', 'sha256', 'sha224', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256',
'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'tlsh', 'x509-fingerprint-sha1', 'x509-fingerprint-md5',
'x509-fingerprint-sha256', 'pehash', 'authentihash', 'impfuzzy'
),
'composite' => array(
'malware-sample', 'filename|md5', 'filename|sha1', 'filename|sha256', 'filename|sha224', 'filename|sha512',
'filename|sha512/224', 'filename|sha512/256', 'filename|ssdeep', 'filename|imphash', 'filename|tlsh',
'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'filename|pehash',
'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384',
'filename|sha3-512', 'filename|ssdeep', 'filename|imphash', 'filename|tlsh', 'filename|pehash',
'filename|authentihash', 'filename|impfuzzy'
)
);
@ -23,18 +23,17 @@ class HashesExport
public function handler($data, $options = array())
{
if ($options['scope'] === 'Attribute') {
if (in_array($data['Attribute']['type'], $this->validTypes['composite'])) {
if (in_array($data['Attribute']['type'], self::VALID_TYPES['composite'], true)) {
return explode('|', $data['Attribute']['value'])[1];
} else if (in_array($data['Attribute']['type'], $this->validTypes['simple'])) {
} else if (in_array($data['Attribute']['type'], self::VALID_TYPES['simple'], true)) {
return $data['Attribute']['value'];
}
}
if ($options['scope'] === 'Event') {
} else if ($options['scope'] === 'Event') {
$result = array();
foreach ($data['Attribute'] as $attribute) {
if (in_array($attribute['type'], $this->validTypes['composite'])) {
if (in_array($attribute['type'], self::VALID_TYPES['composite'], true)) {
$result[] = explode('|', $attribute['value'])[1];
} else if (in_array($attribute['type'], $this->validTypes['simple'])) {
} else if (in_array($attribute['type'], self::VALID_TYPES['simple'], true)) {
$result[] = $attribute['value'];
}
}

108
app/Lib/Export/JsonExport.php
View File

@ -2,7 +2,7 @@
class JsonExport
{
public $non_restrictive_export = true;
public $non_restrictive_export = true;
/**
* @param $data
@ -11,17 +11,17 @@ class JsonExport
*/
public function handler($data, $options = array())
{
if ($options['scope'] === 'Attribute') {
return $this->__attributeHandler($data, $options);
} else if($options['scope'] === 'Event') {
return $this->__eventHandler($data, $options);
} else if($options['scope'] === 'Object') {
if ($options['scope'] === 'Attribute') {
return $this->__attributeHandler($data, $options);
} else if ($options['scope'] === 'Event') {
return $this->__eventHandler($data, $options);
} else if ($options['scope'] === 'Object') {
return $this->__objectHandler($data, $options);
} else if($options['scope'] === 'Sighting') {
return $this->__sightingsHandler($data, $options);
} else if($options['scope'] === 'GalaxyCluster') {
return $this->__galaxyClusterHandler($data, $options);
}
} else if ($options['scope'] === 'Sighting') {
return $this->__sightingsHandler($data, $options);
} else if ($options['scope'] === 'GalaxyCluster') {
return $this->__galaxyClusterHandler($data, $options);
}
}
/**
@ -29,66 +29,68 @@ class JsonExport
* @param array $options
* @return Generator
*/
private function __eventHandler($event, $options = array())
private function __eventHandler($event, $options = array())
{
App::uses('JSONConverterTool', 'Tools');
return JSONConverterTool::streamConvert($event);
}
private function __objectHandler($object, $options = array()) {
App::uses('JSONConverterTool', 'Tools');
return json_encode(JSONConverterTool::convertObject($object, false, true));
return JSONConverterTool::streamConvert($event);
}
private function __attributeHandler($attribute, $options = array())
{
$attribute = array_merge($attribute['Attribute'], $attribute);
unset($attribute['Attribute']);
if (isset($attribute['Object']) && empty($attribute['Object']['id'])) {
unset($attribute['Object']);
}
$tagTypes = array('AttributeTag', 'EventTag');
foreach($tagTypes as $tagType) {
if (isset($attribute[$tagType])) {
foreach ($attribute[$tagType] as $tk => $tag) {
if ($tagType === 'EventTag') {
$attribute[$tagType][$tk]['Tag']['inherited'] = 1;
}
$attribute['Tag'][] = $attribute[$tagType][$tk]['Tag'];
}
unset($attribute[$tagType]);
}
}
unset($attribute['value1']);
unset($attribute['value2']);
return json_encode($attribute);
}
private function __objectHandler($object, $options = array())
{
App::uses('JSONConverterTool', 'Tools');
return JsonTool::encode(JSONConverterTool::convertObject($object, false, true));
}
private function __attributeHandler($attribute, $options = array())
{
$attribute = array_merge($attribute['Attribute'], $attribute);
unset($attribute['Attribute']);
if (isset($attribute['Object']) && empty($attribute['Object']['id'])) {
unset($attribute['Object']);
}
$tagTypes = array('AttributeTag', 'EventTag');
foreach ($tagTypes as $tagType) {
if (isset($attribute[$tagType])) {
foreach ($attribute[$tagType] as $tag) {
if ($tagType === 'EventTag') {
$tag['Tag']['inherited'] = 1;
}
$attribute['Tag'][] = $tag['Tag'];
}
unset($attribute[$tagType]);
}
}
unset($attribute['value1']);
unset($attribute['value2']);
return JsonTool::encode($attribute);
}
private function __sightingsHandler($sighting, $options = array())
{
return json_encode($sighting);
return JsonTool::encode($sighting);
}
private function __galaxyClusterHandler($cluster, $options = array())
{
return json_encode($cluster);
return JsonTool::encode($cluster);
}
public function header($options = array())
{
if ($options['scope'] === 'Attribute') {
return '{"response": {"Attribute": [';
} else {
return '{"response": [';
}
if ($options['scope'] === 'Attribute') {
return '{"response": {"Attribute": [';
} else {
return '{"response": [';
}
}
public function footer($options = array())
{
if ($options['scope'] === 'Attribute') {
return ']}}' . PHP_EOL;
} else {
return ']}' . PHP_EOL;
}
if ($options['scope'] === 'Attribute') {
return ']}}' . PHP_EOL;
} else {
return ']}' . PHP_EOL;
}
}
public function separator()

789
app/Lib/Export/NidsExport.php
View File

@ -7,84 +7,125 @@ class NidsExport
public $classtype = 'trojan-activity';
public $format = ""; // suricata (default), snort
public $supportedObjects = array('network-connection', 'ddos');
public $checkWhitelist = true;
public $checkWhitelist = true;
public $additional_params = array(
'contain' => array(
'Event' => array(
'fields' => array('threat_level_id')
)
),
'flatten' => 1
);
public $additional_params = array(
'contain' => array(
'Event' => array(
'fields' => array('threat_level_id')
)
),
public function handler($data, $options = array())
{
$continue = empty($format);
$this->checkWhitelist = false;
if ($options['scope'] === 'Attribute') {
$this->export(
array($data),
$options['user']['nids_sid'],
$options['returnFormat'],
$continue
);
} else if ($options['scope'] === 'Event') {
if (!empty($data['EventTag'])) {
$data['Event']['EventTag'] = $data['EventTag'];
}
if (!empty($data['Attribute'])) {
$this->__convertFromEventFormat($data['Attribute'], $data, $options, $continue);
}
if (!empty($data['Object'])) {
foreach ($data['Object'] as $object) {
$this->__convertFromEventFormat($object['Attribute'], $data, $options, $continue);
}
}
}
return '';
}
);
private function __convertFromEventFormat($attributes, $event, $options = array(), $continue = false) {
$rearranged = array();
foreach ($attributes as $attribute) {
$attributeTag = array();
if (!empty($attribute['AttributeTag'])) {
$attributeTag = $attribute['AttributeTag'];
unset($attribute['AttributeTag']);
}
$rearranged[] = array(
'Attribute' => $attribute,
'AttributeTag' => $attributeTag,
'Event' => $event['Event']
);
}
$this->export(
$rearranged,
$options['user']['nids_sid'],
$options['returnFormat'],
$continue
);
return true;
public function handler($data, $options = array())
{
$continue = empty($format);
$this->checkWhitelist = false;
if ($options['scope'] === 'Attribute') {
$this->export(
array($data),
$options['user']['nids_sid'],
$options['returnFormat'],
$continue
);
} else if ($options['scope'] === 'Event') {
if (!empty($data['EventTag'])) {
$data['Event']['EventTag'] = $data['EventTag'];
}
if (!empty($data['Attribute'])) {
$this->__convertFromEventFormat($data['Attribute'], $data, $options, $continue);
}
if (!empty($data['Object'])) {
$this->__convertFromEventFormatObject($data['Object'], $data, $options, $continue);
}
}
return '';
}
}
private function __convertFromEventFormat($attributes, $event, $options = array(), $continue = false) {
public function header($options = array())
{
$this->explain();
return '';
}
$rearranged = array();
foreach ($attributes as $attribute) {
$attributeTag = array();
if (!empty($attribute['AttributeTag'])) {
$attributeTag = $attribute['AttributeTag'];
unset($attribute['AttributeTag']);
}
$rearranged[] = array(
'Attribute' => $attribute,
'AttributeTag' => $attributeTag,
'Event' => $event['Event']
);
}
$this->export(
$rearranged,
$options['user']['nids_sid'],
$options['returnFormat'],
$continue
);
return true;
public function footer()
{
return implode ("\n", $this->rules);
}
}
public function separator()
{
return '';
}
private function __convertFromEventFormatObject($objects, $event, $options = array(), $continue = false) {
$rearranged = array();
foreach ($objects as $object) {
if(in_array($object['name'], $this->supportedObjects)){
$objectTag = array();
foreach($object['Attribute'] as $attribute) {
if (!empty($attribute['AttributeTag'])) {
$objectTag = array_merge($objectTag, $attribute['AttributeTag']);
unset($attribute['AttributeTag']);
}
}
$rearranged[] = array(
'Attribute' => $object, // Using 'Attribute' instead of 'Object' to comply with function export
'AttributeTag' => $objectTag, // Using 'AttributeTag' instead of 'ObjectTag' to comply with function export
'Event' => $event['Event']
);
} else { // In case no custom export exists for the object, the approach falls back to the attribute case
$this->__convertFromEventFormat($object['Attribute'], $event, $options, $continue);
}
}
$this->export(
$rearranged,
$options['user']['nids_sid'],
$options['returnFormat'],
$continue
);
return true;
}
public function header($options = array())
{
$this->explain();
return '';
}
public function footer()
{
return implode ("\n", $this->rules);
}
public function separator()
{
return '';
}
public function explain()
{
@ -93,7 +134,7 @@ class NidsExport
$this->rules[] = '# These NIDS rules contain some variables that need to exist in your configuration.';
$this->rules[] = '# Make sure you have set:';
$this->rules[] = '#';
$this->rules[] = '# $HOME_NET - Your internal network range';
$this->rules[] = '# $HOME_NET - Your internal network range';
$this->rules[] = '# $EXTERNAL_NET - The network considered as outside';
$this->rules[] = '# $SMTP_SERVERS - All your internal SMTP servers';
$this->rules[] = '# $HTTP_PORTS - The ports used to contain HTTP traffic (not required with suricata export)';
@ -106,10 +147,10 @@ class NidsExport
public function export($items, $startSid, $format="suricata", $continue = false)
{
$this->format = $format;
if ($this->checkWhitelist && !isset($this->Whitelist)) {
$this->Whitelist = ClassRegistry::init('Whitelist');
$this->whitelist = $this->Whitelist->getBlockedValues();
}
if ($this->checkWhitelist && !isset($this->Whitelist)) {
$this->Whitelist = ClassRegistry::init('Whitelist');
$this->whitelist = $this->Whitelist->getBlockedValues();
}
// output a short explanation
if (!$continue) {
@ -119,20 +160,20 @@ class NidsExport
foreach ($items as $item) {
// retrieve all tags for this item to add them to the msg
$tagsArray = [];
if (!empty($item['AttributeTag'])) {
foreach ($item['AttributeTag'] as $tag_attr) {
if (array_key_exists('name', $tag_attr['Tag'])) {
array_push($tagsArray, $tag_attr['Tag']['name']);
}
}
}
if (!empty($item['Event']['EventTag'])) {
foreach ($item['Event']['EventTag'] as $tag_event) {
if (array_key_exists('name', $tag_event['Tag'])) {
array_push($tagsArray, $tag_event['Tag']['name']);
}
}
}
if (!empty($item['AttributeTag'])) {
foreach ($item['AttributeTag'] as $tag_attr) {
if (array_key_exists('name', $tag_attr['Tag'])) {
array_push($tagsArray, $tag_attr['Tag']['name']);
}
}
}
if (!empty($item['Event']['EventTag'])) {
foreach ($item['Event']['EventTag'] as $tag_event) {
if (array_key_exists('name', $tag_event['Tag'])) {
array_push($tagsArray, $tag_event['Tag']['name']);
}
}
}
$ruleFormatMsgTags = implode(",", $tagsArray);
# proto src_ip src_port direction dst_ip dst_port msg rule_content tag sid rev
@ -142,69 +183,179 @@ class NidsExport
$sid = $startSid + ($item['Attribute']['id'] * 10); // leave 9 possible rules per attribute type
$sid++;
switch ($item['Attribute']['type']) {
// LATER nids - test all the snort attributes
// LATER nids - add the tag keyword in the rules to capture network traffic
// LATER nids - sanitize every $attribute['value'] to not conflict with snort
case 'ip-dst':
$this->ipDstRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'ip-src':
$this->ipSrcRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'ip-dst|port':
$this->ipDstRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'ip-src|port':
$this->ipSrcRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'email':
$this->emailSrcRule($ruleFormat, $item['Attribute'], $sid++);
$this->emailDstRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'email-src':
$this->emailSrcRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'email-dst':
$this->emailDstRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'email-subject':
$this->emailSubjectRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'email-attachment':
$this->emailAttachmentRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'domain':
$this->domainRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'domain|ip':
$this->domainIpRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'hostname':
$this->hostnameRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'url':
$this->urlRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'user-agent':
$this->userAgentRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'ja3-fingerprint-md5':
$this->ja3Rule($ruleFormat, $item['Attribute'], $sid);
break;
case 'ja3s-fingerprint-md5': // Atribute type doesn't exists yet (2020-12-10) but ready when created.
$this->ja3sRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'snort':
$this->snortRule($ruleFormat, $item['Attribute'], $sid, $ruleFormatMsg, $ruleFormatReference);
// no break
default:
break;
if(!empty($item['Attribute']['type'])) { // item is an 'Attribute'
switch ($item['Attribute']['type']) {
// LATER nids - test all the snort attributes
// LATER nids - add the tag keyword in the rules to capture network traffic
// LATER nids - sanitize every $attribute['value'] to not conflict with snort
case 'ip-dst':
$this->ipDstRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'ip-src':
$this->ipSrcRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'ip-dst|port':
$this->ipDstRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'ip-src|port':
$this->ipSrcRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'email':
$this->emailSrcRule($ruleFormat, $item['Attribute'], $sid);
$this->emailDstRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'email-src':
$this->emailSrcRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'email-dst':
$this->emailDstRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'email-subject':
$this->emailSubjectRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'email-attachment':
$this->emailAttachmentRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'domain':
$this->domainRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'domain|ip':
$this->domainIpRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'hostname':
$this->hostnameRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'url':
$this->urlRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'user-agent':
$this->userAgentRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'ja3-fingerprint-md5':
$this->ja3Rule($ruleFormat, $item['Attribute'], $sid);
break;
case 'ja3s-fingerprint-md5': // Atribute type doesn't exists yet (2020-12-10) but ready when created.
$this->ja3sRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'snort':
$this->snortRule($ruleFormat, $item['Attribute'], $sid, $ruleFormatMsg, $ruleFormatReference);
// no break
default:
break;
}
} else if(!empty($item['Attribute']['name'])) { // Item is an 'Object'
switch ($item['Attribute']['name']) {
case 'network-connection':
$this->networkConnectionRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'ddos':
$this->ddosRule($ruleFormat, $item['Attribute'], $sid);
break;
default:
break;
}
}
}
return $this->rules;
}
public function networkConnectionRule($ruleFormat, $object, &$sid)
{
$attributes = NidsExport::getObjectAttributes($object);
if(!array_key_exists('layer4-protocol', $attributes)){
$attributes['layer4-protocol'] = 'ip'; // If layer-4 protocol is unknown, we roll-back to layer-3 ('ip')
}
if(!array_key_exists('ip-src', $attributes)){
$attributes['ip-src'] = '$HOME_NET'; // If ip-src is unknown, we roll-back to $HOME_NET
}
if(!array_key_exists('ip-dst', $attributes)){
$attributes['ip-dst'] = '$HOME_NET'; // If ip-dst is unknown, we roll-back to $HOME_NET
}
if(!array_key_exists('src-port', $attributes)){
$attributes['src-port'] = 'any'; // If src-port is unknown, we roll-back to 'any'
}
if(!array_key_exists('dst-port', $attributes)){
$attributes['dst-port'] = 'any'; // If dst-port is unknown, we roll-back to 'any'
}
$this->rules[] = sprintf(
$ruleFormat,
false,
$attributes['layer4-protocol'], // proto
$attributes['ip-src'], // src_ip
$attributes['src-port'], // src_port
'->', // direction
$attributes['ip-dst'], // dst_ip
$attributes['dst-port'], // dst_port
'Network connection between ' . $attributes['ip-src'] . ' and ' . $attributes['ip-dst'], // msg
'', // rule_content
'', // tag
$sid, // sid
1 // rev
);
}
public function ddosRule($ruleFormat, $object, &$sid)
{
$attributes = NidsExport::getObjectAttributes($object);
if(!array_key_exists('protocol', $attributes)){
$attributes['protocol'] = 'ip'; // If protocol is unknown, we roll-back to 'ip'
}
if(!array_key_exists('ip-src', $attributes)){
$attributes['ip-src'] = '$HOME_NET'; // If ip-src is unknown, we roll-back to $HOME_NET
}
if(!array_key_exists('ip-dst', $attributes)){
$attributes['ip-dst'] = '$HOME_NET'; // If ip-dst is unknown, we roll-back to $HOME_NET
}
if(!array_key_exists('src-port', $attributes)){
$attributes['src-port'] = 'any'; // If src-port is unknown, we roll-back to 'any'
}
if(!array_key_exists('dst-port', $attributes)){
$attributes['dst-port'] = 'any'; // If dst-port is unknown, we roll-back to 'any'
}
$this->rules[] = sprintf(
$ruleFormat,
false,
$attributes['protocol'], // proto
$attributes['ip-src'], // src_ip
$attributes['src-port'], // src_port
'->', // direction
$attributes['ip-dst'], // dst_ip
$attributes['dst-port'], // dst_port
'DDOS attack detected between ' . $attributes['ip-src'] . ' and ' . $attributes['ip-dst'], // msg
'', // rule_content
'', // tag
$sid, // sid
1 // rev
);
}
public static function getObjectAttributes($object)
{
$attributes = array();
foreach ($object['Attribute'] as $attribute) {
$attributes[$attribute['object_relation']] = $attribute['value'];
}
return $attributes;
}
public function domainIpRule($ruleFormat, $attribute, &$sid)
{
$values = explode('|', $attribute['value']);
@ -225,17 +376,17 @@ class NidsExport
$this->rules[] = sprintf(
$ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' : '',
'ip', // proto
'$HOME_NET', // src_ip
'any', // src_port
'->', // direction
$ipport[0], // dst_ip
$ipport[1], // dst_port
'Outgoing To IP: ' . $attribute['value'], // msg
'', // rule_content
'', // tag
$sid, // sid
1 // rev
'ip', // proto
'$HOME_NET', // src_ip
'any', // src_port
'->', // direction
$ipport[0], // dst_ip
$ipport[1], // dst_port
'Outgoing To IP: ' . $attribute['value'], // msg
'', // rule_content
'', // tag
$sid, // sid
1 // rev
);
}
@ -246,17 +397,17 @@ class NidsExport
$this->rules[] = sprintf(
$ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' : '',
'ip', // proto
$ipport[0], // src_ip
$ipport[1], // src_port
'->', // direction
'$HOME_NET', // dst_ip
'any', // dst_port
'Incoming From IP: ' . $attribute['value'], // msg
'', // rule_content
'', // tag
$sid, // sid
1 // rev
'ip', // proto
$ipport[0], // src_ip
$ipport[1], // src_port
'->', // direction
'$HOME_NET', // dst_ip
'any', // dst_port
'Incoming From IP: ' . $attribute['value'], // msg
'', // rule_content
'', // tag
$sid, // sid
1 // rev
);
}
@ -268,17 +419,17 @@ class NidsExport
$this->rules[] = sprintf(
$ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' : '',
'tcp', // proto
'$EXTERNAL_NET', // src_ip
'any', // src_port
'->', // direction
'$SMTP_SERVERS', // dst_ip
'25', // dst_port
'Source Email Address: ' . $attribute['value'], // msg
$content, // rule_content
'tag:session,600,seconds;', // tag
$sid, // sid
1 // rev
'tcp', // proto
'$EXTERNAL_NET', // src_ip
'any', // src_port
'->', // direction
'$SMTP_SERVERS', // dst_ip
'25', // dst_port
'Source Email Address: ' . $attribute['value'], // msg
$content, // rule_content
'tag:session,600,seconds;', // tag
$sid, // sid
1 // rev
);
}
@ -290,17 +441,17 @@ class NidsExport
$this->rules[] = sprintf(
$ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' : '',
'tcp', // proto
'$EXTERNAL_NET', // src_ip
'any', // src_port
'->', // direction
'$SMTP_SERVERS', // dst_ip
'25', // dst_port
'Destination Email Address: ' . $attribute['value'], // msg
$content, // rule_content
'tag:session,600,seconds;', // tag
$sid, // sid
1 // rev
'tcp', // proto
'$EXTERNAL_NET', // src_ip
'any', // src_port
'->', // direction
'$SMTP_SERVERS', // dst_ip
'25', // dst_port
'Destination Email Address: ' . $attribute['value'], // msg
$content, // rule_content
'tag:session,600,seconds;', // tag
$sid, // sid
1 // rev
);
}
@ -313,17 +464,17 @@ class NidsExport
$this->rules[] = sprintf(
$ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' : '',
'tcp', // proto
'$EXTERNAL_NET', // src_ip
'any', // src_port
'->', // direction
'$SMTP_SERVERS', // dst_ip
'25', // dst_port
'Bad Email Subject', // msg
$content, // rule_content
'tag:session,600,seconds;', // tag
$sid, // sid
1 // rev
'tcp', // proto
'$EXTERNAL_NET', // src_ip
'any', // src_port
'->', // direction
'$SMTP_SERVERS', // dst_ip
'25', // dst_port
'Bad Email Subject', // msg
$content, // rule_content
'tag:session,600,seconds;', // tag
$sid, // sid
1 // rev
);
}
@ -336,17 +487,17 @@ class NidsExport
$this->rules[] = sprintf(
$ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' : '',
'tcp', // proto
'$EXTERNAL_NET', // src_ip
'any', // src_port
'->', // direction
'$SMTP_SERVERS', // dst_ip
'25', // dst_port
'Bad Email Attachment', // msg
$content, // rule_content // LATER nids - test and finetune this snort rule https://secure.wikimedia.org/wikipedia/en/wiki/MIME#Content-Disposition
'tag:session,600,seconds;', // tag
$sid, // sid
1 // rev
'tcp', // proto
'$EXTERNAL_NET', // src_ip
'any', // src_port
'->', // direction
'$SMTP_SERVERS', // dst_ip
'25', // dst_port
'Bad Email Attachment', // msg
$content, // rule_content // LATER nids - test and finetune this snort rule https://secure.wikimedia.org/wikipedia/en/wiki/MIME#Content-Disposition
'tag:session,600,seconds;', // tag
$sid, // sid
1 // rev
);
}
@ -358,33 +509,33 @@ class NidsExport
$this->rules[] = sprintf(
$ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' : '',
'udp', // proto
'any', // src_ip
'any', // src_port
'->', // direction
'any', // dst_ip
'53', // dst_port
'Hostname: ' . $attribute['value'], // msg
$content, // rule_content
'', // tag
$sid, // sid
1 // rev
'udp', // proto
'any', // src_ip
'any', // src_port
'->', // direction
'any', // dst_ip
'53', // dst_port
'Hostname: ' . $attribute['value'], // msg
$content, // rule_content
'', // tag
$sid, // sid
1 // rev
);
$sid++;
$this->rules[] = sprintf(
$ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' : '',
'tcp', // proto
'any', // src_ip
'any', // src_port
'->', // direction
'any', // dst_ip
'53', // dst_port
'Hostname: ' . $attribute['value'], // msg
$content. ' flow:established;', // rule_content
'', // tag
$sid, // sid
1 // rev
'tcp', // proto
'any', // src_ip
'any', // src_port
'->', // direction
'any', // dst_ip
'53', // dst_port
'Hostname: ' . $attribute['value'], // msg
$content. ' flow:established;', // rule_content
'', // tag
$sid, // sid
1 // rev
);
$sid++;
// also do http requests
@ -392,17 +543,17 @@ class NidsExport
$this->rules[] = sprintf(
$ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' : '',
'tcp', // proto
'$HOME_NET', // src_ip
'any', // src_port
'->', // direction
'$EXTERNAL_NET', // dst_ip
'$HTTP_PORTS', // dst_port
'Outgoing HTTP Hostname: ' . $attribute['value'], // msg
$content, // rule_content
'tag:session,600,seconds;', // tag
$sid, // sid
1 // rev
'tcp', // proto
'$HOME_NET', // src_ip
'any', // src_port
'->', // direction
'$EXTERNAL_NET', // dst_ip
'$HTTP_PORTS', // dst_port
'Outgoing HTTP Hostname: ' . $attribute['value'], // msg
$content, // rule_content
'tag:session,600,seconds;', // tag
$sid, // sid
1 // rev
);
}
@ -414,33 +565,33 @@ class NidsExport
$this->rules[] = sprintf(
$ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' : '',
'udp', // proto
'any', // src_ip
'any', // src_port
'->', // direction
'any', // dst_ip
'53', // dst_port
'Domain: ' . $attribute['value'], // msg
$content, // rule_content
'', // tag
$sid, // sid
1 // rev
'udp', // proto
'any', // src_ip
'any', // src_port
'->', // direction
'any', // dst_ip
'53', // dst_port
'Domain: ' . $attribute['value'], // msg
$content, // rule_content
'', // tag
$sid, // sid
1 // rev
);
$sid++;
$this->rules[] = sprintf(
$ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' : '',
'tcp', // proto
'any', // src_ip
'any', // src_port
'->', // direction
'any', // dst_ip
'53', // dst_port
'Domain: ' . $attribute['value'], // msg
$content. ' flow:established;', // rule_content
'', // tag
$sid, // sid
1 // rev
'tcp', // proto
'any', // src_ip
'any', // src_port
'->', // direction
'any', // dst_ip
'53', // dst_port
'Domain: ' . $attribute['value'], // msg
$content. ' flow:established;', // rule_content
'', // tag
$sid, // sid
1 // rev
);
$sid++;
// also do http requests,
@ -448,17 +599,17 @@ class NidsExport
$this->rules[] = sprintf(
$ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' : '',
'tcp', // proto
'$HOME_NET', // src_ip
'any', // src_port
'->', // direction
'$EXTERNAL_NET', // dst_ip
'$HTTP_PORTS', // dst_port
'Outgoing HTTP Domain: ' . $attribute['value'], // msg
$content, // rule_content
'tag:session,600,seconds;', // tag
$sid, // sid
1 // rev
'tcp', // proto
'$HOME_NET', // src_ip
'any', // src_port
'->', // direction
'$EXTERNAL_NET', // dst_ip
'$HTTP_PORTS', // dst_port
'Outgoing HTTP Domain: ' . $attribute['value'], // msg
$content, // rule_content
'tag:session,600,seconds;', // tag
$sid, // sid
1 // rev
);
}
@ -473,17 +624,17 @@ class NidsExport
$this->rules[] = sprintf(
$ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' : '',
'tcp', // proto
'$HOME_NET', // src_ip
'any', // src_port
'->', // direction
'$EXTERNAL_NET', // dst_ip
'$HTTP_PORTS', // dst_port
'Outgoing HTTP URL: ' . $attribute['value'], // msg
$content, // rule_content
'tag:session,600,seconds;', // tag
$sid, // sid
1 // rev
'tcp', // proto
'$HOME_NET', // src_ip
'any', // src_port
'->', // direction
'$EXTERNAL_NET', // dst_ip
'$HTTP_PORTS', // dst_port
'Outgoing HTTP URL: ' . $attribute['value'], // msg
$content, // rule_content
'tag:session,600,seconds;', // tag
$sid, // sid
1 // rev
);
}
@ -495,17 +646,17 @@ class NidsExport
$this->rules[] = sprintf(
$ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' : '',
'tcp', // proto
'$HOME_NET', // src_ip
'any', // src_port
'->', // direction
'$EXTERNAL_NET', // dst_ip
'$HTTP_PORTS', // dst_port
'Outgoing User-Agent: ' . $attribute['value'], // msg
$content, // rule_content
'tag:session,600,seconds;', // tag
$sid, // sid
1 // rev
'tcp', // proto
'$HOME_NET', // src_ip
'any', // src_port
'->', // direction
'$EXTERNAL_NET', // dst_ip
'$HTTP_PORTS', // dst_port
'Outgoing User-Agent: ' . $attribute['value'], // msg
$content, // rule_content
'tag:session,600,seconds;', // tag
$sid, // sid
1 // rev
);
}
@ -527,37 +678,37 @@ class NidsExport
$tmpRule = str_replace(array("\r","\n"), " ", $attribute['value']);
// rebuild the rule by overwriting the different keywords using preg_replace()
// sid - '/sid\s*:\s*[0-9]+\s*;/'
// rev - '/rev\s*:\s*[0-9]+\s*;/'
// sid - '/sid\s*:\s*[0-9]+\s*;/'
// rev - '/rev\s*:\s*[0-9]+\s*;/'
// classtype - '/classtype:[a-zA-Z_-]+;/'
// msg - '/msg\s*:\s*".*?"\s*;/'
// msg - '/msg\s*:\s*".*?"\s*;/'
// reference - '/reference\s*:\s*.+?;/'
// tag - '/tag\s*:\s*.+?;/'
// tag - '/tag\s*:\s*.+?;/'
$replaceCount = array();
$tmpRule = preg_replace('/sid\s*:\s*[0-9]+\s*;/', 'sid:' . $sid . ';', $tmpRule, -1, $replaceCount['sid']);
if (null == $tmpRule) {
return false;
} // don't output the rule on error with the regex
} // don't output the rule on error with the regex
$tmpRule = preg_replace('/rev\s*:\s*[0-9]+\s*;/', 'rev:1;', $tmpRule, -1, $replaceCount['rev']);
if (null == $tmpRule) {
return false;
} // don't output the rule on error with the regex
} // don't output the rule on error with the regex
$tmpRule = preg_replace('/classtype:[a-zA-Z_-]+;/', 'classtype:' . $this->classtype . ';', $tmpRule, -1, $replaceCount['classtype']);
if (null == $tmpRule) {
return false;
} // don't output the rule on error with the regex
} // don't output the rule on error with the regex
$tmpRule = preg_replace('/msg\s*:\s*"(.*?)"\s*;/', sprintf($ruleFormatMsg, 'snort-rule | $1') . ';', $tmpRule, -1, $replaceCount['msg']);
if (null == $tmpRule) {
return false;
} // don't output the rule on error with the regex
} // don't output the rule on error with the regex
$tmpRule = preg_replace('/reference\s*:\s*.+?;/', $ruleFormatReference . ';', $tmpRule, -1, $replaceCount['reference']);
if (null == $tmpRule) {
return false;
} // don't output the rule on error with the regex
} // don't output the rule on error with the regex
$tmpRule = preg_replace('/reference\s*:\s*.+?;/', $ruleFormatReference . ';', $tmpRule, -1, $replaceCount['reference']);
if (null == $tmpRule) {
return false;
} // don't output the rule on error with the regex
} // don't output the rule on error with the regex
// FIXME nids - implement priority overwriting
// some values were not replaced, so we need to add them ourselves, and insert them in the rule
@ -667,13 +818,13 @@ class NidsExport
public function checkWhitelist($value)
{
if ($this->checkWhitelist && is_array($this->whitelist)) {
foreach ($this->whitelist as $wlitem) {
if (preg_match($wlitem, $value)) {
return true;
}
}
}
if ($this->checkWhitelist && is_array($this->whitelist)) {
foreach ($this->whitelist as $wlitem) {
if (preg_match($wlitem, $value)) {
return true;
}
}
}
return false;
}
@ -717,4 +868,4 @@ class NidsExport
}
return $ipport;
}
}
}

37
app/Lib/Tools/BackgroundJobsTool.php
View File

@ -7,17 +7,17 @@ App::uses('BackgroundJob', 'Tools/BackgroundJobs');
/**
* BackgroundJobs Tool
*
*
* Utility class to queue jobs, run them and monitor workers.
*
*
* To run a worker manually (debug only):
* $ ./Console/cake start_worker [queue]
*
*
* It is recommended to run these commands with [Supervisor](http://supervisord.org).
* `Supervisor` has an extensive feature set to manage scripts as services,
* such as autorestart, parallel execution, logging, monitoring and much more.
* `Supervisor` has an extensive feature set to manage scripts as services,
* such as autorestart, parallel execution, logging, monitoring and much more.
* All can be managed via the terminal or a XML-RPC API.
*
*
* Use the following configuration as a template for the services:
* /etc/supervisor/conf.d/misp-workers.conf:
* [group:misp-workers]
@ -27,14 +27,14 @@ App::uses('BackgroundJob', 'Tools/BackgroundJobs');
* [program:default]
* command=/var/www/MISP/app/Console/cake start_worker default
* process_name=%(program_name)s_%(process_num)02d
* numprocs=5 ; adjust the amount of parallel workers to your MISP usage
* numprocs=5 ; adjust the amount of parallel workers to your MISP usage
* autostart=true
* autorestart=true
* redirect_stderr=false
* stderr_logfile=/var/www/MISP/app/tmp/logs/misp-workers-errors.log
* stdout_logfile=/var/www/MISP/app/tmp/logs/misp-workers.log
* user=www-data
*
*
*/
class BackgroundJobsTool
{
@ -73,18 +73,21 @@ class BackgroundJobsTool
const
CMD_EVENT = 'event',
CMD_SERVER = 'server',
CMD_ADMIN = 'admin';
CMD_ADMIN = 'admin',
CMD_WORKFLOW = 'workflow';
const ALLOWED_COMMANDS = [
self::CMD_EVENT,
self::CMD_SERVER,
self::CMD_ADMIN
self::CMD_ADMIN,
self::CMD_WORKFLOW,
];
const CMD_TO_SHELL_DICT = [
self::CMD_EVENT => 'EventShell',
self::CMD_SERVER => 'ServerShell',
self::CMD_ADMIN => 'AdminShell'
self::CMD_ADMIN => 'AdminShell',
self::CMD_WORKFLOW => 'WorkflowShell',
];
const JOB_STATUS_PREFIX = 'job_status';
@ -176,7 +179,7 @@ class BackgroundJobsTool
/**
* Enqueue a Job using the CakeResque.
* @deprecated
*
*
* @param string $queue Name of the queue to enqueue the job to.
* @param string $class Class of the job.
* @param array $args Arguments passed to the job.
@ -212,9 +215,9 @@ class BackgroundJobsTool
*
* @param string $queue Queue name, e.g. 'default'.
* @param int $timeout Time to block the read if the queue is empty.
* Must be less than your configured `read_write_timeout`
* Must be less than your configured `read_write_timeout`
* for the redis connection.
*
*
* @throws Exception
*/
public function dequeue($queue, int $timeout = 30)
@ -262,7 +265,7 @@ class BackgroundJobsTool
* Clear all the queue's jobs.
*
* @param string $queue Queue name, e.g. 'default'.
*
*
* @return boolean True on success, false on failure.
*/
public function clearQueue($queue): bool
@ -309,7 +312,7 @@ class BackgroundJobsTool
* Get the number of jobs inside a queue.
*
* @param string $queue Queue name, e.g. 'default'.
*
*
* @return integer Number of jobs.
*/
public function getQueueSize(string $queue): int
@ -327,7 +330,7 @@ class BackgroundJobsTool
* Update job
*
* @param BackgroundJob $job
*
*
* @return void
*/
public function update(BackgroundJob $job)

63
app/Lib/Tools/BetterCakeEventManager.php Normal file
View File

@ -0,0 +1,63 @@
<?php
class BetterCakeEventManager extends CakeEventManager
{
/**
* This method is similar as original dispatch, but do not return newly created event. With returning event, there is
* big memory leak in PHP at least for PHP version 7.4.19.
* @param CakeEvent $event
*/
public function dispatch($event)
{
$listeners = $this->listeners($event->name());
if (empty($listeners)) {
return null;
}
foreach ($listeners as $listener) {
if ($event->isStopped()) {
break;
}
if ($listener['passParams'] === true) {
$result = call_user_func_array($listener['callable'], $event->data);
} else {
$result = $listener['callable']($event);
}
if ($result === false) {
$event->stopPropagation();
}
if ($result !== null) {
$event->result = $result;
}
}
}
/**
* @param $eventKey
* @return array
*/
public function listeners($eventKey)
{
if ($this->_isGlobal) {
$localListeners = [];
} else {
$localListeners = $this->_listeners[$eventKey] ?? [];
}
$globalListeners = static::instance()->prioritisedListeners($eventKey);
$priorities = array_merge(array_keys($globalListeners), array_keys($localListeners));
$priorities = array_unique($priorities, SORT_REGULAR);
asort($priorities);
$result = [];
foreach ($priorities as $priority) {
if (isset($globalListeners[$priority])) {
$result = array_merge($result, $globalListeners[$priority]);
}
if (isset($localListeners[$priority])) {
$result = array_merge($result, $localListeners[$priority]);
}
}
return $result;
}
}

4
app/Lib/Tools/FileAccessTool.php
View File

@ -96,7 +96,7 @@ class FileAccessTool
* @param bool $createFolder
* @throws Exception
*/
public static function writeToFile($file, $content, $createFolder = false)
public static function writeToFile($file, $content, $createFolder = false, $append = false)
{
$dir = dirname($file);
if ($createFolder && !is_dir($dir)) {
@ -105,7 +105,7 @@ class FileAccessTool
}
}
if (file_put_contents($file, $content, LOCK_EX) === false) {
if (file_put_contents($file, $content, LOCK_EX | (!empty($append) ? FILE_APPEND : 0)) === false) {
$freeSpace = disk_free_space($dir);
throw new Exception("An error has occurred while attempt to write to file `$file`. Maybe not enough space? ($freeSpace bytes left)");
}

97
app/Lib/Tools/GraphvizDOTTool.php Normal file
View File

@ -0,0 +1,97 @@
<?php
class GraphvizDOTTool
{
const NODE_STYLE = [
'trigger' => [
'margin' => 0,
'shape' => 'diamond',
],
'logic' => [
'margin' => 0,
'shape' => 'parallelogram',
],
'action' => [
'margin' => 0,
'shape' => 'box',
],
];
const EDGE_STYLE = [
];
/**
* dot Get DOT language format of the provided graph
*
* @return string
*/
public static function dot(array $graph_data)
{
$parsedGraph = self::__parseGraph($graph_data);
$str = self::__header();
$str .= self::__nodes($parsedGraph['nodes']);
$str .= self::__edges($parsedGraph['edges']);
$str .= self::__footer();
return $str;
}
private static function __parseGraph($graph_data)
{
$graphUtil = new GraphUtil($graph_data);
$nodes = $graphUtil->graph;
$edges = $graphUtil->edgeList;
return [
'nodes' => $nodes,
'edges' => $edges,
];
}
private static function __header()
{
return 'digraph G {' . PHP_EOL;
}
private static function __footer()
{
return '}';
}
private static function __nodes($nodes)
{
$str = ' {' . PHP_EOL;
foreach ($nodes as $node) {
$str .= ' ' . self::__node($node);
}
$str .= ' }' . PHP_EOL;
return $str;
}
private static function __node(array $node)
{
$node_attributes = self::NODE_STYLE[$node['data']['module_type']];
$node_attributes['label'] = $node['data']['name'];
$node_attributes_text = self::__arrayToAttributes($node_attributes);
return sprintf('%s [%s]' . PHP_EOL, $node['id'], $node_attributes_text);
}
private static function __edges($edges)
{
$str = '';
foreach ($edges as $source_id => $target_ids) {
foreach ($target_ids as $target_id) {
$str .= ' ' . self::__edge($source_id, $target_id);
}
}
return $str;
}
private static function __edge($source_id, $target_id)
{
return sprintf('%s -> %s [%s]' . PHP_EOL, $source_id, $target_id, self::__arrayToAttributes(self::EDGE_STYLE));
}
private static function __arrayToAttributes(array $list)
{
return implode(', ', array_map(function ($key, $value) {
return sprintf('%s="%s"', $key, $value);
}, array_keys($list), $list));
}
}

83
app/Lib/Tools/MermaidFlowchartTool.php Normal file
View File

@ -0,0 +1,83 @@
<?php
App::uses('FontAwesomeHelper', 'View/Helper');
require_once APP . 'Lib/Tools/WorkflowGraphTool.php';
class MermaidFlowchartTool
{
const NODE_STYLE = [
'trigger' => '{{%s}}',
'logic' => '[/%s/]',
'action' => '[%s]',
];
/**
* dot Get DOT language format of the provided graph
*
* @return string
*/
public static function mermaid(array $graph_data)
{
$parsedGraph = self::__parseGraph($graph_data);
$str = self::__header();
$str .= self::__nodes($parsedGraph['nodes'], $parsedGraph['edges']);
$str .= self::__footer();
return $str;
}
private static function __parseGraph($graph_data)
{
$graphUtil = new GraphUtil($graph_data);
$nodes = Hash::combine($graphUtil->graph, '{n}.id', '{n}');
$edges = $graphUtil->edgeList;
return [
'nodes' => $nodes,
'edges' => $edges,
];
}
private static function __header()
{
return 'flowchart LR' . PHP_EOL;
}
private static function __footer()
{
return '';
}
private static function __nodes($nodes, $edges)
{
$str = '';
foreach ($nodes as $node) {
$str .= self::__node($nodes, $node, $edges[$node['id']]);
}
return $str;
}
private static function __node(array $all_nodes, array $node, array $edges)
{
$str = '';
foreach ($edges as $target_id) {
if (empty($all_nodes[$target_id])) {
continue;
}
$target_node = $all_nodes[$target_id];
$sourceNode = self::__singleNode($node);
$targetNode = self::__singleNode($target_node);
$str .= ' ' . sprintf('%s --> %s', $sourceNode, $targetNode) . PHP_EOL;
}
return $str;
}
private static function __singleNode(array $node)
{
$str = $node['id'];
$icon = sprintf("%s:fa-%s ", FontAwesomeHelper::findNamespace($node['data']['module_data']['icon']), $node['data']['module_data']['icon']);
$node_content = sprintf('"%s%s"',(!empty($node['data']['module_data']['icon']) ? "$icon " : ''), $node['name']);
$str .= sprintf(
self::NODE_STYLE[$node['data']['module_type']],
$node_content
);
return $str;
}
}

13
app/Lib/Tools/ProcessTool.php
View File

@ -1,7 +1,7 @@
<?php
class ProcessException extends Exception
{
/** @var string */
/** @var string|null */
private $stderr;
/** @var string */
@ -10,13 +10,14 @@ class ProcessException extends Exception
/**
* @param string|array $command
* @param int $returnCode
* @param string $stderr
* @param string|null $stderr
* @param string $stdout
*/
public function __construct($command, $returnCode, $stderr, $stdout)
{
$commandForException = is_array($command) ? implode(' ', $command) : $command;
$message = "Command '$commandForException' return error code $returnCode.\nSTDERR: '$stderr'\nSTDOUT: '$stdout'";
$stderrToMessage = $stderr === null ? 'Logged to tmp/logs/exec-errors.log' : "'$stderr'";
$message = "Command '$commandForException' finished with error code $returnCode.\nSTDERR: $stderrToMessage\nSTDOUT: '$stdout'";
$this->stderr = $stderr;
$this->stdout = $stdout;
parent::__construct($message, $returnCode);
@ -48,13 +49,13 @@ class ProcessTool
public static function execute(array $command, $cwd = null, $stderrToFile = false)
{
$descriptorSpec = [
1 => ["pipe", "w"], // stdout
2 => ["pipe", "w"], // stderr
1 => ['pipe', 'w'], // stdout
2 => ['pipe', 'w'], // stderr
];
if ($stderrToFile) {
self::logMessage('Running command ' . implode(' ', $command));
$descriptorSpec[2] = ["file", self::LOG_FILE, 'a'];
$descriptorSpec[2] = ['file', self::LOG_FILE, 'a'];
}
// PHP older than 7.4 do not support proc_open with array, so we need to convert values to string manually

14
app/Lib/Tools/PubSubTool.php
View File

@ -62,9 +62,13 @@ class PubSubTool
if ($response === null) {
throw new Exception("No response from status command returned after 5 seconds.");
}
return json_decode(trim($response[1]), true);
return JsonTool::decode(trim($response[1]));
}
/**
* @return bool
* @throws ProcessException
*/
public function checkIfPythonLibInstalled()
{
$script = APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmqtest.py';
@ -143,6 +147,12 @@ class PubSubTool
return $this->pushToRedis('data:misp_json_warninglist', $warninglist);
}
public function workflow_push(array $data)
{
$topic = 'data:misp_json_workflow';
return $this->pushToRedis($topic, $data);
}
/**
* @param array $data
* @param string $type
@ -301,7 +311,7 @@ class PubSubTool
$pluginConfig = Configure::read('Plugin');
foreach ($settings as $key => $setting) {
$temp = isset($pluginConfig['ZeroMQ_' . $key]) ? $pluginConfig['ZeroMQ_' . $key] : null;
$temp = $pluginConfig['ZeroMQ_' . $key] ?? null;
if ($temp) {
$settings[$key] = $temp;
}

2
app/Lib/Tools/SecurityAudit.php
View File

@ -431,7 +431,7 @@ class SecurityAudit
App::uses('CakeEmail', 'Network/Email');
$email = new CakeEmail();
$emailConfig = $email->config();
if ($emailConfig['transport'] === 'Smtp' && $emailConfig['port'] == 25 && !$emailConfig['tls']) {
if ($emailConfig['transport'] === 'Smtp' && $emailConfig['port'] == 25 && empty($emailConfig['tls'])) {
$output['Email'][] = [
'warning',
__('STARTTLS is not enabled.'),

4
app/Lib/Tools/SendEmail.php
View File

@ -484,6 +484,10 @@ class SendEmail
]);
}
if ($body instanceof SendEmailTemplate && $body->listUnsubscribe()) {
$email->addHeaders(['List-Unsubscribe' => "<{$body->listUnsubscribe()}>"]);
}
$signed = false;
if (Configure::read('GnuPG.sign')) {
if (!$this->gpg) {

15
app/Lib/Tools/SendEmailTemplate.php
View File

@ -10,6 +10,9 @@ class SendEmailTemplate
/** @var string|null */
private $referenceId;
/** @var string */
private $listUnsubscribe;
/** @var string|null */
private $subject;
@ -31,6 +34,18 @@ class SendEmailTemplate
$this->referenceId = $referenceId;
}
/**
* @param string|null $listUnsubscribe
* @return string|void
*/
public function listUnsubscribe($listUnsubscribe = null)
{
if ($listUnsubscribe === null) {
return $this->listUnsubscribe;
}
$this->listUnsubscribe = $listUnsubscribe;
}
/**
* Get subject from template. Must be called after render method.
* @param string|null $subject

58
app/Lib/Tools/ServerSyncTool.php
View File

@ -1,5 +1,6 @@
<?php
App::uses('SyncTool', 'Tools');
App::uses('JsonTool', 'Tools');
class ServerSyncTool
{
@ -8,8 +9,11 @@ class ServerSyncTool
FEATURE_ORG_RULE = 'org_rule',
FEATURE_FILTER_SIGHTINGS = 'filter_sightings',
FEATURE_PROPOSALS = 'proposals',
FEATURE_PROTECTED_EVENT = 'protected_event',
FEATURE_POST_TEST = 'post_test',
FEATURE_PROTECTED_EVENT = 'protected_event';
FEATURE_EDIT_OF_GALAXY_CLUSTER = 'edit_of_galaxy_cluster',
PERM_SYNC = 'perm_sync',
PERM_GALAXY_EDITOR = 'perm_galaxy_editor';
/** @var array */
private $server;
@ -89,6 +93,17 @@ class ServerSyncTool
return $this->get($url);
}
/**
* @param array $events
* @return HttpSocketResponseExtended
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
*/
public function filterEventIdsForPush(array $events)
{
return $this->post('/events/filterEventIdsForPush', $events);
}
/**
* @param array $event
* @return HttpSocketResponseExtended
@ -164,6 +179,39 @@ class ServerSyncTool
return $this->post('/attributes/restSearch.json', $rules);
}
/**
* @param array $rules
* @return HttpSocketResponseExtended
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
*/
public function galaxyClusterSearch(array $rules)
{
return $this->post('/galaxy_clusters/restSearch', $rules);
}
/**
* @param int|string $galaxyClusterId Galaxy Cluster ID or UUID
* @return HttpSocketResponseExtended
* @throws HttpSocketHttpException
*/
public function fetchGalaxyCluster($galaxyClusterId)
{
return $this->get('/galaxy_clusters/view/' . $galaxyClusterId);
}
/**
* @param array $cluster
* @return HttpSocketResponseExtended
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
*/
public function pushGalaxyCluster(array $cluster)
{
$logMessage = "Pushing Galaxy Cluster #{$cluster['GalaxyCluster']['id']} to Server #{$this->serverId()}";
return $this->post('/galaxies/pushCluster', [$cluster], $logMessage);
}
/**
* @param array $params
* @return HttpSocketResponseExtended
@ -332,6 +380,12 @@ class ServerSyncTool
case self::FEATURE_PROTECTED_EVENT:
$version = explode('.', $info['version']);
return $version[0] == 2 && $version[1] == 4 && $version[2] > 155;
case self::FEATURE_EDIT_OF_GALAXY_CLUSTER:
return isset($info['perm_galaxy_editor']);
case self::PERM_SYNC:
return isset($info['perm_sync']) && $info['perm_sync'];
case self::PERM_GALAXY_EDITOR:
return isset($info['perm_galaxy_editor']) && $info['perm_galaxy_editor'];
default:
throw new InvalidArgumentException("Invalid flag `$flag` provided");
}
@ -373,7 +427,7 @@ class ServerSyncTool
private function post($url, $data, $logMessage = null)
{
$protectedMode = !empty($data['Event']['protected']);
$data = json_encode($data, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
$data = JsonTool::encode($data);
if ($logMessage && !empty(Configure::read('Security.sync_audit'))) {
$pushLogEntry = sprintf(

142
app/Lib/Tools/WorkflowFormatConverterTool.php Normal file
View File

@ -0,0 +1,142 @@
<?php
App::uses('JSONConverterTool', 'Tools');
/**
* WorkflowFormatConverterTool convert passed data into the MISP core format with these additional properties:
* - Attributes are encapsulated in the Event they belong to as well as their object (if applicable)
* - Events have an additional key `_AttributeFlattened` which combines both Attribute and ObjectAttribute in the same array
* - Attributes have an additional key `_allTags` which group both AttributeTag and EventTag.
* - Tags in this `_allTags` key have an additional flag `inherited` indicating if the tag has been propagated from the Event to the Attribute
*/
class WorkflowFormatConverterTool
{
private static $fakeSiteAdminUser = ['Role' => ['perm_site_admin' => true]];
public static function convert(array $data, $scope=''): array
{
if (empty($scope)) {
$scope = self::__guessScopeFromData($data);
}
$converted = [];
switch ($scope) {
case 'event':
$converted = self::__convertEvent($data);
break;
case 'attribute':
$converted = self::__convertAttribute($data);
break;
case 'object':
$converted = self::__convertObject($data);
break;
default:
break;
}
$converted = self::__includeFlattenedAttributes($converted);
return $converted;
}
private static function __convertEvent(array $event): array
{
$converted = [];
$converted = JSONConverterTool::convert($event, false, true);
return $converted;
}
private static function __convertObject(array $object): array
{
$converted = [];
$convertedObject = JSONConverterTool::convertObject($object, false, true);
$convertedObject = ['Object' => $convertedObject['Object']];
$converted = self::__encapsulateEntityWithEvent($convertedObject);
return $converted;
}
/**
* __convertAttribute Convert and clean an attribute. May also transform the attribute into an Object if applicable.
* However, the object will not be full and will only contain the attribute
*
* @param array $attribute
* @return array
*/
private static function __convertAttribute(array $attribute): array
{
$allTags = [];
if (!empty($attribute['EventTag'])) {
foreach ($attribute['AttributeTag'] as $attributeTag) {
$attributeTag['Tag']['inherited'] = false;
$allTags[] = $attributeTag['Tag'];
}
foreach ($attribute['EventTag'] as $eventTag) {
$eventTag['Tag']['inherited'] = true;
$allTags[] = $eventTag['Tag'];
}
}
$convertedAttribute = JSONConverterTool::convertAttribute($attribute, true);
$convertedAttribute['Attribute']['_allTags'] = $allTags;
if ($convertedAttribute['Attribute']['object_id'] != 0) {
$objectModel = ClassRegistry::init('MispObject');
$object = $objectModel->fetchObjectSimple(self::$fakeSiteAdminUser, [
'conditions' => [
'Object.id' => $convertedAttribute['Attribute']['object_id'],
],
]);
if (!empty($object)) {
$object = $object[0]['Object'];
$object['Attribute'][] = $convertedAttribute['Attribute'];
$convertedAttribute = ['Object' => $object];
} else {
$convertedAttribute = ['Attribute' => $convertedAttribute['Attribute']];
}
} else {
$convertedAttribute = ['Attribute' => $convertedAttribute['Attribute']];
}
$converted = self::__encapsulateEntityWithEvent($convertedAttribute);
return $converted;
}
private static function __encapsulateEntityWithEvent(array $data): array
{
$eventModel = ClassRegistry::init('Event');
$event = $eventModel->fetchSimpleEvent(self::$fakeSiteAdminUser, $data['Attribute']['event_id'] ?? $data['Object']['event_id'], [
'contain' => [
'EventTag' => ['Tag']
]
]);
if (empty($event)) {
return [];
}
$event = self::__convertEvent($event);
$event = $event['Event'];
reset($data);
$entityType = key($data);
$event[$entityType][] = $data[$entityType];
return ['Event' => $event];
}
private static function __includeFlattenedAttributes(array $event): array
{
$attributes = $event['Event']['Attribute'] ?? [];
$objectAttributes = Hash::extract($event['Event']['Object'] ?? [], '{n}.Attribute.{n}');
$event['Event']['_AttributeFlattened'] = array_merge($attributes, $objectAttributes);
return $event;
}
private static function __guessScopeFromData(array $data)
{
if (isset($data['Object']) && !isset($data['Attribute'])) {
return 'object';
}
if (!isset($data['Attribute'])) {
return 'event';
}
if (!isset($data['Event'])) {
return 'attribute';
}
if (isset($data['RelatedEvent']) || isset($data['Orgc']) || isset($data['Org'])) {
return 'event';
}
if (!empty($data['Attribute'])) {
return 'attribute';
}
}
}

421
app/Lib/Tools/WorkflowGraphTool.php Normal file
View File

@ -0,0 +1,421 @@
<?php
class CyclicGraphException extends Exception {}
class GraphUtil
{
public function __construct($graphData)
{
$this->graph = $graphData;
$this->numberNodes = count($this->graph);
$this->edgeList = $this->_buildEdgeList($graphData);
$this->properties = [];
}
private function _buildEdgeList($graphData): array
{
$list = [];
foreach ($graphData as $node) {
$list[(int)$node['id']] = [];
foreach (($node['outputs'] ?? []) as $output_id => $outputs) {
foreach ($outputs as $connections) {
foreach ($connections as $connection) {
$list[$node['id']][] = (int)$connection['node'];
}
}
}
}
return $list;
}
private function _DFSUtil($node_id, &$color): bool
{
$color[$node_id] = 'GRAY';
foreach ($this->edgeList[$node_id] as $i) {
if ($color[$i] == 'GRAY') {
$this->loopNode = $i;
$this->properties[] = [$node_id, $i, __('Cycle')];
return true;
}
if ($color[$i] == 'WHITE' && $this->_DFSUtil($i, $color)) {
if (!is_null($this->loopNode)) {
$this->properties[] = [$node_id, $i, __('Cycle')];
if ($this->loopNode == $node_id) {
$this->loopNode = null;
}
}
return true;
}
}
$color[$node_id] = 'BLACK';
return false;
}
/**
* isCyclic Return is the graph is cyclic, so if it contains a cycle.
*
* A directed graph G is acyclic if and only if a depth-first search of G yields no back edges.
* Introduction to Algorithms, third edition By Thomas H. Cormen, Charles E. Leiserson, Ronald L. Rivest, Clifford Stein
*
* @return array
*/
public function isCyclic(): array
{
$this->properties = [];
$color = [];
foreach (array_keys($this->edgeList) as $node_id) {
$color[$node_id] = 'WHITE';
}
$this->loopNode = null;
foreach (array_keys($this->edgeList) as $node_id) {
if ($color[$node_id] == 'WHITE') {
if ($this->_DFSUtil($node_id, $color)) {
return [true, $this->properties];
}
}
}
return [false, []];
}
public function hasMultipleOutputConnection(): array
{
$edges = [];
foreach ($this->graph as $node) {
foreach (($node['outputs'] ?? []) as $output_id => $outputs) {
foreach ($outputs as $connections) {
if (count($connections) > 1 && empty($node['data']['multiple_output_connection'])) {
$edges[$node['id']] = array_map(function ($connection) {
return intval($connection['node']);
}, $connections);
}
}
}
}
return [!empty($edges), $edges];
}
}
class GraphWalker
{
private $graph;
private $WorkflowModel;
private $startNodeID;
private $for_path;
private $cursor;
const PATH_TYPE_BLOCKING = 'blocking';
const PATH_TYPE_NON_BLOCKING = 'non-blocking';
const PATH_TYPE_INCLUDE_LOGIC = 'include-logic';
const ALLOWED_PATH_TYPES = [GraphWalker::PATH_TYPE_BLOCKING, GraphWalker::PATH_TYPE_NON_BLOCKING, GraphWalker::PATH_TYPE_INCLUDE_LOGIC];
public function __construct(array $graphData, $WorkflowModel, $startNodeID, $for_path=null)
{
$this->graph = $graphData;
$this->WorkflowModel = $WorkflowModel;
$this->startNodeID = $startNodeID;
$this->for_path = $for_path;
$this->triggersByNodeID = [];
if (empty($this->graph[$startNodeID])) {
throw new Exception(__('Could not find start node %s', $startNodeID));
}
$this->cursor = $startNodeID;
}
private function getModuleClass($node)
{
$moduleClass = $this->loaded_classes[$node['data']['module_type']][$node['data']['id']] ?? null;
return $moduleClass;
}
private function _getPathType($node_id, $path_type)
{
$node = $this->graph[$node_id];
if ($node['data']['module_type'] == 'logic' && $node['data']['id'] == 'concurrent-task') {
return self::PATH_TYPE_NON_BLOCKING;
}
return $path_type;
}
private function _evaluateOutputs($node, WorkflowRoamingData $roamingData, $shouldExecuteLogicNode=true)
{
$allowed_outputs = ($node['outputs'] ?? []);
if ($shouldExecuteLogicNode && $node['data']['module_type'] == 'logic') {
$allowed_outputs = $this->_executeModuleLogic($node, $roamingData);
}
return $allowed_outputs;
}
/**
* _executeModuleLogic function
*
* @param array $node
* @return array
*/
private function _executeModuleLogic(array $node, WorkflowRoamingData $roamingData): array
{
$outputs = ($node['outputs'] ?? []);
if ($node['data']['id'] == 'if') {
$useFirstOutput = $this->_evaluateIFCondition($node, $roamingData);
return $useFirstOutput ? ['output_1' => $outputs['output_1']] : ['output_2' => $outputs['output_2']];
} else if ($node['data']['id'] == 'concurrent-task') {
$this->_evaluateConcurrentTask($node, $roamingData, $outputs['output_1']);
return ['output_1' => []];
} else {
$useFirstOutput = $this->_evaluateCustomLogicCondition($node, $roamingData);
return $useFirstOutput ? ['output_1' => $outputs['output_1']] : ['output_2' => $outputs['output_2']];
}
return $outputs;
}
private function _evaluateIFCondition($node, WorkflowRoamingData $roamingData): bool
{
$result = $this->WorkflowModel->executeNode($node, $roamingData);
return $result;
}
private function _evaluateCustomLogicCondition($node, WorkflowRoamingData $roamingData): bool
{
$result = $this->WorkflowModel->executeNode($node, $roamingData);
return $result;
}
private function _evaluateConcurrentTask($concurrent_node, WorkflowRoamingData $roamingData, array $connections)
{
foreach ($connections['connections'] as $connection) {
$node_id_to_exec = (int)$connection['node'];
$data = $roamingData->getData();
$data['__node_id_to_exec'] = $node_id_to_exec;
$data = $roamingData->setData($data);
$this->WorkflowModel->executeNode($concurrent_node, $roamingData);
}
}
public function _walk($node_id, $path_type=null, array $path_list=[], WorkflowRoamingData $roamingData)
{
$this->cursor = $node_id;
$node = $this->graph[$node_id];
$shouldExecuteLogicNode = $path_type != self::PATH_TYPE_INCLUDE_LOGIC;
if (!$shouldExecuteLogicNode) {
yield ['node' => $node, 'path_type' => $path_type, 'path_list' => $path_list];
} else if ($node['data']['module_type'] != 'trigger' && $node['data']['module_type'] != 'logic') { // trigger and logic nodes should not be returned as they are "control" nodes
yield ['node' => $node, 'path_type' => $path_type, 'path_list' => $path_list];
}
$allowedOutputs = $this->_evaluateOutputs($node, $roamingData, $shouldExecuteLogicNode);
foreach ($allowedOutputs as $output_id => $outputs) {
if ($shouldExecuteLogicNode) {
$path_type = $this->_getPathType($node_id, $path_type);
}
if (is_null($this->for_path) || $path_type == $this->for_path) {
foreach ($outputs as $connections) {
foreach ($connections as $connection_id => $connection) {
$next_node_id = (int)$connection['node'];
$current_path = $this->__genPathList($node_id, $output_id, $connection_id, $next_node_id);
if (in_array($current_path, $path_list)) { // avoid loops
continue;
}
$next_path_list = $path_list;
$next_path_list[] = $current_path;
yield from $this->_walk($next_node_id, $path_type, $next_path_list, $roamingData);
}
}
}
}
}
public function walk(WorkflowRoamingData $roamingData)
{
return $this->_walk($this->cursor, $this->for_path, [], $roamingData);
}
private function __genPathList($source_id, $output_id, $connection_id, $next_node_id)
{
return sprintf('%s:%s:%s:%s', $source_id, $output_id, $connection_id, $next_node_id);
}
public static function parsePathList($pathList): array
{
return array_map(function($path) {
$split = explode(':', $path);
return [
'source_id' => $split[0],
'output_id' => $split[1],
'connection_id' => $split[2],
'next_node_id' => $split[3],
];
}, $pathList);
}
}
class WorkflowRoamingData
{
private $workflow_user;
private $data;
private $workflow;
private $current_node;
public function __construct(array $workflow_user, array $data, array $workflow, int $current_node)
{
$this->workflow_user = $workflow_user;
$this->data = $data;
$this->workflow = $workflow;
$this->current_node = $current_node;
}
public function getUser(): array
{
return $this->workflow_user;
}
public function getData(): array
{
return $this->data;
}
public function getWorkflow(): array
{
return $this->workflow;
}
public function getCurrentNode(): int
{
return $this->current_node;
}
public function setData(array $data)
{
$this->data = $data;
}
public function setCurrentNode(int $current_node)
{
$this->current_node = $current_node;
}
}
class WorkflowGraphTool
{
/**
* extractTriggerFromWorkflow Return the trigger id (or full module) that are specified in the workflow
*
* @param array $workflow
* @param bool $fullNode
* @return int|array|null
*/
public static function extractTriggerFromWorkflow(array $graphData, bool $fullNode = false)
{
$triggers = self::extractTriggersFromWorkflow($graphData, $fullNode);
if (empty($triggers)) {
return null;
}
$node = $triggers[0];
return $node;
}
/**
* extractTriggersFromWorkflow Return the list of triggers id (or full module) that are specified in the workflow
*
* @param array $workflow
* @param bool $fullNode
* @return array
*/
public static function extractTriggersFromWorkflow(array $graphData, bool $fullNode = false): array
{
$triggers = [];
foreach ($graphData as $node) {
if ($node['data']['module_type'] == 'trigger') {
if (!empty($fullNode)) {
$triggers[] = $node;
} else {
$triggers[] = $node['data']['id'];
}
}
}
return $triggers;
}
/**
* extractConcurrentTasksFromWorkflow Return the list of concurrent-tasks's id (or full module) that are included in the workflow
*
* @param array $workflow
* @param bool $fullNode
* @return array
*/
public static function extractConcurrentTasksFromWorkflow(array $graphData, bool $fullNode = false): array
{
$nodes = [];
foreach ($graphData as $node) {
if ($node['data']['module_type'] == 'logic' && $node['data']['id'] == 'concurrent-task') {
if (!empty($fullNode)) {
$nodes[] = $node;
} else {
$nodes[] = $node['data']['id'];
}
}
}
return $nodes;
}
/**
* isAcyclic Return if the graph contains a cycle
*
* @param array $graphData
* @param array $cycles Get a list of cycle
* @return boolean
*/
public static function isAcyclic(array $graphData, array &$cycles=[]): bool
{
$graphUtil = new GraphUtil($graphData);
$result = $graphUtil->isCyclic();
$isCyclic = $result[0];
$cycles = $result[1];
return !$isCyclic;
}
/**
* hasMultipleOutputConnection Return if the graph has multiple connection from a node output
*
* @param array $graphData
* @param array $edges Get a list of edges from the same output
* @return boolean
*/
public static function hasMultipleOutputConnection(array $graphData, array &$edges=[]): bool
{
$graphUtil = new GraphUtil($graphData);
$result = $graphUtil->hasMultipleOutputConnection();
$hasMultipleOutputConnection = $result[0];
$edges = $result[1];
return $hasMultipleOutputConnection;
}
/**
* Undocumented getNodeIdForTrigger
*
* @param array $graphData
* @param string $trigger_id
* @return integer Return the ID of the node for the provided trigger and -1 if no nodes with this id was found.
*/
public static function getNodeIdForTrigger(array $graphData, $trigger_id): int
{
$trigger_node = WorkflowGraphTool::extractTriggerFromWorkflow($graphData, true);
if ($trigger_node['data']['id'] == $trigger_id) {
return $trigger_node['id'];
}
return -1;
}
public static function getRoamingData(array $user=[], array $data=[], array $workflow=[], int $node_id=-1)
{
return new WorkflowRoamingData($user, $data, $workflow, $node_id);
}
public static function getWalkerIterator(array $graphData, $WorkflowModel, $startNodeID, $path_type=null, WorkflowRoamingData $roamingData)
{
if (!in_array($path_type, GraphWalker::ALLOWED_PATH_TYPES)) {
return [];
}
$graphWalker = new GraphWalker($graphData, $WorkflowModel, $startNodeID, $path_type);
return $graphWalker->walk($roamingData);
}
}

23
app/Lib/WorkflowModules/action/Module_blueprint_action_module.php Normal file
View File

@ -0,0 +1,23 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_blueprint_action_module extends WorkflowBaseActionModule
{
public $blocking = false;
public $disabled = true;
public $id = 'blueprint-action-module';
public $name = 'Blueprint action module';
public $description = 'Lorem ipsum dolor, sit amet consectetur adipisicing elit.';
public $icon = 'shapes';
public $inputs = 1;
public $outputs = 1;
public $params = [];
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
{
parent::exec($node, $roamingData, $errors);
// If $this->blocking == true, returning `false` will stop the execution.
$errors[] = __('Execution stopped');
return false;
}
}

23
app/Lib/WorkflowModules/logic/Module_blueprint_logic_module.php Normal file
View File

@ -0,0 +1,23 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_blueprint_logic_module extends WorkflowBaseLogicModule
{
public $disabled = true;
public $id = 'blueprint-logic-module';
public $name = 'Blueprint logic module';
public $description = 'Lorem ipsum dolor, sit amet consectetur adipisicing elit.';
public $icon = 'shapes';
public $inputs = 1;
public $outputs = 2;
public $params = [];
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
{
parent::exec($node, $roamingData, $errors);
$params = $this->getParamsWithValues($node);
$data = $roamingData->getData();
// Returning true will make the execution flow take the first output of this module. Otherwise, the second output will be used.
return true;
}
}

308
app/Model/AppModel.php
View File

@ -25,6 +25,7 @@ App::uses('LogableBehavior', 'Assets.models/behaviors');
App::uses('RandomTool', 'Tools');
App::uses('FileAccessTool', 'Tools');
App::uses('JsonTool', 'Tools');
App::uses('BetterCakeEventManager', 'Tools');
class AppModel extends Model
{
@ -47,15 +48,6 @@ class AppModel extends Model
/** @var AttachmentTool|null */
private $attachmentTool;
public function __construct($id = false, $table = null, $ds = null)
{
parent::__construct($id, $table, $ds);
$this->findMethods['column'] = true;
if (in_array('phar', stream_get_wrappers())) {
stream_wrapper_unregister('phar');
}
}
// deprecated, use $db_changes
// major -> minor -> hotfix -> requires_logout
const OLD_DB_CHANGES = array(
@ -89,10 +81,11 @@ class AppModel extends Model
63 => true, 64 => false, 65 => false, 66 => false, 67 => false, 68 => false,
69 => false, 70 => false, 71 => true, 72 => true, 73 => false, 74 => false,
75 => false, 76 => true, 77 => false, 78 => false, 79 => false, 80 => false,
81 => false, 82 => false, 83 => false, 84 => false, 85 => false, 86 => false
81 => false, 82 => false, 83 => false, 84 => false, 85 => false, 86 => false,
87 => false, 88 => false, 89 => false, 90 => false,
);
public $advanced_updates_description = array(
const ADVANCED_UPDATES_DESCRIPTION = array(
'seenOnAttributeAndObject' => array(
'title' => 'First seen/Last seen Attribute table',
'description' => 'Update the Attribute table to support first_seen and last_seen feature, with a microsecond resolution.',
@ -106,6 +99,15 @@ class AppModel extends Model
),
);
public function __construct($id = false, $table = null, $ds = null)
{
parent::__construct($id, $table, $ds);
$this->findMethods['column'] = true;
if (in_array('phar', stream_get_wrappers(), true)) {
stream_wrapper_unregister('phar');
}
}
public function isAcceptedDatabaseError($errorMessage)
{
if ($this->isMysql()) {
@ -135,8 +137,9 @@ class AppModel extends Model
switch ($command) {
case '2.4.20':
$dbUpdateSuccess = $this->updateDatabase($command);
$this->ShadowAttribute = ClassRegistry::init('ShadowAttribute');
$this->ShadowAttribute->upgradeToProposalCorrelation();
//deprecated
//$this->ShadowAttribute = ClassRegistry::init('ShadowAttribute');
//$this->ShadowAttribute->upgradeToProposalCorrelation();
break;
case '2.4.25':
$dbUpdateSuccess = $this->updateDatabase($command);
@ -225,6 +228,15 @@ class AppModel extends Model
case 48:
$dbUpdateSuccess = $this->__generateCorrelations();
break;
case 89:
$this->__retireOldCorrelationEngine();
$dbUpdateSuccess = true;
break;
case 90:
$dbUpdateSuccess = $this->updateDatabase($command);
$this->Workflow = Classregistry::init('Workflow');
$this->Workflow->enableDefaultModules();
break;
default:
$dbUpdateSuccess = $this->updateDatabase($command);
break;
@ -274,9 +286,9 @@ class AppModel extends Model
$liveOff = false;
$exitOnError = false;
if (isset($this->advanced_updates_description[$command])) {
$liveOff = isset($this->advanced_updates_description[$command]['liveOff']) ? $this->advanced_updates_description[$command]['liveOff'] : $liveOff;
$exitOnError = isset($this->advanced_updates_description[$command]['exitOnError']) ? $this->advanced_updates_description[$command]['exitOnError'] : $exitOnError;
if (isset(self::ADVANCED_UPDATES_DESCRIPTION[$command])) {
$liveOff = isset(self::ADVANCED_UPDATES_DESCRIPTION[$command]['liveOff']) ? self::ADVANCED_UPDATES_DESCRIPTION[$command]['liveOff'] : $liveOff;
$exitOnError = isset(self::ADVANCED_UPDATES_DESCRIPTION[$command]['exitOnError']) ? self::ADVANCED_UPDATES_DESCRIPTION[$command]['exitOnError'] : $exitOnError;
}
$sqlArray = array();
@ -1684,6 +1696,118 @@ class AppModel extends Model
case 86:
$this->__addIndex('attributes', 'timestamp');
break;
case 87:
$sqlArray[] = "CREATE TABLE IF NOT EXISTS `no_acl_correlations` (
`id` int(10) UNSIGNED NOT NULL AUTO_INCREMENT,
`attribute_id` int(10) UNSIGNED NOT NULL,
`1_attribute_id` int(10) UNSIGNED NOT NULL,
`event_id` int(10) UNSIGNED NOT NULL,
`1_event_id` int(10) UNSIGNED NOT NULL,
`value_id` int(10) UNSIGNED NOT NULL,
PRIMARY KEY (`id`),
INDEX `event_id` (`event_id`),
INDEX `1_event_id` (`1_event_id`),
INDEX `attribute_id` (`attribute_id`),
INDEX `1_attribute_id` (`1_attribute_id`),
INDEX `value_id` (`value_id`)
) ENGINE=InnoDB;";
$sqlArray[] = "CREATE TABLE IF NOT EXISTS `default_correlations` (
`id` int(10) UNSIGNED NOT NULL AUTO_INCREMENT,
`attribute_id` int(10) UNSIGNED NOT NULL,
`object_id` int(10) UNSIGNED NOT NULL,
`event_id` int(10) UNSIGNED NOT NULL,
`org_id` int(10) UNSIGNED NOT NULL,
`distribution` tinyint(4) NOT NULL,
`object_distribution` tinyint(4) NOT NULL,
`event_distribution` tinyint(4) NOT NULL,
`sharing_group_id` int(10) UNSIGNED NOT NULL DEFAULT 0,
`object_sharing_group_id` int(10) UNSIGNED NOT NULL DEFAULT 0,
`event_sharing_group_id` int(10) UNSIGNED NOT NULL DEFAULT 0,
`1_attribute_id` int(10) UNSIGNED NOT NULL,
`1_object_id` int(10) UNSIGNED NOT NULL,
`1_event_id` int(10) UNSIGNED NOT NULL,
`1_org_id` int(10) UNSIGNED NOT NULL,
`1_distribution` tinyint(4) NOT NULL,
`1_object_distribution` tinyint(4) NOT NULL,
`1_event_distribution` tinyint(4) NOT NULL,
`1_sharing_group_id` int(10) UNSIGNED NOT NULL DEFAULT 0,
`1_object_sharing_group_id` int(10) UNSIGNED NOT NULL DEFAULT 0,
`1_event_sharing_group_id` int(10) UNSIGNED NOT NULL DEFAULT 0,
`value_id` int(10) UNSIGNED NOT NULL,
PRIMARY KEY (`id`),
INDEX `event_id` (`event_id`),
INDEX `attribute_id` (`attribute_id`),
INDEX `object_id` (`object_id`),
INDEX `org_id` (`org_id`),
INDEX `distribution` (`distribution`),
INDEX `object_distribution` (`object_distribution`),
INDEX `event_distribution` (`event_distribution`),
INDEX `sharing_group_id` (`sharing_group_id`),
INDEX `object_sharing_group_id` (`object_sharing_group_id`),
INDEX `event_sharing_group_id` (`event_sharing_group_id`),
INDEX `1_event_id` (`1_event_id`),
INDEX `1_attribute_id` (`1_attribute_id`),
INDEX `1_object_id` (`1_object_id`),
INDEX `1_org_id` (`1_org_id`),
INDEX `1_distribution` (`1_distribution`),
INDEX `1_object_distribution` (`1_object_distribution`),
INDEX `1_event_distribution` (`1_event_distribution`),
INDEX `1_sharing_group_id` (`1_sharing_group_id`),
INDEX `1_object_sharing_group_id` (`1_object_sharing_group_id`),
INDEX `1_event_sharing_group_id` (`1_event_sharing_group_id`),
INDEX `value_id` (`value_id`)
) ENGINE=InnoDB;";
$sqlArray[] = "CREATE TABLE IF NOT EXISTS `correlation_values` (
`id` int(10) UNSIGNED NOT NULL AUTO_INCREMENT,
`value` varchar(191) NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `value` (`value`(191))
) ENGINE=InnoDB CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;";
$sqlArray[] = "CREATE TABLE IF NOT EXISTS `over_correlating_values` (
`id` int(10) UNSIGNED NOT NULL AUTO_INCREMENT,
`value` text,
`occurrence` int(10) UNSIGNED NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `value` (`value`(191)),
INDEX `occurrence` (`occurrence`)
) ENGINE=InnoDB CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;";
break;
case 88:
$sqlArray[] = 'ALTER TABLE `users` ADD `external_auth_required` tinyint(1) NOT NULL DEFAULT 0;';
$sqlArray[] = 'ALTER TABLE `users` ADD `external_auth_key` text COLLATE utf8_bin;';
break;
case 90:
$sqlArray[] = "CREATE TABLE IF NOT EXISTS `workflows` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`uuid` varchar(40) COLLATE utf8_bin NOT NULL ,
`name` varchar(191) NOT NULL,
`description` varchar(191) NOT NULL,
`timestamp` int(11) NOT NULL DEFAULT 0,
`enabled` tinyint(1) NOT NULL DEFAULT 0,
`counter` int(11) NOT NULL DEFAULT 0,
`trigger_id` varchar(191) COLLATE utf8_bin NOT NULL,
`debug_enabled` tinyint(1) NOT NULL DEFAULT 0,
`data` text,
PRIMARY KEY (`id`),
INDEX `uuid` (`uuid`),
INDEX `name` (`name`),
INDEX `timestamp` (`timestamp`),
INDEX `trigger_id` (`trigger_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;";
$sqlArray[] = "CREATE TABLE IF NOT EXISTS `workflow_blueprints` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`uuid` varchar(40) COLLATE utf8_bin NOT NULL ,
`name` varchar(191) NOT NULL,
`description` varchar(191) NOT NULL,
`timestamp` int(11) NOT NULL DEFAULT 0,
`default` tinyint(1) NOT NULL DEFAULT 0,
`data` text,
PRIMARY KEY (`id`),
INDEX `uuid` (`uuid`),
INDEX `name` (`name`),
INDEX `timestamp` (`timestamp`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;";
break;
case 'fixNonEmptySharingGroupID':
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@ -1765,7 +1889,7 @@ class AppModel extends Model
$total_update_count = $sql_update_count + $index_update_count;
$this->__setUpdateProgress(0, $total_update_count, $command);
$str_index_array = array();
foreach($indexArray as $toIndex) {
foreach ($indexArray as $toIndex) {
$str_index_array[] = __('Indexing %s -> %s', $toIndex[0], $toIndex[1]);
}
$this->__setUpdateCmdMessages(array_merge($sqlArray, $str_index_array));
@ -1773,8 +1897,8 @@ class AppModel extends Model
$errorCount = 0;
// execute test before update. Exit if it fails
if (isset($this->advanced_updates_description[$command]['preUpdate'])) {
$function_name = $this->advanced_updates_description[$command]['preUpdate'];
if (isset(self::ADVANCED_UPDATES_DESCRIPTION[$command]['preUpdate'])) {
$function_name = self::ADVANCED_UPDATES_DESCRIPTION[$command]['preUpdate'];
try {
$this->{$function_name}();
} catch (Exception $e) {
@ -1910,10 +2034,15 @@ class AppModel extends Model
$this->Server->serverSettingsSaveValue('MISP.live', $isLive);
}
// check whether the adminSetting should be updated after the update
private function __postUpdate($command) {
if (isset($this->advanced_updates_description[$command]['record'])) {
if($this->advanced_updates_description[$command]['record']) {
/**
* Check whether the adminSetting should be updated after the update.
* @param string $command
* @return void
*/
private function __postUpdate($command)
{
if (isset(self::ADVANCED_UPDATES_DESCRIPTION[$command]['record'])) {
if (self::ADVANCED_UPDATES_DESCRIPTION[$command]['record']) {
$this->AdminSetting->changeSetting($command, 1);
}
}
@ -2104,7 +2233,7 @@ class AppModel extends Model
'fields' => ['id', 'value'],
]);
if (count($db_version) > 1) {
// we rgan into a bug where we have more than one db_version entry. This bug happened in some rare circumstances around 2.4.50-2.4.57
// we ran into a bug where we have more than one db_version entry. This bug happened in some rare circumstances around 2.4.50-2.4.57
foreach ($db_version as $k => $v) {
if ($k > 0) {
$this->AdminSetting->delete($v['AdminSetting']['id']);
@ -2822,7 +2951,7 @@ class AppModel extends Model
*
* @return false|string
*/
protected function checkMIPSCommit()
public function checkMIPSCommit()
{
static $commit;
if ($commit === null) {
@ -3383,4 +3512,133 @@ class AppModel extends Model
$dataSourceName = $dataSource->config['datasource'];
return $dataSourceName === 'Database/Mysql' || $dataSourceName === 'Database/MysqlObserver' || $dataSourceName === 'Database/MysqlExtended' || $dataSource instanceof Mysql;
}
public function getCorrelationModelName()
{
if (!empty(Configure::read('MISP.correlation_engine'))) {
return Configure::read('MISP.correlation_engine');
}
return 'Default';
}
public function loadCorrelationModel()
{
if (!empty(Configure::read('MISP.correlation_engine'))) {
return ClassRegistry::init(Configure::read('MISP.correlation_engine'));
}
return ClassRegistry::init('Correlation');
}
/**
* executeTrigger
*
* @param string $trigger_id
* @param array $data Data to be passed to the workflow
* @param array $blockingErrors Errors will be appened if any
* @param array $logging If the execution failure should be logged
* @return boolean If the execution for the blocking path was a success
*/
public function executeTrigger($trigger_id, array $data=[], array &$blockingErrors=[], array $logging=[]): bool
{
if ($this->Workflow === null) {
$this->Workflow = ClassRegistry::init('Workflow');
}
if ($this->isTriggerCallable($trigger_id)) {
$success = $this->Workflow->executeWorkflowForTriggerRouter($trigger_id, $data, $blockingErrors, $logging);
if (!empty($logging) && empty($success)) {
$logging['message'] = !empty($logging['message']) ? $logging['message'] : __('Error while executing workflow.');
$errorMessage = implode(', ', $blockingErrors);
$this->loadLog()->createLogEntry('SYSTEM', $logging['action'], $logging['model'], $logging['id'], $logging['message'], __('Returned message: %s', $errorMessage));
}
return $success;
}
return true;
}
public function isTriggerCallable($trigger_id): bool
{
if ($this->Workflow === null) {
$this->Workflow = ClassRegistry::init('Workflow');
}
return $this->Workflow->checkTriggerEnabled($trigger_id) &&
$this->Workflow->checkTriggerListenedTo($trigger_id);
}
public function addPendingLogEntry($logEntry)
{
$logEntries = Configure::read('pendingLogEntries');
$logEntries[] = $logEntry;
Configure::write('pendingLogEntries', $logEntries);
}
/**
* Use different CakeEventManager to fix memory leak
* @return CakeEventManager
*/
public function getEventManager()
{
if (empty($this->_eventManager)) {
$this->_eventManager = new BetterCakeEventManager();
$this->_eventManager->attach($this->Behaviors);
$this->_eventManager->attach($this);
}
return $this->_eventManager;
}
private function __retireOldCorrelationEngine($user = null) {
if ($user === null) {
$user = [
'id' => 0,
'email' => 'SYSTEM',
'Organisation' => [
'name' => 'SYSTEM'
]
];
}
$this->Correlation = ClassRegistry::init('Correlation');
$this->Attribute = ClassRegistry::init('Attribute');
if (!Configure::read('MISP.background_jobs')) {
$this->Correlation->truncate($user, 'Legacy');
$this->Attribute->generateCorrelation();
} else {
$job = ClassRegistry::init('Job');
$jobId = $job->createJob(
'SYSTEM',
Job::WORKER_DEFAULT,
'truncate table',
$this->Correlation->validEngines['Legacy'],
'Job created.'
);
$this->Correlation->Attribute->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[
'truncateTable',
0,
'Legacy',
$jobId
],
true,
$jobId
);
$jobId = $job->createJob(
'SYSTEM',
Job::WORKER_DEFAULT,
'generate correlation',
'All attributes',
'Job created.'
);
$this->Attribute->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[
'jobGenerateCorrelation',
$jobId
],
true,
$jobId
);
}
}
}

337
app/Model/Attribute.php
View File

@ -416,6 +416,7 @@ class Attribute extends AppModel
// update correlation...
if (isset($attribute['deleted']) && $attribute['deleted']) {
$this->Correlation->beforeSaveCorrelation($attribute);
$this->Correlation->advancedCorrelationsUpdate($attribute);
if (isset($attribute['event_id'])) {
$this->__alterAttributeCount($attribute['event_id'], false);
}
@ -438,9 +439,11 @@ class Attribute extends AppModel
) {
$this->Correlation->beforeSaveCorrelation($attribute);
$this->Correlation->afterSaveCorrelation($attribute, false, $passedEvent);
$this->Correlation->advancedCorrelationsUpdate($attribute);
}
} else {
$this->Correlation->afterSaveCorrelation($attribute, false, $passedEvent);
$this->Correlation->advancedCorrelationsUpdate($attribute);
}
}
$result = true;
@ -457,7 +460,8 @@ class Attribute extends AppModel
}
$pubToZmq = $this->pubToZmq('attribute');
$kafkaTopic = $this->kafkaTopic('attribute');
if ($pubToZmq || $kafkaTopic) {
$isTriggerCallable = $this->isTriggerCallable('attribute-after-save');
if ($pubToZmq || $kafkaTopic || $isTriggerCallable) {
$attributeForPublish = $this->fetchAttribute($this->id);
if (!empty($attributeForPublish)) {
$user = array(
@ -486,11 +490,16 @@ class Attribute extends AppModel
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $attributeForPublish, $action);
}
$workflowErrors = [];
$logging = [
'model' => 'Attribute',
'action' => $action,
'id' => $attributeForPublish['Attribute']['id'],
];
$triggerData = $attributeForPublish;
$this->executeTrigger('attribute-after-save', $triggerData, $workflowErrors, $logging);
}
}
if (Configure::read('MISP.enable_advanced_correlations') && in_array($attribute['type'], ['ip-src', 'ip-dst'], true) && strpos($attribute['value'], '/')) {
$this->Correlation->updateCidrList();
}
if ($created && isset($attribute['event_id']) && empty($attribute['skip_auto_increment'])) {
$this->__alterAttributeCount($attribute['event_id']);
}
@ -784,90 +793,6 @@ class Attribute extends AppModel
return $output;
}
public function getRelatedAttributes($user, $attribute, $fields=array(), $includeEventData = false)
{
// LATER getRelatedAttributes($attribute) this might become a performance bottleneck
// exclude these specific categories from being linked
switch ($attribute['category']) {
case 'Antivirus detection':
return null;
}
// exclude these specific types from being linked
switch ($attribute['type']) {
case 'other':
case 'comment':
return null;
}
// prepare the conditions
$conditions = array(
'Attribute.event_id !=' => $attribute['event_id'],
'Attribute.deleted !=' => 1,
);
// prevent issues with empty fields
if (empty($attribute['value1'])) {
return null;
}
if (empty($attribute['value2'])) {
// no value2, only search for value 1
$conditions['OR'] = array(
'Attribute.value1' => $attribute['value1'],
'Attribute.value2' => $attribute['value1'],
);
} else {
// value2 also set, so search for both
$conditions['AND'] = array( // TODO was OR
'Attribute.value1' => array($attribute['value1'],$attribute['value2']),
'Attribute.value2' => array($attribute['value1'],$attribute['value2']),
);
}
$baseConditions = $this->buildConditions($user);
$baseConditions['AND'][] = $conditions;
// do the search
if (empty($fields)) {
$fields = array('Attribute.*');
}
$params = array(
'conditions' => $baseConditions,
'fields' => $fields,
'recursive' => 0,
'group' => array('Attribute.id', 'Attribute.event_id', 'Attribute.object_id', 'Attribute.object_relation', 'Attribute.category', 'Attribute.type', 'Attribute.value', 'Attribute.uuid', 'Attribute.timestamp', 'Attribute.distribution', 'Attribute.sharing_group_id', 'Attribute.to_ids', 'Attribute.comment', 'Event.id', 'Event.uuid', 'Event.threat_level_id', 'Event.analysis', 'Event.info', 'Event.extends_uuid', 'Event.distribution', 'Event.sharing_group_id', 'Event.published', 'Event.date', 'Event.orgc_id', 'Event.org_id', 'Object.id', 'Object.uuid', 'Object.distribution', 'Object.name', 'Object.template_uuid', 'Object.distribution', 'Object.sharing_group_id'),
'order' => 'Attribute.event_id DESC'
);
if (!empty($includeEventData)) {
$params['contain'] = array(
'Event' => array(
'fields' => array(
'Event.id', 'Event.uuid', 'Event.threat_level_id', 'Event.analysis', 'Event.info', 'Event.extends_uuid', 'Event.distribution', 'Event.sharing_group_id', 'Event.published', 'Event.date', 'Event.orgc_id', 'Event.org_id'
)
),
'Object' => array(
'fields' => array(
'Object.id', 'Object.uuid', 'Object.distribution', 'Object.name', 'Object.template_uuid', 'Object.distribution', 'Object.sharing_group_id'
)
)
);
}
$similarEvents = $this->find(
'all',
$params
);
if (!empty($includeEventData)) {
foreach ($similarEvents as $k => $similarEvent) {
$similarEvents[$k] = array_merge(
$similarEvent['Attribute'],
array(
'Event' => $similarEvent['Event']
)
);
}
}
return $similarEvents;
}
public function typeIsMalware($type)
{
return in_array($type, self::ZIPPED_DEFINITION, true);
@ -1611,65 +1536,24 @@ class Attribute extends AppModel
$attributeCount = 0;
if (Configure::read('MISP.background_jobs') && $jobId) {
$this->Job = ClassRegistry::init('Job');
$eventCount = count($eventIds);
} else {
$jobId = false;
}
foreach ($eventIds as $j => $eventId) {
if ($jobId) {
$message = $attributeId ? __('Correlating Attribute %s', $attributeId) : __('Correlating Event %s (%s MB used)', $eventId, intval(memory_get_usage() / 1024 / 1024));
$this->Job->saveProgress($jobId, $message, ($j / $eventCount) * 100);
if (!empty($eventIds)) {
$eventCount = count($eventIds);
foreach ($eventIds as $j => $currentEventId) {
$attributeCount = $this->__iteratedCorrelation(
$jobId,
$full,
$attributeCount,
$attributeId,
$eventCount,
$currentEventId,
$j
);
}
$event = $this->Event->find('first', array(
'recursive' => -1,
'fields' => ['Event.distribution', 'Event.id', 'Event.org_id', 'Event.sharing_group_id', 'Event.disable_correlation'],
'conditions' => array('id' => $eventId),
'order' => false,
));
$attributeConditions = [
'Attribute.event_id' => $eventId,
'Attribute.deleted' => 0,
'Attribute.disable_correlation' => 0,
'NOT' => [
'Attribute.type' => Attribute::NON_CORRELATING_TYPES,
],
];
if ($attributeId) {
$attributeConditions['Attribute.id'] = $attributeId;
}
$query = [
'recursive' => -1,
'conditions' => $attributeConditions,
// fetch just necessary fields to save memory
'fields' => [
'Attribute.id',
'Attribute.type',
'Attribute.value1',
'Attribute.value2',
'Attribute.distribution',
'Attribute.sharing_group_id',
],
'order' => 'Attribute.id',
'limit' => 5000,
'callbacks' => false, // memory leak fix
];
do {
$attributes = $this->find('all', $query);
foreach ($attributes as $attribute) {
$attribute['Attribute']['event_id'] = $eventId;
$this->Correlation->afterSaveCorrelation($attribute['Attribute'], $full, $event);
}
$fetchedAttributes = count($attributes);
unset($attributes);
$attributeCount += $fetchedAttributes;
if ($fetchedAttributes === 5000) { // maximum number of attributes fetched, continue in next loop
$query['conditions']['Attribute.id >'] = $attribute['Attribute']['id'];
} else {
break;
}
} while (true);
// Generating correlations can take long time, so clear CIDR cache after each event to refresh cache
$this->Correlation->clearCidrCache();
} else {
$attributeCount = $this->__iteratedCorrelation($jobId, $full, $attributeCount);
}
if ($jobId) {
$this->Job->saveStatus($jobId, true);
@ -1677,18 +1561,68 @@ class Attribute extends AppModel
return $attributeCount;
}
private function __iteratedCorrelation(
$jobId = false,
$full = false,
$attributeCount = 0,
$attributeId = null,
$eventCount = null,
$eventId = null,
$j = 0
)
{
if ($jobId) {
$message = $attributeId ? __('Correlating Attribute %s', $attributeId) : __('Correlating Event %s (%s MB used)', $eventId, intval(memory_get_usage() / 1024 / 1024));
$this->Job->saveProgress($jobId, $message, ($j / $eventCount) * 100);
}
$attributeConditions = [
'Attribute.deleted' => 0,
'Attribute.disable_correlation' => 0,
'NOT' => [
'Attribute.type' => Attribute::NON_CORRELATING_TYPES,
],
];
if ($eventId) {
$attributeConditions['Attribute.event_id'] = $eventId;
}
if ($attributeId) {
$attributeConditions['Attribute.id'] = $attributeId;
}
$query = [
'recursive' => -1,
'conditions' => $attributeConditions,
// fetch just necessary fields to save memory
'fields' => $this->Correlation->getFieldRules(),
'order' => 'Attribute.id',
'limit' => 5000,
'callbacks' => false, // memory leak fix
];
do {
$attributes = $this->find('all', $query);
foreach ($attributes as $attribute) {
$attribute['Attribute']['event_id'] = $eventId;
if ($full) {
$this->Correlation->beforeSaveCorrelation($attribute['Attribute']);
}
$this->Correlation->afterSaveCorrelation($attribute['Attribute'], $full);
}
$fetchedAttributes = count($attributes);
unset($attributes);
$attributeCount += $fetchedAttributes;
if ($fetchedAttributes === 5000) { // maximum number of attributes fetched, continue in next loop
$query['conditions']['Attribute.id >'] = $attribute['Attribute']['id'];
} else {
break;
}
} while (true);
// Generating correlations can take long time, so clear CIDR cache after each event to refresh cache
$this->Correlation->clearCidrCache();
return $attributeCount;
}
public function purgeCorrelations($eventId = false)
{
if (!$eventId) {
$this->query('TRUNCATE TABLE correlations;');
} else {
$this->Correlation->deleteAll([
'OR' => array(
'Correlation.1_event_id' => $eventId,
'Correlation.event_id' => $eventId,
)
], false);
}
$this->Correlation->purgeCorrelations($eventId);
}
public function reportValidationIssuesAttributes($eventId)
@ -1725,8 +1659,13 @@ class Attribute extends AppModel
return $result;
}
// This method takes a string from an argument with several elements (separated by '&&' and negated by '!') and returns 2 arrays
// array 1 will have all of the non negated terms and array 2 all the negated terms
/**
* This method takes a string from an argument with several elements (separated by '&&' and negated by '!') and returns 2 arrays
* array 1 will have all of the non negated terms and array 2 all the negated terms
*
* @param string|array $args
* @return array[]
*/
public function dissectArgs($args)
{
$result = array(0 => array(), 1 => array(), 2 => array());
@ -1748,7 +1687,7 @@ class Attribute extends AppModel
}
} else {
foreach ($args as $arg) {
if ($arg[0] === '!') {
if (is_string($arg) && $arg[0] === '!') {
$result[1][] = substr($arg, 1);
} else {
$result[0][] = $arg;
@ -2039,7 +1978,6 @@ class Attribute extends AppModel
$params = array(
'conditions' => $this->buildConditions($user),
'fields' => array(),
'recursive' => -1,
'contain' => ['Event', 'Object'], // by default include Event and Object, because it is required for conditions
);
if (isset($options['conditions'])) {
@ -2278,7 +2216,7 @@ class Attribute extends AppModel
$this->attachTagsToAttributes($results, $options);
$proposals_block_attributes = Configure::read('MISP.proposals_block_attributes');
$sgids = $this->SharingGroup->authorizedIds($user);
foreach ($results as &$attribute) {
if (!empty($options['includeContext'])) {
$attribute['Event'] = $eventsById[$attribute['Attribute']['event_id']];
@ -2290,7 +2228,7 @@ class Attribute extends AppModel
}
if (!empty($options['includeCorrelations'])) {
$attributeFields = array('id', 'event_id', 'object_id', 'object_relation', 'category', 'type', 'value', 'uuid', 'timestamp', 'distribution', 'sharing_group_id', 'to_ids', 'comment');
$attribute['Attribute']['RelatedAttribute'] = $this->getRelatedAttributes($user, $attribute['Attribute'], $attributeFields, true);
$attribute['Attribute']['RelatedAttribute'] = $this->Correlation->getRelatedAttributes($user, $sgids, $attribute['Attribute'], $attributeFields, true);
}
if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttribute($attribute['Attribute'])) {
continue;
@ -2467,27 +2405,77 @@ class Attribute extends AppModel
}
$temp = $this->Event->EventTag->find('all', array(
'recursive' => -1,
'contain' => array('Tag'),
'contain' => ['Tag' => ['fields' => ['id', 'name', 'colour', 'numerical_value']]],
'conditions' => $tagConditions,
));
if (empty($temp)) {
$eventTags[$eventId] = [];
} else {
foreach ($temp as $tag) {
$tag['Tag']['inherited'] = true;
$tag['EventTag']['Tag'] = $tag['Tag'];
unset($tag['Tag']);
$eventTags[$eventId][] = $tag['EventTag'];
}
}
}
if (!empty($eventTags)) {
foreach ($eventTags[$eventId] as $eventTag) {
$attribute['EventTag'][] = $eventTag;
}
}
$attribute['EventTag'] = $eventTags[$eventId];
return $attribute;
}
public function touch($attribute_id)
{
$attribute = $this->find('first', [
'conditions' => ['Attribute.id' => $attribute_id],
'recursive' => -1,
]);
$event = $this->Event->find('first', [
'conditions' => ['Event.id' => $attribute['Attribute']['event_id']],
'recursive' => -1,
]);
$timestamp = (new DateTime())->getTimestamp();
$event['Event']['published'] = 0;
$event['Event']['timestamp'] = $timestamp;
$attribute['Attribute']['timestamp'] = $timestamp;
$saveSucces = true;
if ($attribute['Attribute']['object_id'] != 0) {
$saveSucces = $this->Attribute->Object->updateTimestamp($attribute['Attribute']['object_id'], $timestamp);
}
$saveSucces = $saveSucces && $this->save($attribute['Attribute'], true, ['timestamp']);
return $saveSucces && $this->Event->save($event, true, ['timestamp', 'published']);
}
public function attachTagsFromAttributeAndTouch($attribute_id, $event_id, $tags)
{
$touchAttribute = false;
$success = false;
foreach ($tags as $tag_id) {
$nothingToChange = false;
$saveSuccess = $this->AttributeTag->attachTagToAttribute($attribute_id, $event_id, $tag_id, false, $nothingToChange);
$success = $success || !empty($saveSuccess);
$touchAttribute = $touchAttribute || !$nothingToChange;
}
if ($touchAttribute) {
return $this->touch($attribute_id);
}
return $success;
}
public function detachTagsFromAttributeAndTouch($attribute_id, $event_id, $tags)
{
$touchAttribute = false;
$success = false;
foreach ($tags as $tag_id) {
$nothingToChange = false;
$saveSuccess = $this->AttributeTag->detachTagFromAttribute($attribute_id, $event_id, $tag_id, $nothingToChange);
$success = $success || !empty($saveSuccess);
$touchAttribute = $touchAttribute || !$nothingToChange;
}
if ($touchAttribute) {
return $this->touch($attribute_id);
}
return $success;
}
private function __blockAttributeViaProposal($attribute)
{
if (!empty($attribute['ShadowAttribute'])) {
@ -2681,8 +2669,12 @@ class Attribute extends AppModel
public function setTimestampConditions($timestamp, $conditions, $scope = 'Event.timestamp', $returnRaw = false)
{
if (is_array($timestamp)) {
$timestamp[0] = intval($this->Event->resolveTimeDelta($timestamp[0]));
$timestamp[1] = intval($this->Event->resolveTimeDelta($timestamp[1]));
if (count($timestamp) !== 2) {
throw new InvalidArgumentException('Invalid date specification, must be string or array with two elements');
}
$timestamp[0] = intval($this->resolveTimeDelta($timestamp[0]));
$timestamp[1] = intval($this->resolveTimeDelta($timestamp[1]));
if ($timestamp[0] > $timestamp[1]) {
$temp = $timestamp[0];
$timestamp[0] = $timestamp[1];
@ -2691,7 +2683,7 @@ class Attribute extends AppModel
$conditions['AND'][] = array($scope . ' >=' => $timestamp[0]);
$conditions['AND'][] = array($scope . ' <=' => $timestamp[1]);
} else {
$timestamp = intval($this->Event->resolveTimeDelta($timestamp));
$timestamp = intval($this->resolveTimeDelta($timestamp));
$conditions['AND'][] = array($scope . ' >=' => $timestamp);
}
if ($returnRaw) {
@ -2703,8 +2695,8 @@ class Attribute extends AppModel
public function setTimestampSeenConditions($timestamp, $conditions, $scope = 'Attribute.first_seen', $returnRaw = false)
{
if (is_array($timestamp)) {
$timestamp[0] = intval($this->Event->resolveTimeDelta($timestamp[0])) * 1000000; // seen in stored in micro-seconds in the DB
$timestamp[1] = intval($this->Event->resolveTimeDelta($timestamp[1])) * 1000000; // seen in stored in micro-seconds in the DB
$timestamp[0] = intval($this->resolveTimeDelta($timestamp[0])) * 1000000; // seen in stored in micro-seconds in the DB
$timestamp[1] = intval($this->resolveTimeDelta($timestamp[1])) * 1000000; // seen in stored in micro-seconds in the DB
if ($timestamp[0] > $timestamp[1]) {
$temp = $timestamp[0];
$timestamp[0] = $timestamp[1];
@ -2713,7 +2705,7 @@ class Attribute extends AppModel
$conditions['AND'][] = array($scope . ' >=' => $timestamp[0]);
$conditions['AND'][] = array($scope . ' <=' => $timestamp[1]);
} else {
$timestamp = intval($this->Event->resolveTimeDelta($timestamp)) * 1000000; // seen in stored in micro-seconds in the DB
$timestamp = intval($this->resolveTimeDelta($timestamp)) * 1000000; // seen in stored in micro-seconds in the DB
if ($scope == 'Attribute.first_seen') {
$conditions['AND'][] = array($scope . ' >=' => $timestamp);
} else {
@ -3237,7 +3229,6 @@ class Attribute extends AppModel
$filters['wildcard'] = $filters['searchall'];
}
}
$subqueryElements = $this->Event->harvestSubqueryElements($filters);
$filters = $this->Event->addFiltersFromSubqueryElements($filters, $subqueryElements, $user);
$filters = $this->Event->addFiltersFromUserSettings($user, $filters);

8
app/Model/AttributeTag.php
View File

@ -135,7 +135,7 @@ class AttributeTag extends AppModel
* @return bool
* @throws Exception
*/
public function attachTagToAttribute($attribute_id, $event_id, $tag_id, $local = false)
public function attachTagToAttribute($attribute_id, $event_id, $tag_id, $local = false, &$nothingToChange = false)
{
$existingAssociation = $this->hasAny([
'tag_id' => $tag_id,
@ -152,11 +152,13 @@ class AttributeTag extends AppModel
if (!$this->save($data)) {
return false;
}
} else {
$nothingToChange = true;
}
return true;
}
public function detachTagFromAttribute($attribute_id, $event_id, $tag_id)
public function detachTagFromAttribute($attribute_id, $event_id, $tag_id, &$nothingToChange = false)
{
$existingAssociation = $this->find('first', array(
'recursive' => -1,
@ -173,6 +175,8 @@ class AttributeTag extends AppModel
if ($result) {
return true;
}
} else {
$nothingToChange = true;
}
return false;
}

1
app/Model/AuditLog.php
View File

@ -33,6 +33,7 @@ class AuditLog extends AppModel
public $actsAs = [
'Containable',
'LightPaginator'
];
/** @var array|null */

594
app/Model/Behavior/DefaultCorrelationBehavior.php Normal file
View File

@ -0,0 +1,594 @@
<?php
App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
/**
* Default correlation behaviour
*/
class DefaultCorrelationBehavior extends ModelBehavior
{
private $__tableName = 'default_correlations';
private $__config = [
'AttributeFetcher' => [
'fields' => [
'Attribute.event_id',
'Attribute.object_id',
'Attribute.id',
'Attribute.type',
'Attribute.distribution',
'Attribute.sharing_group_id',
'Attribute.value1',
'Attribute.value2',
],
'contain' => [
'Event' => [
'fields' => [
'Event.id',
'Event.org_id',
'Event.distribution',
'Event.sharing_group_id',
'Event.disable_correlation',
]
],
'Object' => [
'fields' => [
'Object.id',
'Object.distribution',
'Object.sharing_group_id',
]
]
],
]
];
public $Correlation = null;
private $deadlockAvoidance = false;
public function setup(Model $Model, $settings = []) {
$Model->useTable = $this->__tableName;
$this->Correlation = $Model;
$this->deadlockAvoidance = $settings['deadlockAvoidance'];
}
public function getTableName(Model $Model)
{
return $this->__tableName;
}
public function createCorrelationEntry(Model $Model, $value, $a, $b) {
$value_id = $this->Correlation->CorrelationValue->getValueId($value);
if ($this->deadlockAvoidance) {
return [
'value_id' => $value_id,
'1_event_id' => $a['Event']['id'],
'1_object_id' => $a['Attribute']['object_id'],
'1_attribute_id' => $a['Attribute']['id'],
'1_org_id' => $a['Event']['org_id'],
'1_distribution' => $a['Attribute']['distribution'],
'1_event_distribution' => $a['Event']['distribution'],
'1_object_distribution' => empty($a['Attribute']['object_id']) ? 0 : $a['Object']['distribution'],
'1_sharing_group_id' => $a['Attribute']['sharing_group_id'],
'1_event_sharing_group_id' => $a['Event']['sharing_group_id'],
'1_object_sharing_group_id' => empty($a['Attribute']['object_id']) ? 0 : $a['Object']['sharing_group_id'],
'event_id' => $b['Event']['id'],
'object_id' => $b['Attribute']['object_id'],
'attribute_id' => $b['Attribute']['id'],
'org_id' => $b['Event']['org_id'],
'distribution' => $b['Attribute']['distribution'],
'event_distribution' => $b['Event']['distribution'],
'object_distribution' => empty($b['Attribute']['object_id']) ? 0 : $b['Object']['distribution'],
'sharing_group_id' => $b['Attribute']['sharing_group_id'],
'event_sharing_group_id' => $b['Event']['sharing_group_id'],
'object_sharing_group_id' => empty($b['Attribute']['object_id']) ? 0 : $b['Object']['sharing_group_id'],
];
} else {
return [
(int) $value_id,
(int) $a['Event']['id'],
(int) $a['Attribute']['object_id'],
(int) $a['Attribute']['id'],
(int) $a['Event']['org_id'],
(int) $a['Attribute']['distribution'],
(int) $a['Event']['distribution'],
(int) empty($a['Attribute']['object_id']) ? 0 : $a['Object']['distribution'],
(int) $a['Attribute']['sharing_group_id'],
(int) $a['Event']['sharing_group_id'],
(int) empty($a['Attribute']['object_id']) ? 0 : $a['Object']['sharing_group_id'],
(int) $b['Event']['id'],
(int) $b['Attribute']['object_id'],
(int) $b['Attribute']['id'],
(int) $b['Event']['org_id'],
(int) $b['Attribute']['distribution'],
(int) $b['Event']['distribution'],
(int) empty($b['Attribute']['object_id']) ? 0 : $b['Object']['distribution'],
(int) $b['Attribute']['sharing_group_id'],
(int) $b['Event']['sharing_group_id'],
(int) empty($b['Attribute']['object_id']) ? 0 : $b['Object']['sharing_group_id']
];
}
}
public function saveCorrelations(Model $Model, $correlations)
{
$fields = [
'value_id',
'1_event_id',
'1_object_id',
'1_attribute_id',
'1_org_id',
'1_distribution',
'1_event_distribution',
'1_object_distribution',
'1_sharing_group_id',
'1_event_sharing_group_id',
'1_object_sharing_group_id',
'event_id',
'object_id',
'attribute_id',
'org_id',
'distribution',
'event_distribution',
'object_distribution',
'sharing_group_id',
'event_sharing_group_id',
'object_sharing_group_id'
];
if ($this->deadlockAvoidance) {
return $this->Correlation->saveMany($correlations, array(
'atomic' => false,
'callbacks' => false,
'deep' => false,
'validate' => false,
'fieldList' => $fields
));
} else {
$db = $this->Correlation->getDataSource();
// Split to chunks datasource is is enabled
if (count($correlations) > 100) {
foreach (array_chunk($correlations, 100) as $chunk) {
$db->insertMulti('default_correlations', $fields, $chunk);
}
return true;
} else {
return $db->insertMulti('default_correlations', $fields, $correlations);
}
}
}
public function runBeforeSaveCorrelation(Model $Model, $attribute) {
// (update-only) clean up the relation of the old value: remove the existing relations related to that attribute, we DO have a reference, the id
// ==> DELETE FROM default_correlations WHERE 1_attribute_id = $a_id OR attribute_id = $a_id; */
// first check if it's an update
if (isset($attribute['id'])) {
$Model->deleteAll([
'OR' => [
'1_attribute_id' => $attribute['id'],
'attribute_id' => $attribute['id']
],
], false);
}
if ($attribute['type'] === 'ssdeep') {
$this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep');
$this->FuzzyCorrelateSsdeep->purge(null, $attribute['id']);
}
}
public function getContainRules(Model $Model, $filter = null)
{
if (empty($filter)) {
return $this->__config['AttributeFetcher']['contain'];
} else {
return empty($this->__config['AttributeFetcher']['contain'][$filter]) ? false : $this->__config['AttributeFetcher']['contain'][$filter];
}
}
public function getFieldRules(Model $Model)
{
return $this->__config['AttributeFetcher']['fields'];
}
private function __collectCorrelations($user, $id, $sgids, $primary)
{
$max_correlations = Configure::read('MISP.max_correlations_per_event') ?: 5000;
$source = $primary ? '' : '1_';
$prefix = $primary ? '1_' : '';
$correlations = $this->Correlation->find('all', array(
'fields' => [
$source . 'attribute_id',
$prefix . 'attribute_id',
$prefix . 'org_id',
$prefix . 'event_id',
$prefix . 'event_distribution',
$prefix . 'event_sharing_group_id',
$prefix . 'object_id',
$prefix . 'object_distribution',
$prefix . 'object_sharing_group_id',
$prefix . 'distribution',
$prefix . 'sharing_group_id'
],
'conditions' => [
'OR' => [
$source . 'event_id' => $id
],
'AND' => [
[
'CorrelationValue.value NOT IN (select value from correlation_exclusions)'
],
[
'CorrelationValue.value NOT IN (select value from over_correlating_values)'
]
]
],
'recursive' => -1,
'contain' => [
'CorrelationValue' => [
'fields' => [
'CorrelationValue.id',
'CorrelationValue.value'
]
]
],
'order' => false,
'limit' => $max_correlations
));
foreach ($correlations as $k => &$correlation) {
if (!$this->checkCorrelationACL($user, $correlation['Correlation'], $sgids, $prefix)) {
unset($correlations[$k]);
}
}
$correlations = array_values($correlations);
return $correlations;
}
public function runGetAttributesRelatedToEvent(Model $Model, $user, $id, $sgids)
{
$temp_correlations = $this->__collectCorrelations($user, $id, $sgids, false);
$temp_correlations_1 = $this->__collectCorrelations($user, $id, $sgids, true);
$correlations = [];
$event_ids = [];
foreach ($temp_correlations as $temp_correlation) {
$correlations[] = [
'id' => $temp_correlation['Correlation']['event_id'],
'attribute_id' => $temp_correlation['Correlation']['attribute_id'],
'parent_id' => $temp_correlation['Correlation']['1_attribute_id'],
'value' => $temp_correlation['CorrelationValue']['value']
];
$event_ids[$temp_correlation['Correlation']['event_id']] = true;
}
foreach ($temp_correlations_1 as $temp_correlation) {
$correlations[] = [
'id' => $temp_correlation['Correlation']['1_event_id'],
'attribute_id' => $temp_correlation['Correlation']['1_attribute_id'],
'parent_id' => $temp_correlation['Correlation']['attribute_id'],
'value' => $temp_correlation['CorrelationValue']['value']
];
$event_ids[$temp_correlation['Correlation']['1_event_id']] = true;
}
if (empty($correlations)) {
return [];
}
$conditions = $Model->Event->createEventConditions($user);
$conditions['Event.id'] = array_keys($event_ids);
$events = $Model->Event->find('all', [
'recursive' => -1,
'conditions' => $conditions,
'fields' => ['Event.id', 'Event.orgc_id', 'Event.info', 'Event.date'],
]);
$events = array_column(array_column($events, 'Event'), null, 'id');
$relatedAttributes = [];
foreach ($correlations as $correlation) {
$eventId = $correlation['id'];
if (!isset($events[$eventId])) {
continue;
}
$event = $events[$eventId];
$correlation['org_id'] = $events[$eventId]['orgc_id'];
$correlation['info'] = $events[$eventId]['info'];
$correlation['date'] = $events[$eventId]['date'];
$parentId = $correlation['parent_id'];
unset($correlation['parent_id']);
$relatedAttributes[$parentId][] = $correlation;
}
return $relatedAttributes;
}
public function runGetRelatedAttributes(Model $Model, $user, $sgids, $attribute, $fields = [], $includeEventData = false)
{
// LATER getRelatedAttributes($attribute) this might become a performance bottleneck
// prepare the conditions
$conditions = [
[
'Correlation.1_event_id !=' => $attribute['event_id'],
'Correlation.attribute_id' => $attribute['id']
],
[
'Correlation.event_id !=' => $attribute['event_id'],
'Correlation.1_attribute_id' => $attribute['id']
]
];
$corr_fields = [
[
'1_attribute_id',
'1_object_id',
'1_event_id',
'1_distribution',
'1_object_distribution',
'1_event_distribution',
'1_sharing_group_id',
'1_object_sharing_group_id',
'1_event_sharing_group_id',
'1_org_id',
'value_id'
],
[
'attribute_id',
'object_id',
'event_id',
'distribution',
'object_distribution',
'event_distribution',
'sharing_group_id',
'object_sharing_group_id',
'event_sharing_group_id',
'org_id',
'value_id'
]
];
$prefixes = ['1_', ''];
$correlated_attribute_ids = [];
foreach ($conditions as $k => $condition) {
$temp_correlations = $Model->find('all', [
'recursive' => -1,
'conditions' => $condition,
'fields' => $corr_fields[$k]
]);
if (!empty($temp_correlations)) {
foreach ($temp_correlations as $temp_correlation) {
if (empty($user['Role']['perm_site_admin'])) {
if (!$this->checkCorrelationACL($user, $temp_correlation, $sgids, $prefixes[$k])) {
continue;
}
}
$correlated_attribute_ids[] = $temp_correlation['Correlation'][$prefixes[$k] . 'attribute_id'];
}
}
}
$contain = [];
if (!empty($includeEventData)) {
$contain['Event'] = [
'fields' => [
'Event.id',
'Event.uuid',
'Event.threat_level_id',
'Event.analysis',
'Event.info',
'Event.extends_uuid',
'Event.distribution',
'Event.sharing_group_id',
'Event.published',
'Event.date',
'Event.orgc_id',
'Event.org_id'
]
];
}
$relatedAttributes = $Model->Attribute->find('all', [
'recursive' => -1,
'conditions' => [
'Attribute.id' => $correlated_attribute_ids
],
'fields' => $fields,
'contain' => $contain
]);
if (!empty($includeEventData)) {
$results = [];
foreach ($relatedAttributes as $k => $attribute) {
$temp = $attribute['Attribute'];
$temp['Event'] = $attribute['Event'];
$results[] = $temp;
}
return $results;
} else {
return $relatedAttributes;
}
}
public function fetchRelatedEventIds(Model $Model, array $user, int $eventId, array $sgids)
{
// search the correlation table for the event ids of the related events
// Rules:
// 1. Event is owned by the user (org_id matches)
// 2. User is allowed to see both the event and the org:
// a. Event:
// i. Event has a distribution between 1-3 (community only, connected communities, all orgs)
// ii. Event has a sharing group that the user is accessible to view
// b. Attribute:
// i. Attribute has a distribution of 5 (inheritance of the event, for this the event check has to pass anyway)
// ii. Atttibute has a distribution between 1-3 (community only, connected communities, all orgs)
// iii. Attribute has a sharing group that the user is accessible to view
$primaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, $sgids, true);
$secondaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, $sgids, false);
return array_unique(array_merge($primaryEventIds,$secondaryEventIds));
}
private function __filterRelatedEvents(Model $Model, array $user, int $eventId, array $sgids, bool $primary)
{
$current = $primary ? '' : '1_';
$prefix = $primary ? '1_' : '';
$correlations = $Model->find('all', [
'recursive' => -1,
'fields' => [
$prefix . 'org_id',
$prefix . 'event_id',
$prefix . 'event_distribution',
$prefix . 'event_sharing_group_id',
$prefix . 'object_id',
$prefix . 'object_distribution',
$prefix . 'object_sharing_group_id',
$prefix . 'distribution',
$prefix . 'sharing_group_id'
],
'conditions' => [
$current . 'event_id' => $eventId
],
'unique' => true,
]);
$eventIds = [];
if (empty($user['Role']['perm_site_admin'])) {
foreach ($correlations as $k => $correlation) {
// if we have already added this event as a valid target, no need to check again.
if (isset($eventIds[$correlation['Correlation'][$prefix . 'event_id']])) {
continue;
}
$correlation = $correlation['Correlation'];
if (!$this->checkCorrelationACL($user, $correlation, $sgids, $prefix)) {
unset($correlations[$k]);
continue;
}
$eventIds[$correlation[$prefix . 'event_id']] = true;
}
return array_keys($eventIds);
} else {
$eventIds = Hash::extract($correlations, '{n}.Correlation.' . $prefix . 'event_id');
return $eventIds;
}
}
private function checkCorrelationACL($user, $correlation, $sgids, $prefix)
{
if ($user['Role']['perm_site_admin']) {
return true;
}
// check if user can see the event
if (isset($correlation['Correlation'])) {
$correlation = $correlation['Correlation'];
}
if (
$correlation[$prefix . 'org_id'] != $user['org_id'] &&
(
$correlation[$prefix . 'event_distribution'] == 0 ||
(
$correlation[$prefix . 'event_distribution'] == 4 &&
!in_array($correlation[$prefix . 'event_sharing_group_id'], $sgids)
)
)
) {
return false;
}
//check if the user can see the object, if we're looking at an object attribute
if (
$correlation[$prefix . 'object_id'] &&
(
$correlation[$prefix . 'object_distribution'] == 0 ||
$correlation[$prefix . 'object_distribution'] == 5 ||
(
$correlation[$prefix . 'object_distribution'] == 4 &&
!in_array($correlation[$prefix . 'object_sharing_group_id'], $sgids)
)
)
) {
return false;
}
//check if the user can see the attribute
if (
(
$correlation[$prefix . 'distribution'] == 0 ||
(
$correlation[$prefix . 'distribution'] == 4 &&
!in_array($correlation[$prefix . 'sharing_group_id'], $sgids)
)
)
) {
return false;
}
return true;
}
public function updateContainedCorrelations(
Model $Model,
array $data,
string $type = 'event',
array $options = []
): bool
{
$updateCorrelation = [];
$updateFields = [
'Correlation.' . $type . '_id',
'Correlation.1_' . $type . '_id'
];
if (
isset($data['distribution']) &&
(
empty($options['fieldList']) ||
in_array('distribution', $options['fieldList'])
)
) {
$updateCorrelation[0]['Correlation.' . $type . '_distribution'] = (int)$data['distribution'];
$updateCorrelation[1]['Correlation.1_' . $type . '_distribution'] = (int)$data['distribution'];
}
if (
isset($data['sharing_group_id']) &&
(
empty($options['fieldList']) ||
in_array('sharing_group_id', $options['fieldList'])
)
) {
$updateCorrelation[0]['Correlation.' . $type . '_sharing_group_id'] = (int)$data['sharing_group_id'];
$updateCorrelation[1]['Correlation.1_' . $type . '_sharing_group_id'] = (int)$data['sharing_group_id'];
}
if (!empty($updateCorrelation)) {
foreach ($updateCorrelation as $k => $side) {
$Model->updateAll(
$side,
[
$updateFields[$k] => (int)$data['id']]
);
}
}
return true;
}
public function purgeCorrelations(Model $Model, $eventId = null): void
{
if (!$eventId) {
$Model->query('TRUNCATE TABLE default_correlations;');
//$Model->query('TRUNCATE TABLE correlation_values;');
//$Model->query('TRUNCATE TABLE over_correlating_values;');
} else {
$Model->deleteAll([
'OR' => array(
'Correlation.1_event_id' => $eventId,
'Correlation.event_id' => $eventId,
)
], false);
}
}
public function purgeByValue(Model $Model, string $value): void
{
$valueIds = $Model->CorrelationValue->find('column', [
'recursive' => -1,
'conditions' => [
'OR' => [
'CorrelationValue.value LIKE' => '%' . $value,
'CorrelationValue.value LIKE' => $value . '%'
]
],
'fields' => [
'CorrelationValue.id'
]
]);
$Model->deleteAll([
'Correlation.value_id' => $valueIds
]);
}
}

19
app/Model/Behavior/LightPaginatorBehavior.php Normal file
View File

@ -0,0 +1,19 @@
<?php
App::uses('LightPagination', 'Model');
/**
* Behavior to change default pagination to a lighter one
*/
class LightPaginatorBehavior extends ModelBehavior
{
// Avoid getting the count of the whole result set
public function paginateCount(
Model $model,
$conditions = null,
$recursive = 0,
$extra = []
) {
return PHP_INT_MAX; // Hack to make PaginatorComponent::paginate() think there is a next page
}
}

389
app/Model/Behavior/NoAclCorrelationBehavior.php Normal file
View File

@ -0,0 +1,389 @@
<?php
App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
/**
* ACL-less correlation behaviour for end-point instances
*/
class NoAclCorrelationBehavior extends ModelBehavior
{
private $__tableName = 'no_acl_correlations';
private $__config = [
'AttributeFetcher' => [
'fields' => [
'Attribute.event_id',
'Attribute.id',
'Attribute.type',
'Attribute.value1',
'Attribute.value2',
],
'contain' => [
'Event' => [
'fields' => [
'Event.id',
'Event.disable_correlation',
]
]
],
]
];
public $Correlation = null;
private $deadlockAvoidance = false;
public function setup(Model $Model, $settings = []) {
$Model->useTable = $this->__tableName;
$this->Correlation = $Model;
$this->deadlockAvoidance = $settings['deadlockAvoidance'];
}
public function getTableName(Model $Model)
{
return $this->__tableName;
}
public function createCorrelationEntry(Model $Model, $value, $a, $b) {
$value_id = $this->Correlation->CorrelationValue->getValueId($value);
if ($this->deadlockAvoidance) {
return [
'value_id' => $value_id,
'1_event_id' => $a['Event']['id'],
'1_attribute_id' => $a['Attribute']['id'],
'event_id' => $b['Event']['id'],
'attribute_id' => $b['Attribute']['id']
];
} else {
return [
(int) $value_id,
(int) $a['Event']['id'],
(int) $a['Attribute']['id'],
(int) $b['Event']['id'],
(int) $b['Attribute']['id']
];
}
}
public function saveCorrelations(Model $Model, $correlations)
{
$fields = [
'value_id',
'1_event_id',
'1_attribute_id',
'event_id',
'attribute_id'
];
if ($this->deadlockAvoidance) {
return $this->Correlation->saveMany($correlations, array(
'atomic' => false,
'callbacks' => false,
'deep' => false,
'validate' => false,
'fieldList' => $fields
));
} else {
$db = $this->Correlation->getDataSource();
// Split to chunks datasource is is enabled
if (count($correlations) > 100) {
foreach (array_chunk($correlations, 100) as $chunk) {
$db->insertMulti('no_acl_correlations', $fields, $chunk);
}
return true;
} else {
return $db->insertMulti('no_acl_correlations', $fields, $correlations);
}
}
}
public function runBeforeSaveCorrelation(Model $Model, $attribute) {
// (update-only) clean up the relation of the old value: remove the existing relations related to that attribute, we DO have a reference, the id
// ==> DELETE FROM no_acl_correlations WHERE 1_attribute_id = $a_id OR attribute_id = $a_id; */
// first check if it's an update
if (isset($attribute['id'])) {
$Model->deleteAll([
'OR' => [
'1_attribute_id' => $attribute['id'],
'attribute_id' => $attribute['id']
],
], false);
}
if ($attribute['type'] === 'ssdeep') {
$this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep');
$this->FuzzyCorrelateSsdeep->purge(null, $attribute['id']);
}
}
public function getContainRules(Model $Model, $filter = null)
{
if (empty($filter)) {
return $this->__config['AttributeFetcher']['contain'];
} else {
return empty($this->__config['AttributeFetcher']['contain'][$filter]) ? false : $this->__config['AttributeFetcher']['contain'][$filter];
}
}
public function getFieldRules(Model $Model)
{
return $this->__config['AttributeFetcher']['fields'];
}
private function __collectCorrelations($user, $id, $primary)
{
$max_correlations = Configure::read('MISP.max_correlations_per_event') ?: 5000;
$source = $primary ? '' : '1_';
$prefix = $primary ? '1_' : '';
$correlations = $this->Correlation->find('all', [
'fields' => [
$source . 'attribute_id',
$prefix . 'attribute_id',
$prefix . 'event_id'
],
'conditions' => [
'OR' => [
$source . 'event_id' => $id
],
'AND' => [
[
'CorrelationValue.value NOT IN (select value from correlation_exclusions)'
],
[
'CorrelationValue.value NOT IN (select value from over_correlating_values)'
]
]
],
'recursive' => -1,
'contain' => [
'CorrelationValue' => [
'fields' => [
'CorrelationValue.id',
'CorrelationValue.value'
]
]
],
'order' => false,
'limit' => $max_correlations
]);
return $correlations;
}
public function runGetAttributesRelatedToEvent(Model $Model, $user, $id)
{
$temp_correlations = $this->__collectCorrelations($user, $id, false);
$temp_correlations_1 = $this->__collectCorrelations($user, $id, true);
$correlations = [];
$event_ids = [];
foreach ($temp_correlations as $temp_correlation) {
$correlations[] = [
'id' => $temp_correlation['Correlation']['event_id'],
'attribute_id' => $temp_correlation['Correlation']['attribute_id'],
'parent_id' => $temp_correlation['Correlation']['1_attribute_id'],
'value' => $temp_correlation['CorrelationValue']['value']
];
$event_ids[$temp_correlation['Correlation']['event_id']] = true;
}
foreach ($temp_correlations_1 as $temp_correlation) {
$correlations[] = [
'id' => $temp_correlation['Correlation']['1_event_id'],
'attribute_id' => $temp_correlation['Correlation']['1_attribute_id'],
'parent_id' => $temp_correlation['Correlation']['attribute_id'],
'value' => $temp_correlation['CorrelationValue']['value']
];
$event_ids[$temp_correlation['Correlation']['1_event_id']] = true;
}
if (empty($correlations)) {
return [];
}
$conditions = [
'Event.id' => array_keys($event_ids)
];
$events = $Model->Event->find('all', array(
'recursive' => -1,
'conditions' => $conditions,
'fields' => ['Event.id', 'Event.orgc_id', 'Event.info', 'Event.date'],
));
$events = array_column(array_column($events, 'Event'), null, 'id');
$relatedAttributes = [];
foreach ($correlations as $correlation) {
$eventId = $correlation['id'];
if (!isset($events[$eventId])) {
continue;
}
$event = $events[$eventId];
$correlation['org_id'] = $events[$eventId]['orgc_id'];
$correlation['info'] = $events[$eventId]['info'];
$correlation['date'] = $events[$eventId]['date'];
$parentId = $correlation['parent_id'];
unset($correlation['parent_id']);
$relatedAttributes[$parentId][] = $correlation;
}
return $relatedAttributes;
}
public function runGetRelatedAttributes(Model $Model, $user, $sgids, $attribute, $fields = [], $includeEventData = false)
{
// LATER getRelatedAttributes($attribute) this might become a performance bottleneck
// prepare the conditions
$conditions = [
[
'Correlation.1_event_id !=' => $attribute['event_id'],
'Correlation.attribute_id' => $attribute['id']
],
[
'Correlation.event_id !=' => $attribute['event_id'],
'Correlation.1_attribute_id' => $attribute['id']
]
];
$corr_fields = [
[
'1_attribute_id',
'1_event_id',
'value_id'
],
[
'attribute_id',
'event_id',
'value_id'
]
];
$prefixes = ['1_', ''];
$correlated_attribute_ids = [];
foreach ($conditions as $k => $condition) {
$temp_correlations = $Model->find('all', [
'recursive' => -1,
'conditions' => $condition,
'fields' => $corr_fields[$k]
]);
if (!empty($temp_correlations)) {
foreach ($temp_correlations as $temp_correlation) {
$correlated_attribute_ids[] = $temp_correlation['Correlation'][$prefixes[$k] . 'attribute_id'];
}
}
}
$contain = [];
if (!empty($includeEventData)) {
$contain['Event'] = [
'fields' => [
'Event.id',
'Event.uuid',
'Event.threat_level_id',
'Event.analysis',
'Event.info',
'Event.extends_uuid',
'Event.distribution',
'Event.sharing_group_id',
'Event.published',
'Event.date',
'Event.orgc_id',
'Event.org_id'
]
];
}
$relatedAttributes = $Model->Attribute->find('all', [
'recursive' => -1,
'conditions' => [
'Attribute.id' => $correlated_attribute_ids
],
'fields' => $fields,
'contain' => $contain
]);
if (!empty($includeEventData)) {
$results = [];
foreach ($relatedAttributes as $k => $attribute) {
$temp = $attribute['Attribute'];
$temp['Event'] = $attribute['Event'];
$results[] = $temp;
}
return $results;
} else {
return $relatedAttributes;
}
}
public function fetchRelatedEventIds(Model $Model, array $user, int $eventId, array $sgids)
{
// search the correlation table for the event ids of the related events
// Rules:
// 1. Event is owned by the user (org_id matches)
// 2. User is allowed to see both the event and the org:
// a. Event:
// i. Event has a distribution between 1-3 (community only, connected communities, all orgs)
// ii. Event has a sharing group that the user is accessible to view
// b. Attribute:
// i. Attribute has a distribution of 5 (inheritance of the event, for this the event check has to pass anyway)
// ii. Atttibute has a distribution between 1-3 (community only, connected communities, all orgs)
// iii. Attribute has a sharing group that the user is accessible to view
$primaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, true);
$secondaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, false);
return array_unique(array_merge($primaryEventIds,$secondaryEventIds));
}
private function __filterRelatedEvents(Model $Model, array $user, int $eventId, bool $primary)
{
$current = $primary ? '' : '1_';
$prefix = $primary ? '1_' : '';
$correlations = $Model->find('all', [
'recursive' => -1,
'fields' => [
$prefix . 'event_id'
],
'conditions' => [
$current . 'event_id' => $eventId
],
'unique' => true,
]);
$eventIds = Hash::extract($correlations, '{n}.Correlation.' . $prefix . 'event_id');
return $eventIds;
}
public function updateContainedCorrelations(
Model $Model,
array $data,
string $type = 'event',
array $options = []
): bool
{
// We don't care. No ACL means nothing to change.
return true;
}
public function purgeCorrelations(Model $Model, $eventId = null): void
{
if (!$eventId) {
$Model->query('TRUNCATE TABLE no_acl_correlations;');
//$Model->query('TRUNCATE TABLE correlation_values;');
//$Model->query('TRUNCATE TABLE over_correlating_values;');
} else {
$Model->deleteAll([
'OR' => array(
'Correlation.1_event_id' => $eventId,
'Correlation.event_id' => $eventId,
)
], false);
}
}
public function purgeByValue(Model $Model, string $value): void
{
$valueIds = $Model->CorrelationValue->find('column', [
'recursive' => -1,
'conditions' => [
'OR' => [
'CorrelationValue.value LIKE' => '%' . $value,
'CorrelationValue.value LIKE' => $value . '%'
]
],
'fields' => [
'CorrelationValue.id'
]
]);
$Model->deleteAll([
'Correlation.value_id' => $valueIds
]);
}
}

488
app/Model/Correlation.php
View File

@ -10,6 +10,8 @@ class Correlation extends AppModel
const CACHE_NAME = 'misp:top_correlations',
CACHE_AGE = 'misp:top_correlations_age';
private $__compositeTypes = [];
public $belongsTo = array(
'Attribute' => [
'className' => 'Attribute',
@ -18,7 +20,25 @@ class Correlation extends AppModel
'Event' => array(
'className' => 'Event',
'foreignKey' => 'event_id'
)
),
'Object' => array(
'className' => 'Object',
'foreignKey' => 'object_id'
),
'CorrelationValue' => [
'className' => 'CorrelationValue',
'foreignKey' => 'value_id'
]
);
public $validEngines = [
'Default' => 'default_correlations',
'NoAcl' => 'no_acl_correlations',
'Legacy' => 'correlations'
];
public $actsAs = array(
'Containable'
);
/** @var array */
@ -39,12 +59,26 @@ class Correlation extends AppModel
/** @var array */
private $cidrListCache;
private $__correlationEngine = 'DefaultCorrelation';
protected $_config = [];
private $__tempContainCache = [];
public $OverCorrelatingValue = null;
public function __construct($id = false, $table = null, $ds = null)
{
parent::__construct($id, $table, $ds);
$this->oldSchema = $this->schema('date') !== null;
$this->deadlockAvoidance = Configure::read('MISP.deadlock_avoidance');
$this->__correlationEngine = $this->getCorrelationModelName();
$this->deadlockAvoidance = Configure::check('MISP.deadlock_avoidance') ? Configure::read('MISP.deadlock_avoidance') : false;
// load the currently used correlation engine
$this->Behaviors->load($this->__correlationEngine . 'Correlation', ['deadlockAvoidance' => false]);
// getTableName() needs to be implemented by the engine - this points us to the table to be used
$this->useTable = $this->getTableName();
$this->advancedCorrelationEnabled = (bool)Configure::read('MISP.enable_advanced_correlations');
// load the overcorrelatingvalue model for chaining
$this->OverCorrelatingValue = ClassRegistry::init('OverCorrelatingValue');
}
public function correlateValueRouter($value)
@ -89,9 +123,9 @@ class Correlation extends AppModel
return null;
}
if (in_array($attribute['type'], ['ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port'], true)) {
if (in_array($attribute['Attribute']['type'], ['ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port'], true)) {
return $this->cidrCorrelation($attribute);
} else if ($attribute['type'] === 'ssdeep' && function_exists('ssdeep_fuzzy_compare')) {
} else if ($attribute['Attribute']['type'] === 'ssdeep' && function_exists('ssdeep_fuzzy_compare')) {
return $this->ssdeepCorrelation($attribute);
}
return null;
@ -102,7 +136,7 @@ class Correlation extends AppModel
if (!$this->advancedCorrelationEnabled) {
return [];
}
$extraConditions = $this->__buildAdvancedCorrelationConditions($correlatingAttribute['Attribute']);
$extraConditions = $this->__buildAdvancedCorrelationConditions($correlatingAttribute);
if (empty($extraConditions)) {
return [];
}
@ -117,25 +151,16 @@ class Correlation extends AppModel
'Attribute.deleted' => 0
],
'recursive' => -1,
'fields' => [
'Attribute.event_id',
'Attribute.id',
'Attribute.distribution',
'Attribute.sharing_group_id',
'Attribute.value1',
'Attribute.value2',
],
'contain' => [
'Event' => [
'fields' => ['Event.id', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id']
]
],
'fields' => $this->getFieldRules(),
'contain' => $this->getContainRules(),
'order' => [],
]);
}
private function __getMatchingAttributes($value)
{
// stupid hack to allow statically retrieving the constants
ClassRegistry::init('Attribute');
$conditions = [
'OR' => [
'Attribute.value1' => $value,
@ -154,20 +179,8 @@ class Correlation extends AppModel
$correlatingAttributes = $this->Attribute->find('all', [
'conditions' => $conditions,
'recursive' => -1,
'fields' => [
'Attribute.event_id',
'Attribute.id',
'Attribute.type',
'Attribute.distribution',
'Attribute.sharing_group_id',
'Attribute.value1',
'Attribute.value2',
],
'contain' => [
'Event' => [
'fields' => ['Event.id', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id']
]
],
'fields' => $this->getFieldRules(),
'contain' => $this->getContainRules(),
'order' => [],
]);
return $correlatingAttributes;
@ -179,35 +192,9 @@ class Correlation extends AppModel
* @param array $b Attribute B
* @return array
*/
private function __addCorrelationEntry($value, $a, $b)
private function __createCorrelationEntry($value, $a, $b)
{
if ($this->deadlockAvoidance) {
return [
'value' => $value,
'1_event_id' => $a['Event']['id'],
'1_attribute_id' => $a['Attribute']['id'],
'event_id' => $b['Event']['id'],
'attribute_id' => $b['Attribute']['id'],
'org_id' => $b['Event']['org_id'],
'distribution' => $b['Event']['distribution'],
'a_distribution' => $b['Attribute']['distribution'],
'sharing_group_id' => $b['Event']['sharing_group_id'],
'a_sharing_group_id' => $b['Attribute']['sharing_group_id'],
];
} else {
return [
$value,
(int) $a['Event']['id'],
(int) $a['Attribute']['id'],
(int) $b['Event']['id'],
(int) $b['Attribute']['id'],
(int) $b['Event']['org_id'],
(int) $b['Event']['distribution'],
(int) $b['Attribute']['distribution'],
(int) $b['Event']['sharing_group_id'],
(int) $b['Attribute']['sharing_group_id'],
];
}
return $this->createCorrelationEntry($value, $a, $b);
}
public function correlateValue($value, $jobId = false)
@ -232,7 +219,7 @@ class Correlation extends AppModel
if ($correlatingAttribute['Attribute']['event_id'] === $correlatingAttribute2['Attribute']['event_id']) {
continue;
}
$correlations[] = $this->__addCorrelationEntry($value, $correlatingAttribute, $correlatingAttribute2);
$correlations[] = $this->__createCorrelationEntry($value, $correlatingAttribute, $correlatingAttribute2);
}
$extraCorrelations = $this->__addAdvancedCorrelations($correlatingAttribute);
if (!empty($extraCorrelations)) {
@ -240,8 +227,8 @@ class Correlation extends AppModel
if ($correlatingAttribute['Attribute']['event_id'] === $extraCorrelation['Attribute']['event_id']) {
continue;
}
$correlations[] = $this->__addCorrelationEntry($value, $correlatingAttribute, $extraCorrelation);
//$correlations = $this->__addCorrelationEntry($value, $extraCorrelation, $correlatingAttribute, $correlations);
$correlations[] = $this->__createCorrelationEntry($value, $correlatingAttribute, $extraCorrelation);
//$correlations = $this->__createCorrelationEntry($value, $extraCorrelation, $correlatingAttribute, $correlations);
}
}
if ($jobId && $k % 100 === 0) {
@ -258,74 +245,38 @@ class Correlation extends AppModel
* @param array $correlations
* @return array|bool|bool[]|mixed
*/
private function __saveCorrelations($correlations)
private function __saveCorrelations(array $correlations)
{
$fields = [
'value', '1_event_id', '1_attribute_id', 'event_id', 'attribute_id', 'org_id',
'distribution', 'a_distribution', 'sharing_group_id', 'a_sharing_group_id',
];
// In older MISP instances, correlations table contains also date and info columns, that stores information
// about correlated event title and date. But because this information can be fetched directly from Event table,
// it is not necessary to keep them there. The problem is that these columns are marked as not null, so they must
// be filled with value and removing these columns can take long time for big instances. So for new installation
// these columns doesn't exists anymore and we don't need to save dummy value into them. Also feel free to remove
// them from your instance.
if ($this->oldSchema) {
$fields[] = 'date';
$fields[] = 'info';
}
if ($this->deadlockAvoidance) {
if ($this->oldSchema) {
foreach ($correlations as &$correlation) {
$correlation['date'] = '1000-01-01'; // Dummy value
$correlation['info'] = ''; // Dummy value
}
}
return $this->saveMany($correlations, array(
'atomic' => false,
'callbacks' => false,
'deep' => false,
'validate' => false,
'fieldList' => $fields
));
} else {
if ($this->oldSchema) {
foreach ($correlations as &$correlation) {
$correlation[] = '1000-01-01'; // Dummy value
$correlation[] = ''; // Dummy value
}
}
$db = $this->getDataSource();
// Split to chunks datasource is is enabled
if (count($correlations) > 100) {
foreach (array_chunk($correlations, 100) as $chunk) {
$db->insertMulti('correlations', $fields, $chunk);
}
return true;
} else {
return $db->insertMulti('correlations', $fields, $correlations);
}
}
return $this->saveCorrelations($correlations);
}
public function beforeSaveCorrelation($attribute)
public function correlateAttribute(array $attribute): void
{
// (update-only) clean up the relation of the old value: remove the existing relations related to that attribute, we DO have a reference, the id
// ==> DELETE FROM correlations WHERE 1_attribute_id = $a_id OR attribute_id = $a_id; */
// first check if it's an update
if (isset($attribute['id'])) {
$this->deleteAll([
'OR' => [
'Correlation.1_attribute_id' => $attribute['id'],
'Correlation.attribute_id' => $attribute['id']
],
], false);
}
if ($attribute['type'] === 'ssdeep') {
$this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep');
$this->FuzzyCorrelateSsdeep->purge(null, $attribute['id']);
$this->runBeforeSaveCorrelation($attribute);
$this->afterSaveCorrelation($attribute);
}
public function beforeSaveCorrelation(array $attribute): void
{
$this->runBeforeSaveCorrelation($attribute);
}
private function __cachedGetContainData($scope, $id)
{
if (!empty($this->getContainRules($scope))) {
if (empty($this->__tempContainCache[$scope][$id])) {
$temp = $this->Attribute->$scope->find('first', array(
'recursive' => -1,
'fields' => $this->getContainRules($scope)['fields'],
'conditions' => ['id' => $id],
'order' => array(),
));
$temp = empty($temp) ? false : $temp[$scope];
$this->__tempContainCache[$scope][$id] = $temp;
return $temp;
} else {
return $this->__tempContainCache[$scope][$id];
}
}
}
@ -337,42 +288,48 @@ class Correlation extends AppModel
*/
public function afterSaveCorrelation($a, $full = false, $event = false)
{
if (!empty($a['disable_correlation']) || Configure::read('MISP.completely_disable_correlation')) {
$a = ['Attribute' => $a];
if (!empty($a['Attribute']['disable_correlation']) || Configure::read('MISP.completely_disable_correlation')) {
return true;
}
// Don't do any correlation if the type is a non correlating type
if (in_array($a['type'], Attribute::NON_CORRELATING_TYPES, true)) {
if (in_array($a['Attribute']['type'], Attribute::NON_CORRELATING_TYPES, true)) {
return true;
}
if (!$event) {
$event = $this->Attribute->Event->find('first', array(
'recursive' => -1,
'fields' => array('Event.distribution', 'Event.id', 'Event.org_id', 'Event.sharing_group_id', 'Event.disable_correlation'),
'conditions' => array('id' => $a['event_id']),
'order' => array(),
));
$a['Event'] = $this->__cachedGetContainData('Event', $a['Attribute']['event_id']);
if (!$a['Event']) {
// orphaned attribute, do not correlate
return true;
}
} else {
$a['Event'] = $event['Event'];
}
if (!empty($event['Event']['disable_correlation'])) {
if (!empty($a['Event']['disable_correlation'])) {
return true;
}
if (!empty($a['Attribute']['object_id'])) {
$a['Object'] = $this->__cachedGetContainData('Object', $a['Attribute']['object_id']);
if (!$a['Object']) {
// orphaned attribute, do not correlate
return true;
}
}
// generate additional correlating attribute list based on the advanced correlations
if (!$this->__preventExcludedCorrelations($a['value1'])) {
if (!$this->__preventExcludedCorrelations($a['Attribute']['value1'])) {
$extraConditions = $this->__buildAdvancedCorrelationConditions($a);
$correlatingValues = [$a['value1']];
$correlatingValues = [$a['Attribute']['value1']];
} else {
$extraConditions = null;
$correlatingValues = [null];
$correlatingValues = [];
}
if (!empty($a['value2']) && !in_array($a['type'], Attribute::PRIMARY_ONLY_CORRELATING_TYPES, true) && !$this->__preventExcludedCorrelations($a['value2'])) {
$correlatingValues[] = $a['value2'];
if (!empty($a['Attribute']['value2']) && !in_array($a['Attribute']['type'], Attribute::PRIMARY_ONLY_CORRELATING_TYPES, true) && !$this->__preventExcludedCorrelations($a['Attribute']['value2'])) {
$correlatingValues[] = $a['Attribute']['value2'];
}
if (empty($correlatingValues)) {
return true;
}
$attributeToProcess = ['Attribute' => $a, 'Event' => $event['Event']];
$correlations = [];
foreach ($correlatingValues as $k => $cV) {
if ($cV === null) {
@ -387,42 +344,48 @@ class Correlation extends AppModel
],
],
'NOT' => [
'Attribute.event_id' => $a['event_id'],
'Attribute.event_id' => $a['Attribute']['event_id'],
'Attribute.type' => Attribute::NON_CORRELATING_TYPES,
],
'Attribute.disable_correlation' => 0,
'Event.disable_correlation' => 0,
'Attribute.deleted' => 0,
];
$fields = ['Attribute.id', 'Attribute.distribution', 'Attribute.sharing_group_id'];
if ($k === 0 && !empty($extraConditions)) {
$conditions['OR'][] = $extraConditions;
// Fetch value field just when fetching attributes also by extra conditions, because then it can be
// not exact match
$fields[] = 'Attribute.value1';
$fields[] = 'Attribute.value2';
}
if ($full) {
$conditions['Attribute.id > '] = $a['id'];
}
$correlationLimit = $this->OverCorrelatingValue->getLimit();
$correlatingAttributes = $this->Attribute->find('all', [
'conditions' => $conditions,
'recursive' => -1,
'fields' => $fields,
'contain' => ['Event.id', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id'],
'fields' => $this->getFieldRules(),
'contain' => $this->getContainRules(),
'order' => [],
'callbacks' => 'before', // memory leak fix
// let's fetch the limit +1 - still allows us to detect overcorrelations, but we'll also never need more
'limit' => empty($correlationLimit) ? null : ($correlationLimit+1)
]);
foreach ($correlatingAttributes as $corr) {
if (isset($corr['Attribute']['value1'])) {
// Let's check if we don't have a case of an over-correlating attribute
$count = count($correlatingAttributes);
if ($count > $correlationLimit) {
// If we have more correlations for the value than the limit, set the block entry and stop the correlation process
$this->OverCorrelatingValue->block($cV, $count);
return true;
} else {
// If we have fewer hits than the limit, proceed with the correlation, but first make sure we remove any existing blockers
$this->OverCorrelatingValue->unblock($cV);
}
foreach ($correlatingAttributes as $b) {
if (isset($b['Attribute']['value1'])) {
// TODO: Currently it is hard to check if value1 or value2 correlated, so we check value2 and if not, it is value1
$value = $cV === $corr['Attribute']['value2'] ? $corr['Attribute']['value2'] : $corr['Attribute']['value1'];
$value = $cV === $b['Attribute']['value2'] ? $b['Attribute']['value2'] : $b['Attribute']['value1'];
} else {
$value = $cV;
}
$correlations[] = $this->__addCorrelationEntry($value, $attributeToProcess, $corr);
$correlations[] = $this->__addCorrelationEntry($cV, $corr, $attributeToProcess);
if ($a['Attribute']['id'] > $b['Attribute']['id']) {
$correlations[] = $this->__createCorrelationEntry($value, $a, $b);
} else {
$correlations[] = $this->__createCorrelationEntry($value, $b, $a);
}
}
}
if (empty($correlations)) {
@ -486,8 +449,8 @@ class Correlation extends AppModel
if (!isset($this->FuzzyCorrelateSsdeep)) {
$this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep');
}
$value = $attribute['value1'];
$fuzzyIds = $this->FuzzyCorrelateSsdeep->query_ssdeep_chunks($value, $attribute['id']);
$value = $attribute['Attribute']['value1'];
$fuzzyIds = $this->FuzzyCorrelateSsdeep->query_ssdeep_chunks($value, $attribute['Attribute']['id']);
if (!empty($fuzzyIds)) {
$ssdeepIds = $this->Attribute->find('list', array(
'recursive' => -1,
@ -517,7 +480,7 @@ class Correlation extends AppModel
private function cidrCorrelation($attribute)
{
$ipValues = array();
$ip = $attribute['value1'];
$ip = $attribute['Attribute']['value1'];
if (strpos($ip, '/') !== false) { // IP is CIDR
list($networkIp, $mask) = explode('/', $ip);
$ip_version = filter_var($networkIp, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) ? 4 : 6;
@ -703,7 +666,7 @@ class Correlation extends AppModel
$maxPage = ceil($maxId / $chunkSize);
for ($page = 0; $page < $maxPage; $page++) {
$correlations = $this->find('column', [
'fields' => ['value'],
'fields' => ['value_id'],
'conditions' => [
'id >' => $page * $chunkSize,
'id <=' => ($page + 1) * $chunkSize
@ -724,7 +687,7 @@ class Correlation extends AppModel
return true;
}
public function findTop(array $query)
public function findTop(array $query): array
{
try {
$redis = $this->setupRedisWithException();
@ -732,13 +695,20 @@ class Correlation extends AppModel
return false;
}
$start = $query['limit'] * ($query['page'] -1);
$end = $query['limit'] * $query['page'];
$end = $query['limit'] * $query['page'] - 1;
$list = $redis->zRevRange(self::CACHE_NAME, $start, $end, true);
$results = [];
foreach ($list as $value => $count) {
$realValue = $this->CorrelationValue->find('first',
[
'recursive' => -1,
'conditions' => ['CorrelationValue.id' => $value],
'fields' => 'CorrelationValue.value'
]
);
$results[] = [
'Correlation' => [
'value' => $value,
'value' => $realValue['CorrelationValue']['value'],
'count' => $count,
'excluded' => $this->__preventExcludedCorrelations($value),
]
@ -757,11 +727,22 @@ class Correlation extends AppModel
return $redis->get(self::CACHE_AGE);
}
/**
* @param array $attribute
* @return void
*/
public function advancedCorrelationsUpdate(array $attribute)
{
if ($this->advancedCorrelationEnabled && in_array($attribute['type'], ['ip-src', 'ip-dst'], true) && strpos($attribute['value'], '/')) {
$this->updateCidrList();
}
}
/**
* Get list of all CIDR for correlation from database
* @return array
*/
private function getCidrListFromDatabase()
private function getCidrListFromDatabase(): array
{
return $this->Attribute->find('column', [
'conditions' => [
@ -779,9 +760,9 @@ class Correlation extends AppModel
/**
* @return array
*/
public function updateCidrList()
public function updateCidrList(): array
{
$redis = $this->setupRedis();
$redis = $this->setupRedisWithException();
$cidrList = [];
$this->cidrListCache = null;
if ($redis) {
@ -804,7 +785,7 @@ class Correlation extends AppModel
/**
* @return void
*/
public function clearCidrCache()
public function clearCidrCache(): void
{
$this->cidrListCache = null;
}
@ -812,13 +793,13 @@ class Correlation extends AppModel
/**
* @return array
*/
public function getCidrList()
public function getCidrList(): array
{
if ($this->cidrListCache !== null) {
return $this->cidrListCache;
}
$redis = $this->setupRedis();
$redis = $this->setupRedisWithException();
if ($redis) {
if (!$redis->exists('misp:cidr_cache_list')) {
$cidrList = $this->updateCidrList();
@ -831,4 +812,155 @@ class Correlation extends AppModel
$this->cidrListCache = $cidrList;
return $cidrList;
}
/**
* @param array $user User array
* @param int $eventIds List of event IDs
* @param array $sgids List of sharing group IDs
* @return array
*/
public function getAttributesRelatedToEvent(array $user, $eventIds, array $sgids): array
{
return $this->runGetAttributesRelatedToEvent($user, $eventIds, $sgids);
}
/**
* @param array $user User array
* @param array $attribute Attribute Array
* @param array $fields List of fields to include
* @param bool $includeEventData Flag to include the event data in the response
* @return array
*/
public function getRelatedAttributes($user, $sgids, $attribute, $fields=[], $includeEventData = false): array
{
if (in_array($attribute['type'], Attribute::NON_CORRELATING_TYPES)) {
return [];
}
return $this->runGetRelatedAttributes($user, $sgids, $attribute, $fields, $includeEventData);
}
/**
* @param array $user User array
* @param int $eventIds List of event IDs
* @param array $sgids List of sharing group IDs
* @return array
*/
public function getRelatedEventIds(array $user, int $eventId, array $sgids): array
{
$relatedEventIds = $this->fetchRelatedEventIds($user, $eventId, $sgids);
if (empty($relatedEventIds)) {
return [];
}
return $relatedEventIds;
}
public function attachExclusionsToOverCorrelations($data)
{
foreach ($data as $k => $v) {
$data[$k]['OverCorrelatingValue']['excluded'] = $this->__preventExcludedCorrelations($data[$k]['OverCorrelatingValue']['value']);
}
return $data;
}
public function setCorrelationExclusion($attribute)
{
if (empty($this->__compositeTypes)) {
$this->__compositeTypes = $this->Attribute->getCompositeTypes();
}
$values = [$attribute['value']];
if (in_array($attribute['type'], $this->__compositeTypes)) {
$values = explode('|', $attribute['value']);
}
if ($this->__preventExcludedCorrelations($values[0])) {
$attribute['correlation_exclusion'] = true;
}
if (!empty($values[1]) && $this->__preventExcludedCorrelations($values[1])) {
$attribute['correlation_exclusion'] = true;
}
if ($this->OverCorrelatingValue->checkValue($values[0])) {
$attribute['over_correlation'] = true;
}
if (!empty($values[1]) && $this->OverCorrelatingValue->checkValue($values[1])) {
$attribute['over_correlation'] = true;
}
return $attribute;
}
public function collectMetrics()
{
$results['engine'] = $this->getCorrelationModelName();
$results['db'] = [
'Default' => [
'name' => __('Default correlation engine'),
'tables' => [
'default_correlations' => [
'id_limit' => 4294967295
],
'correlation_values' => [
'id_limit' => 4294967295
]
]
],
'NoAcl' => [
'name' => __('No ACL correlation engine'),
'tables' => [
'no_acl_correlations' => [
'id_limit' => 4294967295
],
'correlation_values' => [
'id_limit' => 4294967295
]
]
],
'Legacy' => [
'name' => __('Legacy correlation engine (< 2.4.160)'),
'tables' => [
'correlations' => [
'id_limit' => 2147483647
]
]
]
];
$results['over_correlations'] = $this->OverCorrelatingValue->find('count');
$this->CorrelationExclusion = ClassRegistry::init('CorrelationExclusion');
$results['excluded_correlations'] = $this->CorrelationExclusion->find('count');
foreach ($results['db'] as &$result) {
foreach ($result['tables'] as $table_name => &$table_data) {
$size_metrics = $this->query(sprintf('show table status like \'%s\';', $table_name));
if (!empty($size_metrics)) {
$table_data['size_on_disk'] = $this->query(
//'select FILE_SIZE from information_schema.innodb_sys_tablespaces where FILENAME like \'%/' . $table_name . '.ibd\';'
sprintf(
'select TABLE_NAME, ROUND((DATA_LENGTH + INDEX_LENGTH)) AS size FROM information_schema.TABLES where TABLE_SCHEMA="%s" AND TABLE_NAME="%s"',
$this->getDataSource()->config['database'],
$table_name
)
)[0][0]['size'];
$last_id = $this->query(sprintf('select max(id) as max_id from %s;', $table_name));
$table_data['row_count'] = $size_metrics[0]['TABLES']['Rows'];
$table_data['last_id'] = $last_id[0][0]['max_id'];
$table_data['id_saturation'] = round(100 * $table_data['last_id'] / $table_data['id_limit'], 2);
}
}
}
return $results;
}
public function truncate(array $user, string $engine)
{
$table = $this->validEngines[$engine];
$result = $this->query('truncate table ' . $table);
if ($result !== true) {
$this->loadLog()->createLogEntry(
$user,
'truncate',
'Correlation',
0,
'Could not truncate table ' . $table,
'Errors: ' . json_encode($result)
);
}
return $result === true;
}
}

33
app/Model/CorrelationExclusion.php
View File

@ -16,6 +16,8 @@ class CorrelationExclusion extends AppModel
'Containable',
);
private $__redis = null;
public $validate = [
'value' => [
'uniqueValue' => [
@ -52,15 +54,15 @@ class CorrelationExclusion extends AppModel
public function cacheValues()
{
try {
$redis = $this->setupRedisWithException();
$this->__redis = $this->setupRedisWithException();
} catch (Exception $e) {
return false;
}
$redis->del($this->key);
$this->__redis->del($this->key);
$exclusions = $this->find('column', [
'fields' => ['value']
]);
$redis->sAddArray($this->key, $exclusions);
$this->__redis->sAddArray($this->key, $exclusions);
}
public function cleanRouter($user)
@ -97,19 +99,16 @@ class CorrelationExclusion extends AppModel
$this->Job = ClassRegistry::init('Job');
$this->Job->id = $jobId;
}
$query = sprintf(
'DELETE FROM correlations where (%s) or (%s);',
sprintf(
'value IN (%s)',
'SELECT correlation_exclusions.value FROM correlation_exclusions WHERE correlations.value = correlation_exclusions.value'
),
sprintf(
'EXISTS (SELECT NULL FROM correlation_exclusions WHERE (%s) OR (%s))',
"correlations.value LIKE CONCAT('%', correlation_exclusions.value)",
"correlations.value LIKE CONCAT(correlation_exclusions.value, '%')"
)
);
$this->query($query);
$this->Job->saveProgress($jobId, 'Job done.', 100);
$values = $this->find('column', [
'recursive' => -1,
'fields' => ['value']
]);
$this->Correlation = ClassRegistry::init('Correlation');
foreach ($values as $value) {
$this->Correlation->purgeByValue($value);
}
if ($jobId) {
$this->Job->saveProgress($jobId, 'Job done.', 100);
}
}
}

59
app/Model/CorrelationValue.php Normal file
View File

@ -0,0 +1,59 @@
<?php
App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
class CorrelationValue extends AppModel
{
public $recursive = -1;
public $actsAs = array(
'Containable'
);
public $validate = [
];
public function getValueId($value)
{
// index is 191 long, missing the existing value lookup can lead to a duplicate entry
$value = mb_substr($value, 0, 191);
$existingValue = $this->find('first', [
'recursive' => -1,
'conditions' => [
'value' => $value
]
]);
if (empty($existingValue)) {
$this->create();
try {
$this->save(['value' => $value]);
return $this->id;
} catch (Exception $e) {
$existingValue = $this->find('first', [
'recursive' => -1,
'conditions' => [
'value' => $value
]
]);
return $existingValue['ExistingValue']['id'];
}
} else {
return $existingValue['CorrelationValue']['id'];
}
return false;
}
public function getValue($id)
{
$existingValue = $this->find('first', [
'recursive' => -1,
'conditions' => [
'id' => $id
]
]);
if (!empty($existingValue)) {
return $existingValue['CorrelationValue']['value'];
}
return false;
}
}

583
app/Model/Event.php
View File

@ -17,6 +17,7 @@ App::uses('ProcessTool', 'Tools');
* @property ThreatLevel $ThreatLevel
* @property Sighting $Sighting
* @property Organisation $Org
* @property Organisation $Orgc
* @property CryptographicKey $CryptographicKey
*/
class Event extends AppModel
@ -301,6 +302,9 @@ class Event extends AppModel
private $assetCache = [];
/** @var array|null */
private $eventBlockRule;
public function beforeDelete($cascade = true)
{
// blocklist the event UUID if the feature is enabled
@ -434,16 +438,7 @@ class Event extends AppModel
{
$event = $this->data['Event'];
if (!Configure::read('MISP.completely_disable_correlation') && !$created) {
$updateCorrelation = [];
if (isset($event['distribution']) && (empty($options['fieldList']) || in_array('distribution', $options['fieldList']))) {
$updateCorrelation['Correlation.distribution'] = (int)$event['distribution'];
}
if (isset($event['sharing_group_id']) && (empty($options['fieldList']) || in_array('sharing_group_id', $options['fieldList']))) {
$updateCorrelation['Correlation.sharing_group_id'] = (int)$event['sharing_group_id'];
}
if (!empty($updateCorrelation)) {
$this->Attribute->Correlation->updateAll($updateCorrelation, ['Correlation.event_id' => (int)$event['id']]);
}
$this->Attribute->Correlation->updateContainedCorrelations($event, 'event');
}
if (empty($event['unpublishAction']) && empty($event['skip_zmq']) && $this->pubToZmq('event')) {
$pubSubTool = $this->getPubSubTool();
@ -455,6 +450,17 @@ class Event extends AppModel
if (empty($event['unpublishAction']) && empty($event['skip_kafka'])) {
$this->publishKafkaNotification('event', $this->quickFetchEvent($event['id']), $created ? 'add' : 'edit');
}
if ($this->isTriggerCallable('event-after-save')) {
$event = $this->quickFetchEvent($event['id']);
$workflowErrors = [];
$logging = [
'model' => 'Event',
'action' => $created ? 'add' : 'edit',
'id' => $event['Event']['id'],
];
$triggerData = $event;
$this->executeTrigger('event-after-save', $triggerData, $workflowErrors, $logging);
}
}
public function attachTagsToEvents(array $events)
@ -483,6 +489,47 @@ class Event extends AppModel
return $events;
}
public function touch($event_id)
{
$event = $this->find('first', [
'conditions' => ['Event.id' => $event_id],
'recursive' => -1,
]);
$event['Event']['published'] = 0;
$event['Event']['timestamp'] = (new DateTime())->getTimestamp();
return $this->save($event, true, ['timestamp', 'published']);
}
public function attachTagsToEventAndTouch($event_id, $tags)
{
$touchEvent = false;
$success = false;
foreach ($tags as $tagId) {
$nothingToChange = false;
$success = $success || $this->EventTag->attachTagToEvent($event_id, ['id' => $tagId], $nothingToChange);
$touchEvent = $touchEvent || !$nothingToChange;
}
if ($touchEvent) {
return $this->touch($event_id);
}
return $success;
}
public function detachTagsFromEventAndTouch($event_id, $tags)
{
$touchEvent = false;
$success = false;
foreach ($tags as $tagId) {
$nothingToChange = false;
$success = $success || $this->EventTag->detachTagFromEvent($event_id, $tagId, $nothingToChange);
$touchEvent = $touchEvent || !$nothingToChange;
}
if ($touchEvent) {
return $this->touch($event_id);
}
return $success;
}
/**
* Gets the logged in user + an array of events, attaches the correlation count to each
* @param array $user
@ -492,16 +539,8 @@ class Event extends AppModel
public function attachCorrelationCountToEvents(array $user, array $events)
{
$sgids = $this->SharingGroup->authorizedIds($user);
$eventIds = array_column(array_column($events, 'Event'), 'id');
$conditionsCorrelation = $this->__buildEventConditionsCorrelation($user, $eventIds, $sgids);
$this->Attribute->Correlation->virtualFields['count'] = 'count(distinct(Correlation.event_id))';
$correlations = $this->Attribute->Correlation->find('list', array(
'fields' => array('Correlation.1_event_id', 'Correlation.count'),
'conditions' => $conditionsCorrelation,
'group' => array('Correlation.1_event_id'),
));
foreach ($events as &$event) {
$event['Event']['correlation_count'] = isset($correlations[$event['Event']['id']]) ? $correlations[$event['Event']['id']] : 0;
$event['Event']['correlation_count'] = $this->getRelatedEventCount($user, $event['Event']['id'], $sgids);
}
return $events;
}
@ -569,61 +608,12 @@ class Event extends AppModel
return $events;
}
private function __buildEventConditionsCorrelation($user, $eventIds, $sgids)
public function getRelatedEventCount($user, $eventId, $sgids)
{
if (!is_array($eventIds)) {
$eventIds = array($eventIds);
if (!isset($sgids) || empty($sgids)) {
$sgids = array(-1);
}
if (!$user['Role']['perm_site_admin']) {
$conditionsCorrelation = array(
'AND' => array(
'Correlation.1_event_id' => $eventIds,
array(
'OR' => array(
'Correlation.org_id' => $user['org_id'],
'AND' => array(
array(
'OR' => array(
array(
'AND' => array(
'Correlation.distribution >' => 0,
'Correlation.distribution <' => 4,
),
),
array(
'AND' => array(
'Correlation.distribution' => 4,
'Correlation.sharing_group_id' => $sgids
),
),
),
),
array(
'OR' => array(
'Correlation.a_distribution' => 5,
array(
'AND' => array(
'Correlation.a_distribution >' => 0,
'Correlation.a_distribution <' => 4,
),
),
array(
'AND' => array(
'Correlation.a_distribution' => 4,
'Correlation.a_sharing_group_id' => $sgids
),
),
),
),
),
),
),
),
);
} else {
$conditionsCorrelation = array('Correlation.1_event_id' => $eventIds);
}
return $conditionsCorrelation;
return count($this->Attribute->Correlation->getRelatedEventIds($user, $eventId, $sgids));
}
private function getRelatedEvents($user, $eventId, $sgids)
@ -631,48 +621,31 @@ class Event extends AppModel
if (!isset($sgids) || empty($sgids)) {
$sgids = array(-1);
}
// search the correlation table for the event ids of the related events
// Rules:
// 1. Event is owned by the user (org_id matches)
// 2. User is allowed to see both the event and the org:
// a. Event:
// i. Event has a distribution between 1-3 (community only, connected communities, all orgs)
// ii. Event has a sharing group that the user is accessible to view
// b. Attribute:
// i. Attribute has a distribution of 5 (inheritance of the event, for this the event check has to pass anyway)
// ii. Atttibute has a distribution between 1-3 (community only, connected communities, all orgs)
// iii. Attribute has a sharing group that the user is accessible to view
$conditionsCorrelation = $this->__buildEventConditionsCorrelation($user, $eventId, $sgids);
$relatedEventIds = $this->Attribute->Correlation->find('column', array(
'fields' => array('Correlation.event_id'),
'conditions' => $conditionsCorrelation,
'unique' => true,
));
$relatedEventIds = $this->Attribute->Correlation->getRelatedEventIds($user, $eventId, $sgids);
if (empty($relatedEventIds)) {
return [];
}
// now look up the event data for these attributes
$conditions = $this->createEventConditions($user);
$conditions['AND'][] = array('Event.id' => $relatedEventIds);
$fields = array('id', 'date', 'threat_level_id', 'info', 'published', 'uuid', 'analysis', 'timestamp', 'distribution', 'org_id', 'orgc_id');
$orgfields = array('id', 'name', 'uuid');
$relatedEvents = $this->find(
$relatedEvents = $this->find(
'all',
array('conditions' => $conditions,
[
'conditions' => [
'Event.id' => $relatedEventIds
],
'recursive' => -1,
'order' => 'Event.date DESC',
'fields' => $fields,
'contain' => array(
'Org' => array(
'fields' => $orgfields
),
'Orgc' => array(
'fields' => $orgfields
)
)
)
'order' => 'date DESC',
'fields' => [
'id', 'date', 'threat_level_id', 'info', 'published', 'uuid', 'analysis', 'timestamp', 'distribution', 'org_id', 'orgc_id'
],
'contain' => [
'Org' => [
'fields' => ['id', 'name', 'uuid']
],
'Orgc' => [
'fields' => ['id', 'name', 'uuid']
]
]
]
);
$fieldsToRearrange = array('Org', 'Orgc');
foreach ($relatedEvents as $k => $relatedEvent) {
@ -696,8 +669,8 @@ class Event extends AppModel
public function getRelatedAttributes(array $user, $id, $shadowAttribute = false, $scope = 'event')
{
if ($shadowAttribute) {
$parentIdField = '1_shadow_attribute_id';
$correlationModelName = 'ShadowAttributeCorrelation';
// no longer supported
return [];
} else {
$parentIdField = '1_attribute_id';
$correlationModelName = 'Correlation';
@ -705,102 +678,8 @@ class Event extends AppModel
if (!isset($this->{$correlationModelName})) {
$this->{$correlationModelName} = ClassRegistry::init($correlationModelName);
}
if (!$user['Role']['perm_site_admin']) {
$sgids = $this->SharingGroup->authorizedIds($user);
$conditionsCorrelation = array(
'AND' => array(
$correlationModelName . '.1_' . $scope . '_id' => $id,
array(
'OR' => array(
$correlationModelName . '.org_id' => $user['org_id'],
'AND' => array(
array(
'OR' => array(
array(
'AND' => array(
$correlationModelName . '.distribution >' => 0,
$correlationModelName . '.distribution <' => 4,
),
),
array(
'AND' => array(
$correlationModelName . '.distribution' => 4,
$correlationModelName . '.sharing_group_id' => $sgids
),
),
),
),
array(
'OR' => array(
$correlationModelName . '.a_distribution' => 5,
array(
'AND' => array(
$correlationModelName . '.a_distribution >' => 0,
$correlationModelName . '.a_distribution <' => 4,
),
),
array(
'AND' => array(
$correlationModelName . '.a_distribution' => 4,
$correlationModelName . '.a_sharing_group_id' => $sgids
),
),
),
),
),
)
)
)
);
} else {
$conditionsCorrelation = array($correlationModelName . '.1_' . $scope . '_id' => $id);
}
$max_correlations = Configure::read('MISP.max_correlations_per_event') ?: 5000;
$correlations = $this->{$correlationModelName}->find('all', array(
'fields' => ['event_id', 'attribute_id', 'value', $parentIdField],
'conditions' => $conditionsCorrelation,
'recursive' => -1,
'order' => false,
'limit' => $max_correlations
));
if (empty($correlations)) {
return array();
}
$correlations = array_column($correlations, $correlationModelName);
$eventIds = array_unique(array_column($correlations, 'event_id'));
$conditions = $this->createEventConditions($user);
$conditions['Event.id'] = $eventIds;
$events = $this->find('all', array(
'recursive' => -1,
'conditions' => $conditions,
'fields' => ['Event.id', 'Event.orgc_id', 'Event.info', 'Event.date'],
));
$events = array_column(array_column($events, 'Event'), null, 'id');
$relatedAttributes = [];
foreach ($correlations as $correlation) {
// User don't have access to correlated attribute event, skip.
$eventId = $correlation['event_id'];
if (!isset($events[$eventId])) {
continue;
}
$event = $events[$eventId];
$current = array(
'id' => $eventId,
'attribute_id' => $correlation['attribute_id'],
'value' => $correlation['value'],
'org_id' => $event['orgc_id'],
'info' => $event['info'],
'date' => $event['date'],
);
$parentId = $correlation[$parentIdField];
$relatedAttributes[$parentId][] = $current;
}
$sgids = $this->SharingGroup->authorizedIds($user);
$relatedAttributes = $this->{$correlationModelName}->getAttributesRelatedToEvent($user, $id, $sgids);
return $relatedAttributes;
}
@ -836,12 +715,13 @@ class Event extends AppModel
/**
* @param array $event
* @param array $server
* @param ServerSyncTool $serverSync
* @return false|string
* @throws HttpSocketJsonException
* @throws JsonException
* @throws Exception
*/
public function uploadEventToServer(array $event, array $server)
public function uploadEventToServer(array $event, array $server, ServerSyncTool $serverSync)
{
$this->Server = ClassRegistry::init('Server');
@ -852,8 +732,6 @@ class Event extends AppModel
return 'The distribution level of this event blocks it from being pushed.';
}
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
$push = $this->Server->checkVersionCompatibility($server, false, $serverSync);
if (empty($push['canPush'])) {
return 'The remote user is not a sync user - the upload of the event has been blocked.';
@ -2101,7 +1979,7 @@ class Event extends AppModel
if (!empty($options['includeRelatedTags'])) {
$event = $this->includeRelatedTags($event, $options);
}
$event['RelatedShadowAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], true);
//$event['RelatedShadowAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], true);
}
$shadowAttributeByOldId = [];
if (!empty($event['ShadowAttribute'])) {
@ -2147,6 +2025,7 @@ class Event extends AppModel
unset($event['Attribute'][$key]);
continue;
}
$attribute = $this->Attribute->Correlation->setCorrelationExclusion($attribute);
if ($attribute['category'] === 'Financial fraud') {
$attribute = $this->Attribute->attachValidationWarnings($attribute);
}
@ -2753,7 +2632,7 @@ class Event extends AppModel
public function set_filter_published(&$params, $conditions, $options)
{
if (isset($params['published'])) {
if (isset($params['published']) && $params['published'] !== [true, false]) {
$conditions['AND']['Event.published'] = $params['published'];
}
return $conditions;
@ -3131,8 +3010,8 @@ class Event extends AppModel
{
if (Configure::read('MISP.extended_alert_subject')) {
$subject = preg_replace("/\r|\n/", "", $event['Event']['info']);
if (strlen($subject) > 58) {
$subject = substr($subject, 0, 55) . '... - ';
if (mb_strlen($subject) > 58) {
$subject = mb_substr($subject, 0, 55) . '... - ';
} else {
$subject .= " - ";
}
@ -3159,6 +3038,10 @@ class Event extends AppModel
$template->set('tlp', $subjMarkingString);
$template->subject($subject);
$template->referenceId("event-alert|{$event['Event']['id']}");
$unsubscribeLink = $this->__getAnnounceBaseurl() . '/users/unsubscribe/' . $this->User->unsubscribeCode($user);
$template->set('unsubscribe', $unsubscribeLink);
$template->listUnsubscribe($unsubscribeLink);
return $template;
}
@ -3433,17 +3316,18 @@ class Event extends AppModel
return $attributes;
}
public function checkEventBlockRules($event)
/**
* @param array $event
* @return bool
*/
private function checkEventBlockRules(array $event)
{
if (!isset($this->AdminSetting)) {
if (!isset($this->eventBlockRule)) {
$this->AdminSetting = ClassRegistry::init('AdminSetting');
$setting = $this->AdminSetting->getSetting('eventBlockRule');
$this->eventBlockRule = $setting ? json_decode($setting, true) : false;
}
$setting = $this->AdminSetting->getSetting('eventBlockRule');
if (empty($setting)) {
return true;
}
$rules = json_decode($setting, true);
if (empty($rules)) {
if (empty($this->eventBlockRule)) {
return true;
}
if (!empty($rules['tags'])) {
@ -3544,7 +3428,9 @@ class Event extends AppModel
public function _add(array &$data, $fromXml, array $user, $org_id = 0, $passAlong = null, $fromPull = false, $jobId = null, &$created_id = 0, &$validationErrors = array())
{
if (Configure::read('MISP.enableEventBlocklisting') !== false && isset($data['Event']['uuid'])) {
$this->EventBlocklist = ClassRegistry::init('EventBlocklist');
if (!isset($this->EventBlocklist)) {
$this->EventBlocklist = ClassRegistry::init('EventBlocklist');
}
if ($this->EventBlocklist->isBlocked($data['Event']['uuid'])) {
return 'Blocked by blocklist';
}
@ -4215,8 +4101,15 @@ class Event extends AppModel
$failedServers = [];
foreach ($servers as $server) {
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
try {
$this->pushSightingsToServer($server, $event, $sightingsUuidsToPush);
try {
if ($serverSync->eventExists($event) === false) {
continue; // skip if event not exists on remote server
}
} catch (Exception $e) {}
$this->pushSightingsToServer($serverSync, $event, $sightingsUuidsToPush);
} catch (Exception $e) {
$this->logException("Uploading sightings to server {$server['Server']['id']} failed.", $e);
$failedServers[] = $server['Server']['url'];
@ -4229,25 +4122,16 @@ class Event extends AppModel
}
/**
* @param array $server
* @param ServerSyncTool $serverSync
* @param array $event
* @param array $sightingsUuidsToPush
* @throws HttpSocketJsonException
* @throws JsonException
* @throws Exception
*/
private function pushSightingsToServer(array $server, array $event, array $sightingsUuidsToPush = [])
private function pushSightingsToServer(ServerSyncTool $serverSync, array $event, array $sightingsUuidsToPush = [])
{
App::uses('ServerSyncTool', 'Tools');
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
try {
if ($serverSync->eventExists($event) === false) {
return; // skip if event not exists on remote server
}
} catch (Exception $e) {}
$fakeSyncUser = [
'org_id' => $server['Server']['remote_org_id'],
'org_id' => $serverSync->server()['Server']['remote_org_id'],
'Role' => [
'perm_site_admin' => 0,
],
@ -4316,8 +4200,6 @@ class Event extends AppModel
}
$uploaded = true;
$failedServers = [];
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
foreach ($servers as $server) {
if (
@ -4327,7 +4209,7 @@ class Event extends AppModel
}
// Skip servers where the event has come from.
if ($passAlong != $server['Server']['id']) {
$HttpSocket = $syncTool->setupHttpSocket($server);
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
$params = [
'eventid' => $id,
'includeAttachments' => true,
@ -4355,17 +4237,17 @@ class Event extends AppModel
'perm_site_admin' => 0
)
);
$this->Server->syncGalaxyClusters($HttpSocket, $server, $fakeSyncUser, $technique=$event['Event']['id'], $event=$event);
$thisUploaded = $this->uploadEventToServer($event, $server);
$this->Server->syncGalaxyClusters($serverSync, $server, $fakeSyncUser, $technique=$event['Event']['id'], $event=$event);
$thisUploaded = $this->uploadEventToServer($event, $server, $serverSync);
if ($thisUploaded === 'Success') {
try {
$this->pushSightingsToServer($server, $event); // push sighting by method that check for duplicates
$this->pushSightingsToServer($serverSync, $event); // push sighting by method that check for duplicates
} catch (Exception $e) {
$this->logException("Uploading sightings to server {$server['Server']['id']} failed.", $e);
}
}
if (isset($this->data['ShadowAttribute'])) {
$this->Server->syncProposals($HttpSocket, $server, null, $id, $this);
$this->Server->syncProposals(null, $server, null, $id, $this);
}
if (!$thisUploaded) {
$uploaded = !$uploaded ? $uploaded : $thisUploaded;
@ -4438,7 +4320,6 @@ class Event extends AppModel
$jobId
);
}
return $this->publish($id, $passAlong);
}
@ -4483,9 +4364,59 @@ class Event extends AppModel
'recursive' => -1,
'conditions' => array('Event.id' => $id)
));
if (empty($event)) {
return false;
}
$hostOrg = $this->Org->find('first', [
'recursive' => -1,
'conditions' => [
'id' => Configure::read('MISP.host_org_id')
],
]);
if (empty($hostOrg)) {
$hostOrg = $this->Org->find('first', [
'recursive' => -1,
'order' => ['id ASC']
]);
}
$userForPubSub = [
'id' => 0,
'org_id' => $hostOrg['Org']['id'],
'Role' => ['perm_sync' => 0, 'perm_audit' => 0, 'perm_site_admin' => 1],
'Organisation' => $hostOrg['Org']
];
$allowZMQ = Configure::read('Plugin.ZeroMQ_enable');
$kafkaTopic = Configure::read('Plugin.Kafka_event_publish_notifications_topic');
$allowKafka = Configure::read('Plugin.Kafka_enable') &&
Configure::read('Plugin.Kafka_event_publish_notifications_enable') &&
!empty($kafkaTopic);
$triggerCallable = $this->isTriggerCallable('event-publish');
if ($allowZMQ || $allowKafka || $triggerCallable) {
$currentUserId = Configure::read('CurrentUserId');
$userForWorkflow = $this->User->getAuthUser($currentUserId, true);
$userForWorkflow['Role']['perm_site_admin'] = 1;
$fullEvent = $this->fetchEvent($userForWorkflow, [
'eventid' => $id,
'includeAttachments' => 1
]);
}
if ($triggerCallable) {
$workflowErrors = [];
$logging = [
'model' => 'Event',
'action' => 'publish',
'id' => $id,
'message' => __('Publishing stopped by a blocking workflow.'),
];
$success = $this->executeTrigger('event-publish', $fullEvent[0], $workflowErrors, $logging);
if (empty($success)) {
$errorMessage = implode(', ', $workflowErrors);
return $errorMessage;
}
}
if ($jobId) {
$this->Behaviors->unload('SysLogLogable.SysLogLogable');
} else {
@ -4498,50 +4429,11 @@ class Event extends AppModel
$event['Event']['skip_kafka'] = 1;
$this->save($event, array('fieldList' => $fieldList));
}
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable');
$kafkaTopic = Configure::read('Plugin.Kafka_event_publish_notifications_topic');
$pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_event_publish_notifications_enable') && !empty($kafkaTopic);
if ($pubToZmq || $pubToKafka) {
$hostOrgId = Configure::read('MISP.host_org_id');
if (!empty($hostOrgId)) {
$hostOrg = $this->Org->find('first', [
'recursive' => -1,
'conditions' => [
'id' => $hostOrgId
]
]
);
}
if (empty($hostOrg)) {
$hostOrg = $this->Org->find('first', [
'recursive' => -1,
'order' => ['id ASC']
]);
$hostOrgId = $hostOrg['Org']['id'];
}
$user = array('org_id' => $hostOrgId, 'Role' => array('perm_sync' => 0, 'perm_audit' => 0, 'perm_site_admin' => 0), 'Organisation' => $hostOrg['Org']);
if ($pubToZmq) {
$params = array('eventid' => $id);
if (Configure::read('Plugin.ZeroMQ_include_attachments')) {
$params['includeAttachments'] = 1;
}
$fullEvent = $this->fetchEvent($user, $params);
if (!empty($fullEvent)) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->publishEvent($fullEvent[0], 'publish');
}
}
if ($pubToKafka) {
$params = array('eventid' => $id);
if (Configure::read('Plugin.Kafka_include_attachments')) {
$params['includeAttachments'] = 1;
}
$fullEvent = $this->fetchEvent($user, $params);
if (!empty($fullEvent)) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $fullEvent[0], 'publish');
}
}
if ($allowZMQ) {
$this->publishEventToZmq($id, $userForPubSub, $fullEvent);
}
if ($allowKafka) {
$this->publishEventToKafka($id, $userForPubSub, $fullEvent, $kafkaTopic);
}
return $this->uploadEventToServersRouter($id, $passAlong);
}
@ -4753,30 +4645,6 @@ class Event extends AppModel
return $xmlArray;
}
public function removeOlder(array &$events, $scope = 'events')
{
$field = $scope === 'sightings' ? 'sighting_timestamp' : 'timestamp';
$localEvents = $this->find('all', [
'recursive' => -1,
'fields' => ['Event.uuid', 'Event.' . $field, 'Event.locked'],
]);
$localEvents = array_column(array_column($localEvents, 'Event'), null, 'uuid');
foreach ($events as $k => $event) {
// remove all events for the sighting sync if the remote is not aware of the new field yet
if (!isset($event[$field])) {
unset($events[$k]);
} else {
$uuid = $event['uuid'];
if (isset($localEvents[$uuid])
&& ($localEvents[$uuid][$field] >= $event[$field]
|| ($scope === 'events' && !$localEvents[$uuid]['locked'])))
{
unset($events[$k]);
}
}
}
}
public function sharingGroupRequired($field)
{
if ($this->data[$this->alias]['distribution'] == 4) {
@ -4813,7 +4681,6 @@ class Event extends AppModel
if ($filterType) {
$include = true;
/* proposal */
if ($filterType['proposal'] == 0) { // `both`
// pass, do not consider as `both` is selected
@ -5459,7 +5326,7 @@ class Event extends AppModel
'Object.event_id' => $event_id,
'Object.deleted' => 0),
'recursive' => -1,
'fields' => array('Object.id', 'Object.uuid', 'Object.name')
'fields' => array('Object.id', 'Object.uuid', 'Object.name', 'Object.distribution', 'Object.sharing_group_id')
));
if (!empty($initial_object)) {
$initial_attributes = $this->Attribute->find('all', array(
@ -5923,6 +5790,8 @@ class Event extends AppModel
public function enrichmentRouter($options)
{
$result = $this->enrichment($options);
return __('#' . $result . ' attributes have been created during the enrichment process.');
if (Configure::read('MISP.background_jobs')) {
/** @var Job $job */
@ -5964,10 +5833,30 @@ class Event extends AppModel
throw new MethodNotAllowedException(__('%s not set', $option_field));
}
}
$event = $this->fetchEvent($params['user'], array('eventid' => $params['event_id'], 'includeAttachments' => 1, 'flatten' => 1));
$event = [];
if (!empty($params['attribute_uuids'])) {
$attributes = $this->Attribute->fetchAttributes($params['user'], [
'conditions' => [
'Attribute.uuid' => $params['attribute_uuids'],
],
'withAttachments' => 1,
]);
$event = [
[
'Event' => ['id' => $params['event_id']],
'Attribute' => Hash::extract($attributes, '{n}.Attribute')
]
];
} else {
$event = $this->fetchEvent($params['user'], [
'eventid' => $params['event_id'],
'includeAttachments' => 1,
'flatten' => 1,
]);
}
$this->Module = ClassRegistry::init('Module');
$enabledModules = $this->Module->getEnabledModules($params['user']);
if (empty($enabledModules)) {
if (empty($enabledModules) || is_string($enabledModules)) {
return true;
}
$options = array();
@ -5988,6 +5877,9 @@ class Event extends AppModel
$event_id = $event[0]['Event']['id'];
foreach ($event[0]['Attribute'] as $attribute) {
$object_id = $attribute['object_id'];
if ($object_id != '0' && empty($initial_objects[$object_id])) {
$initial_objects[$object_id] = $this->fetchInitialObject($event_id, $object_id);
}
foreach ($enabledModules['modules'] as $module) {
if (in_array($module['name'], $params['modules'])) {
if (in_array($attribute['type'], $module['mispattributes']['input'])) {
@ -5997,15 +5889,19 @@ class Event extends AppModel
}
if (!empty($module['mispattributes']['format']) && $module['mispattributes']['format'] == 'misp_standard') {
$data['attribute'] = $attribute;
if ($object_id != '0' && empty($initial_objects[$object_id])) {
$initial_objects[$object_id] = $this->fetchInitialObject($event_id, $object_id);
}
} else {
$data[$attribute['type']] = $attribute['value'];
}
$result = $this->Module->queryModuleServer($data, false, 'Enrichment');
if (!$result) {
if ($object_id != '0' && !empty($initial_objects[$object_id])) {
$attribute['Object'] = $initial_objects[$object_id]['Object'];
}
$triggerData = $event[0];
$triggerData['Attribute'] = [$attribute];
$result = $this->Module->queryModuleServer($data, false, 'Enrichment', false, $triggerData);
if ($result === false) {
throw new MethodNotAllowedException(h($module['name']) . ' service not reachable.');
} else if (!is_array($result)) {
continue 2;
}
//if (isset($result['error'])) $this->Session->setFlash($result['error']);
if (!is_array($result)) {
@ -7636,4 +7532,51 @@ class Event extends AppModel
),
);
}
private function __prepareEventForPubSub($id, $user, &$fullEvent)
{
if ($fullEvent) {
if (empty(Configure::read('Plugin.ZeroMQ_include_attachments'))) {
foreach ($fullEvent[0]['Attribute'] as $k => $attribute) {
if (isset($attribute['data'])) {
unset($fullEvent[0]['Attribute'][$k]['data']);
}
}
foreach ($fullEvent[0]['Object'] as $k => $object) {
foreach ($object['Attribute'] as $k2 => $attribute) {
if (isset($attribute['data'])) {
unset($fullEvent[0]['Object'][$k]['Attribute'][$k2]['data']);
}
}
}
}
} else {
$params = [
'eventid' => $id
];
if (Configure::read('Plugin.ZeroMQ_include_attachments')) {
$params['includeAttachments'] = 1;
}
$fullEvent = $this->fetchEvent($user, $params);
}
return $fullEvent;
}
public function publishEventToZmq($id, $user, &$fullEvent)
{
$fullEvent = $this->__prepareEventForPubSub($id, $user, $fullEvent);
if (!empty($fullEvent)) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->publishEvent($fullEvent[0], 'publish');
}
}
public function publishEventToKafka($id, $user, &$fullEvent, $kafkaTopic)
{
$fullEvent = $this->__prepareEventForPubSub($id, $user, $fullEvent);
if (!empty($fullEvent)) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $fullEvent[0], 'publish');
}
}
}

242
app/Model/GalaxyCluster.php
View File

@ -628,13 +628,11 @@ class GalaxyCluster extends AppModel
$elevatedUser = array(
'Role' => array(
'perm_site_admin' => 1,
'perm_sync' => 1
'perm_sync' => 1,
'perm_audit' => 0,
),
'org_id' => $clusterOrgcId['GalaxyCluster']['orgc_id']
);
$elevatedUser['Role']['perm_site_admin'] = 1;
$elevatedUser['Role']['perm_sync'] = 1;
$elevatedUser['Role']['perm_audit'] = 0;
$cluster = $this->fetchGalaxyClusters($elevatedUser, array('minimal' => true, 'conditions' => array('id' => $clusterId)), $full=false);
if (empty($cluster)) {
return true;
@ -655,14 +653,10 @@ class GalaxyCluster extends AppModel
return true;
}
$uploaded = false;
$failedServers = array();
App::uses('SyncTool', 'Tools');
foreach ($servers as &$server) {
foreach ($servers as $server) {
if ((!isset($server['Server']['internal']) || !$server['Server']['internal']) && $cluster['GalaxyCluster']['distribution'] < 2) {
continue;
}
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
$fakeSyncUser = array(
'id' => 0,
'email' => 'fakeSyncUser@user.test',
@ -678,14 +672,13 @@ class GalaxyCluster extends AppModel
);
$cluster = $this->fetchGalaxyClusters($fakeSyncUser, array('conditions' => array('GalaxyCluster.id' => $clusterId)), $full=true);
if (empty($cluster)) {
return true;
continue;
}
$cluster = $cluster[0];
$result = $this->uploadClusterToServer($cluster, $server, $HttpSocket, $fakeSyncUser);
if ($result == 'Success') {
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
$result = $this->uploadClusterToServer($cluster, $server, $serverSync, $fakeSyncUser);
if ($result === 'Success') {
$uploaded = true;
} else {
$failedServers[] = $server;
}
}
return $uploaded;
@ -1124,7 +1117,7 @@ class GalaxyCluster extends AppModel
if (!empty($tagsToFetch)) {
$tags = $this->GalaxyClusterRelation->GalaxyClusterRelationTag->Tag->find('all', [
'conditions' => ['id' => array_unique($tagsToFetch)],
'conditions' => ['id' => array_unique($tagsToFetch, SORT_REGULAR)],
'recursive' => -1,
]);
$tags = array_column(array_column($tags, 'Tag'), null, 'id');
@ -1672,79 +1665,29 @@ class GalaxyCluster extends AppModel
/**
* @return string|bool The result of the upload. True if success, a string otherwise
* @throws Exception
*/
public function uploadClusterToServer($cluster, $server, $HttpSocket, $user)
{
$this->Server = ClassRegistry::init('Server');
$this->Log = ClassRegistry::init('Log');
$push = $this->Server->checkVersionCompatibility($server, false);
if (empty($push['canPush']) && empty($push['canPushGalaxyCluster'])) {
return __('The remote user does not have the permission to manipulate galaxies - the upload of the galaxy clusters has been blocked.');
}
$updated = null;
$newLocation = $newTextBody = '';
$result = $this->__executeRestfulGalaxyClusterToServer($cluster, $server, null, $newLocation, $newTextBody, $HttpSocket, $user);
if ($result !== true) {
return $result;
}
if (strlen($newLocation)) { // HTTP/1.1 302 Found and Location: http://<newLocation>
$result = $this->__executeRestfulGalaxyClusterToServer($cluster, $server, $newLocation, $newLocation, $newTextBody, $HttpSocket, $user);
if ($result !== true) {
return $result;
}
}
$uploadFailed = false;
try {
$json = json_decode($newTextBody, true);
} catch (Exception $e) {
$uploadFailed = true;
}
if (!is_array($json) || $uploadFailed) {
$this->Log->createLogEntry($user, 'push', 'GalaxyCluster', $cluster['GalaxyCluster']['id'], 'push', $newTextBody);
}
return 'Success';
}
private function __executeRestfulGalaxyClusterToServer($cluster, $server, $resourceId, &$newLocation, &$newTextBody, $HttpSocket, $user)
{
$result = $this->restfulGalaxyClusterToServer($cluster, $server, $resourceId, $newLocation, $newTextBody, $HttpSocket);
if (is_numeric($result)) {
$error = $this->__resolveErrorCode($result, $cluster, $server, $user);
if ($error) {
return $error . ' Error code: ' . $result;
}
}
return true;
}
/**
* @return string|bool|int The result of the upload.
*/
public function restfulGalaxyClusterToServer($cluster, $server, $urlPath, &$newLocation, &$newTextBody, $HttpSocket = null)
public function uploadClusterToServer(array $cluster, array $server, ServerSyncTool $serverSync, array $user)
{
$cluster = $this->__prepareForPushToServer($cluster, $server);
if (is_numeric($cluster)) {
return $cluster;
}
$url = $server['Server']['url'];
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
$scope = 'galaxies/pushCluster';
$uri = $url . '/' . $scope;
$clusters = array($cluster);
$data = json_encode($clusters);
if (!empty(Configure::read('Security.sync_audit'))) {
$pushLogEntry = sprintf(
"==============================================================\n\n[%s] Pushing Galaxy Cluster #%d to Server #%d:\n\n%s\n\n",
date("Y-m-d H:i:s"),
$cluster['GalaxyCluster']['id'],
$server['Server']['id'],
$data
);
file_put_contents(APP . 'files/scripts/tmp/debug_server_' . $server['Server']['id'] . '.log', $pushLogEntry, FILE_APPEND);
try {
if (!$serverSync->isSupported(ServerSyncTool::PERM_SYNC) || $serverSync->isSupported(ServerSyncTool::PERM_GALAXY_EDITOR)) {
return __('The remote user does not have the permission to manipulate galaxies - the upload of the galaxy clusters has been blocked.');
}
$serverSync->pushGalaxyCluster($cluster)->json();
} catch (Exception $e) {
$title = __('Uploading GalaxyCluster (%s) to Server (%s)', $cluster['GalaxyCluster']['id'], $server['Server']['id']);
$this->loadLog()->createLogEntry($user, 'push', 'GalaxyCluster', $cluster['GalaxyCluster']['id'], $title, $e->getMessage());
$this->logException("Could not push galaxy cluster to remote server {$serverSync->serverId()}", $e);
return $e->getMessage();
}
$response = $HttpSocket->post($uri, $data, $request);
return $this->__handleRestfulGalaxyClusterToServerResponse($response, $newLocation, $newTextBody);
return 'Success';
}
/**
@ -1752,7 +1695,7 @@ class GalaxyCluster extends AppModel
*
* @param array $cluster
* @param array $server
* @return array The cluster ready to be pushed
* @return array|int The cluster ready to be pushed
*/
private function __prepareForPushToServer(array $cluster, array $server)
{
@ -1773,11 +1716,9 @@ class GalaxyCluster extends AppModel
}
$this->Event = ClassRegistry::init('Event');
if ($this->Event->checkDistributionForPush($cluster, $server, 'GalaxyCluster')) {
$cluster = $this->__updateClusterForSync($cluster, $server);
} else {
return 403;
return $this->__updateClusterForSync($cluster, $server);
}
return $cluster;
return 403;
}
/**
@ -1886,90 +1827,39 @@ class GalaxyCluster extends AppModel
return $relation;
}
/**
* @return string|bool|int The result of the upload.
*/
private function __handleRestfulGalaxyClusterToServerResponse($response, &$newLocation, &$newTextBody)
{
switch ($response->code) {
case '200': // 200 (OK) + entity-action-result
if ($response->isOk()) {
$newTextBody = $response->body();
return true;
} else {
try {
$jsonArray = json_decode($response->body, true);
} catch (Exception $e) {
return true;
}
return $jsonArray['name'];
}
// no break
case '302': // Found
$newLocation = $response->headers['Location'];
$newTextBody = $response->body();
return true;
case '404': // Not Found
$newLocation = $response->headers['Location'];
$newTextBody = $response->body();
return 404;
case '405':
return 405;
case '403': // Not authorised
return 403;
}
}
private function __resolveErrorCode($code, &$cluster, &$server, $user)
{
$this->Log = ClassRegistry::init('Log');
$error = false;
switch ($code) {
case 403:
return __('The distribution level of the cluster blocks it from being pushed.');
case 405:
$error = __('The sync user on the remote instance does not have the required privileges to handle this cluster.');
break;
}
if ($error) {
$newTextBody = 'Uploading GalaxyCluster (' . $cluster['GalaxyCluster']['id'] . ') to Server (' . $server['Server']['id'] . ')';
$newTextBody = __('Uploading GalaxyCluster (%s) to Server (%s)', $cluster['GalaxyCluster']['id'], $server['Server']['id']);
$this->Log->createLogEntry($user, 'push', 'GalaxyCluster', $cluster['GalaxyCluster']['id'], 'push', $newTextBody);
}
return $error;
}
/**
* pullGalaxyClusters
*
* @param array $user
* @param array $server
* @param string|int $technique The technique startegy used for pulling
* @param array $user
* @param ServerSyncTool $serverSync
* @param string|int $technique The technique startegy used for pulling
* allowed:
* - int <event id> event containing the clusters to pulled
* - string <full> pull everything
* - string <update> pull updates of cluster present locally
* - string <pull_relevant_clusters> pull clusters based on tags present locally
* @return int The number of pulled clusters
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
*/
public function pullGalaxyClusters(array $user, array $server, $technique = 'full')
public function pullGalaxyClusters(array $user, ServerSyncTool $serverSync, $technique = 'full')
{
$this->Server = ClassRegistry::init('Server');
$compatible = $this->Server->checkVersionCompatibility($server, $user)['supportEditOfGalaxyCluster'];
$compatible = $serverSync->isSupported(ServerSyncTool::FEATURE_EDIT_OF_GALAXY_CLUSTER);
if (!$compatible) {
return 0;
}
$clusterIds = $this->getClusterIdListBasedOnPullTechnique($user, $technique, $server);
$successes = array();
$fails = array();
$clusterIds = $this->getClusterIdListBasedOnPullTechnique($user, $technique, $serverSync);
$successes = 0;
// now process the $clusterIds to pull each of the events sequentially
if (!empty($clusterIds)) {
// download each cluster
foreach ($clusterIds as $k => $clusterId) {
$this->__pullGalaxyCluster($clusterId, $successes, $fails, $server, $user);
foreach ($clusterIds as $clusterId) {
if ($this->__pullGalaxyCluster($clusterId, $serverSync, $user)) {
$successes++;
}
}
}
return count($successes);
return $successes;
}
/**
@ -1977,16 +1867,16 @@ class GalaxyCluster extends AppModel
*
* @param array $user
* @param string|int $technique
* @param array $server
* @param ServerSyncTool $serverSync
* @return array cluster ID list to be pulled
*/
private function getClusterIdListBasedOnPullTechnique(array $user, $technique, array $server)
private function getClusterIdListBasedOnPullTechnique(array $user, $technique, ServerSyncTool $serverSync)
{
$this->Server = ClassRegistry::init('Server');
try {
if ("update" === $technique) {
$localClustersToUpdate = $this->getElligibleLocalClustersToUpdate($user);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = true, $elligibleClusters = $localClustersToUpdate);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($serverSync, $onlyUpdateLocalCluster = true, $elligibleClusters = $localClustersToUpdate);
} elseif ("pull_relevant_clusters" === $technique) {
// Fetch all local custom cluster tags then fetch their corresponding clusters on the remote end
$tagNames = $this->Tag->find('column', array(
@ -2005,55 +1895,39 @@ class GalaxyCluster extends AppModel
}
$localClustersToUpdate = $this->getElligibleLocalClustersToUpdate($user);
$conditions = array('uuid' => array_keys($clusterUUIDs));
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = false, $elligibleClusters = $localClustersToUpdate, $conditions = $conditions);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($serverSync, $onlyUpdateLocalCluster = false, $elligibleClusters = $localClustersToUpdate, $conditions = $conditions);
} elseif (is_numeric($technique)) {
$conditions = array('eventid' => $technique);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = false, $elligibleClusters = array(), $conditions = $conditions);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($serverSync, $onlyUpdateLocalCluster = false, $elligibleClusters = array(), $conditions = $conditions);
} else {
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = false);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($serverSync, $onlyUpdateLocalCluster = false);
}
} catch (HttpSocketHttpException $e) {
if ($e->getCode() === 403) {
return array('error' => array(1, null));
} else {
$this->logException("Could not get eligible cluster IDs from server {$server['Server']['id']} for pull.", $e);
$this->logException("Could not get eligible cluster IDs from server {$serverSync->serverId()} for pull.", $e);
return array('error' => array(2, $e->getMessage()));
}
} catch (Exception $e) {
$this->logException("Could not get eligible cluster IDs from server {$server['Server']['id']} for pull.", $e);
$this->logException("Could not get eligible cluster IDs from server {$serverSync->serverId()} for pull.", $e);
return array('error' => array(2, $e->getMessage()));
}
return $clusterIds;
}
private function __pullGalaxyCluster($clusterId, &$successes, &$fails, $server, $user)
private function __pullGalaxyCluster($clusterId, ServerSyncTool $serverSync, array $user)
{
$cluster = $this->downloadGalaxyClusterFromServer($clusterId, $server);
if (!empty($cluster)) {
$cluster = $this->updatePulledClusterBeforeInsert($cluster, $server, $user);
$result = $this->captureCluster($user, $cluster, $fromPull=true, $orgId=$server['Server']['org_id']);
if ($result['success']) {
$successes[] = $clusterId;
} else {
$fails[$clusterId] = __('Failed because of errors: ') . json_encode($result['errors']);
}
} else {
$fails[$clusterId] = __('failed downloading the galaxy cluster');
try {
$cluster = $serverSync->fetchGalaxyCluster($clusterId)->json();
} catch (Exception $e) {
$this->logException("Could not fetch galaxy cluster $clusterId from server {$serverSync->serverId()}", $e);
return false;
}
return true;
}
public function downloadGalaxyClusterFromServer($clusterId, $server, $HttpSocket=null)
{
$url = $server['Server']['url'];
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
$uri = $url . '/galaxy_clusters/view/' . $clusterId;
$response = $HttpSocket->get($uri, $data = '', $request);
if ($response->isOk()) {
return json_decode($response->body, true);
}
return null;
$cluster = $this->updatePulledClusterBeforeInsert($cluster, $serverSync->server(), $user);
$result = $this->captureCluster($user, $cluster, $fromPull=true, $orgId=$serverSync->server()['Server']['org_id']);
return $result['success'];
}
private function updatePulledClusterBeforeInsert($cluster, $server, $user)

7
app/Model/Log.php
View File

@ -39,11 +39,14 @@ class Log extends AppModel
'enrichment',
'error',
'execute_blueprint',
'execute_workflow',
'exec_module',
'export',
'fetchEvent',
'file_upload',
'galaxy',
'include_formula',
'load_module',
'login',
'login_fail',
'logout',
@ -104,6 +107,8 @@ class Log extends AppModel
'email' => array('values' => array('admin_email'))
);
public $actsAs = ['LightPaginator'];
/**
* Null when not defined, false when not enabled
* @var Syslog|null|false
@ -202,7 +207,7 @@ class Log extends AppModel
*/
public function createLogEntry($user, $action, $model, $modelId = 0, $title = '', $change = '')
{
if (in_array($action, ['tag', 'galaxy', 'publish', 'publish_sightings', 'enable'], true) && Configure::read('MISP.log_new_audit')) {
if (in_array($action, ['tag', 'galaxy', 'publish', 'publish_sightings', 'enable', 'edit'], true) && Configure::read('MISP.log_new_audit')) {
return; // Do not store tag changes when new audit is enabled
}
if ($user === 'SYSTEM') {

31
app/Model/MispObject.php
View File

@ -300,6 +300,31 @@ class MispObject extends AppModel
public function afterSave($created, $options = array())
{
if (!Configure::read('MISP.completely_disable_correlation') && !$created) {
$object = $this->data['Object'];
$this->Attribute->Correlation->updateContainedCorrelations($object, 'object');
}
if (!empty($this->data['Object']['deleted']) && !$created) {
$attributes_to_delete = $this->Attribute->find('all', [
'recursive' => -1,
'conditions' => [
'Attribute.object_id' => $this->id,
'Attribute.deleted' => 0
]
]);
foreach ($attributes_to_delete as &$attribute_to_delete) {
$attribute_to_delete['Attribute']['deleted'] = 1;
unset($attribute_to_delete['Attribute']['timestamp']);
}
$this->Attribute->saveMany($attributes_to_delete);
}
$workflowErrors = [];
$logging = [
'model' => 'Object',
'action' => $created ? 'add' : 'edit',
'id' => $this->data['Object']['id'],
];
$this->executeTrigger('object-after-save', $this->data, $workflowErrors, $logging);
$pubToZmq = $this->pubToZmq('object') && empty($this->data['Object']['skip_zmq']);
$kafkaTopic = $this->kafkaTopic('object');
$pubToKafka = $kafkaTopic && empty($this->data['Object']['skip_kafka']);
@ -1004,6 +1029,9 @@ class MispObject extends AppModel
$objectId = $this->id;
if (!empty($object['Object']['Attribute'])) {
foreach ($object['Object']['Attribute'] as $attribute) {
if (!empty($object['Object']['deleted'])) {
$attribute['deleted'] = 1;
}
$this->Attribute->captureAttribute($attribute, $eventId, $user, $objectId, false, $parentEvent);
}
}
@ -1088,6 +1116,9 @@ class MispObject extends AppModel
}
if (!empty($object['Attribute'])) {
foreach ($object['Attribute'] as $attribute) {
if (!empty($object['deleted'])) {
$attribute['deleted'] = 1;
}
$result = $this->Attribute->editAttribute($attribute, $event, $user, $object['id'], false, $force);
}
}

103
app/Model/Module.php
View File

@ -10,12 +10,14 @@ class Module extends AppModel
'Enrichment' => array('hover', 'expansion'),
'Import' => array('import'),
'Export' => array('export'),
'Action' => array('action'),
'Cortex' => array('cortex')
);
private $__typeToFamily = array(
'Import' => 'Import',
'Export' => 'Export',
'Action' => 'Action',
'hover' => 'Enrichment',
'expansion' => 'Enrichment',
'Cortex' => 'Cortex'
@ -131,6 +133,8 @@ class Module extends AppModel
$output['Import'] = $temp['name'];
} elseif (isset($temp['meta']['module-type']) && in_array('export', $temp['meta']['module-type'])) {
$output['Export'] = $temp['name'];
} elseif (isset($temp['meta']['module-type']) && in_array('action', $temp['meta']['module-type'])) {
$output['Action'] = $temp['name'];
} else {
foreach ($temp['mispattributes']['input'] as $input) {
if (!isset($temp['meta']['module-type']) || (in_array('expansion', $temp['meta']['module-type']) || in_array('cortex', $temp['meta']['module-type']))) {
@ -199,6 +203,53 @@ class Module extends AppModel
return "$url:$port";
}
private function __prepareAndExectureForTrigger($postData, $triggerData=[]): bool
{
$this->Workflow = ClassRegistry::init('Workflow');
$trigger_id = 'enrichment-before-query';
$workflowErrors = [];
$logging = [
'model' => 'Workflow',
'action' => 'execute_workflow',
'id' => 0,
];
if (empty($triggerData) && $this->Workflow->isTriggerCallable($trigger_id) && !empty($postData['attribute_uuid'])) {
$this->User = ClassRegistry::init('User');
$this->Attribute = ClassRegistry::init('Attribute');
$user = $this->User->getAuthUser(Configure::read('CurrentUserId'), true);
$options = [
'conditions' => [
'Attribute.uuid' => $postData['attribute_uuid'],
],
'includeAllTags' => true,
'includeAttributeUuid' => true,
'flatten' => true,
'deleted' => [0, 1],
'withAttachments' => true,
'contain' => ['Event' => ['fields' => ['distribution', 'sharing_group_id']]],
];
$attributes = $this->Attribute->fetchAttributes($user, $options);
$triggerData = !empty($attributes) ? $attributes[0] : [];
$logging['message'] = __('The workflow `%s` prevented attribute `%s` (from event `%s`) to query the module `%s`', $trigger_id, $postData['attribute_uuid'], $triggerData['Attribute']['event_id'], $postData['module']);
} else {
if (isset($triggerData['Attribute'])) {
$logging['message'] = __('The workflow `%s` prevented attribute `%s` (from event `%s`) to query the module `%s`',
$trigger_id,
$triggerData['Attribute']['id'] ?? $triggerData['Attribute'][0]['id'],
$triggerData['Attribute']['event_id'] ?? $triggerData['Attribute'][0]['event_id'],
$postData['module']
);
} else {
$logging['message'] = __('The workflow `%s` prevented attribute `%s` (from event `%s`) to query the module `%s`', $trigger_id, $triggerData['Event']['Attribute'][0]['id'], $triggerData['Event']['id'], $postData['module']);
}
}
if (empty($triggerData)) {
return false;
}
$success = $this->executeTrigger($trigger_id, $triggerData, $workflowErrors, $logging);
return !empty($success);
}
/**
* Send request to `/query` module endpoint.
*
@ -209,8 +260,15 @@ class Module extends AppModel
* @return array|false
* @throws JsonException
*/
public function queryModuleServer(array $postData, $hover = false, $moduleFamily = 'Enrichment', $throwException = false)
public function queryModuleServer(array $postData, $hover = false, $moduleFamily = 'Enrichment', $throwException = false, $triggerData=[])
{
if ($moduleFamily == 'Enrichment') {
$success = $this->__prepareAndExectureForTrigger($postData, $triggerData);
if (!$success) {
$trigger_id = 'enrichment-before-query';
return __('Trigger `%s` blocked enrichment', $trigger_id);
}
}
if ($hover) {
$timeout = Configure::read('Plugin.' . $moduleFamily . '_hover_timeout') ?: 5;
} else {
@ -290,15 +348,42 @@ class Module extends AppModel
foreach ($modules as $module) {
if (array_intersect($this->__validTypes[$moduleFamily], $module['meta']['module-type'])) {
$moduleSettings = [
array('name' => 'enabled', 'type' => 'boolean'),
array('name' => 'restrict', 'type' => 'orgs')
[
'name' => 'enabled',
'type' => 'boolean',
'description' => empty($module['meta']['description']) ? '' : $module['meta']['description']
]
];
if (isset($module['meta']['config'])) {
foreach ($module['meta']['config'] as $key => $value) {
if (is_string($key)) {
$moduleSettings[] = array('name' => $key, 'type' => 'string', 'description' => $value);
} else {
$moduleSettings[] = array('name' => $value, 'type' => 'string');
if ($moduleFamily !== 'Action') {
$moduleSettings[] = [
'name' => 'restrict',
'type' => 'orgs',
'description' => __('Restrict the use of this module to an organisation.')
];
if (isset($module['meta']['config'])) {
foreach ($module['meta']['config'] as $key => $value) {
if (is_array($value)) {
$name = is_string($key) ? $key : $value['name'];
$moduleSettings[] = [
'name' => $name,
'type' => isset($value['type']) ? $value['type'] : 'string',
'test' => isset($value['test']) ? $value['test'] : null,
'description' => isset($value['description']) ? $value['description'] : null,
'null' => isset($value['null']) ? $value['null'] : null,
'test' => isset($value['test']) ? $value['test'] : null,
'bigField' => isset($value['bigField']) ? $value['bigField'] : false,
'cli_only' => isset($value['cli_only']) ? $value['cli_only'] : false,
'redacted' => isset($value['redacted']) ? $value['redacted'] : false
];
} else if (is_string($key)) {
$moduleSettings[] = [
'name' => $key,
'type' => 'string',
'description' => $value
];
} else {
$moduleSettings[] = array('name' => $value, 'type' => 'string');
}
}
}
}

68
app/Model/OverCorrelatingValue.php Normal file
View File

@ -0,0 +1,68 @@
<?php
App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
class OverCorrelatingValue extends AppModel
{
public $recursive = -1;
public $actsAs = array(
'Containable'
);
public $validate = [
];
public function block($value, $count)
{
$this->unblock($value);
$this->create();
$this->save(
[
'value' => $value,
'occurrence' => $count
]
);
}
public function unBlock($value)
{
$this->deleteAll(
[
'OverCorrelatingValue.value' => $value
]
);
}
public function getLimit()
{
return Configure::check('MISP.correlation_limit') ? Configure::read('MISP.correlation_limit') : 20;
}
public function getOverCorrelations($query)
{
$data = $this->find('all', $query);
$limit = $this->getLimit();
foreach ($data as $k => $v) {
if ($v['OverCorrelatingValue']['occurrence'] >= $limit) {
$data[$k]['OverCorrelatingValue']['over_correlation'] = true;
} else {
$data[$k]['OverCorrelatingValue']['over_correlation'] = false;
}
}
return $data;
}
public function checkValue($value)
{
$hit = $this->find('first', [
'recursive' => -1,
'conditions' => ['value' => $value],
'fields' => ['id']
]);
if (empty($hit)) {
return false;
}
return true;
}
}

15
app/Model/Post.php
View File

@ -28,6 +28,21 @@ class Post extends AppModel
),
);
public function afterSave($created, $options = array())
{
$post = $this->data;
if ($this->isTriggerCallable('post-after-save')) {
$workflowErrors = [];
$logging = [
'model' => 'Post',
'action' => $created ? 'add' : 'edit',
'id' => $post['Post']['id'],
];
$triggerData = $post;
$this->executeTrigger('post-after-save', $triggerData, $workflowErrors, $logging);
}
}
public function sendPostsEmailRouter($user_id, $post_id, $event_id, $title, $message)
{
if (Configure::read('MISP.background_jobs')) {

433
app/Model/Server.php
View File

@ -132,6 +132,59 @@ class Server extends AppModel
public $syncTestErrorCodes = array();
const MYSQL_RECOMMENDED_SETTINGS = [
'innodb_buffer_pool_size' => [
'default' => '134217728',
'recommended' => '2147483648',
'explanation' => 'The InnoDB buffer pool is the memory area where caches table and index data reside. It is the most important MySQL setting, in a dedicated server it should be around 3/4 of all the available RAM. In a shared server it should be around 1/2 of the available RAM.',
],
'innodb_dedicated_server' => [
'default' => '0',
'recommended' => '',
'explanation' => 'Set to `1` if the database is running in a dedicated server. The database engine will examine the available memory and dynamically set `innodb_buffer_pool_size`, `innodb_log_file_size`, `innodb_log_files_in_group` and `innodb_flush_method`. It is particularly useful in cloud enviroments that can be auto-scaled.',
],
'innodb_log_file_size' => [
'default' => '100663296',
'recommended' => '629145600',
'explanation' => 'This parameter determines the fixed size for MySQLs redo logs. Tuning this value affects the crash recovery time and also overall system performance.',
],
'innodb_log_files_in_group' => [
'default' => '2',
'recommended' => '2',
'explanation' => 'Defines the number of log files in the log group.',
],
'innodb_change_buffering' => [
'default' => 'none',
'recommended' => 'none',
'explanation' => 'Whether InnoDB performs change buffering, an optimization that delays write operations to secondary indexes so that the I/O operations can be performed sequentially, enabling it causes extremely long shutdown times for upgrades.',
],
'innodb_io_capacity' => [
'default' => '200',
'recommended' => '1000',
'explanation' => 'Defines the number of I/O operations per second (IOPS) available to InnoDB background tasks, such as flushing pages from the buffer pool and merging data from the change buffer.',
],
'innodb_io_capacity_max' => [
'default' => '2000',
'recommended' => '2000',
'explanation' => 'If flushing activity falls behind, InnoDB can flush more aggressively, at a higher rate of I/O operations per second (IOPS) than defined by the `innodb_io_capacity variable`.',
],
'innodb_stats_persistent' => [
'default' => 'ON',
'recommended' => 'ON',
'explanation' => 'Specifies whether InnoDB index statistics are persisted to disk. Otherwise, statistics may be recalculated frequently which can lead to variations in query execution plans.',
],
'innodb_read_io_threads' => [
'default' => '4',
'recommended' => '16',
'explanation' => 'The number of I/O threads for read operations in InnoDB.',
],
'innodb_write_io_threads' => [
'default' => '4',
'recommended' => '4',
'explanation' => 'The number of I/O threads for write operations in InnoDB.',
],
];
public function __construct($id = false, $table = null, $ds = null)
{
parent::__construct($id, $table, $ds);
@ -161,7 +214,12 @@ class Server extends AppModel
'description' => 'If your your database is locked and is not updating, unlock it here.',
'ignore_disabled' => true,
'url' => '/servers/releaseUpdateLock/'
)
),
'normalizeCustomTagsToTaxonomyFormat' => array(
'title' => 'Normalize custom tags to taxonomy format',
'description' => 'Transform all custom tags existing in a taxonomy into the taxonomy version',
'url' => '/taxonomies/normalizeCustomTagsToTaxonomyFormat/'
),
);
public $validEventIndexFilters = array('searchall', 'searchpublished', 'searchorg', 'searchtag', 'searcheventid', 'searchdate', 'searcheventinfo', 'searchthreatlevel', 'searchdistribution', 'searchanalysis', 'searchattribute');
@ -222,25 +280,14 @@ class Server extends AppModel
throw new InvalidArgumentException("Invalid pull technique `$technique`.");
}
private function __checkIfEventIsBlockedBeforePull($event)
{
if (Configure::read('MISP.enableEventBlocklisting') !== false) {
if (!isset($this->EventBlocklist)) {
$this->EventBlocklist = ClassRegistry::init('EventBlocklist');
}
if ($this->EventBlocklist->isBlocked($event['Event']['uuid'])) {
return true;
}
}
return false;
}
/**
* @param array $event
* @param array $server
* @param array $user
* @param array $pullRules
* @param bool $pullRulesEmptiedEvent
*/
private function __updatePulledEventBeforeInsert(array &$event, array $server, array $user, array $pullRules, bool &$pullRulesEmptiedEvent=false)
private function __updatePulledEventBeforeInsert(array &$event, array $server, array $user, array $pullRules, bool &$pullRulesEmptiedEvent = false)
{
// we have an Event array
// The event came from a pull, so it should be locked.
@ -269,14 +316,12 @@ class Server extends AppModel
}
}
$typeFilteringEnabled = !empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) &&
!empty($pullRules['type_attributes']['NOT']);
if (isset($event['Event']['Attribute'])) {
$originalCount = count($event['Event']['Attribute']);
foreach ($event['Event']['Attribute'] as $key => $attribute) {
if (
!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) &&
!empty($pullRules['type_attributes']['NOT']) &&
in_array($attribute['type'], $pullRules['type_attributes']['NOT'])
) {
if ($typeFilteringEnabled && in_array($attribute['type'], $pullRules['type_attributes']['NOT'])) {
unset($event['Event']['Attribute'][$key]);
continue;
}
@ -297,7 +342,7 @@ class Server extends AppModel
}
}
}
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalCount > 0 && count($event['Event']['Attribute']) == 0) {
if ($typeFilteringEnabled && $originalCount > 0 && empty($event['Event']['Attribute'])) {
$pullRulesEmptiedEvent = true;
}
}
@ -323,11 +368,7 @@ class Server extends AppModel
if (isset($object['Attribute'])) {
$originalAttributeCount = count($object['Attribute']);
foreach ($object['Attribute'] as $j => $a) {
if (
!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) &&
!empty($pullRules['type_attributes']['NOT']) &&
in_array($a['type'], $pullRules['type_attributes']['NOT'])
) {
if ($typeFilteringEnabled && in_array($a['type'], $pullRules['type_attributes']['NOT'])) {
unset($event['Event']['Object'][$i]['Attribute'][$j]);
continue;
}
@ -348,13 +389,13 @@ class Server extends AppModel
}
}
}
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalAttributeCount > 0 && empty($event['Event']['Object'][$i]['Attribute'])) {
if ($typeFilteringEnabled && $originalAttributeCount > 0 && empty($event['Event']['Object'][$i]['Attribute'])) {
unset($event['Event']['Object'][$i]); // Object is empty, get rid of it
$pullRulesEmptiedEvent = true;
}
}
}
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalObjectCount > 0 && count($event['Event']['Object']) == 0) {
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalObjectCount > 0 && empty($event['Event']['Object'])) {
$pullRulesEmptiedEvent = true;
}
}
@ -466,7 +507,18 @@ class Server extends AppModel
}
}
private function __pullEvent($eventId, &$successes, &$fails, Event $eventModel, ServerSyncTool $serverSync, $user, $jobId, $force = false)
/**
* @param int|string $eventId Event ID or UUID
* @param array $successes
* @param array $fails
* @param Event $eventModel
* @param ServerSyncTool $serverSync
* @param array $user
* @param int $jobId
* @param bool $force
* @return bool
*/
private function __pullEvent($eventId, array &$successes, array &$fails, Event $eventModel, ServerSyncTool $serverSync, $user, $jobId, $force = false)
{
$params = [
'deleted' => [0, 1],
@ -489,23 +541,16 @@ class Server extends AppModel
return false;
}
if (!empty($event)) {
if ($this->__checkIfEventIsBlockedBeforePull($event)) {
return false;
$pullRulesEmptiedEvent = false;
$this->__updatePulledEventBeforeInsert($event, $serverSync->server(), $user, $serverSync->pullRules(), $pullRulesEmptiedEvent);
if (!$this->__checkIfEventSaveAble($event)) {
if (!$pullRulesEmptiedEvent) { // The event is empty because of the filtering rule. This is not considered a failure
$fails[$eventId] = __('Empty event detected.');
}
$pullRulesEmptiedEvent = false;
$this->__updatePulledEventBeforeInsert($event, $serverSync->server(), $user, $serverSync->pullRules(), $pullRulesEmptiedEvent);
if (!$this->__checkIfEventSaveAble($event)) {
if (!$pullRulesEmptiedEvent) { // The event is empty because of the filtering rule. This is not considered a failure
$fails[$eventId] = __('Empty event detected.');
}
} else {
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $headers, $body);
}
} else {
// error
$fails[$eventId] = __('failed downloading the event');
return false;
}
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $headers, $body);
return true;
}
@ -551,7 +596,7 @@ class Server extends AppModel
if ($jobId) {
$job->saveProgress($jobId, $technique === 'pull_relevant_clusters' ? __('Pulling relevant galaxy clusters.') : __('Pulling galaxy clusters.'));
}
$pulledClusters = $this->GalaxyCluster->pullGalaxyClusters($user, $server, $technique);
$pulledClusters = $this->GalaxyCluster->pullGalaxyClusters($user, $serverSync, $technique);
if ($technique === 'pull_relevant_clusters') {
if ($jobId) {
$job->saveStatus($jobId, true, 'Pulling complete.');
@ -579,8 +624,8 @@ class Server extends AppModel
/** @var Event $eventModel */
$eventModel = ClassRegistry::init('Event');
$successes = array();
$fails = array();
$successes = [];
$fails = [];
// now process the $eventIds to pull each of the events sequentially
if (!empty($eventIds)) {
// download each event
@ -622,7 +667,7 @@ class Server extends AppModel
count($fails)
);
$this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['Server']['id'], 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $email, $change);
return array($successes, $fails, $pulledProposals, $pulledSightings, $pulledClusters);
return [$successes, $fails, $pulledProposals, $pulledSightings, $pulledClusters];
}
public function filterRuleToParameter($filter_rules)
@ -662,30 +707,20 @@ class Server extends AppModel
/**
* fetchCustomClusterIdsFromServer Fetch custom-published remote clusters' UUIDs and versions
*
* @param array $server
* @param HttpSocketExtended|null $HttpSocket
* @param ServerSyncTool $serverSync
* @param array $conditions
* @return array The list of clusters
* @throws JsonException|HttpSocketHttpException|HttpSocketJsonException
*/
private function fetchCustomClusterIdsFromServer(array $server, HttpSocketExtended $HttpSocket=null, array $conditions=array())
private function fetchCustomClusterIdsFromServer(ServerSyncTool $serverSync, array $conditions = [])
{
$url = $server['Server']['url'];
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
$uri = $url . '/galaxy_clusters/restSearch';
$filterRules = [
'published' => 1,
'minimal' => 1,
'custom' => 1,
];
$filterRules = array_merge($filterRules, $conditions);
$response = $HttpSocket->post($uri, json_encode($filterRules), $request);
if (!$response->isOk()) {
throw new HttpSocketHttpException($response);
}
$clusterArray = $response->json();
$clusterArray = $serverSync->galaxyClusterSearch($filterRules)->json();
if (isset($clusterArray['response'])) {
$clusterArray = $clusterArray['response'];
}
@ -705,9 +740,9 @@ class Server extends AppModel
* @throws HttpSocketJsonException
* @throws JsonException
*/
public function getElligibleClusterIdsFromServerForPull(array $server, $HttpSocket=null, $onlyUpdateLocalCluster=true, array $elligibleClusters=array(), array $conditions=array())
public function getElligibleClusterIdsFromServerForPull(ServerSyncTool $serverSyncTool, $onlyUpdateLocalCluster=true, array $elligibleClusters=array(), array $conditions=array())
{
$clusterArray = $this->fetchCustomClusterIdsFromServer($server, $HttpSocket, $conditions=$conditions);
$clusterArray = $this->fetchCustomClusterIdsFromServer($serverSyncTool, $conditions=$conditions);
if (!empty($clusterArray)) {
foreach ($clusterArray as $cluster) {
if (isset($elligibleClusters[$cluster['GalaxyCluster']['uuid']])) {
@ -730,8 +765,7 @@ class Server extends AppModel
/**
* Get an array of cluster_ids that are present on the remote server and returns clusters that should be pushed.
* @param array $server
* @param HttpSocket|null $HttpSocket
* @param ServerSyncTool $serverSync
* @param array $localClusters
* @param array $conditions
* @return array
@ -739,9 +773,9 @@ class Server extends AppModel
* @throws HttpSocketJsonException
* @throws JsonException
*/
public function getElligibleClusterIdsFromServerForPush(array $server, $HttpSocket=null, $localClusters=array(), $conditions=array())
private function getElligibleClusterIdsFromServerForPush(ServerSyncTool $serverSync, array $localClusters=array(), array $conditions=array())
{
$clusterArray = $this->fetchCustomClusterIdsFromServer($server, $HttpSocket, $conditions=$conditions);
$clusterArray = $this->fetchCustomClusterIdsFromServer($serverSync, $conditions=$conditions);
$keyedClusterArray = Hash::combine($clusterArray, '{n}.GalaxyCluster.uuid', '{n}.GalaxyCluster.version');
if (!empty($localClusters)) {
foreach ($localClusters as $k => $localCluster) {
@ -784,13 +818,35 @@ class Server extends AppModel
return $eventIndex;
}
/**
* @param array $events
* @return void
*/
private function removeOlderEvents(array &$events)
{
$conditions = (count($events) > 10000) ? [] : ['Event.uuid' => array_column($events, 'uuid')];
$this->Event = ClassRegistry::init('Event');
$localEvents = $this->Event->find('all', [
'recursive' => -1,
'conditions' => $conditions,
'fields' => ['Event.uuid', 'Event.timestamp', 'Event.locked'],
]);
$localEvents = array_column(array_column($localEvents, 'Event'), null, 'uuid');
foreach ($events as $k => $event) {
$uuid = $event['uuid'];
if (isset($localEvents[$uuid]) && ($localEvents[$uuid]['timestamp'] >= $event['timestamp'] || !$localEvents[$uuid]['locked'])) {
unset($events[$k]);
}
}
}
/**
* Get an array of event UUIDs that are present on the remote server.
*
* @param ServerSyncTool $serverSync
* @param bool $all
* @param bool $ignoreFilterRules
* @param bool $force
* @param bool $ignoreFilterRules Ignore defined server pull rules
* @param bool $force If true, returns all events regardless their update timestamp
* @return array Array of event UUIDs.
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
@ -820,8 +876,7 @@ class Server extends AppModel
}
}
if (!$force) {
$this->Event = ClassRegistry::init('Event');
$this->Event->removeOlder($eventArray);
$this->removeOlderEvents($eventArray);
}
return array_column($eventArray, 'uuid');
}
@ -940,7 +995,7 @@ class Server extends AppModel
// sync custom galaxy clusters if user is capable
if ($push['canEditGalaxyCluster'] && $server['Server']['push_galaxy_clusters'] && "full" == $technique) {
$clustersSuccesses = $this->syncGalaxyClusters($HttpSocket, $this->data, $user, $technique='full');
$clustersSuccesses = $this->syncGalaxyClusters($serverSync, $this->data, $user, $technique='full');
} else {
$clustersSuccesses = array();
}
@ -989,7 +1044,7 @@ class Server extends AppModel
'fields' => array('Event.id', 'Event.timestamp', 'Event.sighting_timestamp', 'Event.uuid', 'Event.orgc_id'), // array of field names
);
$eventIds = $this->Event->find('all', $findParams);
$eventUUIDsFiltered = $this->getEventIdsForPush($server, $HttpSocket, $eventIds);
$eventUUIDsFiltered = $this->getEventIdsForPush($server, $serverSync, $eventIds);
if (!empty($eventUUIDsFiltered)) {
$eventCount = count($eventUUIDsFiltered);
// now process the $eventIds to push each of the events sequentially
@ -1018,11 +1073,11 @@ class Server extends AppModel
$event = $event[0];
$event['Event']['locked'] = 1;
if ($push['canEditGalaxyCluster'] && $server['Server']['push_galaxy_clusters'] && "full" != $technique) {
$clustersSuccesses = $this->syncGalaxyClusters($HttpSocket, $this->data, $user, $technique=$event['Event']['id'], $event=$event);
$clustersSuccesses = $this->syncGalaxyClusters($serverSync, $this->data, $user, $technique=$event['Event']['id'], $event=$event);
} else {
$clustersSuccesses = array();
}
$result = $this->Event->uploadEventToServer($event, $server, $HttpSocket);
$result = $this->Event->uploadEventToServer($event, $server, $serverSync);
if ('Success' === $result) {
$successes[] = $event['Event']['id'];
} else {
@ -1049,7 +1104,7 @@ class Server extends AppModel
if ($push['canPush'] || $push['canSight']) {
$this->Sighting = ClassRegistry::init('Sighting');
$sightingSuccesses =$this->Sighting->pushSightings($user, $serverSync);
$sightingSuccesses = $this->Sighting->pushSightings($user, $serverSync);
} else {
$sightingSuccesses = array();
}
@ -1083,23 +1138,35 @@ class Server extends AppModel
return true;
}
public function getEventIdsForPush(array $server, HttpSocket $HttpSocket, array $eventIds)
/**
* @param array $server
* @param ServerSyncTool $serverSync
* @param array $events
* @return array|false
*/
private function getEventIdsForPush(array $server, ServerSyncTool $serverSync, array $events)
{
foreach ($eventIds as $k => $event) {
if (empty($this->eventFilterPushableServers($event, array($server)))) {
unset($eventIds[$k]);
$request = [];
foreach ($events as $event) {
if (empty($this->eventFilterPushableServers($event, [$server]))) {
continue;
}
unset($eventIds[$k]['Event']['id']);
$request[] = ['Event' => [
'uuid' => $event['Event']['uuid'],
'timestamp' => $event['Event']['timestamp'],
]];
}
$request = $this->setupSyncRequest($server);
$data = json_encode($eventIds);
$uri = $server['Server']['url'] . '/events/filterEventIdsForPush';
$response = $HttpSocket->post($uri, $data, $request);
if ($response->code == '200') {
return $this->jsonDecode($response->body());
if (empty($request)) {
return [];
}
try {
return $serverSync->filterEventIdsForPush($request)->json();
} catch (Exception $e) {
$this->logException("Could not filter events for push when pushing to server {$serverSync->serverId()}", $e);
return false;
}
return false;
}
/**
@ -1112,7 +1179,7 @@ class Server extends AppModel
* @param array|bool $event
* @return array List of successfully pushed clusters
*/
public function syncGalaxyClusters($HttpSocket, array $server, array $user, $technique='full', $event=false)
public function syncGalaxyClusters(ServerSyncTool $serverSync, array $server, array $user, $technique='full', $event=false)
{
$successes = array();
if (!$server['Server']['push_galaxy_clusters']) {
@ -1120,7 +1187,6 @@ class Server extends AppModel
}
$this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
$this->Event = ClassRegistry::init('Event');
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$clusters = array();
if ($technique == 'full') {
$clusters = $this->GalaxyCluster->getElligibleClustersToPush($user, $conditions=array(), $full=true);
@ -1135,13 +1201,13 @@ class Server extends AppModel
}
$localClusterUUIDs = Hash::extract($clusters, '{n}.GalaxyCluster.uuid');
try {
$clustersToPush = $this->getElligibleClusterIdsFromServerForPush($server, $HttpSocket = $HttpSocket, $localClusters = $clusters, $conditions = array('uuid' => $localClusterUUIDs));
$clustersToPush = $this->getElligibleClusterIdsFromServerForPush($serverSync, $localClusters = $clusters, $conditions = array('uuid' => $localClusterUUIDs));
} catch (Exception $e) {
$this->logException("Could not get eligible cluster IDs from server #{$server['Server']['id']} for push.", $e);
return [];
}
foreach ($clustersToPush as $cluster) {
$result = $this->GalaxyCluster->uploadClusterToServer($cluster, $server, $HttpSocket, $user);
$result = $this->GalaxyCluster->uploadClusterToServer($cluster, $server, $serverSync, $user);
if ($result === 'Success') {
$successes[] = __('GalaxyCluster %s', $cluster['GalaxyCluster']['uuid']);
}
@ -1153,10 +1219,10 @@ class Server extends AppModel
{
$saModel = ClassRegistry::init('ShadowAttribute');
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
if ($sa_id == null) {
if ($event_id == null) {
// event_id is null when we are doing a push
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
try {
$ids = $this->getEventIdsFromServer($serverSync, true, true);
} catch (Exception $e) {
@ -1226,7 +1292,7 @@ class Server extends AppModel
public function getCurrentServerSettings()
{
$serverSettings = $this->serverSettings;
$moduleTypes = array('Enrichment', 'Import', 'Export', 'Cortex');
$moduleTypes = array('Enrichment', 'Import', 'Export', 'Action', 'Cortex');
return $this->readModuleSettings($serverSettings, $moduleTypes);
}
@ -1248,6 +1314,15 @@ class Server extends AppModel
$setting['test'] = 'testBool';
$setting['type'] = 'boolean';
$setting['description'] = __('Enable or disable the %s module.', $module);
if (!empty($result['description'])) {
$setting['description'] = sprintf(
"[%s%s%s] %s",
'<span class="bold">',
$setting['description'],
'</span>',
$result['description']
);
}
$setting['value'] = false;
} elseif ($result['type'] === 'orgs') {
$setting['description'] = __('Restrict the %s module to the given organisation.', $module);
@ -1258,15 +1333,29 @@ class Server extends AppModel
return $this->loadLocalOrganisations();
};
} else {
$setting['test'] = 'testForEmpty';
$setting['type'] = 'string';
$setting['test'] = isset($result['test']) ? $result['test'] : 'testForEmpty';
$setting['type'] = isset($result['type']) ? $result['type'] : 'string';
$setting['description'] = isset($result['description']) ? $result['description'] : __('Set this required module specific setting.');
$setting['value'] = '';
$setting['value'] = isset($result['value']) ? $result['value'] : '';
}
$serverSettings['Plugin'][$moduleType . '_' . $module . '_' . $result['name']] = $setting;
}
}
}
if (Configure::read('Plugin.Workflow_enable')) {
$this->Workflow = ClassRegistry::init('Workflow');
$triggerModules = $this->Workflow->getModulesByType('trigger');
foreach ($triggerModules as $triggerModule) {
$setting = [
'level' => 1,
'description' => __('Enable/disable the `%s` trigger', $triggerModule['id']),
'value' => false,
'test' => 'testBool',
'type' => 'boolean'
];
$serverSettings['Plugin']['Workflow_triggers_' . $triggerModule['id']] = $setting;
}
}
}
return $serverSettings;
}
@ -1514,6 +1603,17 @@ class Server extends AppModel
}
}
public function testForCorrelationEngine($value)
{
$defaults = $this->generateServerSettings();
$options = $defaults['MISP']['correlation_engine']['options'];
if (!empty($value) && !in_array($value, array_keys($options))) {
return __('Please select a valid option from the list of available engines: ', implode(', ', array_keys($options)));
} else {
return true;
}
}
public function testLocalOrg($value)
{
if ($value == 0) {
@ -2095,7 +2195,7 @@ class Server extends AppModel
// This is just hack to reset opcache, so for next request cache will be reloaded.
$this->opcacheResetConfig();
if (strpos($settingName, 'Plugin.Enrichment') !== false || strpos($settingName, 'Plugin.Import') !== false || strpos($settingName, 'Plugin.Export') !== false || strpos($settingName, 'Plugin.Cortex') !== false) {
if (strpos($settingName, 'Plugin.Enrichment') !== false || strpos($settingName, 'Plugin.Import') !== false || strpos($settingName, 'Plugin.Export') !== false || strpos($settingName, 'Plugin.Cortex') !== false || strpos($settingName, 'Plugin.Action') !== false || strpos($settingName, 'Plugin.Workflow') !== false) {
$serverSettings = $this->getCurrentServerSettings();
} else {
$serverSettings = $this->serverSettings;
@ -2503,7 +2603,6 @@ class Server extends AppModel
$canPush = isset($remoteVersion['perm_sync']) ? $remoteVersion['perm_sync'] : false;
$canSight = isset($remoteVersion['perm_sighting']) ? $remoteVersion['perm_sighting'] : false;
$supportEditOfGalaxyCluster = isset($remoteVersion['perm_galaxy_editor']);
$canEditGalaxyCluster = isset($remoteVersion['perm_galaxy_editor']) ? $remoteVersion['perm_galaxy_editor'] : false;
$remoteVersionString = $remoteVersion['version'];
$remoteVersion = explode('.', $remoteVersion['version']);
@ -2555,7 +2654,6 @@ class Server extends AppModel
'canPush' => $canPush,
'canSight' => $canSight,
'canEditGalaxyCluster' => $canEditGalaxyCluster,
'supportEditOfGalaxyCluster' => $supportEditOfGalaxyCluster,
'version' => $remoteVersion,
'protectedMode' => $protectedMode,
];
@ -2706,6 +2804,44 @@ class Server extends AppModel
return $schemaDiagnostic;
}
/*
* Get RDBMS configuration values
*/
public function dbConfiguration(): array
{
if ($this->isMysql()) {
$configuration = [];
$dbVariables = $this->query("SHOW VARIABLES;");
$settings = array_keys(self::MYSQL_RECOMMENDED_SETTINGS);
foreach ($dbVariables as $dbVariable) {
// different rdbms have different casing
if (isset($dbVariable['SESSION_VARIABLES'])) {
$dbVariable = $dbVariable['SESSION_VARIABLES'];
} elseif (isset($dbVariable['session_variables'])) {
$dbVariable = $dbVariable['session_variables'];
} else {
continue;
}
if (in_array($dbVariable['Variable_name'], $settings)) {
$configuration[] = [
'name' => $dbVariable['Variable_name'],
'value' => $dbVariable['Value'],
'default' => self::MYSQL_RECOMMENDED_SETTINGS[$dbVariable['Variable_name']]['default'],
'recommended' => self::MYSQL_RECOMMENDED_SETTINGS[$dbVariable['Variable_name']]['recommended'],
'explanation' => self::MYSQL_RECOMMENDED_SETTINGS[$dbVariable['Variable_name']]['explanation'],
];
}
}
return $configuration;
} else {
return [];
}
}
/*
* Work in progress, still needs DEFAULT in the schema for it to work correctly
* Currently only works for missing_column and column_different
@ -3024,13 +3160,6 @@ class Server extends AppModel
$tableIndexDiff = array_diff(array_keys($indexes), array_keys($actualIndex[$tableName])); // check for missing indexes
foreach ($tableIndexDiff as $columnDiff) {
$shouldBeUnique = $indexes[$columnDiff];
if ($shouldBeUnique && !$this->checkIfColumnContainsJustUniqueValues($tableName, $columnDiff)) {
$indexDiff[$tableName][$columnDiff] = array(
'message' => __('Column `%s` should be unique indexed, but contains duplicate values', $columnDiff),
'sql' => '',
);
continue;
}
$message = __('Column `%s` should be indexed', $columnDiff);
$indexDiff[$tableName][$columnDiff] = array(
@ -3058,15 +3187,6 @@ class Server extends AppModel
'sql' => $sql,
);
} else {
if (!$this->checkIfColumnContainsJustUniqueValues($tableName, $column)) {
$message = __('Column `%s` should be unique index, but contains duplicate values', $column);
$indexDiff[$tableName][$column] = array(
'message' => $message,
'sql' => '',
);
continue;
}
$sql = $this->generateSqlDropIndexQuery($tableName, $column);
$sql .= '<br>' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, true);
@ -3309,7 +3429,15 @@ class Server extends AppModel
return 1;
}
$pubSubTool = $this->getPubSubTool();
if (!$pubSubTool->checkIfPythonLibInstalled()) {
try {
$isInstalled = $pubSubTool->checkIfPythonLibInstalled();
} catch (Exception $e) {
$this->logException('ZMQ is not properly installed.', $e, LOG_NOTICE);
$diagnostic_errors++;
return 2;
}
if (!$isInstalled) {
$diagnostic_errors++;
return 2;
}
@ -3391,6 +3519,7 @@ class Server extends AppModel
$sqlResult = $this->query($sql);
if (isset($sqlResult[0][0])) {
$sessionCount = $sqlResult[0][0]['session_count'];
$errorCode = 0;
} else {
$errorCode = 9;
}
@ -3762,8 +3891,7 @@ class Server extends AppModel
public function extensionDiagnostics()
{
try {
$file = new File(APP . DS . 'composer.json');
$composer = $this->jsonDecode($file->read());
$composer = FileAccessTool::readJsonFromFile(APP . DS . 'composer.json');
$extensions = [];
foreach ($composer['require'] as $require => $foo) {
if (substr($require, 0, 4) === 'ext-') {
@ -4738,14 +4866,34 @@ class Server extends AppModel
'type' => 'boolean',
'null' => true
],
'enable_advanced_correlations' => array(
'correlation_engine' => [
'level' => 0,
'description' => __('Choose which correlation engine to use. MISP defaults to the default engine, maintaining all data in the database whilst enforcing ACL rules on any non site-admin user. This is recommended for any MISP instnace with multiple organisations. If you are an endpoint MISP, consider switching to the much leaner and faster No ACL engine.'),
'value' => 'default',
'test' => 'testForCorrelationEngine',
'type' => 'string',
'null' => true,
'options' => [
'Default' => __('Default Correlation Engine'),
'NoAcl' => __('No ACL Engine')
],
],
'correlation_limit' => [
'level' => 0,
'description' => __('Set a value for the maximum number of correlations a value should have before MISP will refuse to correlate it (extremely over-correlating values are rarely useful from a correlation perspective).'),
'value' => 100,
'test' => 'testForNumeric',
'type' => 'numeric',
'null' => true
],
'enable_advanced_correlations' => [
'level' => 0,
'description' => __('Enable some performance heavy correlations (currently CIDR correlation)'),
'value' => false,
'test' => 'testBool',
'type' => 'boolean',
'null' => true
),
],
'server_settings_skip_backup_rotate' => array(
'level' => 1,
'description' => __('Enable this setting to directly save the config.php file without first creating a temporary file and moving it to avoid concurency issues. Generally not recommended, but useful when for example other tools modify/maintain the config.php file.'),
@ -6973,6 +7121,34 @@ class Server extends AppModel
'test' => 'testForEmpty',
'type' => 'numeric'
),
'Action_services_url' => array(
'level' => 1,
'description' => __('The url used to access the action services. By default, it is accessible at http://127.0.0.1:6666'),
'value' => 'http://127.0.0.1',
'test' => 'testForEmpty',
'type' => 'string'
),
'Action_services_port' => array(
'level' => 1,
'description' => __('The port used to access the action services. By default, it is accessible at 127.0.0.1:6666'),
'value' => '6666',
'test' => 'testForPortNumber',
'type' => 'numeric'
),
'Action_services_enable' => array(
'level' => 0,
'description' => __('Enable/disable the action services'),
'value' => false,
'test' => 'testBool',
'type' => 'boolean'
),
'Action_timeout' => array(
'level' => 1,
'description' => __('Set a timeout for the action services'),
'value' => 10,
'test' => 'testForEmpty',
'type' => 'numeric'
),
'Enrichment_hover_enable' => array(
'level' => 0,
'description' => __('Enable/disable the hover over information retrieved from the enrichment modules'),
@ -7008,6 +7184,20 @@ class Server extends AppModel
'test' => 'testForPortNumber',
'type' => 'numeric'
),
'Workflow_enable' => array(
'level' => 1,
'description' => __('Enable/disable workflow feature. [experimental]'),
'value' => false,
'test' => 'testBool',
'type' => 'boolean'
),
'Workflow_debug_url' => array(
'level' => 1,
'description' => __('Set the debug URL where info about workflow execution will be POSTed'),
'value' => 'http://127.0.0.1:27051',
'test' => 'testForEmpty',
'type' => 'string'
),
'Cortex_services_url' => array(
'level' => 1,
'description' => __('The url used to access Cortex. By default, it is accessible at http://cortex-url'),
@ -7273,6 +7463,7 @@ class Server extends AppModel
'Get IPs for user ID' => 'MISP/app/Console/cake Admin UserIP [user_id]',
'Get user ID for user IP' => 'MISP/app/Console/cake Admin IPUser [ip]',
'Generate correlation' => 'MISP/app/Console/cake Admin jobGenerateCorrelation [job_id]',
'Truncate correlation table' => 'MISP/app/Console/cake Admin truncateTable [user_id] [correlation_engine_name] [job_id]',
'Purge correlation' => 'MISP/app/Console/cake Admin jobPurgeCorrelation [job_id]',
'Generate shadow attribute correlation' => 'MISP/app/Console/cake Admin jobGenerateShadowAttributeCorrelation [job_id]',
'Update MISP' => 'MISP/app/Console/cake Admin updateMISP',

6
app/Model/ShadowAttribute.php
View File

@ -190,7 +190,8 @@ class ShadowAttribute extends AppModel
$this->data['ShadowAttribute']['deleted'] = 0;
}
if ($this->data['ShadowAttribute']['deleted']) {
$this->__beforeDeleteCorrelation($this->data['ShadowAttribute']);
// correlations for proposals are deprecated.
//$this->__beforeDeleteCorrelation($this->data['ShadowAttribute']);
}
// convert into utc and micro sec
@ -277,12 +278,15 @@ class ShadowAttribute extends AppModel
$result = $result && $this->saveBase64EncodedAttachment($this->data['ShadowAttribute']);
}
}
/*
* correlations are deprecated for proposals
if ((isset($this->data['ShadowAttribute']['deleted']) && $this->data['ShadowAttribute']['deleted']) || (isset($this->data['ShadowAttribute']['proposal_to_delete']) && $this->data['ShadowAttribute']['proposal_to_delete'])) {
// this is a deletion
// Could be a proposal to delete or flagging a proposal that it was discarded / accepted - either way, we don't want to correlate here for now
} else {
$this->__afterSaveCorrelation($this->data['ShadowAttribute']);
}
*/
if (empty($this->data['ShadowAttribute']['deleted'])) {
$action = $created ? 'add' : 'edit';
$this->publishKafkaNotification('shadow_attribute', $this->data, $action);

3
app/Model/Tag.php
View File

@ -583,7 +583,8 @@ class Tag extends AppModel
$changedTags = $this->AttributeTag->getAffectedRows();
$this->EventTag->updateAll(['tag_id' => $destinationTag['Tag']['id']], ['tag_id' => $sourceTag['Tag']['id']]);
$changedTags += $this->EventTag->getAffectedRows();
$this->GalaxyClusterRelationTag->updateAll(['tag_id' => $destinationTag['Tag']['id']], ['tag_id' => $sourceTag['Tag']['id']]);
$changedTags += $this->GalaxyClusterRelationTag->getAffectedRows();
$this->delete($sourceTag['Tag']['id']);
return [

73
app/Model/Taxonomy.php
View File

@ -770,4 +770,77 @@ class Taxonomy extends AppModel
}
return $splits;
}
private function __craftTaxonomiesTags()
{
$taxonomies = $this->find('all', [
'fields' => ['namespace'],
'contain' => ['TaxonomyPredicate' => ['TaxonomyEntry']],
]);
$allTaxonomyTags = [];
foreach ($taxonomies as $taxonomy) {
$namespace = $taxonomy['Taxonomy']['namespace'];
foreach ($taxonomy['TaxonomyPredicate'] as $predicate) {
if (isset($predicate['TaxonomyEntry']) && !empty($predicate['TaxonomyEntry'])) {
foreach ($predicate['TaxonomyEntry'] as $entry) {
$tag = $namespace . ':' . $predicate['value'] . '="' . $entry['value'] . '"';
$allTaxonomyTags[$tag] = true;
}
} else {
$tag = $namespace . ':' . $predicate['value'];
$allTaxonomyTags[$tag] = true;
}
}
}
return $allTaxonomyTags;
}
/**
* normalizeCustomTagsToTaxonomyFormat Transform all custom tags into their taxonomy version.
*
* @return int The number of converted tag
*/
public function normalizeCustomTagsToTaxonomyFormat(): array
{
$tagConverted = 0;
$rowUpdated = 0;
$craftedTags = $this->__craftTaxonomiesTags();
$allTaxonomyTagsByName = Hash::combine($this->getAllTaxonomyTags(false, false, true, false, true), '{n}.Tag.name', '{n}.Tag.id');
$tagsToMigrate = array_diff_key($allTaxonomyTagsByName, $craftedTags);
foreach ($tagsToMigrate as $tagToMigrate_name => $tagToMigrate_id) {
foreach (array_keys($craftedTags) as $craftedTag) {
if (strcasecmp($craftedTag, $tagToMigrate_name) == 0) {
$result = $this->__updateTagToNormalized(intval($tagToMigrate_id), intval($allTaxonomyTagsByName[$craftedTag]));
$tagConverted += 1;
$rowUpdated += $result['changed'];
}
}
}
return [
'tag_converted' => $tagConverted,
'row_updated' => $rowUpdated,
];
}
/**
* __updateTagToNormalized Change the link of element having $source_id tag attached to them for the $target_id one.
* Updated:
* - event_tags
* - attribute_tags
* - galaxy_cluster_relation_tags
*
* Ignored: As this is defined by users, let them do the migration themselves
* - tag_collection_tags
* - template_tags
* - favorite_tags
*
* @param int $source_id
* @param int $target_id
* @return array
* @throws Exception
*/
private function __updateTagToNormalized($source_id, $target_id): array
{
return $this->Tag->mergeTag($source_id, $target_id);
}
}

48
app/Model/User.php
View File

@ -269,6 +269,27 @@ class User extends AppModel
$passwordHasher = new BlowfishConstantPasswordHasher();
$this->data[$this->alias]['password'] = $passwordHasher->hash($this->data[$this->alias]['password']);
}
$user = $this->data;
$action = empty($this->id) ? 'add' : 'edit';
$user_id = $action == 'add' ? 0 : $user['User']['id'];
$trigger_id = 'user-before-save';
$workflowErrors = [];
$logging = [
'model' => 'User',
'action' => $action,
'id' => $user_id,
'message' => __('The workflow `%s` prevented the saving of user %s', $trigger_id, $user_id),
];
if (
empty($user['User']['action']) ||
(
$user['User']['action'] != 'logout' &&
$user['User']['action'] != 'login'
)
) {
$success = $this->executeTrigger($trigger_id, $user['User'], $workflowErrors, $logging);
return !empty($success);
}
return true;
}
@ -276,6 +297,23 @@ class User extends AppModel
{
$pubToZmq = $this->pubToZmq('user');
$kafkaTopic = $this->kafkaTopic('user');
$action = empty($created) ? 'edit' : 'add';
$user = $this->data;
if (
empty($user['User']['action']) ||
(
$user['User']['action'] != 'logout' &&
$user['User']['action'] != 'login'
)
) {
$workflowErrors = [];
$logging = [
'model' => 'User',
'action' => $action,
'id' => $user['User']['id'],
];
$this->executeTrigger('user-after-save', $user['User'], $workflowErrors, $logging);
}
if ($pubToZmq || $kafkaTopic) {
if (!empty($this->data)) {
$user = $this->data;
@ -1581,4 +1619,14 @@ class User extends AppModel
'conditions' => array('EventDelegation.org_id' => $user['org_id'])
));
}
/**
* Generate code that is used in event alert unsubscribe link.
* @return string
*/
public function unsubscribeCode(array $user)
{
$salt = Configure::read('Security.salt');
return substr(hash('sha256', "{$user['id']}|$salt"), 0, 8);
}
}

1352
app/Model/Workflow.php Normal file
View File

File diff suppressed because it is too large Load Diff

173
app/Model/WorkflowBlueprint.php Normal file
View File

@ -0,0 +1,173 @@
<?php
App::uses('AppModel', 'Model');
App::uses('Folder', 'Utility');
App::uses('File', 'Utility');
class WorkflowBlueprint extends AppModel
{
const REPOSITORY_PATH = APP . 'files' . DS . 'misp-workflow-blueprints' . DS . 'blueprints';
public $recursive = -1;
public $actsAs = [
'AuditLog',
'Containable',
'SysLogLogable.SysLogLogable' => [
'roleModel' => 'Role',
'roleKey' => 'role_id',
'change' => 'full'
],
];
public $belongsTo = [
];
public $validate = [
'value' => [
'stringNotEmpty' => [
'rule' => ['stringNotEmpty']
]
],
'uuid' => [
'uuid' => [
'rule' => 'uuid',
'message' => 'Please provide a valid RFC 4122 UUID'
],
'unique' => [
'rule' => 'isUnique',
'message' => 'The UUID provided is not unique',
'required' => 'create'
]
],
];
const CAPTURE_FIELDS = ['name', 'description', 'timestamp', 'data'];
public function beforeValidate($options = array())
{
parent::beforeValidate();
if (empty($this->data['WorkflowBlueprint']['uuid'])) {
$this->data['WorkflowBlueprint']['uuid'] = CakeText::uuid();
} else {
$this->data['WorkflowBlueprint']['uuid'] = strtolower($this->data['WorkflowBlueprint']['uuid']);
}
if (empty($this->data['WorkflowBlueprint']['data'])) {
$this->data['WorkflowBlueprint']['data'] = [];
}
if (empty($this->data['WorkflowBlueprint']['timestamp'])) {
$this->data['WorkflowBlueprint']['timestamp'] = time();
}
return true;
}
public function afterFind($results, $primary = false)
{
foreach ($results as $k => $result) {
if (empty($result['WorkflowBlueprint']['data'])) {
$result['WorkflowBlueprint']['data'] = '{}';
}
$results[$k]['WorkflowBlueprint']['data'] = JsonTool::decode($result['WorkflowBlueprint']['data']);
$results[$k] = $this->attachModuleDataToBlueprint($results[$k]);
if (!empty($results[$k]['WorkflowBlueprint']['data'])) {
$results[$k]['WorkflowBlueprint']['mermaid'] = $this->getMermaidForData($results[$k]['WorkflowBlueprint']['data']);
}
}
return $results;
}
public function beforeSave($options = [])
{
if (is_array($this->data['WorkflowBlueprint']['data'])) {
$this->data['WorkflowBlueprint']['data'] = JsonTool::encode($this->data['WorkflowBlueprint']['data']);
}
return true;
}
public function attachModuleDataToBlueprint(array $blueprint)
{
$this->Workflow = ClassRegistry::init('Workflow');
foreach ($blueprint['WorkflowBlueprint']['data'] as $i => $node) {
$module = $this->Workflow->getModuleConfigByType($node['data']['module_type'], $node['data']['id']);
$blueprint['WorkflowBlueprint']['data'][$i]['data']['module_data'] = $module;
}
return $blueprint;
}
/**
* __readBlueprintsFromRepo Reads blueprints from the misp-workflow-blueprints repository
*
* @return array
*/
private function __readBlueprintsFromRepo(): array
{
$dir = new Folder(self::REPOSITORY_PATH);
$files = $dir->find('.*\.json');
$blueprints = [];
foreach ($files as $file) {
$blueprints[] = FileAccessTool::readJsonFromFile($dir->pwd() . DS . $file);
}
return $blueprints;
}
/**
* update Update the blueprint using the default repository
*
* @param boolean $force
* @return void
*/
public function update($force=false)
{
$blueprints_from_repo = $this->__readBlueprintsFromRepo();
if (empty($blueprints_from_repo)) {
throw new NotFoundException(__('Default blueprints could not be loaded or `%s` folder is empty', self::REPOSITORY_PATH));
}
$existing_blueprints = $this->find('all', [
'recursive' => -1
]);
$existing_blueprints_by_uuid = Hash::combine($existing_blueprints, '{n}.WorkflowBlueprint.uuid', '{n}.WorkflowBlueprint');
foreach ($blueprints_from_repo as $blueprint_from_repo) {
$blueprint_from_repo = $blueprint_from_repo['WorkflowBlueprint'];
$blueprint_from_repo['default'] = true;
if (!empty($existing_blueprints_by_uuid[$blueprint_from_repo['uuid']])) {
$existing_blueprint = $existing_blueprints_by_uuid[$blueprint_from_repo['uuid']];
if ($force || $blueprint_from_repo['timestamp'] > $existing_blueprint['timestamp']) {
$blueprint_from_repo['id'] = $existing_blueprint['id'];
$this->save($blueprint_from_repo);
continue;
}
} else {
$this->create();
$this->save($blueprint_from_repo);
}
}
}
public function getMermaidForData($workflowBlueprintData)
{
App::uses('MermaidFlowchartTool', 'Tools');
$mermaid = MermaidFlowchartTool::mermaid($workflowBlueprintData);
return $mermaid;
}
public function getDotNotation($id)
{
App::uses('GraphvizDOTTool', 'Tools');
$blueprint = $this->find('first', [
'conditions' => ['id' => $id],
'recursive' => -1,
]);
$dot = GraphvizDOTTool::dot($blueprint['WorkflowBlueprint']['data']);
return $dot;
}
public function getMermaid($id)
{
App::uses('MermaidFlowchartTool', 'Tools');
$blueprint = $this->find('first', [
'conditions' => ['id' => $id],
'recursive' => -1,
]);
$mermaid = MermaidFlowchartTool::mermaid($blueprint['WorkflowBlueprint']['data']);
return $mermaid;
}
}

112
app/Model/WorkflowModules/Module_misp_module.php Normal file
View File

@ -0,0 +1,112 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_misp_module extends WorkflowBaseActionModule
{
public $blocking = false;
public $is_misp_module = true;
public $id = 'misp-module';
public $name = 'misp-module';
public $description = 'misp-module';
public $icon = 'python';
public $icon_class = 'fab';
public $inputs = 1;
public $outputs = 1;
public $support_filters = false;
public $params = [];
/** @var Module */
private $Module;
private $misp_module_config;
public function __construct($misp_module_config)
{
parent::__construct();
$this->id = Inflector::underscore($misp_module_config['name']);
$this->name = $misp_module_config['name'];
$this->description = $misp_module_config['meta']['description'];
if (!empty($misp_module_config['meta']['icon'])) {
$this->icon = $misp_module_config['meta']['icon'];
}
if (!empty($misp_module_config['meta']['icon_class'])) {
$this->icon_class = $misp_module_config['meta']['icon_class'];
}
if (!empty($misp_module_config['meta']['inputs'])) {
$this->inputs = (int)$misp_module_config['meta']['inputs'];
}
if (!empty($misp_module_config['meta']['outputs'])) {
$this->inputs = (int)$misp_module_config['meta']['outputs'];
}
if (!empty($misp_module_config['meta']['config']['blocking'])) {
$this->blocking = !empty($misp_module_config['meta']['config']['blocking']);
}
if (!empty($misp_module_config['meta']['config']['expect_misp_core_format'])) {
$this->expect_misp_core_format = !empty($misp_module_config['meta']['config']['expect_misp_core_format']);
}
if (!empty($misp_module_config['meta']['config']['support_filters'])) {
$this->support_filters = !empty($misp_module_config['meta']['config']['support_filters']);
}
if (!empty($misp_module_config['meta']['config'])) {
foreach ($misp_module_config['meta']['config']['params'] as $paramName => $moduleParam) {
$this->params[] = $this->translateParams($paramName, $moduleParam);
}
}
$this->Module = ClassRegistry::init('Module');
}
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool
{
parent::exec($node, $roamingData);
$postData = ['module' => $this->name];
$rData = $roamingData->getData();
$postData['data'] = $rData;
if ($this->support_filters) {
$filters = $this->getFilters($node);
$extracted = $this->extractData($rData, $filters['selector']);
if ($extracted === false) {
return false;
}
$filteredItems = $this->getItemsMatchingCondition($extracted, $filters['value'], $filters['operator'], $filters['path']);
$postData['filteredItems'] = !empty($filteredItems) ? $filteredItems : $rData;
}
$indexedParams = $this->getParamsWithValues($node);
$postData['params'] = Hash::combine($indexedParams, '{s}.id', '{s}.value');
$params = $this->getParamsWithValues($node);
$matchingData = [];
foreach ($params as $param) {
if (!empty($param['_isHashPath'])) {
$matchingData[$param['label']] = !empty($param['value']) ? $this->extractData($rData, $param['value']) : $rData;
}
}
if (!empty($matchingData)) {
$postData['matchingData'] = $matchingData;
}
$query = $this->Module->queryModuleServer($postData, false, 'Action', false, $postData['data']);
if (!empty($query['error'])) {
$errors[] = $query['error'];
return false;
}
return true;
}
// FIXME: We might want to align the module config with what's currently supported
protected function translateParams($paramName, $moduleParam): array
{
$param = [
'id' => Inflector::slug(Inflector::underscore($paramName)),
'label' => Inflector::humanize($paramName),
'placeholder' => $moduleParam['value'] ?? '',
];
if ($moduleParam['type'] == 'hash_path') {
$param['type'] = 'input';
$param['_isHashPath'] = true;
} elseif ($moduleParam['type'] == 'large_string') {
$param['type'] = 'textarea';
} else {
$param['type'] = 'input';
}
return $param;
}
}

250
app/Model/WorkflowModules/WorkflowBaseModule.php Normal file
View File

@ -0,0 +1,250 @@
<?php
class WorkflowBaseModule
{
public $is_misp_module = false;
public $blocking = false;
public $is_custom = false;
public $expect_misp_core_format = false;
public $id = 'to-override';
public $name = 'to-override';
public $version = '0.1';
public $description = 'to-override';
public $icon = '';
public $icon_class = '';
public $inputs = 0;
public $outputs = 0;
public $multiple_output_connection = false;
public $support_filters = false;
public $saved_filters = [
['text' => 'selector', 'value' => ''],
['text' => 'value', 'value' => ''],
['text' => 'operator', 'value' => ''],
['text' => 'path', 'value' => ''],
];
public $params = [];
private $Event;
/** @var PubSubTool */
private static $loadedPubSubTool;
public function __construct()
{
}
protected function mergeNodeConfigIntoParameters($node): array
{
$fullIndexedParams = [];
foreach ($this->params as $param) {
$param['value'] = $nodeParamByID[$param['id']]['value'] ?? null;
$param['value'] = $node['data']['indexed_params'][$param['id']] ?? null;
$fullIndexedParams[$param['id']] = $param;
}
return $fullIndexedParams;
}
protected function getParamsWithValues($node): array
{
$indexedParams = $this->mergeNodeConfigIntoParameters($node);
foreach ($indexedParams as $id => $param) {
$indexedParams[$id]['value'] = $param['value'] ?? ($param['default'] ?? '');
}
return $indexedParams;
}
protected function filtersEnabled($node): bool
{
$indexedFilters = $this->getFilters($node);
foreach ($indexedFilters as $k => $v) {
if ($v != '') {
return true;
}
}
return false;
}
protected function getFilters($node): array
{
$indexedFilters = [];
$nodeParam = [];
foreach ($node['data']['saved_filters'] as $name => $value) {
$nodeParam[$name] = $value;
}
foreach ($this->saved_filters as $filter) {
$filter['value'] = $nodeParam[$filter['text']] ?? $filter['value'];
$indexedFilters[$filter['text']] = $filter['value'];
}
return $indexedFilters;
}
public function getConfig(): array
{
$reflection = new ReflectionObject($this);
$properties = [];
foreach ($reflection->getProperties() as $property) {
if ($property->isPublic()) {
$properties[$property->getName()] = $property->getValue($this);
}
}
return $properties;
}
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool
{
return true;
}
protected function push_zmq($message)
{
if (!self::$loadedPubSubTool) {
App::uses('PubSubTool', 'Tools');
$pubSubTool = new PubSubTool();
$pubSubTool->initTool();
self::$loadedPubSubTool = $pubSubTool;
}
$pubSubTool = self::$loadedPubSubTool;
$pubSubTool->workflow_push($message);
}
protected function logError($message)
{
$this->Log = ClassRegistry::init('Log');
$this->Log->createLogEntry('SYSTEM', 'exec_module', 'Workflow', $this->id, $message);
}
public function checkLoading()
{
return 'The Factory Must Grow';
}
protected function extractData($data, $path)
{
$extracted = $data;
if (!empty($path)) {
try {
$extracted = Hash::extract($data, $path);
} catch (Exception $e) {
return false;
}
}
return $extracted;
}
protected function extractDataForFilters(array $node, WorkflowRoamingData $roamingData)
{
$rData = $roamingData->getData();
if (empty($this->support_filters)) {
return $rData;
}
$filters = $this->getFilters($node);
if (in_array(null, array_values($filters))) {
return $rData;
}
$extracted = $this->extractData($rData, $filters['selector']);
if ($extracted === false) {
return $rData;
}
$matchingItems = $this->getItemsMatchingCondition($extracted, $filters['value'], $filters['operator'], $filters['path']);
return $matchingItems;
}
protected function evaluateCondition($data, $operator, $value): bool
{
if ($operator == 'in') {
return is_array($data) && in_array($value, $data);
} elseif ($operator == 'not_in') {
return is_array($data) && !in_array($value, $data);
} elseif ($operator == 'equals') {
return !is_array($data) && $data == $value;
} elseif ($operator == 'not_equals') {
return !is_array($data) && $data != $value;
} elseif ($operator == 'in_or' || $operator == 'in_and' || $operator == 'not_in_or' || $operator == 'not_in_and') {
if (!is_array($data) || !is_array($value)) {
return false;
}
$matching = array_filter($data, function($item) use ($value) {
return in_array($item, $value);
});
if ($operator == 'in_or') {
return !empty($matching);
} elseif ($operator == 'in_and') {
return array_values($matching) == array_values($value);
} elseif ($operator == 'not_in_or') {
return empty($matching);
} elseif ($operator == 'not_in_and') {
return array_values($matching) != array_values($value);
}
}
return false;
}
protected function getItemsMatchingCondition($items, $value, $operator, $path)
{
foreach ($items as $i => $item) {
$subItem = $this->extractData($item, $path, $operator);
if (in_array($operator, ['equals', 'not_equals'])) {
$subItem = !empty($subItem) ? $subItem[0] : $subItem;
}
if (!$this->evaluateCondition($subItem, $operator, $value)) {
unset($items[$i]);
}
}
return $items;
}
}
class WorkflowBaseTriggerModule extends WorkflowBaseModule
{
const OVERHEAD_LOW = 1;
const OVERHEAD_MEDIUM = 2;
const OVERHEAD_HIGH = 3;
public $scope = 'others';
public $blocking = false;
public $misp_core_format = false;
public $trigger_overhead = self::OVERHEAD_LOW;
public $trigger_overhead_message = '';
public $inputs = 0;
public $outputs = 1;
/**
* normalizeData Massage the data before entering the workflow
*
* @param array $data
* @return array|false
*/
public function normalizeData(array $data)
{
if (!empty($this->misp_core_format)) {
$converted = $this->convertData($data);
if (empty($converted)) {
return false;
}
return $converted;
}
return $data;
}
/**
* convertData function
*
* @param array $data
* @return array
*/
protected function convertData(array $data): array
{
App::uses('WorkflowFormatConverterTool', 'Tools');
return WorkflowFormatConverterTool::convert($data);
}
}
class WorkflowBaseLogicModule extends WorkflowBaseModule
{
public $blocking = false;
public $inputs = 1;
public $outputs = 2;
}
class WorkflowBaseActionModule extends WorkflowBaseModule
{
}

89
app/Model/WorkflowModules/action/Module_enrich_event.php Normal file
View File

@ -0,0 +1,89 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_enrich_event extends WorkflowBaseActionModule
{
public $id = 'enrich-event';
public $name = 'Enrich Event';
public $description = 'Enrich all Attributes contained in the Event with the provided module.';
public $icon = 'asterisk';
public $inputs = 1;
public $outputs = 1;
public $support_filters = true;
public $expect_misp_core_format = true;
public $params = [];
private $Module;
public function __construct()
{
parent::__construct();
$this->Module = ClassRegistry::init('Module');
$modules = $this->Module->getModules('Enrichment');
$moduleOptions = [];
if (is_array($modules)) {
$moduleOptions = array_merge([''], Hash::combine($modules, '{n}.name', '{n}.name'));
} else {
$moduleOptions[] = $modules;
}
$this->params = [
[
'id' => 'modules',
'label' => 'Modules',
'type' => 'select',
'options' => $moduleOptions,
],
];
}
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
{
parent::exec($node, $roamingData, $errors);
$params = $this->getParamsWithValues($node);
if (empty($params['modules']['value'])) {
$errors[] = __('No enrichmnent module selected');
return false;
}
$rData = $roamingData->getData();
$event_id = $rData['Event']['id'];
$options = [
'user' => $roamingData->getUser(),
'event_id' => $event_id,
'modules' => [$params['modules']['value']]
];
$filters = $this->getFilters($node);
$extracted = $this->extractData($rData, $filters['selector']);
if ($extracted === false) {
return false;
}
$matchingItems = $this->getItemsMatchingCondition($extracted, $filters['value'], $filters['operator'], $filters['path']);
if (!empty($matchingItems)) {
$extractedUUIDs = $this->extractData($matchingItems, '{n}.uuid');
if ($extractedUUIDs === false) {
return false;
}
$options['attribute_uuids'] = $extractedUUIDs;
}
$this->Event = ClassRegistry::init('Event');
$result = $this->Event->enrichment($options);
if ($result === true) {
$this->push_zmq([
'Warning' => __('Error while trying to reach enrichment service or no module available'),
'Attribute added' => 0
]);
} else {
$this->push_zmq([
'Enriching event' => $event_id,
'Attribute added' => $result
]);
$fullEvent = $this->Event->fetchEvent($roamingData->getUser(), [
'eventid' => $event_id,
'includeAttachments' => 1
]);
$roamingData->setData($fullEvent[0]);
}
return true;
}
}

43
app/Model/WorkflowModules/action/Module_push_zmq.php Normal file
View File

@ -0,0 +1,43 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_push_zmq extends WorkflowBaseActionModule
{
public $blocking = false;
public $id = 'push-zmq';
public $name = 'Push to ZMQ';
public $description = 'Push to the ZMQ channel';
public $icon_path = 'zeromq.png';
public $inputs = 1;
public $outputs = 1;
public $params = [];
public function __construct()
{
parent::__construct();
$this->params = [
[
'id' => 'data_extraction_path',
'label' => 'Data extraction path',
'type' => 'input',
'default' => '',
'placeholder' => 'Attribute.{n}.AttributeTag.{n}.Tag.name',
],
];
}
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool
{
parent::exec($node, $roamingData, $errors);
$params = $this->getParamsWithValues($node);
$path = $params['match_condition']['value'];
$data = $roamingData->getData();
$extracted = $this->extractData($data, $path);
if ($extracted === false) {
$errors[] = __('Error while trying to extract data with path `%s`', $path);
return false;
}
$this->push_zmq(JsonTool::encode($extracted));
return true;
}
}

26
app/Model/WorkflowModules/action/Module_stop_execution.php Normal file
View File

@ -0,0 +1,26 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_stop_execution extends WorkflowBaseActionModule
{
public $blocking = true;
public $id = 'stop-execution';
public $name = 'Stop execution';
public $description = 'Essentially stops the execution for blocking workflows. Do nothing for non-blocking ones';
public $icon = 'ban';
public $inputs = 1;
public $outputs = 0;
public $params = [];
public function __construct()
{
parent::__construct();
}
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
{
parent::exec($node, $roamingData, $errors);
$errors[] = __('Execution stopped');
return false;
}
}

136
app/Model/WorkflowModules/action/Module_tag_operation.php Normal file
View File

@ -0,0 +1,136 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_tag_operation extends WorkflowBaseActionModule
{
public $blocking = false;
public $id = 'tag_operation';
public $name = 'Tag operation';
public $description = 'Add or remove tags on Event or Attributes.';
public $icon = 'tags';
public $inputs = 1;
public $outputs = 1;
public $support_filters = true;
public $expect_misp_core_format = true;
public $params = [];
private $Tag;
private $Event;
private $Attribute;
public function __construct()
{
parent::__construct();
$conditions = [
'Tag.is_galaxy' => 0,
];
$this->Tag = ClassRegistry::init('Tag');
$this->Event = ClassRegistry::init('Event');
$this->Attribute = ClassRegistry::init('Attribute');
$tags = $this->Tag->find('all', [
'conditions' => $conditions,
'recursive' => -1,
'order' => ['name asc'],
'fields' => ['Tag.id', 'Tag.name']
]);
$tags = array_column(array_column($tags, 'Tag'), 'name', 'id');
$this->params = [
[
'id' => 'scope',
'label' => 'Scope',
'type' => 'select',
'options' => [
'event' => __('Event'),
'attribute' => __('Attributes'),
],
'default' => 'event',
],
[
'id' => 'action',
'label' => 'Action',
'type' => 'select',
'options' => [
'add' => __('Add Tags'),
'remove' => __('Remove Tags'),
],
'default' => 'add',
],
[
'id' => 'tags',
'label' => 'Tags',
'type' => 'picker',
'multiple' => true,
'options' => $tags,
'placeholder' => __('Pick a tag'),
],
];
}
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
{
parent::exec($node, $roamingData, $errors);
$params = $this->getParamsWithValues($node);
$rData = $roamingData->getData();
if ($this->filtersEnabled($node)) {
$filters = $this->getFilters($node);
$extracted = $this->extractData($rData, $filters['selector']);
if ($extracted === false) {
return false;
}
$matchingItems = $this->getItemsMatchingCondition($extracted, $filters['value'], $filters['operator'], $filters['path']);
} else {
$matchingItems = $rData;
if ($params['scope']['value'] == 'attribute') {
$matchingItems = Hash::extract($matchingItems, '_AttributeFlattened.{n}');
}
}
$result = false;
if ($params['scope']['value'] == 'event') {
if ($params['action']['value'] == 'remove') {
$result = $this->__removeTagsFromEvent($matchingItems, $params['tags']['value']);
} else {
$result = $this->__addTagsToEvent($matchingItems, $params['tags']['value']);
}
} else {
if ($params['action']['value'] == 'remove') {
$result = $this->__removeTagsFromAttributes($matchingItems, $params['tags']['value']);
} else {
$result = $this->__addTagsToAttributes($matchingItems, $params['tags']['value']);
}
}
return $result;
}
private function __addTagsToAttributes(array $attributes, array $tags): bool
{
$success = false;
foreach ($attributes as $attribute) {
$saveSuccess = $this->Attribute->attachTagsFromAttributeAndTouch($attribute['id'], $attribute['event_id'], $tags);
$success = $success || !empty($saveSuccess);
}
return $success;
}
private function __removeTagsFromAttributes(array $attributes,array $tags): bool
{
$success = false;
foreach ($attributes as $attribute) {
$saveSuccess = $this->Attribute->detachTagsFromAttributeAndTouch($attribute['id'], $attribute['event_id'], $tags);
$success = $success || !empty($saveSuccess);
}
return $success;
}
private function __addTagsToEvent(array $event, array $tags): bool
{
return !empty($this->Event->attachTagsToEventAndTouch($event['Event']['id'], $tags));
}
private function __removeTagsFromEvent(array $event, array $tags): bool
{
return !empty($this->Event->detachTagsFromEventAndTouch($event['Event']['id'], $tags));
}
}

108
app/Model/WorkflowModules/action/Module_webhook.php Normal file
View File

@ -0,0 +1,108 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
App::uses('SyncTool', 'Tools');
App::uses('JsonTool', 'Tools');
class Module_webhook extends WorkflowBaseActionModule
{
public $id = 'webhook';
public $name = 'Webhook';
public $description = 'Allow to perform custom callbacks to the provided URL';
public $icon_path = 'webhook.png';
public $inputs = 1;
public $outputs = 1;
public $support_filters = false;
public $params = [];
private $timeout = false;
private $Event;
public function __construct()
{
parent::__construct();
$this->params = [
[
'id' => 'url',
'label' => 'Payload URL',
'type' => 'input',
'placeholder' => 'https://example.com/test',
],
[
'id' => 'content_type',
'label' => 'Content type',
'type' => 'select',
'default' => 'json',
'options' => [
'json' => 'application/json',
'form' => 'application/x-www-form-urlencoded',
],
],
[
'id' => 'data_extraction_path',
'label' => 'Data extraction path',
'type' => 'input',
'default' => '',
'placeholder' => 'Attribute.{n}.AttributeTag.{n}.Tag.name',
],
];
}
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
{
parent::exec($node, $roamingData, $errors);
$params = $this->getParamsWithValues($node);
if (empty($params['url']['value'])) {
$errors[] = __('URL not provided.');
return false;
}
$rData = $roamingData->getData();
$path = $params['data_extraction_path']['value'];
$extracted = !empty($params['data_extraction_path']['value']) ? $this->extractData($rData, $path) : $rData;
try {
$response = $this->doRequest($params['url']['value'], $params['content_type']['value'], $extracted);
if ($response->isOk()) {
return true;
}
if ($response->code === 403 || $response->code === 401) {
$errors[] = __('Authentication failed.');
return false;
}
$errors[] = __('Something went wrong with the request or the remote side is having issues. Body returned: %s', $response->body);
return false;
} catch (SocketException $e) {
$errors[] = __('Something went wrong while sending the request. Error returned: %s', $e->getMessage());
return false;
} catch (Exception $e) {
$errors[] = __('Something went wrong. Error returned: %s', $e->getMessage());
return false;
}
$errors[] = __('Something went wrong with the request or the remote side is having issues.');
return false;
}
private function doRequest($url, $contentType, array $data)
{
$this->Event = ClassRegistry::init('Event'); // We just need a model to use AppModel functions
$version = implode('.', $this->Event->checkMISPVersion());
$commit = $this->Event->checkMIPSCommit();
$request = [
'header' => [
'Accept' => 'application/json',
'Content-Type' => 'application/json',
'User-Agent' => 'MISP ' . $version . (empty($commit) ? '' : ' - #' . $commit),
]
];
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket(null, $this->timeout);
if ($contentType == 'form') {
$request['header']['Content-Type'] = 'application/x-www-form-urlencoded';
$response = $HttpSocket->post($url, $data, $request);
} else {
$response = $HttpSocket->post($url, JsonTool::encode($data), $request);
}
return $response;
}
}

59
app/Model/WorkflowModules/logic/Module_concurrent_task.php Normal file
View File

@ -0,0 +1,59 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
App::uses('BackgroundJobsTool', 'Tools');
class Module_concurrent_task extends WorkflowBaseLogicModule
{
public $id = 'concurrent-task';
public $name = 'Concurrent Task';
public $description = 'Allow breaking the execution process and running concurrent tasks. You can connect multiple nodes the `concurrent` output.';
public $icon = 'random';
public $inputs = 1;
public $outputs = 1;
public $multiple_output_connection = true;
public $html_template = 'concurrent';
public $params = [];
private $Workflow;
private $Job;
public function __construct()
{
parent::__construct();
$this->Workflow = ClassRegistry::init('Workflow');
$this->Job = ClassRegistry::init('Job');
}
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
{
parent::exec($node, $roamingData, $errors);
$data = $roamingData->getData();
$node_id_to_exec = (int)$data['__node_id_to_exec'];
unset($data['__node_id_to_exec']);
$roamingData->setData($data);
$jobId = $this->Job->createJob(
$roamingData->getUser(),
Job::WORKER_PRIO,
'workflowParallelTask',
sprintf('Workflow ID: %s', $roamingData->getWorkflow()['Workflow']['id']),
__('Running workflow parallel tasks.')
);
$this->Job->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::PRIO_QUEUE,
BackgroundJobsTool::CMD_WORKFLOW,
[
'walkGraph',
$roamingData->getWorkflow()['Workflow']['id'],
$node_id_to_exec,
JsonTool::encode($roamingData->getData()),
GraphWalker::PATH_TYPE_NON_BLOCKING,
$jobId
],
true,
$jobId
);
return true;
}
}

128
app/Model/WorkflowModules/logic/Module_distribution_if.php Normal file
View File

@ -0,0 +1,128 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_distribution_if extends WorkflowBaseLogicModule
{
public $id = 'distribution-if';
public $name = 'IF :: Distribution';
public $description = 'Distribution IF / ELSE condition block. The `then` output will be used if the encoded conditions is satisfied, otherwise the `else` output will be used.';
public $icon = 'code-branch';
public $inputs = 1;
public $outputs = 2;
public $html_template = 'if';
public $expect_misp_core_format = true;
public $params = [];
private $Attribute;
private $operators = [
'equals' => 'Is',
'not_equals' => 'Is not',
'more_restrictive_or_equal_than' => 'More restrictive or equal than',
'more_permisive_or_equal_than' => 'More permisive or equal than',
];
public function __construct()
{
parent::__construct();
$this->Attribute = ClassRegistry::init('Attribute');
$distributionLevels = $this->Attribute->shortDist;
unset($distributionLevels[4]);
unset($distributionLevels[5]);
$distribution_param = [];
foreach ($distributionLevels as $i => $text) {
$distribution_param[] = ['name' => $text, 'value' => $i];
}
$this->params = [
[
'id' => 'scope',
'label' => 'Scope',
'type' => 'select',
'options' => [
'attribute' => __('Final distribution of the Attribute'),
'event' => __('Distribution of the Event'),
],
'default' => 'attribute',
],
[
'id' => 'condition',
'label' => 'Condition',
'type' => 'select',
'default' => 'equals',
'options' => $this->operators,
],
[
'id' => 'distribution',
'label' => 'Distribution',
'type' => 'select',
'default' => '0',
'options' => $distribution_param,
'placeholder' => __('Pick a distribution'),
],
];
}
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool
{
parent::exec($node, $roamingData, $errors);
$params = $this->getParamsWithValues($node);
$scope = $params['scope']['value'];
$operator = $params['condition']['value'];
$value = $params['distribution']['value'];
$data = $roamingData->getData();
$final_distribution = $this->__getPropagatedDistribution($data['Event']);
if ($scope == 'attribute') {
$final_distribution = $this->__getPropagatedDistribution(
$data['Event'],
$data['Event']['Attribute'][0]['Object'] ?? [],
$data['Event']['Attribute'][0]
);
}
if ($final_distribution == -1) {
return false; // distribution not supported
}
if ($operator == 'more_restrictive_or_equal_than') {
$operator = 'in';
$distribution_range = range(0, $value);
} else if ($operator == 'more_permisive_or_equal_than') {
$operator = 'in';
$distribution_range = range($value, 3);
} else {
$distribution_range = intval($value);
}
if ($operator == 'more_restrictive_or_equal_than' || $operator == 'more_permisive_or_equal_than') {
$distribution_range = array_diff($value, [4]); // ignore sharing_group for now
}
$eval = $this->evaluateCondition($distribution_range, $operator, $final_distribution);
return !empty($eval);
}
/**
* __getPropagatedDistribution Get the final distribution of the attribute where distribution of its parent (events/objects) is applied
*
* @param array $event
* @param array $object
* @param array $attribute
* @return integer
*/
private function __getPropagatedDistribution(array $event, array $object=[], array $attribute=[]): int
{
$finalDistribution = 5;
if (!empty($attribute)) {
$finalDistribution = intval($attribute['distribution']);
}
if (!empty($object)) { // downgrade based on the object distribution
$finalDistribution = min($finalDistribution, intval($object['distribution']));
}
$finalDistribution = min($finalDistribution, intval($event['distribution'])); // downgrade based on the event distribution
if (!empty($attribute)) {
if ($attribute['distribution'] == 5) { // mirror distribution for the one of the event
$attribute['distribution'] = intval($event['distribution']);
}
}
if ($finalDistribution == 4) { // ignore sharing group for now
$finalDistribution = -1;
}
return $finalDistribution;
}
}

65
app/Model/WorkflowModules/logic/Module_generic_if.php Normal file
View File

@ -0,0 +1,65 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_generic_if extends WorkflowBaseLogicModule
{
public $id = 'generic-if';
public $name = 'IF :: Generic';
public $description = 'Generic IF / ELSE condition block. The `then` output will be used if the encoded conditions is satisfied, otherwise the `else` output will be used.';
public $icon = 'code-branch';
public $inputs = 1;
public $outputs = 2;
public $html_template = 'if';
public $params = [];
private $operators = [
'in' => 'In',
'not_in' => 'Not in',
'equals' => 'Equals',
'not_equals' => 'Not equals',
];
public function __construct()
{
parent::__construct();
$this->params = [
[
'id' => 'value',
'label' => 'Value',
'type' => 'input',
'placeholder' => 'tlp:red',
],
[
'id' => 'operator',
'label' => 'Operator',
'type' => 'select',
'default' => 'in',
'options' => $this->operators,
],
[
'id' => 'hash_path',
'label' => 'Hash path',
'type' => 'input',
'placeholder' => 'Attribute.{n}.Tag',
],
];
}
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool
{
parent::exec($node, $roamingData, $errors);
$params = $this->getParamsWithValues($node);
$path = $params['hash_path']['value'];
$operator = $params['operator']['value'];
$value = $params['value']['value'];
$data = $roamingData->getData();
$extracted = [];
if ($operator == 'equals' || $operator == 'not_equals') {
$extracted = Hash::get($data, $path, []);
} else {
$extracted = Hash::extract($data, $path);
}
$eval = $this->evaluateCondition($extracted, $operator, $value);
return !empty($eval);
}
}

76
app/Model/WorkflowModules/logic/Module_organisation_if.php Normal file
View File

@ -0,0 +1,76 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_organisation_if extends WorkflowBaseLogicModule
{
public $id = 'organisation-if';
public $name = 'IF :: Organisation';
public $description = 'Organisation IF / ELSE condition block. The `then` output will be used if the encoded conditions is satisfied, otherwise the `else` output will be used.';
public $icon = 'code-branch';
public $inputs = 1;
public $outputs = 2;
public $html_template = 'if';
public $expect_misp_core_format = true;
public $params = [];
private $Organisation;
private $operators = [
'equals' => 'Is',
'not_equals' => 'Is not',
];
public function __construct()
{
parent::__construct();
$this->Organisation = ClassRegistry::init('Organisation');
$orgs = $this->Organisation->find('list', [
'fields' => ['id', 'name'],
'order' => 'LOWER(name)'
]);
$this->params = [
[
'id' => 'org_type',
'label' => 'Organisation Type',
'type' => 'select',
'options' => [
'org' => __('Owner Organisation'),
'orgc' => __('Creator Organisation'),
],
'default' => 'orgc',
],
[
'id' => 'condition',
'label' => 'Condition',
'type' => 'select',
'default' => 'equals',
'options' => $this->operators,
],
[
'id' => 'org_id',
'type' => 'picker',
'multiple' => false,
'options' => $orgs,
'default' => 1,
'placeholder' => __('Pick an organisation'),
],
];
}
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool
{
parent::exec($node, $roamingData, $errors);
$params = $this->getParamsWithValues($node);
$org_type = $params['org_type']['value'];
$operator = $params['condition']['value'];
$org_id = $params['org_id']['value'];
$data = $roamingData->getData();
$path = 'Event.org_id';
if ($org_type == 'orgc') {
$path = 'Event.orgc_id';
}
$extracted_org = intval(Hash::get($data, $path)) ?? -1;
$eval = $this->evaluateCondition($extracted_org, $operator, $org_id);
return !empty($eval);
}
}

47
app/Model/WorkflowModules/logic/Module_published_if.php Normal file
View File

@ -0,0 +1,47 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_published_if extends WorkflowBaseLogicModule
{
public $id = 'published-if';
public $name = 'IF :: Published';
public $description = 'Published IF / ELSE condition block. The `then` output will be used if the encoded conditions is satisfied, otherwise the `else` output will be used.';
public $icon = 'code-branch';
public $inputs = 1;
public $outputs = 2;
public $html_template = 'if';
public $expect_misp_core_format = true;
public $params = [];
private $operators = [
'equals' => 'Event is published',
'not_equals' => 'Event is not published',
];
public function __construct()
{
parent::__construct();
$this->params = [
[
'id' => 'condition',
'label' => 'Condition',
'type' => 'select',
'default' => 'equals',
'options' => $this->operators,
],
];
}
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool
{
parent::exec($node, $roamingData, $errors);
$params = $this->getParamsWithValues($node);
$operator = $params['condition']['value'];
$data = $roamingData->getData();
$path = 'Event.published';
$is_published = !empty(Hash::get($data, $path));
$eval = $this->evaluateCondition($is_published, $operator, true);
return $eval;
}
}

94
app/Model/WorkflowModules/logic/Module_tag_if.php Normal file
View File

@ -0,0 +1,94 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_tag_if extends WorkflowBaseLogicModule
{
public $id = 'tag-if';
public $name = 'IF :: Tag';
public $description = 'Tag IF / ELSE condition block. The `then` output will be used if the encoded conditions is satisfied, otherwise the `else` output will be used.';
public $icon = 'code-branch';
public $inputs = 1;
public $outputs = 2;
public $html_template = 'if';
public $expect_misp_core_format = true;
public $params = [];
private $Tag;
private $operators = [
'in_or' => 'Is tagged with any (OR)',
'in_and' => 'Is tagged with all (AND)',
'not_in_or' => 'Is not tagged with any (OR)',
'not_in_and' => 'Is not tagged with all (AND)',
];
public function __construct()
{
parent::__construct();
$conditions = [
'Tag.is_galaxy' => 0,
];
$this->Tag = ClassRegistry::init('Tag');
$tags = $this->Tag->find('all', [
'conditions' => $conditions,
'recursive' => -1,
'order' => ['name asc'],
'fields' => ['Tag.id', 'Tag.name']
]);
$tags = array_column(array_column($tags, 'Tag'), 'name', 'id');
$this->params = [
[
'id' => 'scope',
'label' => 'Scope',
'type' => 'select',
'options' => [
'event' => __('Event'),
'attribute' => __('Attribute'),
'event_attribute' => __('Inherited Attribute'),
],
'default' => 'event',
],
[
'id' => 'condition',
'label' => 'Condition',
'type' => 'select',
'default' => 'in_or',
'options' => $this->operators,
],
[
'id' => 'tags',
'label' => 'Tags',
'type' => 'picker',
'multiple' => true,
'options' => $tags,
'placeholder' => __('Pick a tag'),
],
];
}
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool
{
parent::exec($node, $roamingData, $errors);
$params = $this->getParamsWithValues($node);
$value = $params['tags']['value'];
$operator = $params['condition']['value'];
$scope = $params['scope']['value'];
$data = $roamingData->getData();
$extracted = $this->__getTagFromScope($scope, $data);
$eval = $this->evaluateCondition($extracted, $operator, $value);
return !empty($eval);
}
private function __getTagFromScope($scope, array $data): array
{
$path = '';
if ($scope == 'attribute') {
$path = 'Event._AttributeFlattened.{n}.Tag.{n}.id';
} elseif ($scope == 'event_attribute') {
$path = 'Event._AttributeFlattened.{n}._allTags.{n}.id';
} else {
$path = 'Event.Tag.{n}.id';
}
return Hash::extract($data, $path) ?? [];
}
}

22
app/Model/WorkflowModules/trigger/Module_attribute_after_save.php Normal file
View File

@ -0,0 +1,22 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_attribute_after_save extends WorkflowBaseTriggerModule
{
public $id = 'attribute-after-save';
public $scope = 'attribute';
public $name = 'Attribute After Save';
public $description = 'This trigger is called after an Attribute has been saved in the database';
public $icon = 'cube';
public $inputs = 0;
public $outputs = 1;
public $blocking = false;
public $misp_core_format = true;
public $trigger_overhead = self::OVERHEAD_HIGH;
public function __construct()
{
parent::__construct();
$this->trigger_overhead_message = __('This trigger is called each time an Attribute has been saved. This means that when a large quantity of Attributes are being saved (e.g. Feed pulling or synchronisation), the workflow will be run for as many time as there are Attributes.');
}
}

21
app/Model/WorkflowModules/trigger/Module_enrichment_before_query.php Normal file
View File

@ -0,0 +1,21 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_enrichment_before_query extends WorkflowBaseTriggerModule
{
public $id = 'enrichment-before-query';
public $scope = 'others';
public $name = 'Enrichment Before Query';
public $description = 'This trigger is called just before a query against the enrichment service is done';
public $icon = 'asterisk';
public $inputs = 0;
public $outputs = 1;
public $blocking = true;
public $misp_core_format = true;
public $trigger_overhead = self::OVERHEAD_LOW;
public function __construct()
{
parent::__construct();
}
}

22
app/Model/WorkflowModules/trigger/Module_event_after_save.php Normal file
View File

@ -0,0 +1,22 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_event_after_save extends WorkflowBaseTriggerModule
{
public $id = 'event-after-save';
public $scope = 'event';
public $name = 'Event After Save';
public $description = 'This trigger is called after an Event has been saved in the database';
public $icon = 'envelope';
public $inputs = 0;
public $outputs = 1;
public $blocking = false;
public $misp_core_format = true;
public $trigger_overhead = self::OVERHEAD_HIGH;
public function __construct()
{
parent::__construct();
$this->trigger_overhead_message = __('This trigger is called each time an Event or Attribute have been saved. This means that when a large quantity of Attributes are being saved (e.g. Feed pulling or synchronisation), the workflow will be run for as many time as there are Attributes.');
}
}

21
app/Model/WorkflowModules/trigger/Module_event_publish.php Normal file
View File

@ -0,0 +1,21 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_event_publish extends WorkflowBaseTriggerModule
{
public $id = 'event-publish';
public $scope = 'event';
public $name = 'Event Publish';
public $description = 'This trigger is called just before a MISP Event starts the publishing process';
public $icon = 'upload';
public $inputs = 0;
public $outputs = 1;
public $blocking = true;
public $misp_core_format = true;
public $trigger_overhead = self::OVERHEAD_LOW;
public function __construct()
{
parent::__construct();
}
}

22
app/Model/WorkflowModules/trigger/Module_object_after_save.php Normal file
View File

@ -0,0 +1,22 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_object_after_save extends WorkflowBaseTriggerModule
{
public $id = 'object-after-save';
public $scope = 'object';
public $name = 'Object After Save';
public $description = 'This trigger is called after an Object has been saved in the database';
public $icon = 'cubes';
public $inputs = 0;
public $outputs = 1;
public $blocking = false;
public $misp_core_format = true;
public $trigger_overhead = self::OVERHEAD_HIGH;
public function __construct()
{
parent::__construct();
$this->trigger_overhead_message = __('This trigger is called each time an Object has been saved. This means that when a large quantity of Objects are being saved (e.g. Feed pulling or synchronisation), the workflow will be run for as many time as there are Objects.');
}
}

43
app/Model/WorkflowModules/trigger/Module_post_after_save.php Normal file
View File

@ -0,0 +1,43 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_post_after_save extends WorkflowBaseTriggerModule
{
public $id = 'post-after-save';
public $scope = 'post';
public $name = 'Post After Save';
public $description = 'This trigger is called after a Post has been saved in the database';
public $icon = 'comment';
public $blocking = false;
public $misp_core_format = false;
public $trigger_overhead = self::OVERHEAD_LOW;
private $Thread;
private $Event;
public function __construct()
{
parent::__construct();
}
public function normalizeData(array $data)
{
parent::normalizeData($data);
$this->Thread = ClassRegistry::init('Thread');
$thread = $this->Thread->find('first', [
'recursive' => -1,
'conditions' => ['id' => $data['Post']['thread_id']],
]);
$data['Thread'] = !empty($thread) ? $thread['Thread'] : [];
if (!empty($thread) && !empty($thread['Thread']['event_id'])) {
$this->Event = ClassRegistry::init('Event');
$event = $this->Event->find('first', [
'recursive' => -1,
'conditions' => ['id' => $thread['Thread']['event_id']],
]);
$event = $this->convertData($event);
$data['Event'] = $event['Event'];
}
return $data;
}
}

20
app/Model/WorkflowModules/trigger/Module_user_after_save.php Normal file
View File

@ -0,0 +1,20 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_user_after_save extends WorkflowBaseTriggerModule
{
public $id = 'user-after-save';
public $scope = 'user';
public $name = 'User After Save';
public $description = 'This trigger is called after a user has been saved in the database';
public $icon = 'user-edit';
public $inputs = 0;
public $outputs = 1;
public $blocking = false;
public $trigger_overhead = self::OVERHEAD_LOW;
public function __construct()
{
parent::__construct();
}
}

20
app/Model/WorkflowModules/trigger/Module_user_before_save.php Normal file
View File

@ -0,0 +1,20 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_user_before_save extends WorkflowBaseTriggerModule
{
public $id = 'user-before-save';
public $scope = 'user';
public $name = 'User Before Save';
public $description = 'This trigger is called just before a user is save in the database';
public $icon = 'user-plus';
public $inputs = 0;
public $outputs = 1;
public $blocking = true;
public $trigger_overhead = self::OVERHEAD_LOW;
public function __construct()
{
parent::__construct();
}
}

BIN
app/Plugin/AadAuth/.images/Picture39.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 16 KiB

206
app/Plugin/AadAuth/Controller/Component/Auth/AadAuthenticateAuthenticate.php
View File

@ -2,6 +2,7 @@
App::uses('BaseAuthenticate', 'Controller/Component/Auth');
App::uses('RandomTool', 'Tools');
App::uses('HttpSocket', 'Network/Http');
if (session_status() == PHP_SESSION_NONE) {
session_start();
@ -10,7 +11,8 @@ if (session_status() == PHP_SESSION_NONE) {
// Generating a new session will fail the further flow of AAD.
// session_regenerate_id();
class AadAuthenticateAuthenticate extends BaseAuthenticate {
class AadAuthenticateAuthenticate extends BaseAuthenticate
{
/**
* Holds the application ID
@ -44,45 +46,44 @@ class AadAuthenticateAuthenticate extends BaseAuthenticate {
* Provider authentication URL
*
* @var string
*/
*/
protected static $auth_provider;
/**
* Provider URL for additional user details
*
* @var string
*/
*/
protected static $auth_provider_user;
/**
* Flag that indicates if we need to check for AD groups for defining MISP access
*
* @var bool
*/
*/
protected static $check_ad_groups;
/**
* AD group MISP user
*
* @var string
*/
*/
protected static $misp_user;
/**
* AD group MISP org admin
*
* @var string
*/
*/
protected static $misp_orgadmin;
/**
* AD group MISP siteadmin
*
* @var string
*/
*/
protected static $misp_siteadmin;
public function __construct()
{
self::$client_id = Configure::read('AadAuth.client_id');
@ -99,7 +100,7 @@ class AadAuthenticateAuthenticate extends BaseAuthenticate {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->settings['fields'] = array('username' => 'email');
$this->settings['fields'] = ['username' => 'email'];
}
/**
@ -108,37 +109,51 @@ class AadAuthenticateAuthenticate extends BaseAuthenticate {
* @param string $level Log level
* @param string $logmessage Message to log
* @return bool result of the log action
*/
private function _log($level, $logmessage)
*/
private function _log($level, $logmessage)
{
$log = array(
$log = [
'org' => 'SYSTEM',
'model' => 'User',
'model_id' => 0,
'email' => false,
'action' => 'auth',
'title' => $logmessage
);
$this->Log->save($log);
];
$this->Log->save($log);
CakeLog::write($level, $logmessage);
return true;
}
/**
* Log non 200-ish HTTP responses
*
* @param string $level Log level
* @param string $url Requested URL
* @param HttpSocketResponse $response HTTP response
* @return bool result of the log action
*/
private function _logHttpError(string $level, string $url, HttpSocketResponse $response)
{
$this->_log($level, "POST request to url: {$url} returned HTTP code: {$response->code} with response body: {$response->body}");
return true;
}
/**
* Find the user to authenticate with
*
* @param CakeRequest $request The request that contains login information.
* @return mixed False on login failure. An array of User data on success.
*/
public function getUser(CakeRequest $request)
*/
public function getUser(CakeRequest $request)
{
// we only proceed if called with a request to authenticate via AAD
if (array_key_exists('AzureAD', $request->query) and $request->query['AzureAD'] == 'enable') {
$user = $this->_getUserAad($request);
return $user;
}
elseif (array_key_exists('code', $request->query)) // in the Azure flow
} elseif (array_key_exists('code', $request->query)) // in the Azure flow
{
$user = $this->_getUserAad($request);
return $user;
@ -152,8 +167,8 @@ class AadAuthenticateAuthenticate extends BaseAuthenticate {
* @param CakeRequest $request The request that contains login information.
* @param CakeResponse $response Unused response object.
* @return mixed False on login failure. An array of User data on success.
*/
public function authenticate(CakeRequest $request, CakeResponse $response)
*/
public function authenticate(CakeRequest $request, CakeResponse $response)
{
return self::getUser($request);
}
@ -163,10 +178,10 @@ class AadAuthenticateAuthenticate extends BaseAuthenticate {
*
* @param CakeRequest $request The request that contains login information.
* @return mixed False on login failure. An array of User data on success.
*/
*/
private function _getUserAad(CakeRequest $request)
{
if (!headers_sent()) {
if (!headers_sent()) {
if (!isset($_GET["code"]) and !isset($_GET["error"])) {
$url = self::$auth_provider . self::$ad_tenant . "/oauth2/v2.0/authorize?";
$url .= "state=" . session_id();
@ -179,68 +194,68 @@ class AadAuthenticateAuthenticate extends BaseAuthenticate {
$this->_log("info", "Redirect to Azure for authentication.");
exit; // we need to exit once the header to redirect to Azure is sent
}
elseif (isset($_GET["error"])) { //Second load of this page begins, but hopefully we end up to the next elseif section...
$this->_log("warning", "Return from Aure redirect. Error received at the beginning of second stage. _GET: " . http_build_query($_GET,'',' - '));
} elseif (isset($_GET["error"])) { //Second load of this page begins, but hopefully we end up to the next elseif section...
$this->_log("warning", "Return from Aure redirect. Error received at the beginning of second stage. _GET: " . http_build_query($_GET, '', ' - '));
return false;
}
elseif (strcmp(session_id(), $_GET["state"]) == 0) {
} elseif (strcmp(session_id(), $_GET["state"]) == 0) {
//Verifying the received tokens with Azure and finalizing the authentication part
$content = "grant_type=authorization_code";
$content .= "&client_id=" . self::$client_id;
$content .= "&redirect_uri=" . urlencode(self::$redirect_uri);
$content .= "&code=" . $_GET["code"];
$content .= "&client_secret=" . urlencode(self::$client_secret);
$options = array(
"http" => array( //Use "http" even if you send the request with https
"method" => "POST",
"header" => "Content-Type: application/x-www-form-urlencoded\r\n" .
"Content-Length: " . strlen($content) . "\r\n",
"content" => $content
)
);
$context = stream_context_create($options);
$json = file_get_contents(self::$auth_provider . self::$ad_tenant . "/oauth2/v2.0/token", false, $context);
if ($json === false) {
$this->_log("warning", "Error received during Bearer token fetch (context).");
// For debug : "PHP_Error" => error_get_last(), "\$_GET[]" => $_GET, "HTTP_msg" => $options), "");
return false;
}
$params = [
'grant_type' => 'authorization_code',
'client_id' => self::$client_id,
'redirect_uri' => self::$redirect_uri,
'code' => $_GET["code"],
'client_secret' => self::$client_secret
];
$authdata = json_decode($json, true);
$options = [
'header' => [
'Content-Type' => 'application/x-www-form-urlencoded'
]
];
$url = self::$auth_provider . self::$ad_tenant . "/oauth2/v2.0/token";
$response = (new HttpSocket())->post($url, $params, $options);
if (!$response->isOk()) {
$this->_log("warning", "Error received during Bearer token fetch (context).");
$this->_logHttpError("debug", $url, $response);
return false;
}
$authdata = json_decode($response->body, true);
if (isset($authdata["error"])) {
$this->_log("warning", "Error received during Bearer token fetch (authdata).");
// For debug : "\$authdata[]" => $authdata, "\$_GET[]" => $_GET, "HTTP_msg" => $options), $error_email);
$this->_log("debug", "Response: " . json_encode($authdata["error"]));
return false;
}
$options = array(
"http" => array( //Use "http" even if you send the request with https
"method" => "GET",
"header" => "Accept: application/json\r\n" .
"Authorization: Bearer " . $authdata["access_token"] . "\r\n"
)
);
$context = stream_context_create($options);
$this->_log("info", "Fetching user data from Azure.");
$json = file_get_contents(self::$auth_provider_user . "/v1.0/me", false, $context);
if ($json === false) {
$options = [
'header' => [
'Accept' => 'application/json',
'Authorization' => 'Bearer ' . $authdata["access_token"]
]
];
$url = self::$auth_provider_user . "/v1.0/me";
$response = (new HttpSocket())->get($url, null, $options);
if (!$response->isOk()) {
$this->_log("warning", "Error received during user data fetch.");
// For debug : "PHP_Error" => error_get_last(), "\$_GET[]" => $_GET, "HTTP_msg" => $options), $error_email);
$this->_logHttpError("debug", $url, $response);
return false;
}
$userdata = json_decode($json, true); //This should now contain your logged on user information
if (isset($userdata["error"])){
$userdata = json_decode($response->body, true); //This should now contain your logged on user information
if (isset($userdata["error"])) {
$this->_log("warning", "User data fetch contained an error.");
// For debug : "\$userdata[]" => $userdata, "\$authdata[]" => $authdata, "\$_GET[]" => $_GET, "HTTP_msg" => $options), $error_email);
$this->_log("debug", "Response: " . json_encode($userdata["error"]));
return false;
}
}
$mispUsername = false;
if (isset($userdata["userPrincipalName"])){
if (isset($userdata["userPrincipalName"])) {
$userPrincipalName = $userdata["userPrincipalName"];
/*
@ -251,15 +266,19 @@ class AadAuthenticateAuthenticate extends BaseAuthenticate {
if (self::$check_ad_groups) {
if ($this->_checkAdGroup($authdata)) {
$mispUsername = $userPrincipalName;
$this->_log("info", "Successful AAD group check for for ${mispUsername}");
}
}
else {
} else {
$mispUsername = $userPrincipalName;
}
if ($mispUsername) {
$this->_log("info", "Attempt authentication for ${mispUsername}");
return $this->_findUser($mispUsername);
$this->_log("info", "Attempt AAD authentication for ${mispUsername}");
$user = $this->_findUser($mispUsername);
if ($user) {
$this->_log("info", "AAD authentication successful for ${mispUsername}");
}
return $user;
}
}
}
@ -274,39 +293,39 @@ class AadAuthenticateAuthenticate extends BaseAuthenticate {
*
* @param array $authdata The authdata array received from Azure
* @return mixed False if no MISP groups have been found; String if a group was found
*/
private function _checkAdGroup($authdata)
*/
private function _checkAdGroup($authdata)
{
$options = array(
"http" => array( //Use "http" even if you send the request with https
"method" => "GET",
"header" => "Accept: application/json\r\n" .
"Authorization: Bearer " . $authdata["access_token"] . "\r\n"
)
);
$context = stream_context_create($options);
$this->_log("info", "Fetching user group data from Azure.");
$json = file_get_contents(self::$auth_provider_user . "/v1.0/me/memberOf", false, $context);
if ($json === false) {
$this->_log("warning", "Error received during user group data fetch.");
// For debug : "PHP_Error" => error_get_last(), "\$_GET[]" => $_GET, "HTTP_msg" => $options), $error_email);
return false;
}
$options = [
'header' => [
'Accept' => 'application/json',
'Authorization' => 'Bearer ' . $authdata["access_token"]
]
];
$url = self::$auth_provider_user . "/v1.0/me/memberOf";
$groupdata = json_decode($json, true); //This should now contain your logged on user memberOf (groups) information
$response = (new HttpSocket())->get($url, null, $options);
if (!$response->isOk()) {
$this->_log("warning", "Error received during user group data fetch.");
$this->_logHttpError("debug", $url, $response);
return false;
}
$groupdata = json_decode($response->body, true); //This should now contain your logged on user memberOf (groups) information
if (isset($groupdata["error"])) {
$this->_log("warning", "Group data fetch contained an error.");
// For debug : "\$groupdata[]" => $groupdata, "\$authdata[]" => $authdata, "\$_GET[]" => $_GET, "HTTP_msg" => $options), $error_email);
$this->_log("debug", "Response: " . json_encode($groupdata["error"]));
return false;
}
}
// Now check if the user has any of the MISP AAD groups enabled
foreach ($groupdata["value"] as $group) {
$groupdisplayName = $group["displayName"];
if ($groupdisplayName == self::$misp_siteadmin) {
return self::$misp_siteadmin;
}
}
if ($groupdisplayName == self::$misp_orgadmin) {
return self::$misp_orgadmin;
}
@ -315,7 +334,8 @@ class AadAuthenticateAuthenticate extends BaseAuthenticate {
}
}
$this->_log("warning", "The user is not a member of any of the MISP AAD groups.");
return false;
}
}

11
app/Plugin/AadAuth/README.md
View File

@ -128,4 +128,15 @@ Add the information we made a note of earlier when creating the `App Registation
All fields need to match explicitly without any leading or trailing whitespace, if you added the groups these are case sensitive.
### Disable users password change
By default MISP will still create a password for the user, when enrolling a new user on MISP, uncheck the _"Send credentials automatically"_ checkbox.
![Send credentials automatically](.images/Picture39.png)
Additionally, it is recommended to set the following settings in the MISP config:
* `MISP.disableUserSelfManagement => true`: Removes the ability of users to change their user settings and reset their authentication keys.
* `MISP.disable_user_login_change => true`: Removes the ability of users to change their username (email), except for site admins.
* `MISP.disable_user_password_change => true`: Removes the ability of users to change their own password.
This way users will not be able to change their passwords and by-pass the AAD login flow.

26
app/View/Attributes/ajax/recorrelationConfirmation.ctp Normal file
View File

@ -0,0 +1,26 @@
<div class="confirmation">
<?php
echo $this->Form->create('Attribute', ['style' => 'margin:0px;', 'id' => 'PromptForm', 'url' => $baseurl . '/attributes/generateCorrelation']);
$message = __('Recorrelate instance');
$buttonTitle = __('Recorrelate');
?>
<legend><?= $message ?></legend>
<div style="padding-left:5px;padding-right:5px;padding-bottom:5px;">
<?php
echo '<p>' . __('Are you sure you wish to start a recorrelation for the currently active correlation engine?') . '</p>';
echo '<p>' . __('Depending on the system and the amount of attributes, this might take a long time.') . '</p>';
?>
<table>
<tr>
<td style="vertical-align:top">
<button role="button" tabindex="0" aria-label="<?= $buttonTitle ?>" title="<?= $buttonTitle ?>" id="PromptYesButton" class="btn btn-primary"><?= __('Yes') ?></button>
</td>
<td style="width:100%;"></td>
<td style="vertical-align:top;">
<span role="button" tabindex="0" aria-label="<?= __('Cancel');?>" title="<?= __('Cancel');?>" class="btn btn-inverse" id="PromptNoButton" onclick="cancelPrompt()"><?= __('No');?></span>
</td>
</tr>
</table>
</div>
<?= $this->Form->end(); ?>
</div>

1
app/View/Attributes/index.ctp
View File

@ -2,7 +2,6 @@
$modules = isset($modules) ? $modules : null;
$cortex_modules = isset($cortex_modules) ? $cortex_modules : null;
echo '<div class="index">';
echo $this->element('/genericElements/IndexTable/index_table', [
'data' => [

20
app/View/AuditLogs/admin_index.ctp
View File

@ -228,20 +228,20 @@
<div class="pagination">
<ul>
<?php
$paginator = $this->Paginator->prev('&laquo; ' . __('previous'), array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'prev disabled', 'escape' => false, 'disabledTag' => 'span'));
$paginator .= $this->Paginator->numbers(array('modulus' => 20, 'separator' => '', 'tag' => 'li', 'currentClass' => 'active', 'currentTag' => 'span'));
$paginator .= $this->Paginator->next(__('next') . ' &raquo;', array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'next disabled', 'escape' => false, 'disabledTag' => 'span'));
$paginator = $this->LightPaginator->prev('&laquo; ' . __('previous'), array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'prev disabled', 'escape' => false, 'disabledTag' => 'span'));
$paginator .= $this->LightPaginator->numbers(array('modulus' => 20, 'separator' => '', 'tag' => 'li', 'currentClass' => 'active', 'currentTag' => 'span'));
$paginator .= $this->LightPaginator->next(__('next') . ' &raquo;', array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'next disabled', 'escape' => false, 'disabledTag' => 'span'));
echo $paginator;
?>
</ul>
</div>
<table class="table table-striped table-hover table-condensed">
<tr>
<th><?= $this->Paginator->sort('created') ?></th>
<th><?= $this->Paginator->sort('user_id', __('User')) ?></th>
<th><?= $this->Paginator->sort('ip', __('IP')) ?></th>
<th><?= $this->Paginator->sort('org_id', __('Org')) ?></th>
<th><?= $this->Paginator->sort('action') ?></th>
<th><?= $this->LightPaginator->sort('created') ?></th>
<th><?= $this->LightPaginator->sort('user_id', __('User')) ?></th>
<th><?= $this->LightPaginator->sort('ip', __('IP')) ?></th>
<th><?= $this->LightPaginator->sort('org_id', __('Org')) ?></th>
<th><?= $this->LightPaginator->sort('action') ?></th>
<th><?= __('Model') ?></th>
<th><?= __('Title') ?></th>
<th><?= __('Change') ?></th>
@ -287,10 +287,6 @@
<?php endforeach; ?>
</table>
<p>
<?= $this->Paginator->counter(array(
'format' => __('Page {:page} of {:pages}, showing {:current} records out of {:count} total, starting on record {:start}, ending on {:end}')
));
?>
</p>
<div class="pagination">
<ul>

2
app/View/CorrelationExclusions/top_correlations.ctp
View File

@ -2,7 +2,7 @@
echo sprintf('<div%s>', empty($ajax) ? ' class="index"' : '');
echo $this->element('genericElements/IndexTable/index_table', [
'data' => [
'stupid_pagination' => 1,
'light_paginator' => 1,
'data' => $data,
'top_bar' => [
'pull' => 'right',

25
app/View/Correlations/ajax/switch_engine_confirmation.ctp Normal file
View File

@ -0,0 +1,25 @@
<div class="confirmation">
<?php
echo $this->Form->create('Correlation', ['style' => 'margin:0px;', 'id' => 'PromptForm', 'url' => $baseurl . '/correlations/switchEngine/' . urlencode($engine)]);
$message = __('Switch Engine');
$buttonTitle = __('Switch');
?>
<legend><?= $message ?></legend>
<div style="padding-left:5px;padding-right:5px;padding-bottom:5px;">
<?php
echo '<p>' . __('Are you sure you want to switch to the given correlation engine (' . h($engine) . ')? If so, it is highly recommended that you recorrelate afterwards.') . '</p>';
?>
<table>
<tr>
<td style="vertical-align:top">
<button role="button" tabindex="0" aria-label="<?= $buttonTitle ?>" title="<?= $buttonTitle ?>" id="PromptYesButton" class="btn btn-primary"><?= __('Yes') ?></button>
</td>
<td style="width:100%;"></td>
<td style="vertical-align:top;">
<span role="button" tabindex="0" aria-label="<?= __('Cancel');?>" title="<?= __('Cancel');?>" class="btn btn-inverse" id="PromptNoButton" onclick="cancelPrompt()"><?= __('No');?></span>
</td>
</tr>
</table>
</div>
<?= $this->Form->end(); ?>
</div>

25
app/View/Correlations/ajax/truncate_confirmation.ctp Normal file
View File

@ -0,0 +1,25 @@
<div class="confirmation">
<?php
echo $this->Form->create('Correlation', ['style' => 'margin:0px;', 'id' => 'PromptForm', 'url' => $baseurl . '/correlations/truncate/' . urlencode($engine)]);
$message = __('Truncate database of the associated correlation engine');
$buttonTitle = __('Truncate');
?>
<legend><?= $message ?></legend>
<div style="padding-left:5px;padding-right:5px;padding-bottom:5px;">
<?php
echo '<p>' . __('Are you sure you want to truncate the correlation table (' . h($table_name) . ') used by the disabled correlation engine ' . h($engine) . '?') . '</p>';
?>
<table>
<tr>
<td style="vertical-align:top">
<button role="button" tabindex="0" aria-label="<?= $buttonTitle ?>" title="<?= $buttonTitle ?>" id="PromptYesButton" class="btn btn-primary"><?= __('Yes') ?></button>
</td>
<td style="width:100%;"></td>
<td style="vertical-align:top;">
<span role="button" tabindex="0" aria-label="<?= __('Cancel');?>" title="<?= __('Cancel');?>" class="btn btn-inverse" id="PromptNoButton" onclick="cancelPrompt()"><?= __('No');?></span>
</td>
</tr>
</table>
</div>
<?= $this->Form->end(); ?>
</div>

84
app/View/Correlations/over_correlations.ctp Normal file
View File

@ -0,0 +1,84 @@
<?php
echo sprintf('<div%s>', empty($ajax) ? ' class="index"' : '');
echo $this->element('genericElements/IndexTable/index_table', [
'data' => [
'light_paginator' => 1,
'data' => $data,
'top_bar' => [
'children' => [
[
'children' => [
[
'type' => 'simple',
'url' => $baseurl . '/correlations/overCorrelations',
'text' => __('All')
],
[
'type' => 'simple',
'url' => $baseurl . '/correlations/overCorrelations/scope:over_correlating',
'text' => __('Over-correlating')
],
[
'type' => 'simple',
'url' => $baseurl . '/correlations/overCorrelations/scope:not_over_correlating',
'text' => __('Not over-correlating')
],
]
]
]
],
'fields' => [
[
'name' => 'Value',
'element' => 'postlink',
'data_path' => 'OverCorrelatingValue.value',
'url' => '/attributes/search/results',
'payload_paths' => [
'value' => 'OverCorrelatingValue.value'
]
],
[
'name' => 'Occurrences',
'data_path' => 'OverCorrelatingValue.occurrence',
'class' => 'shortish'
],
[
'name' => 'Blocked by Threshold',
'data_path' => 'OverCorrelatingValue.over_correlation',
'class' => 'shortish',
'element' => 'boolean'
],
[
'name' => 'Excluded by Exclusion List',
'data_path' => 'OverCorrelatingValue.excluded',
'class' => 'shortish',
'element' => 'boolean'
]
],
'title' => empty($ajax) ? $title_for_layout : false,
'description' => empty($ajax) ? __('The values with the most correlation entries.') : false,
'pull' => 'right',
'actions' => [
[
'onclick' => sprintf(
'openGenericModal(\'%s/correlation_exclusions/add/redirect_controller:correlations/redirect:top/value:[onclick_params_data_path]\');',
$baseurl
),
'onclick_params_data_path' => 'OverCorrelatingValue.value',
'icon' => 'trash',
'title' => __('Add exclusion entry for value'),
'complex_requirement' => [
'function' => function (array $row) {
return !$row['OverCorrelatingValue']['excluded'];
}
]
]
]
]
]);
echo '</div>';
if (empty($ajax)) {
echo $this->element('/genericElements/SideMenu/side_menu', $menuData);
}

2
app/View/Correlations/top.ctp
View File

@ -2,7 +2,7 @@
echo sprintf('<div%s>', empty($ajax) ? ' class="index"' : '');
echo $this->element('genericElements/IndexTable/index_table', [
'data' => [
'stupid_pagination' => 1,
'light_paginator' => 1,
'data' => $data,
'top_bar' => [
'children' => [

2
app/View/DecayingModel/decaying_tool.ctp
View File

@ -167,7 +167,7 @@
<?php
echo $this->element('genericElements/assetLoader', array(
'css' => array('decayingTool'),
'js' => array('d3', 'Chart.min', 'decayingTool')
'js' => array('d3', 'Chart.min', 'decayingTool', 'jquery-ui.min')
));
?>

134
app/View/Elements/Events/View/attribute_correlations.ctp
View File

@ -1,57 +1,81 @@
<?php
$i = 0;
$linkColour = ($scope == 'Attribute') ? 'red' : 'white';
$withPivot = isset($withPivot) ? $withPivot : false;
// remove duplicates
$relatedEvents = array();
foreach ($event['Related' . $scope][$object['id']] as $k => $relatedAttribute) {
if (isset($relatedEvents[$relatedAttribute['id']])) {
unset($event['Related' . $scope][$object['id']][$k]);
} else {
$relatedEvents[$relatedAttribute['id']] = true;
}
}
$count = count($event['Related' . $scope][$object['id']]);
foreach ($event['Related' . $scope][$object['id']] as $relatedAttribute) {
if ($i == 4 && $count > 5) {
$expandButton = __('Show %s more...', $count - 4);
echo sprintf(
'<li class="no-side-padding correlation-expand-button useCursorPointer linkButton %s">%s</li>',
$linkColour,
$expandButton
);
}
$relatedData = array(
'Orgc' => !empty($orgTable[$relatedAttribute['org_id']]) ? $orgTable[$relatedAttribute['org_id']] : 'N/A',
'Date' => isset($relatedAttribute['date']) ? $relatedAttribute['date'] : 'N/A',
'Event' => $relatedAttribute['info'],
'Correlating Value' => $relatedAttribute['value']
);
$popover = '';
foreach ($relatedData as $k => $v) {
$popover .= '<span class="bold black">' . h($k) . '</span>: <span class="blue">' . h($v) . '</span><br>';
}
$link = $this->Html->link(
$relatedAttribute['id'],
$withPivot ?
['controller' => 'events', 'action' => 'view', $relatedAttribute['id'], true, $event['Event']['id']] :
['controller' => 'events', 'action' => 'view', $relatedAttribute['id']],
['class' => ($relatedAttribute['org_id'] == $me['org_id']) ? $linkColour : 'blue']
);
echo sprintf(
'<li class="no-side-padding %s" %s data-toggle="popover" data-content="%s" data-trigger="hover">%s&nbsp;</li>',
($i > 4 || $i == 4 && $count > 5) ? 'correlation-expanded-area' : '',
($i > 4 || $i == 4 && $count > 5) ? 'style="display:none;"' : '',
h($popover),
$link
);
if (!empty($event['Related' . $scope][$object['id']])) {
$i = 0;
$linkColour = ($scope == 'Attribute') ? 'red' : 'white';
$withPivot = isset($withPivot) ? $withPivot : false;
// remove duplicates
$relatedEvents = array();
foreach ($event['Related' . $scope][$object['id']] as $k => $relatedAttribute) {
if (isset($relatedEvents[$relatedAttribute['id']])) {
unset($event['Related' . $scope][$object['id']][$k]);
} else {
$relatedEvents[$relatedAttribute['id']] = true;
}
}
$count = count($event['Related' . $scope][$object['id']]);
foreach ($event['Related' . $scope][$object['id']] as $relatedAttribute) {
if ($i == 4 && $count > 5) {
$expandButton = __('Show %s more...', $count - 4);
echo sprintf(
'<li class="no-side-padding correlation-expand-button useCursorPointer linkButton %s">%s</li>',
$linkColour,
$expandButton
);
}
$relatedData = array(
'Orgc' => !empty($orgTable[$relatedAttribute['org_id']]) ? $orgTable[$relatedAttribute['org_id']] : 'N/A',
'Date' => isset($relatedAttribute['date']) ? $relatedAttribute['date'] : 'N/A',
'Event' => $relatedAttribute['info'],
'Correlating Value' => $relatedAttribute['value']
);
$popover = '';
foreach ($relatedData as $k => $v) {
$popover .= '<span class="bold black">' . h($k) . '</span>: <span class="blue">' . h($v) . '</span><br>';
}
$link = $this->Html->link(
$relatedAttribute['id'],
$withPivot ?
['controller' => 'events', 'action' => 'view', $relatedAttribute['id'], true, $event['Event']['id']] :
['controller' => 'events', 'action' => 'view', $relatedAttribute['id']],
['class' => ($relatedAttribute['org_id'] == $me['org_id']) ? $linkColour : 'blue']
);
echo sprintf(
'<li class="no-side-padding %s" %s data-toggle="popover" data-content="%s" data-trigger="hover">%s&nbsp;</li>',
($i > 4 || $i == 4 && $count > 5) ? 'correlation-expanded-area' : '',
($i > 4 || $i == 4 && $count > 5) ? 'style="display:none;"' : '',
h($popover),
$link
);
$i++;
}
if ($i > 5) {
echo sprintf(
'<li class="no-side-padding correlation-collapse-button useCursorPointer linkButton %s" style="display:none;">%s</li>',
$linkColour,
__('Collapse…')
);
}
$i++;
}
if ($i > 5) {
echo sprintf(
'<li class="no-side-padding correlation-collapse-button useCursorPointer linkButton %s" style="display:none;">%s</li>',
$linkColour,
__('Collapse…')
);
}
}
if (!empty($object['correlation_exclusion'])) {
echo sprintf(
'<span class="bold red" title="%s">%s</span> ',
__('The attribute value matches a correlation exclusion rule defined by a site-administrator for this instance. Click the magnifying glass to search for all occurrences of the value.'),
__('Excluded.')
);
} else if (!empty($object['over_correlation'])) {
echo sprintf(
'<span class="bold red" title="%s">%s</span> ',
__('The instance threshold for the maximum number of correlations for the given attribute value has been exceeded. Click the magnifying glass to search for all occurrences of the value.'),
__('Too many correlations.')
);
}
echo $this->Html->link(
'',
['controller' => 'attributes', 'action' => 'search', 'value' => $object['value']],
[
'class' => 'fa fa-search black',
'title' => __('Search for value'),
'aria-label' => __('Search for value')
]
);

760
app/View/Elements/Events/View/row_attribute.ctp
View File

@ -1,396 +1,396 @@
<?php
$tr_class = '';
if (empty($context)) {
$context = 'event';
}
if ($event['Event']['id'] != $object['event_id']) {
if (!$isSiteAdmin && $event['extensionEvents'][$object['event_id']]['Orgc']['id'] != $me['org_id']) {
$mayModify = false;
}
}
$editScope = ($isSiteAdmin || $mayModify) ? 'Attribute' : 'ShadowAttribute';
if (!empty($child)) {
if ($child === 'last' && empty($object['ShadowAttribute'])) {
$tr_class .= ' tableHighlightBorderBottom borderBlue';
} else {
$tr_class .= ' tableHighlightBorderCenter borderBlue';
}
if (!empty($object['ShadowAttribute'])) {
$tr_class .= ' tableInsetOrangeFirst';
}
} else {
$child = false;
if (!empty($object['ShadowAttribute'])) {
$tr_class .= ' tableHighlightBorderTop borderOrange';
}
}
if (!empty($object['deleted'])) {
$tr_class .= ' deleted-attribute';
}
if (!empty($k)) {
$tr_class .= ' row_' . h($k);
}
$tr_class = '';
if (empty($context)) {
$context = 'event';
}
if ($event['Event']['id'] != $object['event_id']) {
if (!$isSiteAdmin && $event['extensionEvents'][$object['event_id']]['Orgc']['id'] != $me['org_id']) {
$mayModify = false;
}
}
$editScope = ($isSiteAdmin || $mayModify) ? 'Attribute' : 'ShadowAttribute';
if (!empty($child)) {
if ($child === 'last' && empty($object['ShadowAttribute'])) {
$tr_class .= ' tableHighlightBorderBottom borderBlue';
} else {
$tr_class .= ' tableHighlightBorderCenter borderBlue';
}
if (!empty($object['ShadowAttribute'])) {
$tr_class .= ' tableInsetOrangeFirst';
}
} else {
$child = false;
if (!empty($object['ShadowAttribute'])) {
$tr_class .= ' tableHighlightBorderTop borderOrange';
}
}
if (!empty($object['deleted'])) {
$tr_class .= ' deleted-attribute';
}
if (!empty($k)) {
$tr_class .= ' row_' . h($k);
}
$objectId = (int) $object['id'];
$objectId = (int) $object['id'];
$isNonCorrelatingType = in_array($object['type'], Attribute::NON_CORRELATING_TYPES, true);
$correlationDisabled = $object['disable_correlation'] || $isNonCorrelatingType;
$correlationButtonEnabled = $mayChangeCorrelation &&
empty($event['Event']['disable_correlation']) &&
!$isNonCorrelatingType;
$isNonCorrelatingType = in_array($object['type'], Attribute::NON_CORRELATING_TYPES, true);
$correlationDisabled = $object['disable_correlation'] || $isNonCorrelatingType;
$correlationButtonEnabled = $mayChangeCorrelation &&
empty($event['Event']['disable_correlation']) &&
!$isNonCorrelatingType;
$quickEdit = function($fieldName) use ($mayModify, $object) {
if (!$mayModify) {
return ''; // currently it is not supported to create proposals trough quick edit
}
if ($object['deleted']) {
return ''; // deleted attributes are not editable
}
if (($fieldName === 'value' || $fieldName === 'type') && ($object['type'] === 'attachment' || $object['type'] === 'malware-sample')) {
return '';
}
return " data-edit-field=\"$fieldName\"";
}
$quickEdit = function($fieldName) use ($mayModify, $object) {
if (!$mayModify) {
return ''; // currently it is not supported to create proposals trough quick edit
}
if ($object['deleted']) {
return ''; // deleted attributes are not editable
}
if (($fieldName === 'value' || $fieldName === 'type') && ($object['type'] === 'attachment' || $object['type'] === 'malware-sample')) {
return '';
}
return " data-edit-field=\"$fieldName\"";
}
?>
<tr id="Attribute_<?= $objectId ?>_tr" data-primary-id="<?= $objectId ?>" class="<?php echo $tr_class; ?>" tabindex="0">
<?php if (($mayModify || !empty($extended)) && empty($disable_multi_select)): ?>
<td style="width:10px">
<?php if ($mayModify):?>
<input class="select_attribute" type="checkbox" data-id="<?= $objectId ?>" aria-label="<?php echo __('Select attribute');?>">
?>
<tr id="Attribute_<?= $objectId ?>_tr" data-primary-id="<?= $objectId ?>" class="<?php echo $tr_class; ?>" tabindex="0">
<?php if (($mayModify || !empty($extended)) && empty($disable_multi_select)): ?>
<td style="width:10px">
<?php if ($mayModify):?>
<input class="select_attribute" type="checkbox" data-id="<?= $objectId ?>" aria-label="<?php echo __('Select attribute');?>">
<?php endif; ?>
</td>
<?php endif; ?>
<td class="short context hidden"><?= $objectId ?></td>
<td class="short context hidden uuid quickSelect"><?php echo h($object['uuid']); ?></td>
<td class="short context hidden">
<?php echo $this->element('/Events/View/seen_field', array('object' => $object)); ?>
</td>
<?php endif; ?>
<td class="short context hidden"><?= $objectId ?></td>
<td class="short context hidden uuid quickSelect"><?php echo h($object['uuid']); ?></td>
<td class="short context hidden">
<?php echo $this->element('/Events/View/seen_field', array('object' => $object)); ?>
</td>
<td class="short timestamp"><?= $this->Time->date($object['timestamp']) ?></td>
<?php
if (!empty($extended)):
?>
<td class="short">
<?php
$event_info = sprintf('title="%s%s"',
__('Event info') . ':&#10; ',
$object['event_id'] != $event['Event']['id'] ? h($event['extensionEvents'][$object['event_id']]['info']) : h($event['Event']['info'])
);
?>
<?php echo '<a href="' . $baseurl . '/events/view/' . h($object['event_id']) . '" ' . $event_info . '>' . h($object['event_id']) . '</a>'; ?>
</td>
<?php
endif;
?>
<td class="short">
<td class="short timestamp"><?= $this->Time->date($object['timestamp']) ?></td>
<?php
if (!empty($extended)):
if ($object['event_id'] != $event['Event']['id']):
$extensionOrg = $event['extensionEvents'][$object['event_id']]['Orgc'];
echo $this->OrgImg->getOrgLogo($extensionOrg, 24);
else:
echo $this->OrgImg->getOrgLogo($event['Orgc'], 24);
endif;
endif;
?>
</td>
<td class="short"<?= $quickEdit('category') ?>>
<div class="inline-field-solid">
<?php echo h($object['category']); ?>
</div>
</td>
<td class="short"<?= $quickEdit('type') ?>>
<?php if (!empty($object['object_relation'])):?>
<div class="bold"><?php echo h($object['object_relation']); ?>:</div>
<?php endif; ?>
<div class="inline-field-solid">
<?php echo h($object['type']); ?>
</div>
</td>
<td id="Attribute_<?= $objectId ?>_container" class="showspaces limitedWidth shortish"<?= $quickEdit('value') ?>>
<div class="inline-field-solid">
<?php
$value = $this->element('/Events/View/value_field', array('object' => $object));
if (Configure::read('Plugin.Enrichment_hover_enable') && isset($modules) && isset($modules['hover_type'][$object['type']])) {
$commonDataFields = sprintf('data-object-type="Attribute" data-object-id="%s"', $objectId);
$spanExtra = Configure::read('Plugin.Enrichment_hover_popover_only') ? '' : sprintf(' class="eventViewAttributeHover" %s', $commonDataFields);
$popupButton = sprintf('<i class="fa fa-search-plus useCursorPointer eventViewAttributePopup noPrint" title="%s" %s></i>', __('Show hover enrichment'), $commonDataFields);
echo sprintf(
'<span%s>%s</span> %s',
$spanExtra,
$value,
$popupButton
);
} else {
echo $value;
}
?>
</div>
</td>
<td class="short">
<div class="attributeTagContainer">
<?php echo $this->element('ajaxTags', array(
'attributeId' => $objectId,
'tags' => $object['AttributeTag'],
'tagAccess' => ($isSiteAdmin || $mayModify),
'localTagAccess' => ($isSiteAdmin || $mayModify || $me['org_id'] == $event['Event']['org_id'] || (int)$me['org_id'] === Configure::read('MISP.host_org_id')),
'context' => $context,
'scope' => 'attribute',
'tagConflicts' => isset($object['tagConflicts']) ? $object['tagConflicts'] : array()
)
); ?>
</div>
</td>
<?php
if (!empty($includeRelatedTags)) {
$element = '';
if (!empty($object['RelatedTags'])) {
$element = $this->element('ajaxAttributeTags', array('attributeId' => $objectId, 'attributeTags' => $object['RelatedTags'], 'tagAccess' => false));
}
echo sprintf(
'<td class="shortish"><div %s>%s</div></td>',
'class="attributeRelatedTagContainer" id="#Attribute_' . $objectId . 'Related_tr .attributeTagContainer"',
$element
);
}
?>
<td class="short" id="attribute_<?= $objectId ?>_galaxy">
<?php
echo $this->element('galaxyQuickViewNew', array(
'mayModify' => $mayModify,
'isAclTagger' => $isAclTagger,
'data' => (!empty($object['Galaxy']) ? $object['Galaxy'] : array()),
'event' => $event,
'target_id' => $objectId,
'target_type' => 'attribute',
));
?>
</td>
<td class="showspaces bitwider"<?= $quickEdit('comment') ?>>
<div class="inline-field-solid">
<?php echo nl2br(h($object['comment']), false); ?>
</div>
</td>
<td class="short" style="padding-top:3px;">
<input
id="correlation_toggle_<?= $objectId ?>"
class="correlation-toggle"
aria-label="<?php echo __('Toggle correlation');?>"
title="<?php echo __('Toggle correlation');?>"
type="checkbox"
<?php
echo $correlationDisabled ? '' : ' checked';
echo $correlationButtonEnabled ? '' : ' disabled';
?>
>
</td>
<td class="shortish">
<?php
if (!empty($event['RelatedAttribute'][$objectId])) {
echo '<ul class="inline" style="margin:0">';
echo $this->element('Events/View/attribute_correlations', array(
'scope' => 'Attribute',
'object' => $object,
'event' => $event,
'withPivot' => true,
));
echo '</ul>';
}
?>
</td>
<td class="shortish">
<ul class="inline correlations">
<?php
if (isset($object['Feed'])) {
foreach ($object['Feed'] as $feed) {
$relatedData = array(
__('Name') => h($feed['name']),
__('Provider') => h($feed['provider']),
);
if (isset($feed['event_uuids'])) {
$relatedData[__('Event UUIDs')] = implode('<br>', array_map('h', $feed['event_uuids']));
}
$popover = '';
foreach ($relatedData as $k => $v) {
$popover .= '<span class="bold black">' . $k . '</span>: <span class="blue">' . $v . '</span><br>';
}
if ($isSiteAdmin || $hostOrgUser) {
if ($feed['source_format'] === 'misp') {
$liContents = sprintf(
'<form action="%s/feeds/previewIndex/%s" method="post" style="margin:0;line-height:auto;">%s%s</form>',
$baseurl,
h($feed['id']),
sprintf(
'<input type="hidden" name="data[Feed][eventid]" value="%s">',
h(json_encode($feed['event_uuids']))
),
sprintf(
'<input type="submit" class="linkButton useCursorPointer" value="%s" data-toggle="popover" data-content="%s" data-trigger="hover" style="margin-right:3px;line-height:normal;vertical-align: text-top;">',
h($feed['id']),
h($popover)
)
);
} else {
$liContents = sprintf(
'<a href="%s/feeds/previewIndex/%s" data-toggle="popover" data-content="%s" data-trigger="hover">%s</a>',
$baseurl,
h($feed['id']),
h($popover),
h($feed['id'])
);
}
} else {
$liContents = sprintf(
'<span>%s</span>',
h($feed['id'])
);
}
echo "<li>$liContents</li>";
}
}
if (isset($object['Server'])) {
foreach ($object['Server'] as $server) {
$popover = '';
foreach ($server as $k => $v) {
if ($k == 'id') continue;
if (is_array($v)) {
foreach ($v as $k2 => $v2) {
$v[$k2] = h($v2);
}
$v = implode('<br />', $v);
} else {
$v = h($v);
}
$popover .= '<span class=\'bold black\'>' . Inflector::humanize(h($k)) . '</span>: <span class="blue">' . $v . '</span><br />';
}
foreach ($server['event_uuids'] as $k => $event_uuid) {
$liContents = '';
if ($isSiteAdmin) {
$liContents .= sprintf(
'<a href="%s/servers/previewEvent/%s/%s" data-toggle="popover" data-content="%s" data-trigger="hover">%s</a>&nbsp;',
$baseurl,
h($server['id']),
h($event_uuid),
h($popover),
'S' . h($server['id']) . ':' . ($k + 1)
);
} else {
$liContents .= sprintf(
'<span>%s</span>',
'S' . h($server['id']) . ':' . ($k + 1)
);
}
echo "<li>$liContents</li>";
}
}
}
?>
</ul>
</td>
<td class="short">
<input type="checkbox" class="toids-toggle" id="toids_toggle_<?= $objectId ?>" aria-label="<?= __('Toggle IDS flag') ?>" title="<?= __('Toggle IDS flag') ?>"<?= $object['to_ids'] ? ' checked' : ''; ?><?= $mayModify ? '' : ' disabled' ?>>
</td>
<td class="short"<?= $quickEdit('distribution') ?>>
<div class="inline-field-solid<?= $object['distribution'] == 0 ? ' red' : '' ?>">
<?php
if ($object['distribution'] == 4):
?>
<a href="<?php echo $baseurl;?>/sharing_groups/view/<?php echo h($object['sharing_group_id']); ?>"><?php echo h($object['SharingGroup']['name']);?></a>
<?php
else:
echo h($shortDist[$object['distribution']]);
endif;
?>
</div>
</td>
<?php
echo $this->element('/Events/View/sighting_field', array(
'object' => $object,
));
if (!empty($includeSightingdb)) {
echo $this->element('/Events/View/sightingdb_field', array(
'object' => $object,
));
}
if (!empty($includeDecayScore)): ?>
<td class="decayingScoreField">
<div class="inline-field-solid">
<?php echo $this->element('DecayingModels/View/attribute_decay_score', array('scope' => 'object', 'object' => $object, 'uselink' => true)); ?>
</div>
</td>
<?php
endif;
?>
<td class="short action-links">
<?php
if ($object['deleted']):
if ($isSiteAdmin || $mayModify):
?>
<span class="fas fa-redo useCursorPointer" title="<?php echo __('Restore attribute');?>" role="button" tabindex="0" aria-label="<?php echo __('Restore attribute');?>" onclick="deleteObject('attributes', 'restore', '<?= $objectId ?>')"></span>
<span class="fa fa-trash useCursorPointer" title="<?php echo __('Permanently delete attribute');?>" role="button" tabindex="0" aria-label="<?php echo __('Permanently delete attribute');?>" onclick="deleteObject('attributes', 'delete', '<?= $objectId . '/true'; ?>')"></span>
<?php
endif;
else:
if ($isAclAdd && ($isSiteAdmin || !$mayModify)):
if (isset($modules) && isset($modules['types'][$object['type']])):
?>
<span class="fas fa-asterisk useCursorPointer" role="button" tabindex="0" aria-label="<?php echo __('Query enrichment');?>" onclick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/ShadowAttribute');" title="<?php echo __('Propose enrichment');?>">&nbsp;</span>
<?php
endif;
if (isset($cortex_modules) && isset($cortex_modules['types'][$object['type']])):
?>
<span class="icon-eye-open useCursorPointer" role="button" tabindex="0" aria-label="<?php echo __('Query Cortex');?>" onclick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/ShadowAttribute/Cortex');" title="<?php echo __('Propose enrichment through Cortex');?>"></span>
<?php
endif;
?>
<a href="<?php echo $baseurl;?>/shadow_attributes/edit/<?= $objectId ?>" title="<?php echo __('Propose Edit');?>" aria-label="<?php echo __('Propose Edit');?>" class="fa fa-comment"></a>
<span class="fa fa-trash useCursorPointer" title="<?php echo __('Propose Deletion');?>" role="button" tabindex="0" aria-label="Propose deletion" onclick="deleteObject('shadow_attributes', 'delete', '<?= $objectId ?>')"></span>
<?php
if ($isSiteAdmin):
?>
<span class="verticalSeparator">&nbsp;</span>
<?php endif;
endif;
if ($isSiteAdmin || $mayModify):
if (isset($modules) && isset($modules['types'][$object['type']])):
?>
<span class="fas fa-asterisk useCursorPointer" onclick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/Attribute');" title="<?php echo __('Add enrichment');?>" role="button" tabindex="0" aria-label="<?php echo __('Add enrichment');?>">&nbsp;</span>
<?php
endif;
if (isset($cortex_modules) && isset($cortex_modules['types'][$object['type']])):
?>
<span class="icon-eye-open useCursorPointer" onclick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/Attribute/Cortex');" title="<?php echo __('Add enrichment');?>" role="button" tabindex="0" aria-label="<?php echo __('Add enrichment via Cortex');?>"></span>
<?php
endif;
?>
<a href="<?php echo $baseurl;?>/attributes/edit/<?= $objectId ?>" title="<?php echo __('Edit');?>" aria-label="<?php echo __('Edit');?>" class="fa fa-edit"></a>
<td class="short">
<?php
if (empty($event['Event']['publish_timestamp'])):
$event_info = sprintf('title="%s%s"',
__('Event info') . ':&#10; ',
$object['event_id'] != $event['Event']['id'] ? h($event['extensionEvents'][$object['event_id']]['info']) : h($event['Event']['info'])
);
?>
<span class="fa fa-trash useCursorPointer" title="<?php echo __('Permanently delete attribute');?>" role="button" tabindex="0" aria-label="<?php echo __('Permanently delete attribute');?>" onclick="deleteObject('attributes', 'delete', '<?= $objectId . '/true'; ?>')"></span>
<?php
<?php echo '<a href="' . $baseurl . '/events/view/' . h($object['event_id']) . '" ' . $event_info . '>' . h($object['event_id']) . '</a>'; ?>
</td>
<?php
endif;
?>
<td class="short">
<?php
if (!empty($extended)):
if ($object['event_id'] != $event['Event']['id']):
$extensionOrg = $event['extensionEvents'][$object['event_id']]['Orgc'];
echo $this->OrgImg->getOrgLogo($extensionOrg, 24);
else:
?>
<span class="fa fa-trash useCursorPointer" title="<?php echo __('Soft-delete attribute');?>" role="button" tabindex="0" aria-label="<?php echo __('Soft-delete attribute');?>" onclick="deleteObject('attributes', 'delete', '<?= $objectId ?>')"></span>
<?php
echo $this->OrgImg->getOrgLogo($event['Orgc'], 24);
endif;
endif;
endif;
?>
</td>
</tr>
<?php
if (!empty($object['ShadowAttribute'])) {
end($object['ShadowAttribute']);
$lastElement = key($object['ShadowAttribute']);
foreach ($object['ShadowAttribute'] as $propKey => $proposal) {
echo $this->element('/Events/View/row_' . $proposal['objectType'], array(
'object' => $proposal,
?>
</td>
<td class="short"<?= $quickEdit('category') ?>>
<div class="inline-field-solid">
<?php echo h($object['category']); ?>
</div>
</td>
<td class="short"<?= $quickEdit('type') ?>>
<?php if (!empty($object['object_relation'])):?>
<div class="bold"><?php echo h($object['object_relation']); ?>:</div>
<?php endif; ?>
<div class="inline-field-solid">
<?php echo h($object['type']); ?>
</div>
</td>
<td id="Attribute_<?= $objectId ?>_container" class="showspaces limitedWidth shortish"<?= $quickEdit('value') ?>>
<div class="inline-field-solid">
<?php
$value = $this->element('/Events/View/value_field', array('object' => $object));
if (Configure::read('Plugin.Enrichment_hover_enable') && isset($modules) && isset($modules['hover_type'][$object['type']])) {
$commonDataFields = sprintf('data-object-type="Attribute" data-object-id="%s"', $objectId);
$spanExtra = Configure::read('Plugin.Enrichment_hover_popover_only') ? '' : sprintf(' class="eventViewAttributeHover" %s', $commonDataFields);
$popupButton = sprintf('<i class="fa fa-search-plus useCursorPointer eventViewAttributePopup noPrint" title="%s" %s></i>', __('Show hover enrichment'), $commonDataFields);
echo sprintf(
'<span%s>%s</span> %s',
$spanExtra,
$value,
$popupButton
);
} else {
echo $value;
}
?>
</div>
</td>
<td class="short">
<div class="attributeTagContainer">
<?php echo $this->element('ajaxTags', array(
'attributeId' => $objectId,
'tags' => $object['AttributeTag'],
'tagAccess' => ($isSiteAdmin || $mayModify),
'localTagAccess' => ($isSiteAdmin || $mayModify || $me['org_id'] == $event['Event']['org_id'] || (int)$me['org_id'] === Configure::read('MISP.host_org_id')),
'context' => $context,
'scope' => 'attribute',
'tagConflicts' => isset($object['tagConflicts']) ? $object['tagConflicts'] : array()
)
); ?>
</div>
</td>
<?php
if (!empty($includeRelatedTags)) {
$element = '';
if (!empty($object['RelatedTags'])) {
$element = $this->element('ajaxAttributeTags', array('attributeId' => $objectId, 'attributeTags' => $object['RelatedTags'], 'tagAccess' => false));
}
echo sprintf(
'<td class="shortish"><div %s>%s</div></td>',
'class="attributeRelatedTagContainer" id="#Attribute_' . $objectId . 'Related_tr .attributeTagContainer"',
$element
);
}
?>
<td class="short" id="attribute_<?= $objectId ?>_galaxy">
<?php
echo $this->element('galaxyQuickViewNew', array(
'mayModify' => $mayModify,
'mayChangeCorrelation' => $mayChangeCorrelation,
'fieldCount' => $fieldCount,
'child' => $propKey == $lastElement ? 'last' : true,
'objectContainer' => $child
));
}
}
'isAclTagger' => $isAclTagger,
'data' => (!empty($object['Galaxy']) ? $object['Galaxy'] : array()),
'event' => $event,
'target_id' => $objectId,
'target_type' => 'attribute',
));
?>
</td>
<td class="showspaces bitwider"<?= $quickEdit('comment') ?>>
<div class="inline-field-solid">
<?php echo nl2br(h($object['comment']), false); ?>
</div>
</td>
<td class="short" style="padding-top:3px;">
<input
id="correlation_toggle_<?= $objectId ?>"
class="correlation-toggle"
aria-label="<?php echo __('Toggle correlation');?>"
title="<?php echo __('Toggle correlation');?>"
type="checkbox"
<?php
echo $correlationDisabled ? '' : ' checked';
echo $correlationButtonEnabled ? '' : ' disabled';
?>
>
</td>
<td class="shortish">
<?php
//if (!empty($event['RelatedAttribute'][$objectId])) {
echo '<ul class="inline" style="margin:0">';
echo $this->element('Events/View/attribute_correlations', array(
'scope' => 'Attribute',
'object' => $object,
'event' => $event,
'withPivot' => true,
));
echo '</ul>';
//}
?>
</td>
<td class="shortish">
<ul class="inline correlations">
<?php
if (isset($object['Feed'])) {
foreach ($object['Feed'] as $feed) {
$relatedData = array(
__('Name') => h($feed['name']),
__('Provider') => h($feed['provider']),
);
if (isset($feed['event_uuids'])) {
$relatedData[__('Event UUIDs')] = implode('<br>', array_map('h', $feed['event_uuids']));
}
$popover = '';
foreach ($relatedData as $k => $v) {
$popover .= '<span class="bold black">' . $k . '</span>: <span class="blue">' . $v . '</span><br>';
}
if ($isSiteAdmin || $hostOrgUser) {
if ($feed['source_format'] === 'misp') {
$liContents = sprintf(
'<form action="%s/feeds/previewIndex/%s" method="post" style="margin:0;line-height:auto;">%s%s</form>',
$baseurl,
h($feed['id']),
sprintf(
'<input type="hidden" name="data[Feed][eventid]" value="%s">',
h(json_encode($feed['event_uuids']))
),
sprintf(
'<input type="submit" class="linkButton useCursorPointer" value="%s" data-toggle="popover" data-content="%s" data-trigger="hover" style="margin-right:3px;line-height:normal;vertical-align: text-top;">',
h($feed['id']),
h($popover)
)
);
} else {
$liContents = sprintf(
'<a href="%s/feeds/previewIndex/%s" data-toggle="popover" data-content="%s" data-trigger="hover">%s</a>',
$baseurl,
h($feed['id']),
h($popover),
h($feed['id'])
);
}
} else {
$liContents = sprintf(
'<span>%s</span>',
h($feed['id'])
);
}
echo "<li>$liContents</li>";
}
}
if (isset($object['Server'])) {
foreach ($object['Server'] as $server) {
$popover = '';
foreach ($server as $k => $v) {
if ($k == 'id') continue;
if (is_array($v)) {
foreach ($v as $k2 => $v2) {
$v[$k2] = h($v2);
}
$v = implode('<br />', $v);
} else {
$v = h($v);
}
$popover .= '<span class=\'bold black\'>' . Inflector::humanize(h($k)) . '</span>: <span class="blue">' . $v . '</span><br />';
}
foreach ($server['event_uuids'] as $k => $event_uuid) {
$liContents = '';
if ($isSiteAdmin) {
$liContents .= sprintf(
'<a href="%s/servers/previewEvent/%s/%s" data-toggle="popover" data-content="%s" data-trigger="hover">%s</a>&nbsp;',
$baseurl,
h($server['id']),
h($event_uuid),
h($popover),
'S' . h($server['id']) . ':' . ($k + 1)
);
} else {
$liContents .= sprintf(
'<span>%s</span>',
'S' . h($server['id']) . ':' . ($k + 1)
);
}
echo "<li>$liContents</li>";
}
}
}
?>
</ul>
</td>
<td class="short">
<input type="checkbox" class="toids-toggle" id="toids_toggle_<?= $objectId ?>" aria-label="<?= __('Toggle IDS flag') ?>" title="<?= __('Toggle IDS flag') ?>"<?= $object['to_ids'] ? ' checked' : ''; ?><?= $mayModify ? '' : ' disabled' ?>>
</td>
<td class="short"<?= $quickEdit('distribution') ?>>
<div class="inline-field-solid<?= $object['distribution'] == 0 ? ' red' : '' ?>">
<?php
if ($object['distribution'] == 4):
?>
<a href="<?php echo $baseurl;?>/sharing_groups/view/<?php echo h($object['sharing_group_id']); ?>"><?php echo h($object['SharingGroup']['name']);?></a>
<?php
else:
echo h($shortDist[$object['distribution']]);
endif;
?>
</div>
</td>
<?php
echo $this->element('/Events/View/sighting_field', array(
'object' => $object,
));
if (!empty($includeSightingdb)) {
echo $this->element('/Events/View/sightingdb_field', array(
'object' => $object,
));
}
if (!empty($includeDecayScore)): ?>
<td class="decayingScoreField">
<div class="inline-field-solid">
<?php echo $this->element('DecayingModels/View/attribute_decay_score', array('scope' => 'object', 'object' => $object, 'uselink' => true)); ?>
</div>
</td>
<?php
endif;
?>
<td class="short action-links">
<?php
if ($object['deleted']):
if ($isSiteAdmin || $mayModify):
?>
<span class="fas fa-redo useCursorPointer" title="<?php echo __('Restore attribute');?>" role="button" tabindex="0" aria-label="<?php echo __('Restore attribute');?>" onclick="deleteObject('attributes', 'restore', '<?= $objectId ?>')"></span>
<span class="fa fa-trash useCursorPointer" title="<?php echo __('Permanently delete attribute');?>" role="button" tabindex="0" aria-label="<?php echo __('Permanently delete attribute');?>" onclick="deleteObject('attributes', 'delete', '<?= $objectId . '/true'; ?>')"></span>
<?php
endif;
else:
if ($isAclAdd && ($isSiteAdmin || !$mayModify)):
if (isset($modules) && isset($modules['types'][$object['type']])):
?>
<span class="fas fa-asterisk useCursorPointer" role="button" tabindex="0" aria-label="<?php echo __('Query enrichment');?>" onclick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/ShadowAttribute');" title="<?php echo __('Propose enrichment');?>">&nbsp;</span>
<?php
endif;
if (isset($cortex_modules) && isset($cortex_modules['types'][$object['type']])):
?>
<span class="icon-eye-open useCursorPointer" role="button" tabindex="0" aria-label="<?php echo __('Query Cortex');?>" onclick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/ShadowAttribute/Cortex');" title="<?php echo __('Propose enrichment through Cortex');?>"></span>
<?php
endif;
?>
<a href="<?php echo $baseurl;?>/shadow_attributes/edit/<?= $objectId ?>" title="<?php echo __('Propose Edit');?>" aria-label="<?php echo __('Propose Edit');?>" class="fa fa-comment"></a>
<span class="fa fa-trash useCursorPointer" title="<?php echo __('Propose Deletion');?>" role="button" tabindex="0" aria-label="Propose deletion" onclick="deleteObject('shadow_attributes', 'delete', '<?= $objectId ?>')"></span>
<?php
if ($isSiteAdmin):
?>
<span class="verticalSeparator">&nbsp;</span>
<?php endif;
endif;
if ($isSiteAdmin || $mayModify):
if (isset($modules) && isset($modules['types'][$object['type']])):
?>
<span class="fas fa-asterisk useCursorPointer" onclick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/Attribute');" title="<?php echo __('Add enrichment');?>" role="button" tabindex="0" aria-label="<?php echo __('Add enrichment');?>">&nbsp;</span>
<?php
endif;
if (isset($cortex_modules) && isset($cortex_modules['types'][$object['type']])):
?>
<span class="icon-eye-open useCursorPointer" onclick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/Attribute/Cortex');" title="<?php echo __('Add enrichment');?>" role="button" tabindex="0" aria-label="<?php echo __('Add enrichment via Cortex');?>"></span>
<?php
endif;
?>
<a href="<?php echo $baseurl;?>/attributes/edit/<?= $objectId ?>" title="<?php echo __('Edit');?>" aria-label="<?php echo __('Edit');?>" class="fa fa-edit"></a>
<?php
if (empty($event['Event']['publish_timestamp'])):
?>
<span class="fa fa-trash useCursorPointer" title="<?php echo __('Permanently delete attribute');?>" role="button" tabindex="0" aria-label="<?php echo __('Permanently delete attribute');?>" onclick="deleteObject('attributes', 'delete', '<?= $objectId . '/true'; ?>')"></span>
<?php
else:
?>
<span class="fa fa-trash useCursorPointer" title="<?php echo __('Soft-delete attribute');?>" role="button" tabindex="0" aria-label="<?php echo __('Soft-delete attribute');?>" onclick="deleteObject('attributes', 'delete', '<?= $objectId ?>')"></span>
<?php
endif;
endif;
endif;
?>
</td>
</tr>
<?php
if (!empty($object['ShadowAttribute'])) {
end($object['ShadowAttribute']);
$lastElement = key($object['ShadowAttribute']);
foreach ($object['ShadowAttribute'] as $propKey => $proposal) {
echo $this->element('/Events/View/row_' . $proposal['objectType'], array(
'object' => $proposal,
'mayModify' => $mayModify,
'mayChangeCorrelation' => $mayChangeCorrelation,
'fieldCount' => $fieldCount,
'child' => $propKey == $lastElement ? 'last' : true,
'objectContainer' => $child
));
}
}

Some files were not shown because too many files have changed in this diff Show More