MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'develop' into 2.4

pull/8619/head
iglocska 2022-09-26 13:10:51 +02:00
parent 0b68c508a3 eae483d776
commit 40ce38efec
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
74 changed files with 1878 additions and 1868 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
VERSION.json
View File

@ -1 +1 @@
{"major":2, "minor":4, "hotfix":162}
{"major":2, "minor":4, "hotfix":163}

27
app/Console/Command/AdminShell.php
View File

@ -22,8 +22,6 @@ class AdminShell extends AppShell
'Role', 'Feed', 'SharingGroupBlueprint', 'Correlation', 'OverCorrelatingValue'
];
public $tasks = ['ConfigLoad'];
public function getOptionParser()
{
$parser = parent::getOptionParser();
@ -105,7 +103,6 @@ class AdminShell extends AppShell
public function jobGenerateCorrelation()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Generate correlation'] . PHP_EOL);
}
@ -116,7 +113,6 @@ class AdminShell extends AppShell
public function jobGenerateOccurrences()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Generate over-correlation occurrences'] . PHP_EOL);
}
@ -138,7 +134,6 @@ class AdminShell extends AppShell
public function jobGenerateShadowAttributeCorrelation()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Generate shadow attribute correlation'] . PHP_EOL);
}
@ -152,14 +147,12 @@ class AdminShell extends AppShell
public function updateMISP()
{
$this->ConfigLoad->execute();
$status = array('branch' => '2.4');
echo $this->Server->update($status) . PHP_EOL;
}
public function updateAfterPull()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Update after pull'] . PHP_EOL);
}
@ -186,7 +179,6 @@ class AdminShell extends AppShell
$this->error('This method does nothing when SimpleBackgroundJobs are enabled.');
}
$this->ConfigLoad->execute();
$this->Server->restartWorkers();
echo PHP_EOL . 'Workers restarted.' . PHP_EOL;
}
@ -197,7 +189,6 @@ class AdminShell extends AppShell
$this->error('This method does nothing when SimpleBackgroundJobs are enabled.');
}
$this->ConfigLoad->execute();
if (empty($this->args[0]) || !is_numeric($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['worker_management_tasks']['data']['Restart a worker'] . PHP_EOL);
}
@ -223,7 +214,6 @@ class AdminShell extends AppShell
$this->error('This method does nothing when SimpleBackgroundJobs are enabled.');
}
$this->ConfigLoad->execute();
if (empty($this->args[0]) || !is_numeric($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['worker_management_tasks']['data']['Kill a worker'] . PHP_EOL);
}
@ -244,7 +234,6 @@ class AdminShell extends AppShell
$this->error('This method does nothing when SimpleBackgroundJobs are enabled.');
}
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['worker_management_tasks']['data']['Start a worker'] . PHP_EOL);
}
@ -276,7 +265,6 @@ class AdminShell extends AppShell
public function updateGalaxies()
{
$this->ConfigLoad->execute();
// The following is 7.x upwards only
//$value = $this->args[0] ?? $this->args[0] ?? 0;
$value = empty($this->args[0]) ? null : $this->args[0];
@ -347,7 +335,6 @@ class AdminShell extends AppShell
public function updateNoticeLists()
{
$this->ConfigLoad->execute();
$result = $this->Noticelist->update();
if ($result) {
echo 'Notice lists updated' . PHP_EOL;
@ -359,7 +346,6 @@ class AdminShell extends AppShell
# FIXME: Fails to pass userId/orgId properly, global update works.
public function updateObjectTemplates()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Update object templates'] . PHP_EOL);
} else {
@ -392,7 +378,6 @@ class AdminShell extends AppShell
public function jobUpgrade24()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Job upgrade'] . PHP_EOL);
}
@ -410,7 +395,6 @@ class AdminShell extends AppShell
public function prune_update_logs()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Prune update logs'] . PHP_EOL);
}
@ -429,7 +413,6 @@ class AdminShell extends AppShell
public function getWorkers()
{
$this->ConfigLoad->execute();
$result = $this->Server->workerDiagnostics($workerIssueCount);
$query = 'all';
if (!empty($this->args[0])) {
@ -501,7 +484,6 @@ class AdminShell extends AppShell
public function setDatabaseVersion()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Set database version'] . PHP_EOL);
} else {
@ -580,7 +562,6 @@ class AdminShell extends AppShell
public function setDefaultRole()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || !is_numeric($this->args[0])) {
$roles = $this->Role->find('list', array(
'fields' => array('id', 'name')
@ -615,7 +596,6 @@ class AdminShell extends AppShell
*/
public function change_authkey()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
echo 'MISP apikey command line tool' . PHP_EOL . 'To assign a new random API key for a user: ' . APP . 'Console/cake Admin change_authkey [user_email]' . PHP_EOL . 'To assign a fixed API key: ' . APP . 'Console/cake Admin change_authkey [user_email] [authkey]' . PHP_EOL;
die();
@ -646,7 +626,6 @@ class AdminShell extends AppShell
public function recoverSinceLastSuccessfulUpdate()
{
$this->ConfigLoad->execute();
$this->loadModel('Log');
$logs = $this->Log->find('all', array(
'conditions' => array(
@ -685,7 +664,6 @@ class AdminShell extends AppShell
public function cleanCaches()
{
$this->ConfigLoad->execute();
echo 'Cleaning caches...' . PHP_EOL;
$this->Server->cleanCacheFiles();
echo '...caches lost in time, like tears in rain.' . PHP_EOL;
@ -693,7 +671,6 @@ class AdminShell extends AppShell
public function resetSyncAuthkeys()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
echo sprintf(
__("MISP mass sync authkey reset command line tool" . PHP_EOL . "Usage: %sConsole/cake Admin resetSyncAuthkeys [user_id]" . PHP_EOL), APP
@ -719,7 +696,6 @@ class AdminShell extends AppShell
public function purgeFeedEvents()
{
$this->ConfigLoad->execute();
if (
(empty($this->args[0]) || !is_numeric($this->args[0])) ||
(empty($this->args[1]) || !is_numeric($this->args[1]))
@ -759,7 +735,6 @@ class AdminShell extends AppShell
*/
public function UserIP()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get IPs for user ID'] . PHP_EOL);
}
@ -787,7 +762,6 @@ class AdminShell extends AppShell
*/
public function IPUser()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get user ID for user IP'] . PHP_EOL);
}
@ -1207,7 +1181,6 @@ class AdminShell extends AppShell
public function truncateTable()
{
$this->ConfigLoad->execute();
if (!isset($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Truncate table correlation'] . PHP_EOL);
}

3
app/Console/Command/AppShell.php
View File

@ -32,9 +32,10 @@ class AppShell extends Shell
public function initialize()
{
parent::initialize();
$this->ConfigLoad = $this->Tasks->load('ConfigLoad');
$this->ConfigLoad->execute();
parent::initialize();
}
public function perform()

3
app/Console/Command/AuthkeyShell.php
View File

@ -10,13 +10,10 @@ class AuthkeyShell extends AppShell {
public $uses = array('User', 'Log');
public $tasks = array('ConfigLoad');
public function main()
{
$this->err('This method is deprecated. Next time please use `cake user change_authkey [user] [authkey]` command.');
$this->ConfigLoad->execute();
if (!isset($this->args[0]) || empty($this->args[0])) echo 'MISP authkey reset command line tool.' . PHP_EOL . 'To assign a new authkey for a user:' . PHP_EOL . APP . 'Console/cake Authkey [email] [auth_key | optional]' . PHP_EOL;
else {
// get the users that need their password hashed

1
app/Console/Command/BaseurlShell.php
View File

@ -13,7 +13,6 @@ class BaseurlShell extends AppShell {
{
$this->err('This method is deprecated. Next time please use `cake admin setSetting MISP.baseurl [baseurl]` command.');
$this->ConfigLoad->execute();
$baseurl = $this->args[0];
$result = $this->Server->testBaseURL($baseurl);
if (true !== $result) {

13
app/Console/Command/EventShell.php
View File

@ -15,7 +15,6 @@ require_once 'AppShell.php';
class EventShell extends AppShell
{
public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Allowedlist', 'Server', 'Organisation', 'Correlation', 'Tag');
public $tasks = array('ConfigLoad');
public function getOptionParser()
{
@ -121,7 +120,6 @@ class EventShell extends AppShell
public function doPublish()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Do publish'] . PHP_EOL);
}
@ -157,7 +155,6 @@ class EventShell extends AppShell
public function correlateValue()
{
$this->ConfigLoad->execute();
$value = $this->args[0];
if (!empty($this->args[1])) {
@ -182,7 +179,6 @@ class EventShell extends AppShell
public function cache()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Cache event'] . PHP_EOL);
}
@ -223,7 +219,6 @@ class EventShell extends AppShell
private function __runCaching($user, $typeData, $id, $export_type, $subType = '')
{
$this->ConfigLoad->execute();
$export_type = strtolower($typeData['type']);
$final = $this->{$typeData['scope']}->restSearch($user, $typeData['params']['returnFormat'], $typeData['params'], false, $id);
$dir = new Folder(APP . 'tmp/cached_exports/' . $export_type, true, 0750);
@ -240,7 +235,6 @@ class EventShell extends AppShell
public function cachebro()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1])) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Cache bro'] . PHP_EOL);
}
@ -281,7 +275,6 @@ class EventShell extends AppShell
public function alertemail()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Alert email'] . PHP_EOL);
}
@ -314,7 +307,6 @@ class EventShell extends AppShell
public function postsemail()
{
$this->ConfigLoad->execute();
if (
empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) ||
empty($this->args[3]) || empty($this->args[4]) || empty($this->args[5])
@ -348,7 +340,6 @@ class EventShell extends AppShell
public function enqueueCaching()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Enqueue caching'] . PHP_EOL);
}
@ -405,7 +396,6 @@ class EventShell extends AppShell
public function publish()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[2]) || empty($this->args[3])) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Publish event'] . PHP_EOL);
}
@ -465,7 +455,6 @@ class EventShell extends AppShell
public function publish_galaxy_clusters()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || !array_key_exists(3, $this->args)) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Publish Galaxy clusters'] . PHP_EOL);
}
@ -492,7 +481,6 @@ class EventShell extends AppShell
public function enrichment()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Run enrichment'] . PHP_EOL);
}
@ -588,7 +576,6 @@ class EventShell extends AppShell
public function recoverEvent()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1])) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Recover event'] . PHP_EOL);
}

1
app/Console/Command/LiveShell.php
View File

@ -10,7 +10,6 @@ class LiveShell extends AppShell {
public function main()
{
$this->ConfigLoad->execute();
$live = $this->args[0];
if ($live != 0 && $live != 1) {
echo 'Invalid parameters. Usage: /var/www/MISP/app/Console/cake Live [0|1]';

2
app/Console/Command/PasswordShell.php
View File

@ -14,7 +14,6 @@ class PasswordShell extends AppShell {
{
$this->err('This method is deprecated. Next time please use `cake user change_pw [user] [password]` command.');
$this->ConfigLoad->execute();
if (!isset($this->args[0]) || empty($this->args[0]) || !isset($this->args[1]) || empty($this->args[1])) echo 'MISP password reset command line tool.' . PHP_EOL . 'To assign a new password for a user:' . PHP_EOL . APP . 'Console/cake Password [email] [password]' . PHP_EOL;
else {
// get the users that need their password hashed
@ -42,7 +41,6 @@ class PasswordShell extends AppShell {
public function getOptionParser()
{
$this->ConfigLoad->execute();
$parser = parent::getOptionParser();
$parser->addOption('override_password_change', array(
'short' => 'o',

31
app/Console/Command/ServerShell.php
View File

@ -14,6 +14,20 @@ class ServerShell extends AppShell
{
public $uses = array('Server', 'Task', 'Job', 'User', 'Feed');
public function getOptionParser()
{
$parser = parent::getOptionParser();
$parser->addSubcommand('fetchIndex', [
'help' => __('Fetch remote instance event index.'),
'parser' => array(
'arguments' => array(
'server_id' => ['help' => __('Remote server ID.'), 'required' => true],
),
)
]);
return $parser;
}
public function list()
{
$servers = $this->Server->find('all', [
@ -55,9 +69,17 @@ class ServerShell extends AppShell
echo $this->json($res) . PHP_EOL;
}
public function fetchIndex()
{
$serverId = intval($this->args[0]);
$server = $this->getServer($serverId);
$serverSync = new ServerSyncTool($server, $this->Server->setupSyncRequest($server));
$index = $this->Server->getEventIndexFromServer($serverSync);
echo $this->json($index) . PHP_EOL;
}
public function pullAll()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['PullAll'] . PHP_EOL);
}
@ -367,7 +389,6 @@ class ServerShell extends AppShell
public function enqueuePull()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Enqueue pull'] . PHP_EOL);
}
@ -430,7 +451,6 @@ class ServerShell extends AppShell
public function enqueueFeedFetch()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Enqueue feed fetch'] . PHP_EOL);
}
@ -480,7 +500,6 @@ class ServerShell extends AppShell
public function enqueueFeedCache()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Enqueue feed cache'] . PHP_EOL);
}
@ -537,7 +556,6 @@ class ServerShell extends AppShell
public function enqueuePush()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Enqueue push'] . PHP_EOL);
}
@ -580,7 +598,6 @@ class ServerShell extends AppShell
public function sendPeriodicSummaryToUsers()
{
$this->ConfigLoad->execute();
$periods = $this->__getPeriodsForToday();
$start_time = time();
echo __n('Started periodic summary generation for the %s period', 'Started periodic summary generation for periods: %s', count($periods), implode(', ', $periods)) . PHP_EOL;
@ -593,7 +610,7 @@ class ServerShell extends AppShell
$this->User->sendEmail($user, $emailTemplate, false, null);
}
}
echo __('All reports sent. Task took %s secondes', time() - $start_time) . PHP_EOL;
echo __('All reports sent. Task took %s seconds', time() - $start_time) . PHP_EOL;
}
private function __getPeriodsForToday(): array

3
app/Console/Command/WorkflowShell.php
View File

@ -6,11 +6,9 @@ require_once 'AppShell.php';
class WorkflowShell extends AppShell {
public $uses = ['Job', 'Workflow'];
public $tasks = ['ConfigLoad'];
public function executeWorkflowForTrigger()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || empty($this->args[3])) {
die(__('Invalid number of arguments.'));
}
@ -39,7 +37,6 @@ class WorkflowShell extends AppShell {
public function walkGraph()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || empty($this->args[3])) {
die(__('Invalid number of arguments.'));
}

8
app/Controller/AppController.php
View File

@ -49,14 +49,6 @@ class AppController extends Controller
public $restResponsePayload = null;
// Used for _isAutomation(), a check that returns true if the controller & action combo matches an action that is a non-xml and non-json automation method
// This is used to allow authentication via headers for methods not covered by _isRest() - as that only checks for JSON and XML formats
public $automationArray = array(
'events' => array('csv', 'nids', 'hids', 'xml', 'restSearch', 'stix', 'updateGraph', 'downloadOpenIOCEvent'),
'attributes' => array('text', 'downloadAttachment', 'returnAttributes', 'restSearch', 'rpz', 'bro'),
'objects' => array('restSearch')
);
protected $_legacyParams = array();
/** @var array */
public $userRole;

44
app/Controller/AttributesController.php
View File

@ -71,7 +71,8 @@ class AttributesController extends AppController
public function index()
{
$this->paginate['conditions']['AND'][] = $this->Attribute->buildConditions($this->Auth->user());
$user = $this->Auth->user();
$this->paginate['conditions']['AND'][] = $this->Attribute->buildConditions($user);
$attributes = $this->paginate();
if ($this->_isRest()) {
@ -84,13 +85,13 @@ class AttributesController extends AppController
'fields' => ['Orgc.id', 'Orgc.name', 'Orgc.uuid'],
]);
$orgTable = Hash::combine($orgTable, '{n}.Orgc.id', '{n}.Orgc');
$sgids = $this->Attribute->SharingGroup->authorizedIds($this->Auth->user());
$sgids = $this->Attribute->SharingGroup->authorizedIds($user);
foreach ($attributes as &$attribute) {
if (isset($orgTable[$attribute['Event']['orgc_id']])) {
$attribute['Event']['Orgc'] = $orgTable[$attribute['Event']['orgc_id']];
}
$temp = $this->Attribute->Correlation->getRelatedAttributes(
$this->Auth->user(),
$user,
$sgids,
$attribute['Attribute'],
[],
@ -104,7 +105,7 @@ class AttributesController extends AppController
$attribute['Event']['RelatedAttribute'][$attribute['Attribute']['id']] = $temp;
}
list($attributes, $sightingsData) = $this->__searchUI($attributes);
list($attributes, $sightingsData) = $this->__searchUI($attributes, $user);
$this->set('isSearch', 0);
$this->set('sightingsData', $sightingsData);
$this->set('orgTable', array_column($orgTable, 'name', 'id'));
@ -1540,6 +1541,7 @@ class AttributesController extends AppController
public function search($continue = false)
{
$user = $this->Auth->user();
$exception = null;
$filters = $this->__getSearchFilters($exception);
if ($this->request->is('post') || !empty($this->request->params['named']['tags'])) {
@ -1571,7 +1573,7 @@ class AttributesController extends AppController
}
if (!empty($filters)) {
$filters['includeCorrelations'] = 1;
$params = $this->Attribute->restSearch($this->Auth->user(), 'json', $filters, true);
$params = $this->Attribute->restSearch($user, 'json', $filters, true);
if (!isset($params['conditions']['Attribute.deleted'])) {
$params['conditions']['Attribute.deleted'] = 0;
}
@ -1589,7 +1591,7 @@ class AttributesController extends AppController
'fields' => ['Orgc.id', 'Orgc.name', 'Orgc.uuid'],
]);
$orgTable = array_column(array_column($orgTable, 'Orgc'), null, 'id');
$sgids = $this->Attribute->SharingGroup->authorizedIds($this->Auth->user());
$sgids = $this->Attribute->SharingGroup->authorizedIds($user);
foreach ($attributes as &$attribute) {
if (isset($orgTable[$attribute['Event']['orgc_id']])) {
$attribute['Event']['Orgc'] = $orgTable[$attribute['Event']['orgc_id']];
@ -1599,7 +1601,7 @@ class AttributesController extends AppController
}
if (isset($filters['includeCorrelations'])) {
$temp = $this->Attribute->Correlation->getRelatedAttributes(
$this->Auth->user(),
$user,
$sgids,
$attribute['Attribute'],
[],
@ -1617,7 +1619,7 @@ class AttributesController extends AppController
return $this->RestResponse->viewData($attributes, $this->response->type());
}
list($attributes, $sightingsData) = $this->__searchUI($attributes);
list($attributes, $sightingsData) = $this->__searchUI($attributes, $user);
$this->set('sightingsData', $sightingsData);
if (isset($filters['tags']) && !empty($filters['tags'])) {
@ -1653,7 +1655,12 @@ class AttributesController extends AppController
}
}
private function __searchUI(array $attributes)
/**
* @param array $attributes
* @param array $user
* @return array|array[]
*/
private function __searchUI(array $attributes, array $user)
{
if (empty($attributes)) {
return [[], []];
@ -1663,11 +1670,8 @@ class AttributesController extends AppController
$this->loadModel('Sighting');
$this->loadModel('AttachmentScan');
$user = $this->Auth->user();
$attributeIds = [];
$galaxyTags = [];
foreach ($attributes as &$attribute) {
$attributeIds[] = $attribute['Attribute']['id'];
if ($this->Attribute->isImage($attribute['Attribute'])) {
if (extension_loaded('gd')) {
// if extension is loaded, the data is not passed to the view because it is asynchronously fetched
@ -1687,8 +1691,8 @@ class AttributesController extends AppController
$attribute['Attribute']['AttributeTag'] = $attribute['AttributeTag'];
foreach ($attribute['Attribute']['AttributeTag'] as $at) {
if (substr($at['Tag']['name'], 0, 12) === 'misp-galaxy:') {
$galaxyTags[] = $at['Tag']['name'];
if ($at['Tag']['is_galaxy']) {
$galaxyTags[$at['Tag']['id']] = $at['Tag']['name'];
}
}
unset($attribute['AttributeTag']);
@ -1698,15 +1702,12 @@ class AttributesController extends AppController
// Fetch galaxy clusters in one query
if (!empty($galaxyTags)) {
$this->loadModel('GalaxyCluster');
$clusters = $this->GalaxyCluster->getClusters($galaxyTags, $user, true, false);
$clusters = $this->GalaxyCluster->getClustersByTags($galaxyTags, $user, true, false);
$clusters = array_column(array_column($clusters, 'GalaxyCluster'), null, 'tag_id');
} else {
$clusters = [];
}
// Fetch correlations in one query
$correlations = $this->Attribute->Event->getRelatedAttributes($user, $attributeIds, false, 'attribute');
// `attachFeedCorrelations` method expects different attribute format, so we need to transform that, then process
// and then take information back to original attribute structure.
$fakeEventArray = [];
@ -1721,7 +1722,7 @@ class AttributesController extends AppController
}
$cluster = $clusters[$attributeTag['Tag']['id']];
$galaxyId = $cluster['Galaxy']['id'];
$cluster['local'] = isset($attributeTag['local']) ? $attributeTag['local'] : false;
$cluster['local'] = $attributeTag['local'] ?? false;
if (isset($attribute['Attribute']['Galaxy'][$galaxyId])) {
unset($cluster['Galaxy']);
$galaxies[$galaxyId]['GalaxyCluster'][] = $cluster;
@ -1737,12 +1738,9 @@ class AttributesController extends AppController
if (isset($attributesWithFeedCorrelations[$k]['Feed'])) {
$attributes[$k]['Attribute']['Feed'] = $attributesWithFeedCorrelations[$k]['Feed'];
}
if (isset($correlations[$attribute['Attribute']['id']])) {
$attributes[$k]['Attribute']['RelatedAttribute'] = $correlations[$attribute['Attribute']['id']];
}
}
$sightingsData = $this->Sighting->attributesStatistics($attributes, $user);
return array($attributes, $sightingsData);
return [$attributes, $sightingsData];
}
public function checkComposites()

10
app/Controller/Component/IndexFilterComponent.php
View File

@ -10,6 +10,14 @@ class IndexFilterComponent extends Component
public $Controller;
public $isRest = null;
// Used for isApiFunction(), a check that returns true if the controller & action combo matches an action that is a non-xml and non-json automation method
// This is used to allow authentication via headers for methods not covered by _isRest() - as that only checks for JSON and XML formats
const AUTOMATION_ARRAY = array(
'events' => array('csv', 'nids', 'hids', 'xml', 'restSearch', 'stix', 'updateGraph', 'downloadOpenIOCEvent'),
'attributes' => array('text', 'downloadAttachment', 'returnAttributes', 'restSearch', 'rpz', 'bro'),
'objects' => array('restSearch'),
);
public function initialize(Controller $controller)
{
$this->Controller = $controller;
@ -121,6 +129,6 @@ class IndexFilterComponent extends Component
*/
public function isApiFunction($controller, $action)
{
return isset($this->Controller->automationArray[$controller]) && in_array($action, $this->Controller->automationArray[$controller], true);
return isset(self::AUTOMATION_ARRAY[$controller]) && in_array($action, self::AUTOMATION_ARRAY[$controller], true);
}
}

51
app/Controller/CorrelationsController.php
View File

@ -83,33 +83,38 @@ class CorrelationsController extends AppController
'page' => 1,
'order' => 'occurrence desc'
];
foreach (array_keys($query) as $custom_param) {
if (isset($this->params['named'][$custom_param])) {
$query[$custom_param] = $this->params['named'][$custom_param];
}
}
if (isset($this->params['named']['scope'])) {
$limit = $this->Correlation->OverCorrelatingValue->getLimit();
if ($this->params['named']['scope'] === 'over_correlating') {
$query['conditions'][] = ['occurrence >=' => $limit];
} else if ($this->params['named']['scope'] === 'not_over_correlating') {
$query['conditions'][] = ['occurrence <' => $limit];
foreach ($query as $customParam => $foo) {
if (isset($this->request->params['named'][$customParam])) {
$query[$customParam] = $this->request->params['named'][$customParam];
}
}
if (isset($this->request->params['named']['scope'])) {
$limit = $this->Correlation->OverCorrelatingValue->getLimit();
if ($this->request->params['named']['scope'] === 'over_correlating') {
$scope = 'over_correlating';
$query['conditions'][] = ['occurrence >=' => $limit];
} else if ($this->request->params['named']['scope'] === 'not_over_correlating') {
$query['conditions'][] = ['occurrence <' => $limit];
$scope = 'not_over_correlating';
}
} else {
$scope = 'all';
}
$data = $this->Correlation->OverCorrelatingValue->getOverCorrelations($query);
$data = $this->Correlation->attachExclusionsToOverCorrelations($data);
if ($this->_isRest()) {
return $this->RestResponse->viewData($data, 'json');
} else {
$this->__setPagingParams($query['page'], $query['limit'], count($data), 'named');
$this->set('data', $data);
$this->set('title_for_layout', __('Index of over correlating values'));
$this->set('menuData', [
'menuList' => 'correlationExclusions',
'menuItem' => 'over'
]);
}
$this->__setPagingParams($query['page'], $query['limit'], count($data), 'named');
$this->set('data', $data);
$this->set('scope', $scope);
$this->set('title_for_layout', __('Index of over correlating values'));
$this->set('menuData', [
'menuList' => 'correlationExclusions',
'menuItem' => 'over'
]);
}
public function switchEngine(string $engine)
@ -205,17 +210,15 @@ class CorrelationsController extends AppController
{
$this->loadModel('OverCorrelatingValue');
$this->OverCorrelatingValue->generateOccurrencesRouter();
$message = __('Job queued.');
if (Configure::read('MISP.background_jobs')) {
$message = __('Job queued.');
} else {
$message = __('Over-correlations counted successfully.');
}
if (!$this->_isRest()) {
$this->Flash->info($message);
$this->redirect(['controller' => 'correlations', 'action' => 'overCorrelations']);
} else {
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('Correlations', 'generateOccurrences', false, $this->response->type(), $message);
}
$this->Flash->info($message);
$this->redirect(['controller' => 'correlations', 'action' => 'overCorrelations']);
}
}

178
app/Controller/EventsController.php
View File

@ -999,13 +999,15 @@ class EventsController extends AppController
}
$possibleColumns[] = 'attribute_count';
$possibleColumns[] = 'timestamp';
$possibleColumns[] = 'publish_timestamp';
if (Configure::read('MISP.showCorrelationsOnIndex')) {
$possibleColumns[] = 'correlations';
}
if (Configure::read('MISP.showEventReportCountOnIndex')) {
$possibleColumns[] = 'report_count';
}
if (Configure::read('MISP.showSightingsCountOnIndex')) {
$possibleColumns[] = 'sightings';
}
@ -1014,10 +1016,6 @@ class EventsController extends AppController
$possibleColumns[] = 'proposals';
}
if (Configure::read('MISP.showEventReportCountOnIndex')) {
$possibleColumns[] = 'report_count';
}
if (Configure::read('MISP.showDiscussionsCountOnIndex')) {
$possibleColumns[] = 'discussion';
}
@ -1026,6 +1024,9 @@ class EventsController extends AppController
$possibleColumns[] = 'creator_user';
}
$possibleColumns[] = 'timestamp';
$possibleColumns[] = 'publish_timestamp';
$userDisabledColumns = $this->User->UserSetting->getValueForUser($this->Auth->user()['id'], 'event_index_hide_columns');
if ($userDisabledColumns === null) {
$userDisabledColumns = self::DEFAULT_HIDDEN_INDEX_COLUMNS;
@ -4238,29 +4239,28 @@ class EventsController extends AppController
throw new NotFoundException(__('Event not found or you are not authorised to view it.'));
}
$id = $event['Event']['id'];
// #TODO i18n
$exports = array(
'xml' => array(
'url' => $this->baseurl . '/events/restSearch/xml/eventid:' . $id . '.xml',
'text' => 'MISP XML (metadata + all attributes)',
'requiresPublished' => false,
'checkbox' => true,
'checkbox_text' => 'Encode Attachments',
'checkbox_set' => $this->baseurl . '/events/restSearch/xml/eventid:' . $id . '/withAttachments:1.xml',
'checkbox_default' => true
),
'json' => array(
'url' => $this->baseurl . '/events/restSearch/json/eventid:' . $id . '.json',
'text' => 'MISP JSON (metadata + all attributes)',
'text' => __('MISP JSON (metadata + all attributes)'),
'requiresPublished' => false,
'checkbox' => true,
'checkbox_text' => 'Encode Attachments',
'checkbox_text' => __('Encode Attachments'),
'checkbox_set' => $this->baseurl . '/events/restSearch/json/withAttachments:1/eventid:' . $id . '.json',
'checkbox_default' => true
'checkbox_default' => true,
),
'xml' => array(
'url' => $this->baseurl . '/events/restSearch/xml/eventid:' . $id . '.xml',
'text' => __('MISP XML (metadata + all attributes)'),
'requiresPublished' => false,
'checkbox' => true,
'checkbox_text' => __('Encode Attachments'),
'checkbox_set' => $this->baseurl . '/events/restSearch/xml/eventid:' . $id . '/withAttachments:1.xml',
'checkbox_default' => true,
),
'openIOC' => array(
'url' => $this->baseurl . '/events/restSearch/openioc/to_ids:1/published:1/eventid:' . $id . '.json',
'text' => 'OpenIOC (all indicators marked to IDS)',
'text' => __('OpenIOC (all indicators marked to IDS)'),
'requiresPublished' => false,
'checkbox' => false,
),
@ -4269,73 +4269,73 @@ class EventsController extends AppController
'text' => 'CSV',
'requiresPublished' => false,
'checkbox' => true,
'checkbox_text' => 'Include non-IDS marked attributes',
'checkbox_set' => $this->baseurl . '/events/restSearch/returnFormat:csv/to_ids:1||0/published:1||0/includeContext:0/eventid:' . $id
'checkbox_text' => __('Include non-IDS marked attributes'),
'checkbox_set' => $this->baseurl . '/events/restSearch/returnFormat:csv/to_ids:1||0/published:1||0/includeContext:0/eventid:' . $id,
),
'csv_with_context' => array(
'url' => $this->baseurl . '/events/restSearch/returnFormat:csv/to_ids:1/published:1/includeContext:1/eventid:' . $id,
'text' => 'CSV with additional context',
'text' => __('CSV with additional context'),
'requiresPublished' => false,
'checkbox' => true,
'checkbox_text' => 'Include non-IDS marked attributes',
'checkbox_set' => $this->baseurl . '/events/restSearch/returnFormat:csv/to_ids:1||0/published:1||0/includeContext:1/eventid:' . $id
'checkbox_text' => __('Include non-IDS marked attributes'),
'checkbox_set' => $this->baseurl . '/events/restSearch/returnFormat:csv/to_ids:1||0/published:1||0/includeContext:1/eventid:' . $id,
),
'stix_xml' => array(
'url' => $this->baseurl . '/events/restSearch/stix/eventid:' . $id,
'text' => 'STIX 1 XML (metadata + all attributes)',
'text' => __('STIX 1 XML (metadata + all attributes)'),
'requiresPublished' => false,
'checkbox' => true,
'checkbox_text' => 'Encode Attachments',
'checkbox_set' => $this->baseurl . '/events/restSearch/stix/eventid:' . $id . '/withAttachments:1'
'checkbox_text' => __('Encode Attachments'),
'checkbox_set' => $this->baseurl . '/events/restSearch/stix/eventid:' . $id . '/withAttachments:1',
),
'stix_json' => array(
'url' => $this->baseurl . '/events/restSearch/stix-json/eventid:' . $id,
'text' => 'STIX 1 JSON (metadata + all attributes)',
'text' => __('STIX 1 JSON (metadata + all attributes)'),
'requiresPublished' => false,
'checkbox' => true,
'checkbox_text' => 'Encode Attachments',
'checkbox_set' => $this->baseurl . '/events/restSearch/stix-json/withAttachments:1/eventid:' . $id
'checkbox_text' => __('Encode Attachments'),
'checkbox_set' => $this->baseurl . '/events/restSearch/stix-json/withAttachments:1/eventid:' . $id,
),
'stix2_json' => array(
'url' => $this->baseurl . '/events/restSearch/stix2/eventid:' . $id,
'text' => 'STIX 2',
'requiresPublished' => false,
'checkbox' => true,
'checkbox_text' => 'Encode Attachments',
'checkbox_set' => $this->baseurl . '/events/restSearch/stix2/eventid:' . $id . '/withAttachments:1'
'checkbox_text' => __('Encode Attachments'),
'checkbox_set' => $this->baseurl . '/events/restSearch/stix2/eventid:' . $id . '/withAttachments:1',
),
'rpz' => array(
'url' => $this->baseurl . '/attributes/restSearch/returnFormat:rpz/published:1||0/eventid:' . $id,
'text' => 'RPZ Zone file',
'text' => __('RPZ Zone file'),
'requiresPublished' => false,
'checkbox' => false,
),
'suricata' => array(
'url' => $this->baseurl . '/events/restSearch/returnFormat:suricata/published:1||0/eventid:' . $id,
'text' => 'Download Suricata rules',
'text' => __('Suricata rules'),
'requiresPublished' => false,
'checkbox' => false,
),
'snort' => array(
'url' => $this->baseurl . '/events/restSearch/returnFormat:snort/published:1||0/eventid:' . $id,
'text' => 'Download Snort rules',
'text' => __('Snort rules'),
'requiresPublished' => false,
'checkbox' => false,
),
'bro' => array(
'url' => $this->baseurl . '/attributes/bro/download/all/false/' . $id,
// 'url' => '/attributes/restSearch/returnFormat:bro/published:1||0/eventid:' . $id,
'text' => 'Download Bro rules',
'text' => __('Bro rules'),
'requiresPublished' => false,
'checkbox' => false
'checkbox' => false,
),
'text' => array(
'text' => 'Export all attribute values as a text file',
'text' => __('Export all attribute values as a text file'),
'url' => $this->baseurl . '/attributes/restSearch/returnFormat:text/published:1||0/eventid:' . $id,
'requiresPublished' => false,
'checkbox' => true,
'checkbox_text' => 'Include non-IDS marked attributes',
'checkbox_set' => $this->baseurl . '/attributes/restSearch/returnFormat:text/published:1||0/to_ids:1||0/eventid:' . $id
'checkbox_text' => __('Include non-IDS marked attributes'),
'checkbox_set' => $this->baseurl . '/attributes/restSearch/returnFormat:text/published:1||0/to_ids:1||0/eventid:' . $id,
),
);
if ($event['Event']['published'] == 0) {
@ -4346,9 +4346,9 @@ class EventsController extends AppController
}
$exports['csv'] = array(
'url' => $this->baseurl . '/events/restSearch/returnFormat:csv/includeContext:0/eventid:' . $id,
'text' => 'CSV (event not published, IDS flag ignored)',
'text' => __('CSV (event not published, IDS flag ignored)'),
'requiresPublished' => false,
'checkbox' => false
'checkbox' => false,
);
}
$this->loadModel('Module');
@ -4370,73 +4370,73 @@ class EventsController extends AppController
public function importChoice($id = false, $scope = 'event')
{
if ($scope == 'event') {
if ($scope === 'event') {
$event = $this->Event->fetchSimpleEvent($this->Auth->user(), $id);
if (empty($event)) {
throw new NotFoundException(__('Event not found or you are not authorised to view it.'));
}
$imports = array(
'MISP JSON' => array(
'url' => $this->baseurl . '/events/populate/'.$id,
'text' => __('Populate using a JSON file containing MISP event content data'),
'ajax' => false
),
'freetext' => array(
'url' => $this->baseurl . '/events/freeTextImport/' . $id,
'text' => __('Freetext Import'),
'ajax' => true,
'target' => 'popover_form'
),
'template' => array(
'url' => $this->baseurl . '/templates/templateChoices/' . $id,
'text' => __('Populate using a Template'),
'ajax' => true,
'target' => 'popover_form'
),
'OpenIOC' => array(
'url' => $this->baseurl . '/events/addIOC/' . $id,
'text' => __('OpenIOC Import'),
'ajax' => false,
),
'ThreatConnect' => array(
'url' => $this->baseurl . '/attributes/add_threatconnect/' . $id,
'text' => __('ThreatConnect Import'),
'ajax' => false
),
'Forensic analysis' => array(
'url' => $this->baseurl . '/events/upload_analysis_file/'.$id,
'text' => __('(Experimental) Forensic analysis - Mactime'),
'ajax' => false,
)
'MISP JSON' => array(
'url' => $this->baseurl . '/events/populate/'.$id,
'text' => __('Populate using a JSON file containing MISP event content data'),
'ajax' => false
),
'freetext' => array(
'url' => $this->baseurl . '/events/freeTextImport/' . $id,
'text' => __('Freetext Import'),
'ajax' => true,
'target' => 'popover_form'
),
'template' => array(
'url' => $this->baseurl . '/templates/templateChoices/' . $id,
'text' => __('Populate using a Template'),
'ajax' => true,
'target' => 'popover_form'
),
'OpenIOC' => array(
'url' => $this->baseurl . '/events/addIOC/' . $id,
'text' => __('OpenIOC Import'),
'ajax' => false,
),
'ThreatConnect' => array(
'url' => $this->baseurl . '/attributes/add_threatconnect/' . $id,
'text' => __('ThreatConnect Import'),
'ajax' => false
),
'Forensic analysis' => array(
'url' => $this->baseurl . '/events/upload_analysis_file/'.$id,
'text' => __('(Experimental) Forensic analysis - Mactime'),
'ajax' => false,
)
);
$this->loadModel('Module');
$modules = $this->Module->getEnabledModules($this->Auth->user(), false, 'Import');
if (is_array($modules) && !empty($modules)) {
foreach ($modules['modules'] as $module) {
$imports[$module['name']] = array(
'url' => $this->baseurl . '/events/importModule/' . $module['name'] . '/' . $id,
'text' => Inflector::humanize($module['name']),
'ajax' => false
'url' => $this->baseurl . '/events/importModule/' . $module['name'] . '/' . $id,
'text' => Inflector::humanize($module['name']),
'ajax' => false,
);
}
}
} else {
$imports = array(
'MISP' => array(
'url' => $this->baseurl . '/events/add_misp_export',
'text' => __('MISP standard (recommended exchange format - lossless)'),
'ajax' => false,
'bold' => true
'url' => $this->baseurl . '/events/add_misp_export',
'text' => __('MISP standard (recommended exchange format - lossless)'),
'ajax' => false,
'bold' => true,
),
'STIX' => array(
'url' => $this->baseurl . '/events/upload_stix',
'text' => __('STIX 1.1.1 format (lossy)'),
'ajax' => false,
'url' => $this->baseurl . '/events/upload_stix',
'text' => __('STIX 1.1.1 format (lossy)'),
'ajax' => false,
),
'STIX2' => array(
'url' => $this->baseurl . '/events/upload_stix/2',
'text' => __('STIX 2.0 format (lossy)'),
'ajax' => false,
'url' => $this->baseurl . '/events/upload_stix/2',
'text' => __('STIX 2.0 format (lossy)'),
'ajax' => false,
)
);
}

92
app/Controller/GalaxyClustersController.php
View File

@ -9,30 +9,30 @@ class GalaxyClustersController extends AppController
public $components = array('Session', 'RequestHandler');
public $paginate = array(
'limit' => 60,
'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page.
'recursive' => -1,
'order' => array(
'GalaxyCluster.version' => 'DESC',
'GalaxyCluster.value' => 'ASC'
'limit' => 60,
'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page.
'recursive' => -1,
'order' => array(
'GalaxyCluster.version' => 'DESC',
'GalaxyCluster.value' => 'ASC'
),
'contain' => array(
'Tag' => array(
'fields' => array('Tag.id'),
/*
'EventTag' => array(
'fields' => array('EventTag.event_id')
),
'AttributeTag' => array(
'fields' => array('AttributeTag.event_id', 'AttributeTag.attribute_id')
)
*/
),
'contain' => array(
'Tag' => array(
'fields' => array('Tag.id'),
/*
'EventTag' => array(
'fields' => array('EventTag.event_id')
),
'AttributeTag' => array(
'fields' => array('AttributeTag.event_id', 'AttributeTag.attribute_id')
)
*/
),
'GalaxyElement' => array(
'conditions' => array('GalaxyElement.key' => 'synonyms'),
'fields' => array('value')
),
)
'GalaxyElement' => array(
'conditions' => array('GalaxyElement.key' => 'synonyms'),
'fields' => array('value')
),
)
);
public function index($galaxyId)
@ -165,7 +165,6 @@ class GalaxyClustersController extends AppController
*/
public function view($id)
{
$id = $this->Toolbox->findIdByUuid($this->GalaxyCluster, $id);
$cluster = $this->GalaxyCluster->fetchIfAuthorized($this->Auth->user(), $id, 'view', $throwErrors=true, $full=true);
$tag = $this->GalaxyCluster->Tag->find('first', array(
'conditions' => array(
@ -181,29 +180,30 @@ class GalaxyClustersController extends AppController
}
if ($this->_isRest()) {
return $this->RestResponse->viewData($cluster, $this->response->type());
} else {
$clusters = [$cluster];
$this->GalaxyCluster->attachExtendByInfo($this->Auth->user(), $clusters);
$cluster = $clusters[0];
$cluster = $this->GalaxyCluster->attachExtendFromInfo($this->Auth->user(), $cluster);
$this->set('id', $id);
$this->set('galaxy', ['Galaxy' => $cluster['GalaxyCluster']['Galaxy']]);
$this->set('galaxy_id', $cluster['GalaxyCluster']['galaxy_id']);
$this->set('cluster', $cluster);
$this->set('defaultCluster', $cluster['GalaxyCluster']['default']);
if (!empty($cluster['GalaxyCluster']['extended_from'])) {
$newVersionAvailable = $cluster['GalaxyCluster']['extended_from']['GalaxyCluster']['version'] > $cluster['GalaxyCluster']['extends_version'];
} else {
$newVersionAvailable = false;
}
$this->set('newVersionAvailable', $newVersionAvailable);
$this->loadModel('Attribute');
$distributionLevels = $this->Attribute->distributionLevels;
$this->set('distributionLevels', $distributionLevels);
if (!$cluster['GalaxyCluster']['default'] && !$cluster['GalaxyCluster']['published'] && $cluster['GalaxyCluster']['orgc_id'] == $this->Auth->user()['org_id']) {
$this->Flash->warning(__('This cluster is not published. Users will not be able to use it'));
}
}
$clusters = [$cluster];
$this->GalaxyCluster->attachExtendByInfo($this->Auth->user(), $clusters);
$cluster = $clusters[0];
$cluster = $this->GalaxyCluster->attachExtendFromInfo($this->Auth->user(), $cluster);
$this->set('id', $cluster['GalaxyCluster']['id']);
$this->set('galaxy', ['Galaxy' => $cluster['GalaxyCluster']['Galaxy']]);
$this->set('galaxy_id', $cluster['GalaxyCluster']['galaxy_id']);
$this->set('cluster', $cluster);
$this->set('defaultCluster', $cluster['GalaxyCluster']['default']);
if (!empty($cluster['GalaxyCluster']['extended_from'])) {
$newVersionAvailable = $cluster['GalaxyCluster']['extended_from']['GalaxyCluster']['version'] > $cluster['GalaxyCluster']['extends_version'];
} else {
$newVersionAvailable = false;
}
$this->set('newVersionAvailable', $newVersionAvailable);
$this->loadModel('Attribute');
$distributionLevels = $this->Attribute->distributionLevels;
$this->set('distributionLevels', $distributionLevels);
if (!$cluster['GalaxyCluster']['default'] && !$cluster['GalaxyCluster']['published'] && $cluster['GalaxyCluster']['orgc_id'] == $this->Auth->user()['org_id']) {
$this->Flash->warning(__('This cluster is not published. Users will not be able to use it'));
}
$this->set('title_for_layout', __('Galaxy cluster %s', $cluster['GalaxyCluster']['value']));
}
/**

114
app/Controller/ObjectReferencesController.php
View File

@ -9,10 +9,10 @@ class ObjectReferencesController extends AppController
public $components = array('RequestHandler', 'Session');
public $paginate = array(
'limit' => 20,
'order' => array(
'ObjectReference.id' => 'desc'
),
'limit' => 20,
'order' => array(
'ObjectReference.id' => 'desc'
),
);
public function add($objectId = false)
@ -23,7 +23,7 @@ class ObjectReferencesController extends AppController
}
}
if (empty($objectId)) {
throw new MethodNotAllowedException('No object defined.');
throw new NotFoundException('No object defined.');
}
if (Validation::uuid($objectId)) {
$conditions = ['Object.uuid' => $objectId];
@ -91,65 +91,63 @@ class ObjectReferencesController extends AppController
} else {
if ($this->_isRest()) {
return $this->RestResponse->describe('ObjectReferences', 'add', false, $this->response->type());
} else {
$events = $this->ObjectReference->Object->Event->find('all', array(
'conditions' => array(
'OR' => array(
'Event.id' => $object['Event']['id'],
'AND' => array(
'Event.uuid' => $object['Event']['extends_uuid'],
$this->ObjectReference->Object->Event->createEventConditions($this->Auth->user())
)
),
}
$events = $this->ObjectReference->Object->Event->find('all', array(
'conditions' => array(
'OR' => array(
'Event.id' => $object['Event']['id'],
'AND' => array(
'Event.uuid' => $object['Event']['extends_uuid'],
$this->ObjectReference->Object->Event->createEventConditions($this->Auth->user())
)
),
'recursive' => -1,
'fields' => array('Event.id'),
'contain' => array(
),
'recursive' => -1,
'fields' => array('Event.id'),
'contain' => array(
'Attribute' => array(
'conditions' => array('Attribute.deleted' => 0, 'Attribute.object_id' => 0),
'fields' => array('Attribute.id', 'Attribute.uuid', 'Attribute.type', 'Attribute.category', 'Attribute.value', 'Attribute.to_ids')
),
'Object' => array(
'conditions' => array('NOT' => array('Object.id' => $object['Object']['id']), 'Object.deleted' => 0),
'fields' => array('Object.id', 'Object.uuid', 'Object.name', 'Object.meta-category'),
'Attribute' => array(
'conditions' => array('Attribute.deleted' => 0, 'Attribute.object_id' => 0),
'conditions' => array('Attribute.deleted' => 0),
'fields' => array('Attribute.id', 'Attribute.uuid', 'Attribute.type', 'Attribute.category', 'Attribute.value', 'Attribute.to_ids')
),
'Object' => array(
'conditions' => array('NOT' => array('Object.id' => $object['Object']['id']), 'Object.deleted' => 0),
'fields' => array('Object.id', 'Object.uuid', 'Object.name', 'Object.meta-category'),
'Attribute' => array(
'conditions' => array('Attribute.deleted' => 0),
'fields' => array('Attribute.id', 'Attribute.uuid', 'Attribute.type', 'Attribute.category', 'Attribute.value', 'Attribute.to_ids')
)
)
)
));
if (!empty($events)) {
$event = $events[0];
}
for ($i=1; $i < count($events); $i++) {
$event['Attribute'] = array_merge($event['Attribute'], $events[$i]['Attribute']);
$event['Object'] = array_merge($event['Object'], $events[$i]['Object']);
}
$toRearrange = array('Attribute', 'Object');
foreach ($toRearrange as $d) {
if (!empty($event[$d])) {
$temp = array();
foreach ($event[$d] as $data) {
$temp[$data['uuid']] = $data;
}
$event[$d] = $temp;
}
}
$this->loadModel('ObjectRelationship');
$relationships = $this->ObjectRelationship->find('column', array(
'recursive' => -1,
'fields' => ['name'],
));
$relationships = array_combine($relationships, $relationships);
$relationships['custom'] = 'custom';
ksort($relationships);
$this->set('relationships', $relationships);
$this->set('event', $event);
$this->set('objectId', $object['Object']['id']);
$this->layout = false;
$this->render('ajax/add');
)
));
$event = $events[0];
for ($i = 1; $i < count($events); $i++) {
$event['Attribute'] = array_merge($event['Attribute'], $events[$i]['Attribute']);
$event['Object'] = array_merge($event['Object'], $events[$i]['Object']);
}
$toRearrange = array('Attribute', 'Object');
foreach ($toRearrange as $d) {
if (!empty($event[$d])) {
$temp = array();
foreach ($event[$d] as $data) {
$temp[$data['uuid']] = $data;
}
$event[$d] = $temp;
}
}
$this->loadModel('ObjectRelationship');
$relationships = $this->ObjectRelationship->find('column', array(
'recursive' => -1,
'fields' => ['name'],
));
$relationships = array_combine($relationships, $relationships);
$relationships['custom'] = 'custom';
ksort($relationships);
$this->set('relationships', $relationships);
$this->set('event', $event);
$this->set('objectId', $object['Object']['id']);
$this->layout = false;
$this->render('ajax/add');
}
}

83
app/Controller/ObjectsController.php
View File

@ -356,19 +356,20 @@ class ObjectsController extends AppController
public function edit($id, $update_template_available=false, $onlyAddNewAttribute=false)
{
$object = $this->MispObject->fetchObjects($this->Auth->user(), array(
$user = $this->Auth->user();
$object = $this->MispObject->fetchObjects($user, array(
'conditions' => $this->__objectIdToConditions($id),
));
if (empty($object)) {
throw new NotFoundException(__('Invalid object.'));
}
$object = $object[0];
$event = $this->MispObject->Event->fetchSimpleEvent($this->Auth->user(), $object['Event']['id']);
$event = $this->MispObject->Event->fetchSimpleEvent($user, $object['Event']['id']);
if (!$this->__canModifyEvent($event)) {
throw new ForbiddenException(__('Insufficient permissions to edit this object.'));
}
if (!$this->_isRest()) {
$this->MispObject->Event->insertLock($this->Auth->user(), $object['Event']['id']);
$this->MispObject->Event->insertLock($user, $object['Event']['id']);
}
if (!empty($object['Object']['template_uuid']) && !empty($object['Object']['template_version'])) {
$template = $this->MispObject->ObjectTemplate->find('first', array(
@ -425,7 +426,7 @@ class ObjectsController extends AppController
unset($this->request->data['Object']);
}
$objectToSave = $this->MispObject->attributeCleanup($this->request->data);
$objectToSave = $this->MispObject->deltaMerge($object, $objectToSave, $onlyAddNewAttribute, $this->Auth->user());
$objectToSave = $this->MispObject->deltaMerge($object, $objectToSave, $onlyAddNewAttribute, $user);
$error_message = __('Object could not be saved.');
$savedObject = array();
if (!is_numeric($objectToSave)) {
@ -435,10 +436,10 @@ class ObjectsController extends AppController
}
$error_message = __('Object could not be saved.') . PHP_EOL . implode(PHP_EOL, $object_validation_errors);
} else {
$savedObject = $this->MispObject->fetchObjects($this->Auth->user(), array('conditions' => array('Object.id' => $object['Object']['id'])));
$savedObject = $this->MispObject->fetchObjects($user, array('conditions' => array('Object.id' => $object['Object']['id'])));
if (isset($this->request->data['deleted']) && $this->request->data['deleted']) {
$this->MispObject->deleteObject($savedObject[0], $hard=false, $unpublish=false);
$savedObject = $this->MispObject->fetchObjects($this->Auth->user(), array('conditions' => array('Object.id' => $object['Object']['id']))); // make sure the object is deleted
$savedObject = $this->MispObject->fetchObjects($user, array('conditions' => array('Object.id' => $object['Object']['id']))); // make sure the object is deleted
}
}
// we pre-validate the attributes before we create an object at this point
@ -480,15 +481,15 @@ class ObjectsController extends AppController
$enabledRows = array();
$this->request->data['Object'] = $object['Object'];
foreach ($template['ObjectTemplateElement'] as $k => $element) {
foreach ($object['Attribute'] as $k2 => $attribute) {
if ($attribute['object_relation'] == $element['object_relation']) {
foreach ($object['Attribute'] as $attribute) {
if ($attribute['object_relation'] === $element['object_relation']) {
$enabledRows[] = $k;
$this->request->data['Attribute'][$k] = $attribute;
if (!empty($element['values_list'])) {
$this->request->data['Attribute'][$k]['value_select'] = $attribute['value'];
} else {
if (!empty($element['sane_default'])) {
if (in_array($attribute['value'], $element['sane_default'])) {
if (in_array($attribute['value'], $element['sane_default'], true)) {
$this->request->data['Attribute'][$k]['value_select'] = $attribute['value'];
} else {
$this->request->data['Attribute'][$k]['value_select'] = 'Enter value manually';
@ -500,7 +501,7 @@ class ObjectsController extends AppController
}
}
$this->set('enabledRows', $enabledRows);
$distributionData = $this->MispObject->Event->Attribute->fetchDistributionData($this->Auth->user());
$distributionData = $this->MispObject->Event->Attribute->fetchDistributionData($user);
$this->set('distributionData', $distributionData);
$this->set('event', $event);
$this->set('ajax', false);
@ -646,10 +647,9 @@ class ObjectsController extends AppController
// Construct a template with valid object attributes to add to an object
public function quickFetchTemplateWithValidObjectAttributes($id)
{
$fields = array('template_uuid', 'template_version', 'id');
$params = array(
'conditions' => array('Object.id' => $id),
'fields' => $fields,
'fields' => array('template_uuid', 'template_version', 'id'),
'flatten' => 1,
);
// fetchObjects restrict access based on user
@ -664,11 +664,10 @@ class ObjectsController extends AppController
$object = $object[0];
}
// get object attributes already set
$objectRelation = array();
foreach($object['Attribute'] as $attr) {
$objectRelation[$attr['object_relation']] = 1;
$existsObjectRelation = array();
foreach ($object['Attribute'] as $attr) {
$existsObjectRelation[$attr['object_relation']] = true;
}
$objectRelation = array_keys($objectRelation);
// get object attribute defined in the object's template
$template = $this->MispObject->ObjectTemplate->find('first', array(
'conditions' => array(
@ -687,8 +686,8 @@ class ObjectsController extends AppController
}
}
// unset object invalid object attribute
foreach($template['ObjectTemplateElement'] as $i => $objAttr) {
if (in_array($objAttr['object_relation'], $objectRelation) && !$objAttr['multiple']) {
foreach ($template['ObjectTemplateElement'] as $i => $objAttr) {
if (isset($existsObjectRelation[$objAttr['object_relation']]) && !$objAttr['multiple']) {
unset($template['ObjectTemplateElement'][$i]);
}
}
@ -762,13 +761,13 @@ class ObjectsController extends AppController
throw new NotFoundException(__('Invalid template'));
}
if (empty($template['ObjectTemplateElement'])) {
throw new NotFoundException(__('Invalid fields') . ' `' . h($fieldName) . '`');
throw new NotFoundException(__('Invalid field `%s`', h($fieldName)));
}
// check if fields can be added
foreach($object['Attribute'] as $objAttr) {
foreach ($object['Attribute'] as $objAttr) {
$objectAttrFromTemplate = $template['ObjectTemplateElement'][0];
if ($objAttr['object_relation'] == $fieldName && !$objectAttrFromTemplate['multiple']) {
if ($objAttr['object_relation'] === $fieldName && !$objectAttrFromTemplate['multiple']) {
throw new NotFoundException(__('Invalid field'));
}
}
@ -1130,16 +1129,17 @@ class ObjectsController extends AppController
$this->set('unmapped', $unmappedAttributes);
}
function proposeObjectsFromAttributes($event_id, $selected_attributes='[]')
public function proposeObjectsFromAttributes($eventId, $selectedAttributes='[]')
{
if (!$this->request->is('ajax')) {
throw new MethodNotAllowedException(__('This action can only be reached via AJAX.'));
}
$selected_attributes = json_decode($selected_attributes, true);
$res = $this->MispObject->validObjectsFromAttributeTypes($this->Auth->user(), $event_id, $selected_attributes);
$potential_templates = $res['templates'];
$attribute_types = $res['types'];
usort($potential_templates, function($a, $b) {
$selectedAttributes = $this->_jsonDecode($selectedAttributes);
$res = $this->MispObject->validObjectsFromAttributeTypes($this->Auth->user(), $eventId, $selectedAttributes);
$potentialTemplates = $res['templates'];
$attributeTypes = $res['types'];
usort($potentialTemplates, function($a, $b) {
if ($a['ObjectTemplate']['id'] == $b['ObjectTemplate']['id']) {
return 0;
} else if (is_array($a['ObjectTemplate']['compatibility']) && is_array($b['ObjectTemplate']['compatibility'])) {
@ -1152,13 +1152,17 @@ class ObjectsController extends AppController
return count($a['ObjectTemplate']['invalidTypes']) > count($b['ObjectTemplate']['invalidTypes']) ? 1 : -1;
}
});
$this->set('potential_templates', $potential_templates);
$this->set('selected_types', $attribute_types);
$this->set('event_id', $event_id);
$this->set('potential_templates', $potentialTemplates);
$this->set('selected_types', $attributeTypes);
$this->set('event_id', $eventId);
}
public function groupAttributesIntoObject($event_id, $selected_template, $selected_attribute_ids='[]')
{
if (!$this->request->is('ajax')) {
throw new MethodNotAllowedException(__('This action can only be reached via AJAX.'));
}
$event = $this->MispObject->Event->find('first', array(
'recursive' => -1,
'fields' => array('Event.id', 'Event.uuid', 'Event.orgc_id', 'Event.user_id', 'Event.publish_timestamp'),
@ -1171,9 +1175,6 @@ class ObjectsController extends AppController
throw new ForbiddenException(__('You do not have permission to do that.'));
}
$hard_delete_attribute = $event['Event']['publish_timestamp'] == 0;
if (!$this->request->is('ajax')) {
throw new MethodNotAllowedException(__('This action can only be reached via AJAX.'));
}
if ($this->request->is('post')) {
$template = $this->MispObject->ObjectTemplate->find('first', array(
'recursive' => -1,
@ -1183,22 +1184,22 @@ class ObjectsController extends AppController
throw new NotFoundException(__('Invalid template.'));
}
$distribution = $this->request->data['Object']['distribution'];
$sharing_group_id = $this->request->data['Object']['sharing_group_id'];
$sharingGroupId = $this->request->data['Object']['sharing_group_id'] ?? 0;
$comment = $this->request->data['Object']['comment'];
$selected_attribute_ids = json_decode($this->request->data['Object']['selectedAttributeIds'], true);
$selected_object_relation_mapping = json_decode($this->request->data['Object']['selectedObjectRelationMapping'], true);
$selected_attribute_ids = $this->_jsonDecode($this->request->data['Object']['selectedAttributeIds']);
$selected_object_relation_mapping = $this->_jsonDecode($this->request->data['Object']['selectedObjectRelationMapping']);
if ($distribution == 4) {
$sg = $this->MispObject->SharingGroup->fetchSG($sharing_group_id, $this->Auth->user());
$sg = $this->MispObject->SharingGroup->fetchSG($sharingGroupId, $this->Auth->user());
if (empty($sg)) {
throw new NotFoundException(__('Invalid sharing group.'));
}
} else {
$sharing_group_id = 0;
$sharingGroupId = 0;
}
$object = array(
'Object' => array(
'distribution' => $distribution,
'sharing_group_id' => $sharing_group_id,
'sharing_group_id' => $sharingGroupId,
'comment' => $comment,
),
'Attribute' => array()
@ -1212,7 +1213,7 @@ class ObjectsController extends AppController
return $this->RestResponse->saveFailResponse('Objects', 'Created from Attributes', false, $error, $this->response->type());
}
} else {
$selected_attribute_ids = json_decode($selected_attribute_ids, true);
$selected_attribute_ids = $this->_jsonDecode($selected_attribute_ids);
$selected_attributes = $this->MispObject->Attribute->fetchAttributes($this->Auth->user(), array('conditions' => array(
'Attribute.id' => $selected_attribute_ids,
'Attribute.event_id' => $event_id,
@ -1232,7 +1233,7 @@ class ObjectsController extends AppController
$conformity_result = $this->MispObject->ObjectTemplate->checkTemplateConformityBasedOnTypes($template, $selected_attributes);
$skipped_attributes = 0;
foreach ($selected_attributes as $i => $attribute) {
if (in_array($attribute['Attribute']['type'], $conformity_result['invalidTypes'])) {
if (in_array($attribute['Attribute']['type'], $conformity_result['invalidTypes'], true)) {
unset($selected_attributes[$i]);
$array_position = array_search($attribute['Attribute']['id'], $selected_attribute_ids);
unset($selected_attribute_ids[$array_position]);

41
app/Controller/UserSettingsController.php
View File

@ -213,27 +213,28 @@ class UserSettingsController extends AppController
if ($this->_isRest()) {
// GET request via the API should describe the endpoint
return $this->RestResponse->describe('UserSettings', 'setSetting', false, $this->response->type());
} else {
// load the valid settings from the model
if ($this->_isSiteAdmin()) {
$users = $this->UserSetting->User->find('list', array(
'fields' => array('User.id', 'User.email')
));
} else if ($this->_isAdmin()) {
$users = $this->UserSetting->User->find('list', array(
'conditions' => array('User.org_id' => $this->Auth->user('org_id')),
'fields' => array('User.id', 'User.email')
));
} else {
$users = array($this->Auth->user('id') => $this->Auth->user('email'));
}
if (!empty($user_id) && $this->request->is('get')) {
$this->request->data['UserSetting']['user_id'] = $user_id;
}
$this->set('setting', $setting);
$this->set('users', $users);
$this->set('validSettings', $this->UserSetting->settingPlaceholders($this->Auth->user()));
}
// load the valid settings from the model
if ($this->_isSiteAdmin()) {
$users = $this->UserSetting->User->find('list', array(
'fields' => array('User.id', 'User.email')
));
} else if ($this->_isAdmin()) {
$users = $this->UserSetting->User->find('list', array(
'conditions' => array('User.org_id' => $this->Auth->user('org_id')),
'fields' => array('User.id', 'User.email')
));
} else {
$users = array($this->Auth->user('id') => $this->Auth->user('email'));
}
if (!empty($user_id) && $this->request->is('get')) {
$this->request->data['UserSetting']['user_id'] = $user_id;
}
$this->set('setting', $setting);
$this->set('users', $users);
$this->set('validSettings', $this->UserSetting->settingPlaceholders($this->Auth->user()));
$this->set('title_for_layout', __('Set User Setting'));
}
public function getSetting($userId = null, $setting = null)

45
app/Controller/UsersController.php
View File

@ -2766,21 +2766,12 @@ class UsersController extends AppController
public function notificationSettings()
{
$user_id = $this->Auth->user('id');
$user = $this->User->find('first', [
'recursive' => -1,
'conditions' => ['User.id' => $user_id],
'contain' => [
'UserSetting',
]
]);
if (empty($user)) {
throw new NotFoundException(__('Invalid user'));
}
$user = $this->Auth->user();
if ($this->request->is('post') || $this->request->is('put')) {
$success = $this->User->saveNotificationSettings($user_id, $this->request->data);
if (!empty($success)) {
$success = $this->User->saveNotificationSettings($user['id'], $this->request->data);
if ($success) {
$this->_refreshAuth();
$message = __('Notification settings saved');
$this->Flash->success($message);
$this->redirect(['action' => 'view', 'me']);
@ -2789,32 +2780,38 @@ class UsersController extends AppController
$this->Flash->error($message);
}
}
$user['periodic_settings'] = $this->User->extractPeriodicSettingForUser($user);
$this->request->data = $user;
$this->set('user', $user);
$this->request->data = [
'User' => $user,
'periodic_settings' => $this->User->fetchPeriodicSettingForUser($user['id']),
];
$this->loadModel('Attribute');
$distributionData = $this->Attribute->fetchDistributionData($this->Auth->user());
$distributionData = $this->Attribute->fetchDistributionData($user);
unset($distributionData['levels'][5]);
$this->set('sharingGroups', $distributionData['sgs']);
$this->set('distributionLevels', $distributionData['levels']);
$this->loadModel('Organisation');
$orgs = $this->Organisation->find('list', [
'conditions' => ['local' => 1],
$conditions = $this->User->Organisation->createConditions($user);
$conditions['local'] = true;
$orgs = $this->User->Organisation->find('list', [
'conditions' => $conditions,
'fields' => ['id', 'name'],
'order' => 'name',
]);
$this->set('orgs', $orgs);
$this->set('user', $user);
$this->set('title_for_layout', __('Notification settings'));
}
public function viewPeriodicSummary(string $period)
{
$summary = $this->User->generatePeriodicSummary($this->Auth->user('id'), $period);
$periodic_settings = $this->User->extractPeriodicSettingForUser($this->Auth->user('id'));
$notification_settings = $this->User->getUsablePeriodicSettingForUser($periodic_settings, $period);
$this->set('periodic_settings', $periodic_settings);
$userId = $this->Auth->user('id');
$summary = $this->User->generatePeriodicSummary($userId, $period);
$periodicSettings = $this->User->fetchPeriodicSettingForUser($userId);
$this->set('periodic_settings', $periodicSettings);
$this->set('summary', $summary);
$this->set('period', $period);
$this->set('title_for_layout', __('Periodic summary'));
}
private function __canChangePassword()

2
app/Lib/Export/AttackExport.php
View File

@ -127,7 +127,7 @@ class AttackExport
$result['galaxyId'] = $this->__galaxy_id;
$matrixGalaxies = $this->__GalaxyModel->getAllowedMatrixGalaxies();
$result['matrixGalaxies'] = $matrixGalaxies;
return json_encode($result);
return JsonTool::encode($result);
}
public function separator()

118
app/Lib/Export/ContextExport.php
View File

@ -1,8 +1,6 @@
<?php
class ContextExport
{
private $__attack_export_tool = null;
public $additional_params = [
'flatten' => 1,
'includeEventTags' => 1,
@ -12,21 +10,43 @@ class ContextExport
'noShadowAttributes' => true,
'sgReferenceOnly' => true,
'includeEventCorrelations' => false,
'fetchFullClusters' => false,
];
private $__eventTags = [];
/** @var array Tag name => Galaxy */
private $__eventGalaxies = [];
private $__aggregatedTags = [];
private $__aggregatedClusters = [];
private $__taxonomyFetched = [];
private $__galaxyFetched = [];
private $__passedOptions = [];
public $non_restrictive_export = true;
public $renderView = 'context_view';
/** @var AttackExport */
private $AttackExport;
/** @var Taxonomy */
private $Taxonomy;
/** @var Galaxy */
private $Galaxy;
public function header($options = array())
{
$this->Taxonomy = ClassRegistry::init('Taxonomy');
$this->Galaxy = ClassRegistry::init('Galaxy');
App::uses('AttackExport', 'Export');
$this->AttackExport = new AttackExport();
$this->__passedOptions = $options;
return '';
}
public function handler($data, $options = array())
{
$this->__aggregate($data, Hash::extract($data, 'EventTag.{n}.Tag'));
@ -35,32 +55,20 @@ class ContextExport
$this->__aggregate($attribute, Hash::extract($attribute, 'AttributeTag.{n}.Tag'));
}
}
$this->__attack_export_tool->handler($data, $options);
return '';
}
public function header($options = array())
{
$this->__TaxonomyModel = ClassRegistry::init('Taxonomy');
$this->__GalaxyModel = ClassRegistry::init('Galaxy');
App::uses('AttackExport', 'Export');
$this->__attack_export_tool = new AttackExport();
$this->__attack_export_tool->handler($options);
$this->__passedOptions = $options;
$this->AttackExport->handler($data, $options);
return '';
}
public function footer()
{
$attackFinal = $this->__attack_export_tool->footer();
$attackFinal = $this->AttackExport->footer();
$this->__aggregateTagsPerTaxonomy();
$this->__aggregateClustersPerGalaxy();
$attackData = json_decode($attackFinal, true);
$attackData = $attackFinal === '' ? [] : JsonTool::decode($attackFinal);
if (!empty($this->__passedOptions['filters']['staticHtml'])) {
$attackData['static'] = true;
}
return json_encode([
return JsonTool::encode([
'attackData' => $attackData,
'tags' => $this->__aggregatedTags,
'clusters' => $this->__aggregatedClusters,
@ -69,17 +77,15 @@ class ContextExport
public function separator()
{
$this->__attack_export_tool->separator();
return '';
}
private function __aggregate($entity, $tags)
private function __aggregate(array $entity, array $tags)
{
if (!empty($entity['Galaxy'])) {
foreach ($entity['Galaxy'] as $galaxy) {
foreach ($galaxy['GalaxyCluster'] as $galaxyCluster) {
$this->__eventGalaxies[$galaxyCluster['tag_name']] = $galaxyCluster;
$this->fetchGalaxyForTag($galaxyCluster['tag_name']);
}
}
}
@ -94,42 +100,50 @@ class ContextExport
}
}
private function fetchTaxonomyForTag($tagname)
/**
* @param string $tagName
* @return void
* @throws RedisException
*/
private function fetchTaxonomyForTag($tagName)
{
$splits = $this->__TaxonomyModel->splitTagToComponents($tagname);
$splits = $this->Taxonomy->splitTagToComponents($tagName);
if ($splits === null) {
return; // tag is not taxonomy tag
}
if (!isset($this->__taxonomyFetched[$splits['namespace']])) {
$fetchedTaxonomy = $this->__TaxonomyModel->getTaxonomyForTag($tagname, false, true);
$fetchedTaxonomy = $this->Taxonomy->getTaxonomyForTag($tagName, false, true);
if (!empty($fetchedTaxonomy)) {
$this->__taxonomyFetched[$splits['namespace']]['Taxonomy'] = $fetchedTaxonomy['Taxonomy'];
$this->__taxonomyFetched[$splits['namespace']]['TaxonomyPredicate'] = [];
$fetched = [
'Taxonomy' => $fetchedTaxonomy['Taxonomy'],
'TaxonomyPredicate' => [],
];
foreach ($fetchedTaxonomy['TaxonomyPredicate'] as $predicate) {
$this->__taxonomyFetched[$splits['namespace']]['TaxonomyPredicate'][$predicate['value']] = $predicate;
$fetched['TaxonomyPredicate'][$predicate['value']] = $predicate;
if (!empty($predicate['TaxonomyEntry'])) {
$this->__taxonomyFetched[$splits['namespace']]['TaxonomyPredicate'][$predicate['value']]['TaxonomyEntry'] = [];
$fetched['TaxonomyPredicate'][$predicate['value']]['TaxonomyEntry'] = [];
foreach ($predicate['TaxonomyEntry'] as $entry) {
$this->__taxonomyFetched[$splits['namespace']]['TaxonomyPredicate'][$predicate['value']]['TaxonomyEntry'][$entry['value']] = $entry;
$fetched['TaxonomyPredicate'][$predicate['value']]['TaxonomyEntry'][$entry['value']] = $entry;
}
}
}
$this->__taxonomyFetched[$splits['namespace']] = $fetched;
} else {
// Do not try to fetch non existing taxonomy again
$this->__taxonomyFetched[$splits['namespace']] = false;
}
}
}
private function fetchGalaxyForTag($tagname)
{
$splits = $this->__TaxonomyModel->splitTagToComponents($tagname);
$galaxy = $this->__GalaxyModel->find('first', array(
'recursive' => -1,
'conditions' => array('Galaxy.type' => $splits['predicate'])
));
$this->__galaxyFetched[$splits['predicate']] = $galaxy;
}
private function __aggregateTagsPerTaxonomy()
{
ksort($this->__eventTags);
foreach ($this->__eventTags as $tagname => $tagData) {
$splits = $this->__TaxonomyModel->splitTagToComponents($tagname);
$splits = $this->Taxonomy->splitTagToComponents($tagname);
if ($splits === null) {
$this->__aggregatedTags['Custom Tags'][]['Tag'] = $tagData;
continue;
}
$taxonomy = [];
if (!empty($this->__taxonomyFetched[$splits['namespace']])) {
$taxonomy = $this->__taxonomyFetched[$splits['namespace']];
@ -137,7 +151,7 @@ class ContextExport
if (!empty($taxonomy['TaxonomyPredicate'][$splits['predicate']])) {
$predicate = $taxonomy['TaxonomyPredicate'][$splits['predicate']];
$entry = null;
if (!empty($splits['value'])) {
if (!empty($splits['value']) && isset($predicate['TaxonomyEntry'][$splits['value']])) {
$entry = $predicate['TaxonomyEntry'][$splits['value']];
}
unset($predicate['TaxonomyEntry']);
@ -155,12 +169,24 @@ class ContextExport
private function __aggregateClustersPerGalaxy()
{
$galaxyTypes = [];
foreach ($this->__eventGalaxies as $tagName => $foo) {
$splits = $this->Taxonomy->splitTagToComponents($tagName);
$galaxyTypes[$splits['predicate']] = true;
}
$fetchedGalaxies = $this->Galaxy->find('all', [
'recursive' => -1,
'conditions' => array('Galaxy.type' => array_keys($galaxyTypes)),
]);
$fetchedGalaxies = array_column(array_column($fetchedGalaxies, 'Galaxy'), null, 'type');
ksort($this->__eventGalaxies);
foreach ($this->__eventGalaxies as $tagname => $cluster) {
$splits = $this->__TaxonomyModel->splitTagToComponents($tagname);
$galaxy = $this->__galaxyFetched[$splits['predicate']];
foreach ($this->__eventGalaxies as $tagName => $cluster) {
$splits = $this->Taxonomy->splitTagToComponents($tagName);
$galaxy = $fetchedGalaxies[$splits['predicate']];
$this->__aggregatedClusters[$splits['predicate']][] = [
'Galaxy' => $galaxy['Galaxy'],
'Galaxy' => $galaxy,
'GalaxyCluster' => $cluster,
];
}

13
app/Lib/Export/NidsExport.php
View File

@ -1,6 +1,6 @@
<?php
class NidsExport
abstract class NidsExport
{
public $rules = array();
@ -858,15 +858,16 @@ class NidsExport
}
}
/**
* @param array $attribute
* @return array|string[]
*/
public static function getIpPort($attribute)
{
$ipport = array();
if (strpos($attribute['type'], 'port') !== false) {
$ipport = explode('|', $attribute['value']);
return explode('|', $attribute['value']);
} else {
$ipport[0] = $attribute['value'];
$ipport[1] = 'any';
return [$attribute['value'], 'any'];
}
return $ipport;
}
}

5
app/Lib/Tools/AttributeValidationTool.php
View File

@ -635,11 +635,14 @@ class AttributeValidationTool
}
/**
* @param $value
* @param string $value
* @return bool
*/
private static function isSsdeep($value)
{
if (strpos($value, "\n") !== false) {
return false;
}
$parts = explode(':', $value);
if (count($parts) !== 3) {
return false;

18
app/Lib/Tools/ColourPaletteTool.php
View File

@ -1,8 +1,10 @@
<?php
class ColourPaletteTool
{
// pass the number of distinct colours to receive an array of colours
/**
* @param int $count Pass the number of distinct colours to receive an array of colours
* @return array
*/
public function createColourPalette($count)
{
$interval = 1 / $count;
@ -13,6 +15,10 @@ class ColourPaletteTool
return $colours;
}
/**
* @param array $hsv
* @return string
*/
public function HSVtoRGB(array $hsv)
{
list($H, $S, $V) = $hsv;
@ -50,12 +56,16 @@ class ColourPaletteTool
return $this->convertToHex(array($R, $G, $B));
}
/**
* @param array $channels
* @return string
*/
public function convertToHex($channels)
{
$colour = '#';
foreach ($channels as $channel) {
$channel = strval(dechex(round($channel*255)));
if (strlen($channel) == 1) {
$channel = dechex(round($channel*255));
if (strlen($channel) === 1) {
$channel = '0' . $channel;
}
$colour .= $channel;

8
app/Lib/Tools/HttpSocketExtended.php
View File

@ -65,6 +65,14 @@ class HttpSocketJsonException extends Exception
class HttpSocketResponseExtended extends HttpSocketResponse
{
/**
* @return bool
*/
public function isNotModified()
{
return $this->code == 304;
}
/**
* @param string $message
* @throws SocketException

84
app/Lib/Tools/RedisTool.php Normal file
View File

@ -0,0 +1,84 @@
<?php
class RedisTool
{
/** @var Redis|null */
private static $connection;
/** @var string */
private static $serializer;
/**
* @return Redis
* @throws Exception
*/
public static function init()
{
if (self::$connection) {
return self::$connection;
}
if (!class_exists('Redis')) {
throw new Exception("Class Redis doesn't exists. Please install redis extension for PHP.");
}
$host = Configure::read('MISP.redis_host') ?: '127.0.0.1';
$port = Configure::read('MISP.redis_port') ?: 6379;
$database = Configure::read('MISP.redis_database') ?: 13;
$pass = Configure::read('MISP.redis_password');
$redis = new Redis();
if (!$redis->connect($host, (int) $port)) {
throw new Exception("Could not connect to Redis: {$redis->getLastError()}");
}
if (!empty($pass)) {
if (!$redis->auth($pass)) {
throw new Exception("Could not authenticate to Redis: {$redis->getLastError()}");
}
}
if (!$redis->select($database)) {
throw new Exception("Could not select Redis database $database: {$redis->getLastError()}");
}
self::$connection = $redis;
return $redis;
}
/**
* @param mixed $data
* @return string
* @throws JsonException
*/
public static function serialize($data)
{
if (self::$serializer === null) {
self::$serializer = Configure::read('MISP.redis_serializer') ?: false;
}
if (self::$serializer === 'igbinary') {
return igbinary_serialize($data);
} else {
return JsonTool::encode($data);
}
}
/**
* @param string $string
* @return mixed
* @throws JsonException
*/
public static function deserialize($string)
{
if ($string === false) {
return false;
}
if (self::$serializer === null) {
self::$serializer = Configure::read('MISP.redis_serializer') ?: false;
}
if (self::$serializer === 'igbinary') {
return igbinary_unserialize($string);
} else {
return JsonTool::decode($string);
}
}
}

7
app/Lib/Tools/SendEmailTemplate.php
View File

@ -24,7 +24,7 @@ class SendEmailTemplate
/**
* This value will be used for grouping emails in mail client.
* @param string|null $referenceId
* @return string
* @return string|void
*/
public function referenceId($referenceId = null)
{
@ -49,7 +49,7 @@ class SendEmailTemplate
/**
* Get subject from template. Must be called after render method.
* @param string|null $subject
* @return string
* @return string|void
*/
public function subject($subject = null)
{
@ -84,7 +84,6 @@ class SendEmailTemplate
$View->set($this->viewVars);
$View->set('hideDetails', $hideDetails);
$View->viewPath = $View->layoutPath = 'Emails' . DS . 'html';
try {
$View->viewPath = $View->layoutPath = 'Emails' . DS . 'html' . DS . 'Custom';
$html = $View->render($this->viewName); // Attempt to load a custom template if it exists
@ -93,7 +92,7 @@ class SendEmailTemplate
try {
$html = $View->render($this->viewName);
} catch (MissingViewException $e) {
$html = null; // HTMl template is optional
$html = null; // HTML template is optional
}
}

27
app/Lib/Tools/ServerSyncTool.php
View File

@ -72,12 +72,14 @@ class ServerSyncTool
/**
* @param array $params
* @param string|null $etag
* @return HttpSocketResponseExtended
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
*/
public function eventIndex($params = [])
public function eventIndex($params = [], $etag = null)
{
return $this->post('/events/index', $params);
return $this->post('/events/index', $params, null, $etag);
}
/**
@ -420,11 +422,13 @@ class ServerSyncTool
* @param string $url Relative URL
* @param mixed $data
* @param string|null $logMessage
* @param string|null $etag
* @return HttpSocketResponseExtended
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
* @throws JsonException
*/
private function post($url, $data, $logMessage = null)
private function post($url, $data, $logMessage = null, $etag = null)
{
$protectedMode = !empty($data['Event']['protected']);
$data = JsonTool::encode($data);
@ -445,6 +449,16 @@ class ServerSyncTool
$request['header']['x-pgp-signature'] = $this->signEvent($data);
}
if ($etag) {
// Remove compression marks that adds Apache for compressed content
$etagWithoutQuotes = trim($etag, '"');
$dashPos = strrpos($etagWithoutQuotes, '-');
if ($dashPos && in_array(substr($etagWithoutQuotes, $dashPos + 1), ['br', 'gzip'], true)) {
$etag = '"' . substr($etagWithoutQuotes, 0, $dashPos) . '"';
}
$request['header']['If-None-Match'] = $etag;
}
if (strlen($data) > 1024) { // do not compress small body
if ($this->isSupported(self::FEATURE_BR) && function_exists('brotli_compress')) {
$request['header']['Content-Encoding'] = 'br';
@ -458,6 +472,9 @@ class ServerSyncTool
$start = microtime(true);
$response = $this->socket->post($url, $data, $request);
$this->log($start, 'POST', $url, $response);
if ($etag && $response->isNotModified()) {
return $response; // if etag was provided and response code is 304, it is valid response
}
if (!$response->isOk()) {
throw new HttpSocketHttpException($response, $url);
}
@ -515,7 +532,7 @@ class ServerSyncTool
}
/**
* @param float $start
* @param float $start Microtime when request was send
* @param string $method HTTP method
* @param string $url
* @param HttpSocketResponse $response
@ -525,7 +542,7 @@ class ServerSyncTool
$duration = round(microtime(true) - $start, 3);
$responseSize = strlen($response->body);
$ce = $response->getHeader('Content-Encoding');
$logEntry = '[' . date("Y-m-d H:i:s") . "] \"$method $url\" {$response->code} $responseSize $duration $ce\n";
$logEntry = '[' . date('Y-m-d H:i:s', intval($start)) . "] \"$method $url\" {$response->code} $responseSize $duration $ce\n";
file_put_contents(APP . 'tmp/logs/server-sync.log', $logEntry, FILE_APPEND | LOCK_EX);
}
}

6
app/Lib/Tools/TrendingTool.php
View File

@ -44,7 +44,7 @@ class TrendingTool
}
$allTags[$tag] = true;
$trendAnalysis[$timestamp][$tag] = [
'occurence' => round($amount / $eventNumberPerRollingWindow[$timestamp], 2),
'occurrence' => round($amount / $eventNumberPerRollingWindow[$timestamp], 2),
'raw_change' => $rawChange,
'percent_change' => $percentChange,
'change_sign' => $rawChange > 0 ? 1 : ($rawChange < 0 ? -1 : 0),
@ -54,9 +54,9 @@ class TrendingTool
foreach (array_keys($trendAnalysis[$timestamp]) as $tag) {
if (empty($trendAnalysis[$previousTimestamp][$tag])) {
$trendAnalysis[$previousTimestamp][$tag] = [
'occurence' => 0,
'occurrence' => 0,
'raw_change' => -$amount,
'percent_change' => 100 * (-$amount / $amount),
'percent_change' => round(100 * (-$amount / $amount), 2),
'change_sign' => -$amount > 0 ? 1 : (-$amount < 0 ? -1 : 0),
];
}

85
app/Model/AppModel.php
View File

@ -25,6 +25,7 @@ App::uses('LogableBehavior', 'Assets.models/behaviors');
App::uses('RandomTool', 'Tools');
App::uses('FileAccessTool', 'Tools');
App::uses('JsonTool', 'Tools');
App::uses('RedisTool', 'Tools');
App::uses('BetterCakeEventManager', 'Tools');
class AppModel extends Model
@ -38,9 +39,6 @@ class AppModel extends Model
/** @var BackgroundJobsTool */
private static $loadedBackgroundJobsTool;
/** @var null|Redis */
private static $__redisConnection;
private $__profiler = array();
public $elasticSearchClient;
@ -48,6 +46,9 @@ class AppModel extends Model
/** @var AttachmentTool|null */
private $attachmentTool;
/** @var Workflow|null */
private $Workflow;
// deprecated, use $db_changes
// major -> minor -> hotfix -> requires_logout
const OLD_DB_CHANGES = array(
@ -83,7 +84,7 @@ class AppModel extends Model
75 => false, 76 => true, 77 => false, 78 => false, 79 => false, 80 => false,
81 => false, 82 => false, 83 => false, 84 => false, 85 => false, 86 => false,
87 => false, 88 => false, 89 => false, 90 => false, 91 => false, 92 => false,
93 => false, 94 => false, 95 => true, 96 => false, 97 => true,
93 => false, 94 => false, 95 => true, 96 => false, 97 => true, 98 => false,
);
const ADVANCED_UPDATES_DESCRIPTION = array(
@ -1873,6 +1874,9 @@ class AppModel extends Model
ADD COLUMN `notification_monthly` tinyint(1) NOT NULL DEFAULT 0
;";
break;
case 98:
$this->__addIndex('object_template_elements', 'object_template_id');
break;
case 'fixNonEmptySharingGroupID':
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@ -2621,7 +2625,7 @@ class AppModel extends Model
return $remainingTime > 0 || $failThresholdReached;
}
public function getUpdateFailNumber()
private function getUpdateFailNumber()
{
$this->AdminSetting = ClassRegistry::init('AdminSetting');
$updateFailNumber = $this->AdminSetting->getSetting('update_fail_number');
@ -2634,7 +2638,7 @@ class AppModel extends Model
$this->AdminSetting->changeSetting('update_fail_number', 0);
}
public function __increaseUpdateFailNumber()
private function __increaseUpdateFailNumber()
{
$this->AdminSetting = ClassRegistry::init('AdminSetting');
$updateFailNumber = $this->AdminSetting->getSetting('update_fail_number');
@ -2736,7 +2740,7 @@ class AppModel extends Model
return true;
}
public function removeDuplicatedUUIDs()
private function removeDuplicatedUUIDs()
{
$removedResults = array(
'Event' => $this->removeDuplicateEventUUIDs(),
@ -2781,7 +2785,7 @@ class AppModel extends Model
return $counter;
}
public function removeDuplicateAttributeUUIDs()
private function removeDuplicateAttributeUUIDs()
{
$this->Attribute = ClassRegistry::init('Attribute');
$this->Log = ClassRegistry::init('Log');
@ -2835,7 +2839,7 @@ class AppModel extends Model
return $counter;
}
public function removeDuplicateEventUUIDs()
private function removeDuplicateEventUUIDs()
{
$this->Event = ClassRegistry::init('Event');
$this->Log = ClassRegistry::init('Log');
@ -2882,37 +2886,11 @@ class AppModel extends Model
* Similar method as `setupRedis`, but this method throw exception if Redis cannot be reached.
* @return Redis
* @throws Exception
* @deprecated
*/
public function setupRedisWithException()
{
if (self::$__redisConnection) {
return self::$__redisConnection;
}
if (!class_exists('Redis')) {
throw new Exception("Class Redis doesn't exists. Please install redis extension for PHP.");
}
$host = Configure::read('MISP.redis_host') ?: '127.0.0.1';
$port = Configure::read('MISP.redis_port') ?: 6379;
$database = Configure::read('MISP.redis_database') ?: 13;
$pass = Configure::read('MISP.redis_password');
$redis = new Redis();
if (!$redis->connect($host, (int) $port)) {
throw new Exception("Could not connect to Redis: {$redis->getLastError()}");
}
if (!empty($pass)) {
if (!$redis->auth($pass)) {
throw new Exception("Could not authenticate to Redis: {$redis->getLastError()}");
}
}
if (!$redis->select($database)) {
throw new Exception("Could not select Redis database $database: {$redis->getLastError()}");
}
self::$__redisConnection = $redis;
return $redis;
return RedisTool::init();
}
/**
@ -2924,7 +2902,7 @@ class AppModel extends Model
public function setupRedis()
{
try {
return $this->setupRedisWithException();
return RedisTool::init();
} catch (Exception $e) {
return false;
}
@ -3282,7 +3260,7 @@ class AppModel extends Model
return $filter;
}
public function convert_to_memory_limit_to_mb($val)
protected function convert_to_memory_limit_to_mb($val)
{
$val = trim($val);
if ($val == -1) {
@ -3736,22 +3714,6 @@ class AppModel extends Model
return $dataSourceName === 'Database/Mysql' || $dataSourceName === 'Database/MysqlObserver' || $dataSourceName === 'Database/MysqlExtended' || $dataSource instanceof Mysql;
}
public function getCorrelationModelName()
{
if (!empty(Configure::read('MISP.correlation_engine'))) {
return Configure::read('MISP.correlation_engine');
}
return 'Default';
}
public function loadCorrelationModel()
{
if (!empty(Configure::read('MISP.correlation_engine'))) {
return ClassRegistry::init(Configure::read('MISP.correlation_engine'));
}
return ClassRegistry::init('Correlation');
}
/**
* executeTrigger
*
@ -3763,9 +3725,6 @@ class AppModel extends Model
*/
public function executeTrigger($trigger_id, array $data=[], array &$blockingErrors=[], array $logging=[]): bool
{
if ($this->Workflow === null) {
$this->Workflow = ClassRegistry::init('Workflow');
}
if ($this->isTriggerCallable($trigger_id)) {
$success = $this->Workflow->executeWorkflowForTriggerRouter($trigger_id, $data, $blockingErrors, $logging);
if (!empty($logging) && empty($success)) {
@ -3787,13 +3746,6 @@ class AppModel extends Model
$this->Workflow->checkTriggerListenedTo($trigger_id);
}
public function addPendingLogEntry($logEntry)
{
$logEntries = Configure::read('pendingLogEntries');
$logEntries[] = $logEntry;
Configure::write('pendingLogEntries', $logEntries);
}
/**
* Use different CakeEventManager to fix memory leak
* @return CakeEventManager
@ -3808,7 +3760,8 @@ class AppModel extends Model
return $this->_eventManager;
}
private function __retireOldCorrelationEngine($user = null) {
private function __retireOldCorrelationEngine($user = null)
{
if ($user === null) {
$user = [
'id' => 0,

404
app/Model/Attribute.php
View File

@ -107,18 +107,19 @@ class Attribute extends AppModel
const UPLOAD_DEFINITIONS = ['attachment'];
// skip Correlation for the following types
const NON_CORRELATING_TYPES = array(
const NON_CORRELATING_TYPES = [
'comment',
'http-method',
'aba-rtn',
'gender',
'counter',
'float',
'port',
'nationality',
'cortex',
'boolean',
'anonymised'
);
];
const PRIMARY_ONLY_CORRELATING_TYPES = array(
'ip-src|port',
@ -509,21 +510,26 @@ class Attribute extends AppModel
public function beforeDelete($cascade = true)
{
// delete attachments from the disk
$this->read(); // first read the attribute from the db
if ($this->typeIsAttachment($this->data['Attribute']['type'])) {
$this->loadAttachmentTool()->delete($this->data['Attribute']['event_id'], $this->data['Attribute']['id']);
$attribute = $this->find('first', [
'recursive' => -1,
'conditions' => [
'id' => $this->id,
]
]);
if ($this->typeIsAttachment($attribute['Attribute']['type'])) {
$this->loadAttachmentTool()->delete($attribute['Attribute']['event_id'], $attribute['Attribute']['id']);
}
// update correlation..
$this->Correlation->beforeSaveCorrelation($this->data['Attribute']);
if (!empty($this->data['Attribute']['id'])) {
$this->Correlation->beforeSaveCorrelation($attribute['Attribute']);
if (!empty($attribute['Attribute']['id'])) {
if ($this->pubToZmq('attribute')) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->attribute_save($this->data, 'delete');
$pubSubTool->attribute_save($attribute, 'delete');
}
$kafkaTopic = $this->kafkaTopic('attribute');
if ($kafkaTopic) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $this->data, 'delete');
$kafkaPubTool->publishJson($kafkaTopic, $attribute, 'delete');
}
}
}
@ -902,7 +908,7 @@ class Attribute extends AppModel
if ($maxWidth == $defaultMaxSize && $maxHeight == $defaultMaxSize) {
$thumbnailInRedis = Configure::read('MISP.thumbnail_in_redis');
if ($thumbnailInRedis) {
$redis = $this->setupRedisWithException();
$redis = RedisTool::init();
if ($data = $redis->get("misp:thumbnail:attribute:{$attribute['Attribute']['id']}:$outputFormat")) {
return $data;
}
@ -924,7 +930,7 @@ class Attribute extends AppModel
// Save just when requested default thumbnail size
if ($maxWidth == $defaultMaxSize && $maxHeight == $defaultMaxSize) {
if ($thumbnailInRedis) {
$redis->set("misp:thumbnail:attribute:{$attribute['Attribute']['id']}:$outputFormat", $imageData, 3600);
$redis->setex("misp:thumbnail:attribute:{$attribute['Attribute']['id']}:$outputFormat", 3600, $imageData);
} else {
$this->loadAttachmentTool()->save($attribute['Attribute']['event_id'], $attribute['Attribute']['id'], $imageData, $suffix);
}
@ -1018,163 +1024,6 @@ class Attribute extends AppModel
return $data;
}
public function hids($user, $type, $tags = '', $from = false, $to = false, $last = false, $jobId = false, $enforceWarninglist = false)
{
if (empty($user)) {
throw new MethodNotAllowedException(__('Could not read user.'));
}
// check if it's a valid type
if ($type != 'md5' && $type != 'sha1' && $type != 'sha256') {
throw new UnauthorizedException(__('Invalid hash type.'));
}
$conditions = array();
$typeArray = array($type, 'filename|' . $type);
if ($type == 'md5') {
$typeArray[] = 'malware-sample';
}
$rules = array();
$eventIds = $this->Event->fetchEventIds($user, [
'from' => $from,
'to' => $to,
'last' => $last
]);
if (!empty($tags)) {
$tag = ClassRegistry::init('Tag');
$args = $this->dissectArgs($tags);
$tagArray = $tag->fetchEventTagIds($args[0], $args[1]);
if (!empty($tagArray[0])) {
foreach ($eventIds as $k => $v) {
if (!in_array($v['Event']['id'], $tagArray[0])) {
unset($eventIds[$k]);
}
}
}
if (!empty($tagArray[1])) {
foreach ($eventIds as $k => $v) {
if (in_array($v['Event']['id'], $tagArray[1])) {
unset($eventIds[$k]);
}
}
}
}
App::uses('HidsExport', 'Export');
$continue = false;
$eventCount = count($eventIds);
if ($jobId) {
$this->Job = ClassRegistry::init('Job');
$this->Job->id = $jobId;
if (!$this->Job->exists()) {
$jobId = false;
}
}
foreach ($eventIds as $k => $event) {
$conditions['AND'] = array('Attribute.to_ids' => 1, 'Event.published' => 1, 'Attribute.type' => $typeArray, 'Attribute.event_id' => $event['Event']['id']);
$options = array(
'conditions' => $conditions,
'group' => array('Attribute.type', 'Attribute.value1'),
'enforceWarninglist' => $enforceWarninglist,
'flatten' => true
);
$items = $this->fetchAttributes($user, $options);
if (empty($items)) {
continue;
}
$export = new HidsExport();
$rules = array_merge($rules, $export->export($items, strtoupper($type), $continue));
$continue = true;
if ($jobId && ($k % 10 == 0)) {
$this->Job->saveField('progress', $k * 80 / $eventCount);
}
}
return $rules;
}
public function nids($user, $format, $id = false, $continue = false, $tags = false, $from = false, $to = false, $last = false, $type = false, $enforceWarninglist = false, $includeAllTags = false)
{
if (empty($user)) {
throw new MethodNotAllowedException(__('Could not read user.'));
}
$eventIds = $this->Event->fetchEventIds($user, [
'from' => $from,
'to' => $to,
'last' => $last
]);
// If we sent any tags along, load the associated tag names for each attribute
if ($tags) {
$tag = ClassRegistry::init('Tag');
$args = $this->dissectArgs($tags);
$tagArray = $tag->fetchEventTagIds($args[0], $args[1]);
if (!empty($tagArray[0])) {
foreach ($eventIds as $k => $v) {
if (!in_array($v['Event']['id'], $tagArray[0])) {
unset($eventIds[$k]);
}
}
}
if (!empty($tagArray[1])) {
foreach ($eventIds as $k => $v) {
if (in_array($v['Event']['id'], $tagArray[1])) {
unset($eventIds[$k]);
}
}
}
}
if ($id) {
foreach ($eventIds as $k => $v) {
if ($v['Event']['id'] !== $id) {
unset($eventIds[$k]);
}
}
}
if ($format == 'suricata') {
App::uses('NidsSuricataExport', 'Export');
} else {
App::uses('NidsSnortExport', 'Export');
}
$rules = array();
foreach ($eventIds as $event) {
$conditions['AND'] = array('Attribute.to_ids' => 1, "Event.published" => 1, 'Attribute.event_id' => $event['Event']['id']);
$valid_types = array('ip-dst', 'ip-src', 'ip-dst|port', 'ip-src|port', 'eppn', 'email', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'domain', 'domain|ip', 'hostname', 'url', 'user-agent', 'snort');
$conditions['AND']['Attribute.type'] = $valid_types;
if (!empty($type)) {
$conditions['AND'][] = array('Attribute.type' => $type);
}
$params = array(
'conditions' => $conditions, // array of conditions
'recursive' => -1, // int
'fields' => array('Attribute.id', 'Attribute.event_id', 'Attribute.type', 'Attribute.value'),
'contain' => array('Event'=> array('fields' => array('Event.id', 'Event.threat_level_id'))),
'group' => array('Attribute.type', 'Attribute.value1'), // fields to GROUP BY
'enforceWarninglist' => $enforceWarninglist,
'includeAllTags' => $includeAllTags,
'flatten' => true
);
$items = $this->fetchAttributes($user, $params);
if (empty($items)) {
continue;
}
// export depending on the requested type
switch ($format) {
case 'suricata':
$export = new NidsSuricataExport();
break;
case 'snort':
$export = new NidsSnortExport();
break;
}
$rules = array_merge($rules, $export->export($items, $user['nids_sid'], $format, $continue));
// Only prepend the comments once
$continue = true;
}
return $rules;
}
public function set_filter_tags(&$params, $conditions, $options)
{
if (empty($params['tags']) && empty($params['event_tags'])) {
@ -1309,212 +1158,6 @@ class Attribute extends AppModel
return $conditions;
}
public function text($user, $type, $tags = false, $eventId = false, $allowNonIDS = false, $from = false, $to = false, $last = false, $enforceWarninglist = false, $allowNotPublished = false)
{
//permissions are taken care of in fetchAttributes()
$conditions['AND'] = array();
if ($allowNonIDS === false) {
$conditions['AND']['Attribute.to_ids'] = 1;
if ($allowNotPublished === false) {
$conditions['AND']['Event.published'] = 1;
}
}
if (!is_array($type) && $type !== 'all') {
$conditions['AND']['Attribute.type'] = $type;
}
if ($from) {
$conditions['AND']['Event.date >='] = $from;
}
if ($to) {
$conditions['AND']['Event.date <='] = $to;
}
if ($last) {
$conditions['AND']['Event.publish_timestamp >='] = $last;
}
if ($eventId !== false) {
$conditions['AND'][] = array('Event.id' => $eventId);
} elseif ($tags !== false) {
$passed_param = array('tags' => $tags);
$conditions = $this->set_filter_tags($passed_param, $conditions, array('scope' => 'Attribute'));
}
$attributes = $this->fetchAttributes($user, array(
'conditions' => $conditions,
'order' => 'Attribute.value1 ASC',
'fields' => array('value'),
'contain' => array('Event' => array(
'fields' => array('Event.id', 'Event.published', 'Event.date', 'Event.publish_timestamp'),
)),
'enforceWarninglist' => $enforceWarninglist,
'flatten' => 1
));
return $attributes;
}
public function rpz($user, $tags = false, $eventId = false, $from = false, $to = false, $enforceWarninglist = false)
{
// we can group hostname and domain as well as ip-src and ip-dst in this case
$conditions['AND'] = array('Attribute.to_ids' => 1, 'Event.published' => 1);
$typesToFetch = array('ip' => array('ip-src', 'ip-dst'), 'domain' => array('domain'), 'hostname' => array('hostname'));
if ($from) {
$conditions['AND']['Event.date >='] = $from;
}
if ($to) {
$conditions['AND']['Event.date <='] = $to;
}
if ($eventId !== false) {
$conditions['AND'][] = array('Event.id' => $eventId);
}
if ($tags !== false) {
// If we sent any tags along, load the associated tag names for each attribute
$tag = ClassRegistry::init('Tag');
$args = $this->dissectArgs($tags);
$tagArray = $tag->fetchEventTagIds($args[0], $args[1]);
$temp = array();
foreach ($tagArray[0] as $accepted) {
$temp['OR'][] = array('Event.id' => $accepted);
}
$conditions['AND'][] = $temp;
$temp = array();
foreach ($tagArray[1] as $rejected) {
$temp['AND'][] = array('Event.id !=' => $rejected);
}
$conditions['AND'][] = $temp;
}
$values = array();
foreach ($typesToFetch as $k => $v) {
$tempConditions = $conditions;
$tempConditions['type'] = $v;
$temp = $this->fetchAttributes(
$user,
array(
'conditions' => $tempConditions,
'fields' => array('Attribute.value'), // array of field names
'enforceWarninglist' => $enforceWarninglist,
'flatten' => 1
)
);
if (empty($temp)) {
continue;
}
if ($k == 'hostname') {
foreach ($temp as $value) {
$found = false;
if (isset($values['domain'])) {
foreach ($values['domain'] as $domain) {
if (strpos($value['Attribute']['value'], $domain) != 0) {
$found = true;
}
}
}
if (!$found) {
$values[$k][] = $value['Attribute']['value'];
}
}
} else {
foreach ($temp as $value) {
$values[$k][] = $value['Attribute']['value'];
}
}
unset($temp);
}
return $values;
}
public function bro($user, $type, $tags = false, $eventId = false, $from = false, $to = false, $last = false, $enforceWarninglist = false, $skipHeader = false)
{
App::uses('BroExport', 'Export');
$export = new BroExport();
if ($type == 'all') {
$types = array_keys($export->mispTypes);
} else {
$types = array($type);
}
$intel = array();
foreach ($types as $type) {
//restricting to non-private or same org if the user is not a site-admin.
$conditions['AND'] = array('Attribute.to_ids' => 1, 'Event.published' => 1);
if ($from) {
$conditions['AND']['Event.date >='] = $from;
}
if ($to) {
$conditions['AND']['Event.date <='] = $to;
}
if ($last) {
$conditions['AND']['Event.publish_timestamp >='] = $last;
}
if ($eventId !== false) {
$temp = array();
$args = $this->dissectArgs($eventId);
foreach ($args[0] as $accepted) {
$temp['OR'][] = array('Event.id' => $accepted);
}
$conditions['AND'][] = $temp;
$temp = array();
foreach ($args[1] as $rejected) {
$temp['AND'][] = array('Event.id !=' => $rejected);
}
$conditions['AND'][] = $temp;
}
if ($tags !== false) {
// If we sent any tags along, load the associated tag names for each attribute
$tag = ClassRegistry::init('Tag');
$args = $this->dissectArgs($tags);
$tagArray = $tag->fetchEventTagIds($args[0], $args[1]);
$temp = array();
foreach ($tagArray[0] as $accepted) {
$temp['OR'][] = array('Event.id' => $accepted);
}
$conditions['AND'][] = $temp;
$temp = array();
foreach ($tagArray[1] as $rejected) {
$temp['AND'][] = array('Event.id !=' => $rejected);
}
$conditions['AND'][] = $temp;
}
$this->Allowedlist = ClassRegistry::init('Allowedlist');
$this->allowedlist = $this->Allowedlist->getBlockedValues();
$instanceString = 'MISP';
if (Configure::read('MISP.host_org_id') && Configure::read('MISP.host_org_id') > 0) {
$this->Event->Orgc->id = Configure::read('MISP.host_org_id');
if ($this->Event->Orgc->exists()) {
$instanceString = $this->Event->Orgc->field('name') . ' MISP';
}
}
$mispTypes = $export->getMispTypes($type);
foreach ($mispTypes as $mispType) {
$conditions['AND']['Attribute.type'] = $mispType[0];
$intel = array_merge($intel, $this->__bro($user, $conditions, $mispType[1], $export, $this->allowedlist, $instanceString, $enforceWarninglist));
}
}
natsort($intel);
$intel = array_unique($intel);
if (empty($skipHeader)) {
array_unshift($intel, $export->header);
}
return $intel;
}
private function __bro($user, $conditions, $valueField, $export, $allowedlist, $instanceString, $enforceWarninglist)
{
$attributes = $this->fetchAttributes(
$user,
array(
'conditions' => $conditions, // array of conditions
'order' => 'Attribute.value' . $valueField . ' ASC',
'recursive' => -1, // int
'fields' => array('Attribute.id', 'Attribute.event_id', 'Attribute.type', 'Attribute.category', 'Attribute.comment', 'Attribute.to_ids', 'Attribute.value', 'Attribute.value' . $valueField),
'contain' => array('Event' => array('fields' => array('Event.id', 'Event.threat_level_id', 'Event.orgc_id', 'Event.uuid'))),
'enforceWarninglist' => $enforceWarninglist,
'flatten' => 1
)
);
$orgs = $this->Event->Orgc->find('list', array(
'fields' => array('Orgc.id', 'Orgc.name')
));
return $export->export($attributes, $orgs, $valueField, $allowedlist, $instanceString);
}
/**
* @param int|false $jobId
* @param int|false $eventId
@ -1561,7 +1204,8 @@ class Attribute extends AppModel
);
}
} else {
$attributeCount = $this->__iteratedCorrelation($jobId, $full, $attributeCount);
// Not sure why that line was added. If there are no events, there are no correlations to save
// $attributeCount = $this->__iteratedCorrelation($jobId, $full, $attributeCount);
}
if ($jobId) {
$this->Job->saveStatus($jobId, true);
@ -1581,7 +1225,7 @@ class Attribute extends AppModel
{
if ($jobId) {
$message = $attributeId ? __('Correlating Attribute %s', $attributeId) : __('Correlating Event %s (%s MB used)', $eventId, intval(memory_get_usage() / 1024 / 1024));
$this->Job->saveProgress($jobId, $message, ($j / $eventCount) * 100);
$this->Job->saveProgress($jobId, $message, !empty($eventCount) ? ($j / $eventCount) * 100 : 0);
}
$attributeConditions = [
'Attribute.deleted' => 0,
@ -2149,13 +1793,13 @@ class Attribute extends AppModel
if ($options['includeDecayScore']) {
$options['includeEventTags'] = true;
}
if (!$user['Role']['perm_sync'] || !isset($options['deleted']) || !$options['deleted']) {
$params['conditions']['AND']['Attribute.deleted'] = 0;
} else {
if (isset($options['deleted'])) {
if ($options['deleted'] === "only") {
$options['deleted'] = 1;
}
$params['conditions']['AND']['(Attribute.deleted + 0)'] = $options['deleted'];
} elseif (!$user['Role']['perm_sync'] || !isset($options['deleted']) || !$options['deleted']) {
$params['conditions']['AND']['Attribute.deleted'] = 0;
}
if (isset($options['group'])) {
$params['group'] = !empty($options['group']) ? $options['group'] : false;
@ -2371,7 +2015,7 @@ class Attribute extends AppModel
$tags = $this->AttributeTag->Tag->find('all', [
'conditions' => $conditions,
'fields' => ['id', 'name', 'colour', 'numerical_value'],
'fields' => ['id', 'name', 'colour', 'numerical_value', 'is_galaxy'],
'recursive' => -1,
]);
$tags = array_column(array_column($tags, 'Tag'), null, 'id');

42
app/Model/Behavior/DefaultCorrelationBehavior.php
View File

@ -58,12 +58,18 @@ class DefaultCorrelationBehavior extends ModelBehavior
return self::TABLE_NAME;
}
/**
* @param Model $Model
* @param string $value
* @param array $a
* @param array $b
* @return array
*/
public function createCorrelationEntry(Model $Model, $value, $a, $b)
{
$valueId = $this->Correlation->CorrelationValue->getValueId($value);
if ($this->deadlockAvoidance) {
return [
'value_id' => $valueId,
'value_id' => $value,
'1_event_id' => $a['Event']['id'],
'1_object_id' => $a['Attribute']['object_id'],
'1_attribute_id' => $a['Attribute']['id'],
@ -87,7 +93,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
];
} else {
return [
(int) $valueId,
$value,
(int) $a['Event']['id'],
(int) $a['Attribute']['object_id'],
(int) $a['Attribute']['id'],
@ -112,7 +118,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
}
}
public function saveCorrelations(Model $Model, $correlations)
public function saveCorrelations(Model $Model, array $correlations)
{
$fields = [
'value_id',
@ -138,14 +144,16 @@ class DefaultCorrelationBehavior extends ModelBehavior
'object_sharing_group_id'
];
$this->Correlation->CorrelationValue->replaceValueWithId($correlations, $this->deadlockAvoidance ? 'value_id' : 0);
if ($this->deadlockAvoidance) {
return $this->Correlation->saveMany($correlations, array(
return $this->Correlation->saveMany($correlations, [
'atomic' => false,
'callbacks' => false,
'deep' => false,
'validate' => false,
'fieldList' => $fields
));
'fieldList' => $fields,
]);
} else {
$db = $this->Correlation->getDataSource();
// Split to chunks datasource is is enabled
@ -196,7 +204,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
/**
* Fetch correlations for given event.
* @param array $user
* @param int $eventId
* @param int|array $eventId
* @param array $sgids
* @param bool $primary
* @return array
@ -237,7 +245,6 @@ class DefaultCorrelationBehavior extends ModelBehavior
'contain' => [
'CorrelationValue' => [
'fields' => [
'CorrelationValue.id',
'CorrelationValue.value'
]
]
@ -256,7 +263,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
/**
* @param Correlation $Model
* @param array $user
* @param int $id Event ID
* @param int|array $id Event ID
* @param array $sgids
* @return array
*/
@ -329,7 +336,6 @@ class DefaultCorrelationBehavior extends ModelBehavior
[
'1_attribute_id',
'1_object_id',
'1_event_id',
'1_distribution',
'1_object_distribution',
'1_event_distribution',
@ -337,12 +343,10 @@ class DefaultCorrelationBehavior extends ModelBehavior
'1_object_sharing_group_id',
'1_event_sharing_group_id',
'1_org_id',
'value_id'
],
[
'attribute_id',
'object_id',
'event_id',
'distribution',
'object_distribution',
'event_distribution',
@ -350,11 +354,10 @@ class DefaultCorrelationBehavior extends ModelBehavior
'object_sharing_group_id',
'event_sharing_group_id',
'org_id',
'value_id'
]
];
$prefixes = ['1_', ''];
$correlated_attribute_ids = [];
$correlatedAttributeIds = [];
foreach ($conditions as $k => $condition) {
$temp_correlations = $Model->find('all', [
'recursive' => -1,
@ -368,10 +371,15 @@ class DefaultCorrelationBehavior extends ModelBehavior
continue;
}
}
$correlated_attribute_ids[] = $temp_correlation['Correlation'][$prefixes[$k] . 'attribute_id'];
$correlatedAttributeIds[] = $temp_correlation['Correlation'][$prefixes[$k] . 'attribute_id'];
}
}
}
if (empty($correlatedAttributeIds)) {
return [];
}
$contain = [];
if (!empty($includeEventData)) {
$contain['Event'] = [
@ -394,7 +402,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
$relatedAttributes = $Model->Attribute->find('all', [
'recursive' => -1,
'conditions' => [
'Attribute.id' => $correlated_attribute_ids
'Attribute.id' => $correlatedAttributeIds
],
'fields' => $fields,
'contain' => $contain

224
app/Model/Behavior/NoAclCorrelationBehavior.php
View File

@ -1,17 +1,14 @@
<?php
App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
/**
* ACL-less correlation behaviour for end-point instances
*/
class NoAclCorrelationBehavior extends ModelBehavior
{
const TABLE_NAME = 'no_acl_correlations';
private $__tableName = 'no_acl_correlations';
private $__config = [
const CONFIG = [
'AttributeFetcher' => [
'fields' => [
'Attribute.event_id',
@ -31,26 +28,36 @@ class NoAclCorrelationBehavior extends ModelBehavior
]
];
public $Correlation = null;
/** @var Correlation */
public $Correlation;
/** @var bool */
private $deadlockAvoidance = false;
public function setup(Model $Model, $settings = []) {
$Model->useTable = $this->__tableName;
public function setup(Model $Model, $settings = [])
{
$Model->useTable = self::TABLE_NAME;
$this->Correlation = $Model;
$this->deadlockAvoidance = $settings['deadlockAvoidance'];
}
public function getTableName(Model $Model)
{
return $this->__tableName;
return self::TABLE_NAME;
}
public function createCorrelationEntry(Model $Model, $value, $a, $b) {
$value_id = $this->Correlation->CorrelationValue->getValueId($value);
/**
* @param Model $Model
* @param string $value
* @param array $a
* @param array $b
* @return array|int[]
*/
public function createCorrelationEntry(Model $Model, $value, $a, $b)
{
if ($this->deadlockAvoidance) {
return [
'value_id' => $value_id,
'value_id' => $value,
'1_event_id' => $a['Event']['id'],
'1_attribute_id' => $a['Attribute']['id'],
'event_id' => $b['Event']['id'],
@ -58,7 +65,7 @@ class NoAclCorrelationBehavior extends ModelBehavior
];
} else {
return [
(int) $value_id,
$value,
(int) $a['Event']['id'],
(int) $a['Attribute']['id'],
(int) $b['Event']['id'],
@ -67,7 +74,12 @@ class NoAclCorrelationBehavior extends ModelBehavior
}
}
public function saveCorrelations(Model $Model, $correlations)
/**
* @param Model $Model
* @param array $correlations
* @return bool
*/
public function saveCorrelations(Model $Model, array $correlations)
{
$fields = [
'value_id',
@ -77,29 +89,32 @@ class NoAclCorrelationBehavior extends ModelBehavior
'attribute_id'
];
$this->Correlation->CorrelationValue->replaceValueWithId($correlations, $this->deadlockAvoidance ? 'value_id' : 0);
if ($this->deadlockAvoidance) {
return $this->Correlation->saveMany($correlations, array(
return $this->Correlation->saveMany($correlations, [
'atomic' => false,
'callbacks' => false,
'deep' => false,
'validate' => false,
'fieldList' => $fields
));
'fieldList' => $fields,
]);
} else {
$db = $this->Correlation->getDataSource();
// Split to chunks datasource is is enabled
if (count($correlations) > 100) {
foreach (array_chunk($correlations, 100) as $chunk) {
$db->insertMulti('no_acl_correlations', $fields, $chunk);
$db->insertMulti(self::TABLE_NAME, $fields, $chunk);
}
return true;
} else {
return $db->insertMulti('no_acl_correlations', $fields, $correlations);
return $db->insertMulti(self::TABLE_NAME, $fields, $correlations);
}
}
}
public function runBeforeSaveCorrelation(Model $Model, $attribute) {
public function runBeforeSaveCorrelation(Model $Model, $attribute)
{
// (update-only) clean up the relation of the old value: remove the existing relations related to that attribute, we DO have a reference, the id
// ==> DELETE FROM no_acl_correlations WHERE 1_attribute_id = $a_id OR attribute_id = $a_id; */
// first check if it's an update
@ -117,26 +132,37 @@ class NoAclCorrelationBehavior extends ModelBehavior
}
}
/**
* @param Model $Model
* @param string|null $filter
* @return false|mixed
*/
public function getContainRules(Model $Model, $filter = null)
{
if (empty($filter)) {
return $this->__config['AttributeFetcher']['contain'];
return self::CONFIG['AttributeFetcher']['contain'];
} else {
return empty($this->__config['AttributeFetcher']['contain'][$filter]) ? false : $this->__config['AttributeFetcher']['contain'][$filter];
return self::CONFIG['AttributeFetcher']['contain'][$filter] ?? false;
}
}
public function getFieldRules(Model $Model)
{
return $this->__config['AttributeFetcher']['fields'];
return self::CONFIG['AttributeFetcher']['fields'];
}
private function __collectCorrelations($user, $id, $primary)
/**
* @param array $user
* @param int $eventId
* @param bool $primary
* @return array
*/
private function __collectCorrelations(array $user, $eventId, $primary)
{
$max_correlations = Configure::read('MISP.max_correlations_per_event') ?: 5000;
$source = $primary ? '' : '1_';
$prefix = $primary ? '1_' : '';
$correlations = $this->Correlation->find('all', [
return $this->Correlation->find('all', [
'fields' => [
$source . 'attribute_id',
$prefix . 'attribute_id',
@ -144,7 +170,7 @@ class NoAclCorrelationBehavior extends ModelBehavior
],
'conditions' => [
'OR' => [
$source . 'event_id' => $id
$source . 'event_id' => $eventId
],
'AND' => [
[
@ -159,7 +185,6 @@ class NoAclCorrelationBehavior extends ModelBehavior
'contain' => [
'CorrelationValue' => [
'fields' => [
'CorrelationValue.id',
'CorrelationValue.value'
]
]
@ -167,38 +192,45 @@ class NoAclCorrelationBehavior extends ModelBehavior
'order' => false,
'limit' => $max_correlations
]);
return $correlations;
}
public function runGetAttributesRelatedToEvent(Model $Model, $user, $id)
/**
* @param Model $Model
* @param array $user
* @param int|array $id Event ID
* @return array
*/
public function runGetAttributesRelatedToEvent(Model $Model, array $user, $id)
{
$temp_correlations = $this->__collectCorrelations($user, $id, false);
$temp_correlations_1 = $this->__collectCorrelations($user, $id, true);
$correlations = [];
$event_ids = [];
$eventIds = [];
$temp_correlations = $this->__collectCorrelations($user, $id, false);
foreach ($temp_correlations as $temp_correlation) {
$correlations[] = [
'id' => $temp_correlation['Correlation']['event_id'],
'attribute_id' => $temp_correlation['Correlation']['attribute_id'],
'parent_id' => $temp_correlation['Correlation']['1_attribute_id'],
'value' => $temp_correlation['CorrelationValue']['value']
'value' => $temp_correlation['CorrelationValue']['value'],
];
$event_ids[$temp_correlation['Correlation']['event_id']] = true;
$eventIds[$temp_correlation['Correlation']['event_id']] = true;
}
foreach ($temp_correlations_1 as $temp_correlation) {
$temp_correlations = $this->__collectCorrelations($user, $id, true);
foreach ($temp_correlations as $temp_correlation) {
$correlations[] = [
'id' => $temp_correlation['Correlation']['1_event_id'],
'attribute_id' => $temp_correlation['Correlation']['1_attribute_id'],
'parent_id' => $temp_correlation['Correlation']['attribute_id'],
'value' => $temp_correlation['CorrelationValue']['value']
'value' => $temp_correlation['CorrelationValue']['value'],
];
$event_ids[$temp_correlation['Correlation']['1_event_id']] = true;
$eventIds[$temp_correlation['Correlation']['1_event_id']] = true;
}
if (empty($correlations)) {
return [];
}
$conditions = [
'Event.id' => array_keys($event_ids)
'Event.id' => array_keys($eventIds)
];
$events = $Model->Event->find('all', array(
'recursive' => -1,
@ -214,9 +246,9 @@ class NoAclCorrelationBehavior extends ModelBehavior
continue;
}
$event = $events[$eventId];
$correlation['org_id'] = $events[$eventId]['orgc_id'];
$correlation['info'] = $events[$eventId]['info'];
$correlation['date'] = $events[$eventId]['date'];
$correlation['org_id'] = $event['orgc_id'];
$correlation['info'] = $event['info'];
$correlation['date'] = $event['date'];
$parentId = $correlation['parent_id'];
unset($correlation['parent_id']);
$relatedAttributes[$parentId][] = $correlation;
@ -224,47 +256,40 @@ class NoAclCorrelationBehavior extends ModelBehavior
return $relatedAttributes;
}
/**
* @param Correlation $Model
* @param array $user Not used
* @param array $sgids Not used
* @param array $attribute
* @param array $fields Attribute fields to fetch
* @param bool $includeEventData
* @return array
*/
public function runGetRelatedAttributes(Model $Model, $user, $sgids, $attribute, $fields = [], $includeEventData = false)
{
// LATER getRelatedAttributes($attribute) this might become a performance bottleneck
// prepare the conditions
$conditions = [
[
$correlatedAttributeIds = $Model->find('column', [
'conditions' => [
'Correlation.1_event_id !=' => $attribute['event_id'],
'Correlation.attribute_id' => $attribute['id']
'Correlation.attribute_id' => $attribute['id'],
],
[
'fields' => ['1_attribute_id'],
]);
$correlatedAttributeIds2 = $Model->find('column', [
'conditions' => [
'Correlation.event_id !=' => $attribute['event_id'],
'Correlation.1_attribute_id' => $attribute['id']
]
];
$corr_fields = [
[
'1_attribute_id',
'1_event_id',
'value_id'
'Correlation.1_attribute_id' => $attribute['id'],
],
[
'attribute_id',
'event_id',
'value_id'
]
];
$prefixes = ['1_', ''];
$correlated_attribute_ids = [];
foreach ($conditions as $k => $condition) {
$temp_correlations = $Model->find('all', [
'recursive' => -1,
'conditions' => $condition,
'fields' => $corr_fields[$k]
]);
if (!empty($temp_correlations)) {
foreach ($temp_correlations as $temp_correlation) {
$correlated_attribute_ids[] = $temp_correlation['Correlation'][$prefixes[$k] . 'attribute_id'];
}
}
'fields' => ['attribute_id'],
]);
foreach ($correlatedAttributeIds2 as $tempCorrelation) {
$correlatedAttributeIds[] = $tempCorrelation;
}
$contain = [];
if (empty($correlatedAttributeIds)) {
return [];
}
if (!empty($includeEventData)) {
$contain['Event'] = [
'fields' => [
@ -282,18 +307,21 @@ class NoAclCorrelationBehavior extends ModelBehavior
'Event.org_id'
]
];
} else {
$contain = [];
}
$relatedAttributes = $Model->Attribute->find('all', [
'recursive' => -1,
'conditions' => [
'Attribute.id' => $correlated_attribute_ids
'Attribute.id' => $correlatedAttributeIds
],
'fields' => $fields,
'contain' => $contain
]);
if (!empty($includeEventData)) {
$results = [];
foreach ($relatedAttributes as $k => $attribute) {
foreach ($relatedAttributes as $attribute) {
$temp = $attribute['Attribute'];
$temp['Event'] = $attribute['Event'];
$results[] = $temp;
@ -304,30 +332,31 @@ class NoAclCorrelationBehavior extends ModelBehavior
}
}
/**
* @param Correlation $Model
* @param array $user Not used
* @param int $eventId
* @param array $sgids Not used
* @return array
*/
public function fetchRelatedEventIds(Model $Model, array $user, int $eventId, array $sgids)
{
// search the correlation table for the event ids of the related events
// Rules:
// 1. Event is owned by the user (org_id matches)
// 2. User is allowed to see both the event and the org:
// a. Event:
// i. Event has a distribution between 1-3 (community only, connected communities, all orgs)
// ii. Event has a sharing group that the user is accessible to view
// b. Attribute:
// i. Attribute has a distribution of 5 (inheritance of the event, for this the event check has to pass anyway)
// ii. Atttibute has a distribution between 1-3 (community only, connected communities, all orgs)
// iii. Attribute has a sharing group that the user is accessible to view
$primaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, true);
$secondaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, false);
return array_unique(array_merge($primaryEventIds,$secondaryEventIds));
$primaryEventIds = $this->__filterRelatedEvents($Model, $eventId, true);
$secondaryEventIds = $this->__filterRelatedEvents($Model, $eventId, false);
return array_unique(array_merge($primaryEventIds, $secondaryEventIds), SORT_REGULAR);
}
private function __filterRelatedEvents(Model $Model, array $user, int $eventId, bool $primary)
/**
* @param Correlation $Model
* @param int $eventId
* @param bool $primary
* @return array
*/
private function __filterRelatedEvents(Model $Model, int $eventId, bool $primary)
{
$current = $primary ? '' : '1_';
$prefix = $primary ? '1_' : '';
$correlations = $Model->find('all', [
return $Model->find('column', [
'recursive' => -1,
'fields' => [
$prefix . 'event_id'
@ -337,8 +366,6 @@ class NoAclCorrelationBehavior extends ModelBehavior
],
'unique' => true,
]);
$eventIds = Hash::extract($correlations, '{n}.Correlation.' . $prefix . 'event_id');
return $eventIds;
}
public function updateContainedCorrelations(
@ -368,6 +395,11 @@ class NoAclCorrelationBehavior extends ModelBehavior
}
}
/**
* @param Correlation $Model
* @param string $value
* @return void
*/
public function purgeByValue(Model $Model, string $value)
{
$valueIds = $Model->CorrelationValue->find('column', [

157
app/Model/Correlation.php
View File

@ -6,7 +6,8 @@ App::uses('AppModel', 'Model');
* @property Event $Event
* @property CorrelationValue $CorrelationValue
* @method saveCorrelations(array $correlations)
* @method runBeforeSaveCorrelation
* @method createCorrelationEntry(string $value, array $a, array $b)
* @method runBeforeSaveCorrelation(array $attribute)
* @method fetchRelatedEventIds(array $user, int $eventId, array $sgids)
* @method getFieldRules
* @method getContainRules($filter = null)
@ -16,8 +17,6 @@ class Correlation extends AppModel
const CACHE_NAME = 'misp:top_correlations',
CACHE_AGE = 'misp:top_correlations_age';
private $__compositeTypes = [];
public $belongsTo = array(
'Attribute' => [
'className' => 'Attribute',
@ -56,9 +55,6 @@ class Correlation extends AppModel
/** @var array */
private $cidrListCache;
/** @var string */
private $__correlationEngine;
private $__tempContainCache = [];
/** @var OverCorrelatingValue */
@ -67,10 +63,10 @@ class Correlation extends AppModel
public function __construct($id = false, $table = null, $ds = null)
{
parent::__construct($id, $table, $ds);
$this->__correlationEngine = $this->getCorrelationModelName();
$deadlockAvoidance = Configure::check('MISP.deadlock_avoidance') ? Configure::read('MISP.deadlock_avoidance') : false;
$correlationEngine = $this->getCorrelationModelName();
$deadlockAvoidance = Configure::read('MISP.deadlock_avoidance') ?: false;
// load the currently used correlation engine
$this->Behaviors->load($this->__correlationEngine . 'Correlation', ['deadlockAvoidance' => $deadlockAvoidance]);
$this->Behaviors->load($correlationEngine . 'Correlation', ['deadlockAvoidance' => $deadlockAvoidance]);
// getTableName() needs to be implemented by the engine - this points us to the table to be used
$this->useTable = $this->getTableName();
$this->advancedCorrelationEnabled = (bool)Configure::read('MISP.enable_advanced_correlations');
@ -183,17 +179,6 @@ class Correlation extends AppModel
return $correlatingAttributes;
}
/**
* @param string $value
* @param array $a Attribute A
* @param array $b Attribute B
* @return array
*/
private function __createCorrelationEntry($value, $a, $b)
{
return $this->createCorrelationEntry($value, $a, $b);
}
public function correlateValue($value, $jobId = false)
{
$correlatingAttributes = $this->__getMatchingAttributes($value);
@ -216,7 +201,7 @@ class Correlation extends AppModel
if ($correlatingAttribute['Attribute']['event_id'] === $correlatingAttribute2['Attribute']['event_id']) {
continue;
}
$correlations[] = $this->__createCorrelationEntry($value, $correlatingAttribute, $correlatingAttribute2);
$correlations[] = $this->createCorrelationEntry($value, $correlatingAttribute, $correlatingAttribute2);
}
$extraCorrelations = $this->__addAdvancedCorrelations($correlatingAttribute);
if (!empty($extraCorrelations)) {
@ -224,8 +209,8 @@ class Correlation extends AppModel
if ($correlatingAttribute['Attribute']['event_id'] === $extraCorrelation['Attribute']['event_id']) {
continue;
}
$correlations[] = $this->__createCorrelationEntry($value, $correlatingAttribute, $extraCorrelation);
//$correlations = $this->__createCorrelationEntry($value, $extraCorrelation, $correlatingAttribute, $correlations);
$correlations[] = $this->createCorrelationEntry($value, $correlatingAttribute, $extraCorrelation);
//$correlations = $this->createCorrelationEntry($value, $extraCorrelation, $correlatingAttribute, $correlations);
}
}
if ($jobId && $k % 100 === 0) {
@ -252,17 +237,16 @@ class Correlation extends AppModel
}
}
public function correlateAttribute(array $attribute)
{
$this->runBeforeSaveCorrelation($attribute);
$this->afterSaveCorrelation($attribute);
}
public function beforeSaveCorrelation(array $attribute)
{
$this->runBeforeSaveCorrelation($attribute);
}
/**
* @param string $scope
* @param int $id
* @return false|array
*/
private function __cachedGetContainData($scope, $id)
{
if (!empty($this->getContainRules($scope))) {
@ -374,13 +358,13 @@ class Correlation extends AppModel
// If we have more correlations for the value than the limit, set the block entry and stop the correlation process
$this->OverCorrelatingValue->block($cV);
return true;
} else {
} else if ($count !== 0) {
// If we have fewer hits than the limit, proceed with the correlation, but first make sure we remove any existing blockers
$this->OverCorrelatingValue->unblock($cV);
}
foreach ($correlatingAttributes as $b) {
// On a full correlation, only correlate with attributes that have a higher ID to avoid duplicate correlations
if ($full && $b['Attribute']['id'] < $b['Attribute']['id']) {
if ($full && $a['Attribute']['id'] < $b['Attribute']['id']) {
continue;
}
if (isset($b['Attribute']['value1'])) {
@ -390,9 +374,9 @@ class Correlation extends AppModel
$value = $cV;
}
if ($a['Attribute']['id'] > $b['Attribute']['id']) {
$correlations[] = $this->__createCorrelationEntry($value, $a, $b);
$correlations[] = $this->createCorrelationEntry($value, $a, $b);
} else {
$correlations[] = $this->__createCorrelationEntry($value, $b, $a);
$correlations[] = $this->createCorrelationEntry($value, $b, $a);
}
}
}
@ -842,7 +826,7 @@ class Correlation extends AppModel
*/
public function getRelatedAttributes($user, $sgids, $attribute, $fields=[], $includeEventData = false)
{
if (in_array($attribute['type'], Attribute::NON_CORRELATING_TYPES)) {
if (in_array($attribute['type'], Attribute::NON_CORRELATING_TYPES, true)) {
return [];
}
return $this->runGetRelatedAttributes($user, $sgids, $attribute, $fields, $includeEventData);
@ -850,7 +834,7 @@ class Correlation extends AppModel
/**
* @param array $user User array
* @param int $eventId List of event IDs
* @param int $eventId Event ID
* @param array $sgids List of sharing group IDs
* @return array
*/
@ -877,16 +861,19 @@ class Correlation extends AppModel
*/
public function attachCorrelationExclusion(array $attributes)
{
if (!isset($this->__compositeTypes)) {
$this->__compositeTypes = $this->Attribute->getCompositeTypes();
}
$compositeTypes = $this->Attribute->getCompositeTypes();
$valuesToCheck = [];
foreach ($attributes as &$attribute) {
if (in_array($attribute['type'], $this->__compositeTypes, true)) {
if ($attribute['disable_correlation'] || in_array($attribute['type'],Attribute::NON_CORRELATING_TYPES, true)) {
continue;
}
$primaryOnly = in_array($attribute['type'], Attribute::PRIMARY_ONLY_CORRELATING_TYPES, true);
if (in_array($attribute['type'], $compositeTypes, true)) {
$values = explode('|', $attribute['value']);
$valuesToCheck[$values[0]] = true;
$valuesToCheck[$values[1]] = true;
if (!$primaryOnly) {
$valuesToCheck[$values[1]] = true;
}
} else {
$values = [$attribute['value']];
$valuesToCheck[$values[0]] = true;
@ -894,7 +881,7 @@ class Correlation extends AppModel
if ($this->__preventExcludedCorrelations($values[0])) {
$attribute['correlation_exclusion'] = true;
} elseif (!empty($values[1]) && $this->__preventExcludedCorrelations($values[1])) {
} elseif (!empty($values[1]) && !$primaryOnly && $this->__preventExcludedCorrelations($values[1])) {
$attribute['correlation_exclusion'] = true;
}
}
@ -903,16 +890,20 @@ class Correlation extends AppModel
unset($valuesToCheck);
foreach ($attributes as &$attribute) {
if (in_array($attribute['type'], $this->__compositeTypes, true)) {
$values = explode('|', $attribute['value']);
} else {
$values = [$attribute['value']];
if ($attribute['disable_correlation'] || in_array($attribute['type'],Attribute::NON_CORRELATING_TYPES, true)) {
continue;
}
$primaryOnly = in_array($attribute['type'], Attribute::PRIMARY_ONLY_CORRELATING_TYPES, true);
if (in_array($attribute['type'], $compositeTypes, true)) {
$values = explode('|', $attribute['value']);
$values = OverCorrelatingValue::truncateValues($values);
} else {
$values = [OverCorrelatingValue::truncate($attribute['value'])];
}
$values = $this->OverCorrelatingValue->truncateValues($values);
if (isset($overCorrelatingValues[$values[0]])) {
$attribute['over_correlation'] = true;
} elseif (!empty($values[1]) && isset($overCorrelatingValues[$values[1]])) {
} elseif (!empty($values[1]) && !$primaryOnly && isset($overCorrelatingValues[$values[1]])) {
$attribute['over_correlation'] = true;
}
}
@ -922,40 +913,42 @@ class Correlation extends AppModel
public function collectMetrics()
{
$results['engine'] = $this->getCorrelationModelName();
$results['db'] = [
'Default' => [
'name' => __('Default correlation engine'),
'tables' => [
'default_correlations' => [
'id_limit' => 4294967295
],
'correlation_values' => [
'id_limit' => 4294967295
$results = [
'engine' => $this->getCorrelationModelName(),
'db' => [
'Default' => [
'name' => __('Default correlation engine'),
'tables' => [
'default_correlations' => [
'id_limit' => 4294967295
],
'correlation_values' => [
'id_limit' => 4294967295
]
]
],
'NoAcl' => [
'name' => __('No ACL correlation engine'),
'tables' => [
'no_acl_correlations' => [
'id_limit' => 4294967295
],
'correlation_values' => [
'id_limit' => 4294967295
]
]
],
'Legacy' => [
'name' => __('Legacy correlation engine (< 2.4.160)'),
'tables' => [
'correlations' => [
'id_limit' => 2147483647
]
]
]
],
'NoAcl' => [
'name' => __('No ACL correlation engine'),
'tables' => [
'no_acl_correlations' => [
'id_limit' => 4294967295
],
'correlation_values' => [
'id_limit' => 4294967295
]
]
],
'Legacy' => [
'name' => __('Legacy correlation engine (< 2.4.160)'),
'tables' => [
'correlations' => [
'id_limit' => 2147483647
]
]
]
'over_correlations' => $this->OverCorrelatingValue->find('count'),
];
$results['over_correlations'] = $this->OverCorrelatingValue->find('count');
$this->CorrelationExclusion = ClassRegistry::init('CorrelationExclusion');
$results['excluded_correlations'] = $this->CorrelationExclusion->find('count');
foreach ($results['db'] as &$result) {
@ -996,4 +989,12 @@ class Correlation extends AppModel
}
return $result === true;
}
/**
* @return string
*/
private function getCorrelationModelName()
{
return Configure::read('MISP.correlation_engine') ?: 'Default';
}
}

56
app/Model/CorrelationValue.php
View File

@ -5,6 +5,62 @@ class CorrelationValue extends AppModel
{
public $recursive = -1;
/**
* @param array $correlations
* @param string|int $valueIndex
* @return void
*/
public function replaceValueWithId(array &$correlations, $valueIndex)
{
$values = array_column($correlations, $valueIndex);
$valueIds = $this->getIds($values);
foreach ($correlations as &$correlation) {
$value = mb_substr($correlation[$valueIndex], 0, 191);
$correlation[$valueIndex] = $valueIds[$value];
}
}
/**
* @param array $values
* @return array Value in key, value ID in value
*/
private function getIds(array $values)
{
foreach ($values as &$value) {
$value = mb_substr($value, 0, 191);
}
$values = array_unique($values, SORT_REGULAR); // Remove duplicate values
$existingValues = $this->find('list', [
'recursive' => -1,
'callbacks' => false,
'fields' => ['value', 'id'],
'conditions' => [
'value' => $values,
],
]);
$notExistValues = array_diff($values, array_keys($existingValues));
if (!empty($notExistValues)) {
$this->getDataSource()->begin();
foreach ($notExistValues as $notExistValue) {
$this->create();
try {
$this->save(['value' => $notExistValue], [
'callbacks' => false,
'validate' => false,
]);
$existingValues[$notExistValue] = $this->id;
} catch (Exception $e) {
$existingValues[$notExistValue] = $this->getValueId($notExistValue);
}
}
$this->getDataSource()->commit();
}
return $existingValues;
}
/**
* @param string $value
* @return int

6
app/Model/CryptographicKey.php
View File

@ -89,13 +89,13 @@ class CryptographicKey extends AppModel
}
/**
* @return string Instance key fingerprint
* @return string|false Instance key fingerprint or false is no key is set
* @throws Crypt_GPG_BadPassphraseException
* @throws Crypt_GPG_Exception
*/
public function ingestInstanceKey()
{
// If instance just key stored just in GPG homedir, use that key.
// If instance key is stored just in GPG homedir, use that key.
if (Configure::read('MISP.download_gpg_from_homedir')) {
if (!$this->gpg) {
throw new Exception("Could not initiate GPG");
@ -130,7 +130,7 @@ class CryptographicKey extends AppModel
try {
$this->gpg->importKey($instanceKey);
} catch (Crypt_GPG_NoDataException $e) {
throw new MethodNotAllowedException("Could not import the instance key..");
throw new MethodNotAllowedException("Could not import the instance key.");
}
$fingerprint = $this->gpg->getFingerprint(Configure::read('GnuPG.email'));
if ($redis) {

37
app/Model/DecayingModel.php
View File

@ -19,6 +19,9 @@ class DecayingModel extends AppModel
private $__registered_model_classes = array(); // Proxy for already instantiated classes
public $allowed_overrides = array('threshold' => 1, 'lifetime' => 1, 'decay_speed' => 1);
/** @var array */
private $defaultModelsCache;
public function afterFind($results, $primary = false) {
foreach ($results as $k => $v) {
if (!empty($v['DecayingModel']['parameters'])) {
@ -207,7 +210,7 @@ class DecayingModel extends AppModel
return $default_models;
}
public function fetchAllAllowedModels($user, $full=true, $filters=array(), $additionnal_conditions=array())
public function fetchAllAllowedModels($user, $full=true, $filters=array(), $additionalConditions=array())
{
$conditions = array();
if (!$user['Role']['perm_site_admin']) {
@ -223,10 +226,10 @@ class DecayingModel extends AppModel
$conditions[] = array('not' => array('DecayingModel.uuid' => null));
}
}
$conditions[] = array('AND' => $additionnal_conditions);
$conditions[] = array('AND' => $additionalConditions);
$decayingModels = $this->find('all', array(
'conditions' => $conditions,
'include' => $full ? 'DecayingModelMapping' :''
'include' => $full ? 'DecayingModelMapping' : ''
));
foreach ($decayingModels as $i => $decayingModel) { // includes both model default mapping and user mappings
if ($full) {
@ -638,17 +641,29 @@ class DecayingModel extends AppModel
return $attribute;
}
public function attachScoresToEvent($user, $event, $model_id=false, $model_overrides=array(), $include_full_model=0)
/**
* @param array $user
* @param array $event
* @param int|bool $modelId
* @param array $modelOverrides
* @param bool $includeFullModel
* @return array
*/
public function attachScoresToEvent(array $user, array $event, $modelId = false, $modelOverrides = [], $includeFullModel = false)
{
$models = [];
if ($model_id === false) { // fetch all allowed and associated models
$models = $this->fetchAllAllowedModels($user, false, [], ['DecayingModel.enabled' => true]);
if ($modelId === false) { // fetch all allowed and associated models
if (isset($this->defaultModelsCache[$user['id']])) {
$models = $this->defaultModelsCache[$user['id']];
} else {
$models = $this->fetchAllAllowedModels($user, false, [], ['DecayingModel.enabled' => true]);
$this->defaultModelsCache[$user['id']] = $models;
}
} else {
$models = $this->fetchModels($user, $model_id, false, array());
$models = $this->fetchModels($user, $modelId, false, array());
}
foreach ($models as $model) {
if (!empty($model_overrides)) {
$model = $this->overrideModelParameters($model, $model_overrides);
if (!empty($modelOverrides)) {
$model = $this->overrideModelParameters($model, $modelOverrides);
}
$eventScore = $this->getScoreForEvent($event, $model);
$decayed = $this->isEventDecayed($model, $eventScore['score']);
@ -661,7 +676,7 @@ class DecayingModel extends AppModel
'name' => $model['DecayingModel']['name']
]
];
if ($include_full_model) {
if ($includeFullModel) {
$to_attach['DecayingModel'] = $model['DecayingModel'];
}
$event['event_scores'][] = $to_attach;

96
app/Model/Event.php
View File

@ -689,26 +689,18 @@ class Event extends AppModel
}
/**
* Get related attributes for event
* @param array $user
* @param int|array $id Event ID when $scope is 'event', Attribute ID when $scope is 'attribute'
* @param bool $shadowAttribute
* @param string $scope 'event' or 'attribute'
* @param int|array $eventIds Event IDs
* @return array
*/
public function getRelatedAttributes(array $user, $id, $shadowAttribute = false, $scope = 'event')
public function getRelatedAttributes(array $user, $eventIds)
{
if ($shadowAttribute) {
// no longer supported
return [];
} else {
$parentIdField = '1_attribute_id';
$correlationModelName = 'Correlation';
}
if (!isset($this->{$correlationModelName})) {
$this->{$correlationModelName} = ClassRegistry::init($correlationModelName);
if (!isset($this->Correlation)) {
$this->Correlation = ClassRegistry::init('Correlation');
}
$sgids = $this->SharingGroup->authorizedIds($user);
$relatedAttributes = $this->{$correlationModelName}->getAttributesRelatedToEvent($user, $id, $sgids);
$relatedAttributes = $this->Correlation->getAttributesRelatedToEvent($user, $eventIds, $sgids);
return $relatedAttributes;
}
@ -2068,7 +2060,9 @@ class Event extends AppModel
$event['Attribute'] = $this->__attachSharingGroups($event['Attribute'], $sharingGroupData);
}
$event['Attribute'] = $this->Attribute->Correlation->attachCorrelationExclusion($event['Attribute']);
if (!empty($options['includeGranularCorrelations'])) {
$event['Attribute'] = $this->Attribute->Correlation->attachCorrelationExclusion($event['Attribute']);
}
// move all object attributes to a temporary container
$tempObjectAttributeContainer = array();
@ -2182,7 +2176,7 @@ class Event extends AppModel
}
$this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
$clusters = $this->GalaxyCluster->getClusters($galaxyTags, $user, true, $fetchFullCluster);
$clusters = $this->GalaxyCluster->getClustersByTags($galaxyTags, $user, true, $fetchFullCluster);
if (empty($clusters)) {
return;
@ -2200,7 +2194,7 @@ class Event extends AppModel
if (isset($clustersByTagIds[$tagId])) {
$cluster = $clustersByTagIds[$tagId];
$galaxyId = $cluster['Galaxy']['id'];
$cluster['local'] = isset($eventTag['local']) ? $eventTag['local'] : false;
$cluster['local'] = $eventTag['local'] ?? false;
if (isset($event['Galaxy'][$galaxyId])) {
unset($cluster['Galaxy']);
$event['Galaxy'][$galaxyId]['GalaxyCluster'][] = $cluster;
@ -2224,7 +2218,7 @@ class Event extends AppModel
if (isset($clustersByTagIds[$tagId])) {
$cluster = $clustersByTagIds[$tagId];
$galaxyId = $cluster['Galaxy']['id'];
$cluster['local'] = isset($attributeTag['local']) ? $attributeTag['local'] : false;
$cluster['local'] = $attributeTag['local'] ?? false;
if (isset($attribute['Galaxy'][$galaxyId])) {
unset($cluster['Galaxy']);
$attribute['Galaxy'][$galaxyId]['GalaxyCluster'][] = $cluster;
@ -4521,46 +4515,6 @@ class Event extends AppModel
}
}
public function generateLocked()
{
$this->User = ClassRegistry::init('User');
$this->User->recursive = -1;
$localOrgs = array();
$conditions = array();
$orgs = $this->User->find('all', array('fields' => array('DISTINCT org_id')));
foreach ($orgs as $k => $org) {
$orgs[$k]['User']['count'] = $this->User->getOrgMemberCount($orgs[$k]['User']['org_id']);
if ($orgs[$k]['User']['count'] > 1) {
$localOrgs[] = $orgs[$k]['User']['org_id'];
$conditions['AND'][] = array('orgc !=' => $orgs[$k]['User']['org_id']);
} elseif ($orgs[$k]['User']['count'] == 1) {
// If we only have a single user for an org, check if that user is a sync user. If not, then it is a valid local org and the events created by him/her should be unlocked.
$this->User->recursive = 1;
$user = ($this->User->find('first', array(
'fields' => array('id', 'role_id'),
'conditions' => array('org_id' => $org['User']['org_id']),
'contain' => array('Role' => array(
'fields' => array('id', 'perm_sync'),
))
)));
if (!$user['Role']['perm_sync']) {
$conditions['AND'][] = array('orgc !=' => $orgs[$k]['User']['org_id']);
}
}
}
// Don't lock stuff that's already locked
$conditions['AND'][] = array('locked !=' => true);
$this->recursive = -1;
$toBeUpdated = $this->find('count', array(
'conditions' => $conditions
));
$this->updateAll(
array('Event.locked' => 1),
$conditions
);
return $toBeUpdated;
}
public function reportValidationIssuesEvents()
{
$this->Behaviors->detach('Regexp');
@ -4583,20 +4537,6 @@ class Event extends AppModel
return array($result, $k);
}
public function generateThreatLevelFromRisk()
{
$risk = array('Undefined' => 4, 'Low' => 3, 'Medium' => 2, 'High' => 1);
$events = $this->find('all', array('recursive' => -1));
$k = 0;
foreach ($events as $k => $event) {
if ($event['Event']['threat_level_id'] == 0 && isset($event['Event']['risk'])) {
$event['Event']['threat_level_id'] = $risk[$event['Event']['risk']];
$this->save($event);
}
}
return $k;
}
// check two version strings. If version 1 is older than 2, return -1, if they are the same return 0, if version 2 is older return 1
public function compareVersions($version1, $version2)
{
@ -4925,7 +4865,9 @@ class Event extends AppModel
) {
$object['category'] = $object['meta-category'];
$include = empty($filterType['attributeFilter']) || $filterType['attributeFilter'] === 'all' || $filterType['attributeFilter'] === 'object' || $object['meta-category'] === $filterType['attributeFilter'];
$include = empty($filterType['attributeFilter']) ||
in_array($filterType['attributeFilter'], array('all', 'object', 'correlation', 'proposal', 'warning')) ||
$object['meta-category'] === $filterType['attributeFilter'];
if (!$include) {
return null;
@ -6891,7 +6833,7 @@ class Event extends AppModel
}
}
$notCachedTags = array_diff_key($tagIds, isset($this->assetCache['tags']) ? $this->assetCache['tags'] : []);
$notCachedTags = array_diff_key($tagIds, $this->assetCache['tags'] ?? []);
if (empty($notCachedTags)) {
return;
}
@ -7087,7 +7029,7 @@ class Event extends AppModel
*/
private function __clusterEventIds($exportTool, $eventIds)
{
$memory_in_mb = $this->Attribute->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
$memory_in_mb = $this->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
$default_attribute_memory_coefficient = Configure::check('MISP.default_attribute_memory_coefficient') ? Configure::read('MISP.default_attribute_memory_coefficient') : 80;
$default_event_memory_divisor = Configure::check('MISP.default_event_memory_multiplier') ? Configure::read('MISP.default_event_memory_divisor') : 3;
$memory_scaling_factor = isset($exportTool->memory_scaling_factor) ? $exportTool->memory_scaling_factor : $default_attribute_memory_coefficient;
@ -7656,6 +7598,10 @@ class Event extends AppModel
public function getTrendsForTagsFromEvents(array $events, int $baseDayRange, int $rollingWindows=3, $tagFilterPrefixes=null): array
{
$oldestTimestamp = $this->resolveTimeDelta($baseDayRange + $baseDayRange * $rollingWindows . 'd');
$events = array_filter($events, function($event) use ($oldestTimestamp) { // Filter out events having old modification compared to their publish_timestamp
return $event['Event']['timestamp'] >= $oldestTimestamp;
});
App::uses('TrendingTool', 'Tools');
$trendingTool = new TrendingTool($this);
$trendAnalysis = $trendingTool->getTrendsForTags($events, $baseDayRange, $rollingWindows, $tagFilterPrefixes);

10
app/Model/EventLock.php
View File

@ -82,7 +82,7 @@ class EventLock extends AppModel
public function deleteBackgroundJobLock($eventId, $jobId)
{
try {
$redis = $this->setupRedisWithException();
$redis = RedisTool::init();
} catch (Exception $e) {
return false;
}
@ -100,7 +100,7 @@ class EventLock extends AppModel
public function checkLock(array $user, $eventId)
{
try {
$redis = $this->setupRedisWithException();
$redis = RedisTool::init();
} catch (Exception $e) {
return [];
}
@ -113,7 +113,7 @@ class EventLock extends AppModel
$output = [];
$now = time();
foreach ($keys as $value) {
$value = $this->jsonDecode($value);
$value = RedisTool::deserialize($value);
if ($value['timestamp'] + self::DEFAULT_TTL > $now) {
$output[] = $value;
}
@ -130,13 +130,13 @@ class EventLock extends AppModel
private function insertLockToRedis($eventId, $lockId, array $data)
{
try {
$redis = $this->setupRedisWithException();
$redis = RedisTool::init();
} catch (Exception $e) {
return false;
}
$pipeline = $redis->pipeline();
$pipeline->hSet(self::PREFIX . $eventId, $lockId, json_encode($data));
$pipeline->hSet(self::PREFIX . $eventId, $lockId, RedisTool::serialize($data));
$pipeline->expire(self::PREFIX . $eventId, self::DEFAULT_TTL); // prolong TTL
return $pipeline->exec()[0] !== false;
}

64
app/Model/GalaxyCluster.php
View File

@ -967,21 +967,17 @@ class GalaxyCluster extends AppModel
}
/**
* @param array $namesOrIds Cluster tag names or cluster IDs
* @param array $tagNames Cluster tag names with tag ID in key
* @param array $user
* @param bool $postProcess If true, self::postprocess method will be called.
* @param bool $fetchFullCluster
* @return array
*/
public function getClusters(array $namesOrIds, array $user, $postProcess = true, $fetchFullCluster = true)
public function getClustersByTags(array $tagNames, array $user, $postProcess = true, $fetchFullCluster = true)
{
if (count(array_filter($namesOrIds, 'is_numeric')) === count($namesOrIds)) { // all elements are numeric
$conditions = array('GalaxyCluster.id' => $namesOrIds);
} else {
$conditions = array('GalaxyCluster.tag_name' => $namesOrIds);
}
$options = ['conditions' => $conditions];
$options = [
'conditions' => ['GalaxyCluster.tag_name' => $tagNames],
];
if (!$fetchFullCluster) {
$options['contain'] = ['Galaxy', 'GalaxyElement'];
}
@ -989,17 +985,10 @@ class GalaxyCluster extends AppModel
$clusters = $this->fetchGalaxyClusters($user, $options, $fetchFullCluster);
if (!empty($clusters) && $postProcess) {
$tagNames = array_map('strtolower', array_column(array_column($clusters, 'GalaxyCluster'), 'tag_name'));
$tagIds = $this->Tag->find('list', [
'conditions' => ['LOWER(Tag.name)' => $tagNames],
'recursive' => -1,
'fields' => array('Tag.name', 'Tag.id'),
]);
$tagIds = array_change_key_case($tagIds);
$tagIds = array_change_key_case(array_flip($tagNames));
foreach ($clusters as $k => $cluster) {
$tagName = strtolower($cluster['GalaxyCluster']['tag_name']);
$clusters[$k] = $this->postprocess($cluster, isset($tagIds[$tagName]) ? $tagIds[$tagName] : null);
$clusters[$k] = $this->postprocess($cluster, $tagIds[$tagName] ?? null);
}
}
@ -1142,7 +1131,7 @@ class GalaxyCluster extends AppModel
}
$this->Event = ClassRegistry::init('Event');
$sharingGroupData = $this->Event->__cacheSharingGroupData($user, false);
$sharingGroupData = $this->Event->__cacheSharingGroupData($user, true);
foreach ($clusters as $i => $cluster) {
if (!empty($cluster['GalaxyCluster']['sharing_group_id']) && isset($sharingGroupData[$cluster['GalaxyCluster']['sharing_group_id']])) {
$clusters[$i]['SharingGroup'] = $sharingGroupData[$cluster['GalaxyCluster']['sharing_group_id']];
@ -1404,48 +1393,39 @@ class GalaxyCluster extends AppModel
}
/**
* fetchClusterById Simple ACL-aware method to fetch a cluster by Id or UUID
* Simple ACL-aware method to fetch a cluster by Id or UUID
*
* @param array $user
* @param int|string $clusterId
* @param bool $full
* @param array $user
* @param int|string $clusterId Cluster ID or UUID
* @param bool $throwErrors
* @param bool $full
* @return array
*/
public function fetchClusterById(array $user, $clusterId, $throwErrors=true, $full=false)
{
$alias = $this->alias;
if (Validation::uuid($clusterId)) {
$temp = $this->find('first', array(
'recursive' => -1,
'fields' => array("${alias}.id", "${alias}.uuid"),
'conditions' => array("${alias}.uuid" => $clusterId)
));
if (empty($temp)) {
if ($throwErrors) {
throw new NotFoundException(__('Invalid galaxy cluster'));
}
return array();
}
$clusterId = $temp[$alias]['id'];
} elseif (!is_numeric($clusterId)) {
$conditions = array("${alias}.uuid" => $clusterId);
} elseif (is_numeric($clusterId)) {
$conditions = array("${alias}.id" => $clusterId);
} else{
if ($throwErrors) {
throw new NotFoundException(__('Invalid galaxy cluster'));
}
return array();
}
$conditions = array('conditions' => array("${alias}.id" => $clusterId));
$cluster = $this->fetchGalaxyClusters($user, $conditions, $full=$full);
return $cluster;
return $this->fetchGalaxyClusters($user, ['conditions' => $conditions], $full=$full);
}
/**
* fetchIfAuthorized Fetches a cluster and checks if the user has the authorization to perform the requested operation
* Fetches a cluster and checks if the user has the authorization to perform the requested operation
*
* @param array $user
* @param int|string|array $cluster
* @param mixed $authorizations the requested actions to be performed on the cluster
* @param bool $throwErrors Should the function throws excpetion if users is not allowed to perform the action
* @param bool $throwErrors Should the function throws exception if users is not allowed to perform the action
* @param bool $full
* @return array The cluster or an error message
*/
@ -1474,7 +1454,7 @@ class GalaxyCluster extends AppModel
return $cluster;
}
if (in_array('view', $authorizations) && count($authorizations) == 1) {
if (in_array('view', $authorizations) && count($authorizations) === 1) {
return $cluster;
} else {
if (!$user['Role']['perm_galaxy_editor']) {

130
app/Model/MispObject.php
View File

@ -452,7 +452,7 @@ class MispObject extends AppModel
return false;
}
public function saveObject($object, $eventId, $template = false, $user, $errorBehaviour = 'drop', $breakOnDuplicate = false)
public function saveObject(array $object, $eventId, $template = false, $user, $errorBehaviour = 'drop', $breakOnDuplicate = false)
{
$templateFields = array(
'name' => 'name',
@ -482,7 +482,6 @@ class MispObject extends AppModel
}
}
$this->create();
$result = false;
if ($this->save($object)) {
$result = $this->id;
foreach ($object['Attribute'] as $k => $attribute) {
@ -538,7 +537,6 @@ class MispObject extends AppModel
$params = array(
'conditions' => $this->buildConditions($user),
'fields' => array(),
'recursive' => -1
);
if (isset($options['conditions'])) {
$params['conditions']['AND'][] = $options['conditions'];
@ -692,12 +690,14 @@ class MispObject extends AppModel
return $results;
}
/*
/**
* Prepare the template form view's data, setting defaults, sorting elements
* @param array $template
* @param array $request
* @return array
*/
public function prepareTemplate($template, $request = array())
public function prepareTemplate(array $template, array $request = array())
{
$temp = array();
usort($template['ObjectTemplateElement'], function ($a, $b) {
return $a['ui-priority'] < $b['ui-priority'];
});
@ -709,26 +709,28 @@ class MispObject extends AppModel
$request_rearranged[$attribute['object_relation']][] = $attribute;
}
}
foreach ($template_object_elements as $k => $v) {
$typeDefinitions = $this->Event->Attribute->typeDefinitions;
$categoryDefinitions = $this->Event->Attribute->categoryDefinitions;
foreach ($template_object_elements as $v) {
if (empty($request_rearranged[$v['object_relation']])) {
if (isset($this->Event->Attribute->typeDefinitions[$v['type']])) {
$v['default_category'] = $this->Event->Attribute->typeDefinitions[$v['type']]['default_category'];
$v['to_ids'] = $this->Event->Attribute->typeDefinitions[$v['type']]['to_ids'];
if (isset($typeDefinitions[$v['type']])) {
$v['default_category'] = $typeDefinitions[$v['type']]['default_category'];
$v['to_ids'] = $typeDefinitions[$v['type']]['to_ids'];
if (empty($v['categories'])) {
$v['categories'] = array();
foreach ($this->Event->Attribute->categoryDefinitions as $catk => $catv) {
if (in_array($v['type'], $catv['types'])) {
foreach ($categoryDefinitions as $catk => $catv) {
if (in_array($v['type'], $catv['types'], true)) {
$v['categories'][] = $catk;
}
}
}
$template['ObjectTemplateElement'][] = $v;
} else {
$template['warnings'][] = 'Missing attribute type "' . $v['type'] . '" found. Omitted template element ("' . $template_object_elements[$k]['object_relation'] . '") that would not pass validation due to this.';
$template['warnings'][] = 'Missing attribute type "' . $v['type'] . '" found. Omitted template element ("' . $v['object_relation'] . '") that would not pass validation due to this.';
}
} else {
foreach ($request_rearranged[$v['object_relation']] as $request_item) {
if (isset($this->Event->Attribute->typeDefinitions[$v['type']])) {
if (isset($typeDefinitions[$v['type']])) {
$v['default_category'] = $request_item['category'];
$v['value'] = $request_item['value'];
$v['to_ids'] = $request_item['to_ids'];
@ -741,8 +743,8 @@ class MispObject extends AppModel
}
if (empty($v['categories'])) {
$v['categories'] = array();
foreach ($this->Event->Attribute->categoryDefinitions as $catk => $catv) {
if (in_array($v['type'], $catv['types'])) {
foreach ($categoryDefinitions as $catk => $catv) {
if (in_array($v['type'], $catv['types'], true)) {
$v['categories'][] = $catk;
}
}
@ -751,7 +753,7 @@ class MispObject extends AppModel
$template['ObjectTemplateElement'][] = $v;
unset($v['uuid']); // force creating a new attribute if template element entry gets reused
} else {
$template['warnings'][] = 'Missing attribute type "' . $v['type'] . '" found. Omitted template element ("' . $template_object_elements[$k]['object_relation'] . '") that would not pass validation due to this.';
$template['warnings'][] = 'Missing attribute type "' . $v['type'] . '" found. Omitted template element ("' . $v['object_relation'] . '") that would not pass validation due to this.';
}
}
}
@ -1215,36 +1217,40 @@ class MispObject extends AppModel
return count($orphans);
}
public function validObjectsFromAttributeTypes($user, $event_id, $selected_attribute_ids)
/**
* @param array $user
* @param int $eventId
* @param array $selectedAttributeIds
* @return array|array[]
* @throws Exception
*/
public function validObjectsFromAttributeTypes(array $user, $eventId, array $selectedAttributeIds)
{
$attributes = $this->Attribute->fetchAttributes($user,
array(
'conditions' => array(
'Attribute.id' => $selected_attribute_ids,
'Attribute.event_id' => $event_id,
'Attribute.object_id' => 0
),
)
);
$attributes = $this->Attribute->fetchAttributesSimple($user, [
'conditions' => [
'Attribute.id' => $selectedAttributeIds,
'Attribute.event_id' => $eventId,
'Attribute.object_id' => 0,
],
]);
if (empty($attributes)) {
return array('templates' => array(), 'types' => array());
}
$attribute_types = array();
$attributeTypes = array();
foreach ($attributes as $i => $attribute) {
$attribute_types[$attribute['Attribute']['type']] = 1;
$attributeTypes[$attribute['Attribute']['type']] = true;
$attributes[$i]['Attribute']['object_relation'] = $attribute['Attribute']['type'];
}
$attribute_types = array_keys($attribute_types);
$attributeTypes = array_keys($attributeTypes);
$potential_templates = $this->ObjectTemplate->find('list', array(
$potentialTemplateIds = $this->ObjectTemplate->find('column', array(
'recursive' => -1,
'fields' => array(
'ObjectTemplate.id',
'COUNT(ObjectTemplateElement.type) as type_count'
),
'conditions' => array(
'ObjectTemplate.active' => true,
'ObjectTemplateElement.type' => $attribute_types
'ObjectTemplateElement.type' => $attributeTypes,
),
'joins' => array(
array(
@ -1256,15 +1262,13 @@ class MispObject extends AppModel
)
),
'group' => 'ObjectTemplate.id',
'order' => 'type_count DESC'
));
$potential_template_ids = array_keys($potential_templates);
$templates = $this->ObjectTemplate->find('all', array(
$templates = $this->ObjectTemplate->find('all', [
'recursive' => -1,
'conditions' => array('id' => $potential_template_ids),
'contain' => 'ObjectTemplateElement'
));
'conditions' => ['id' => $potentialTemplateIds],
'contain' => ['ObjectTemplateElement' => ['fields' => ['object_relation', 'type', 'multiple']]]
]);
foreach ($templates as $i => $template) {
$res = $this->ObjectTemplate->checkTemplateConformityBasedOnTypes($template, $attributes);
@ -1272,21 +1276,16 @@ class MispObject extends AppModel
$templates[$i]['ObjectTemplate']['invalidTypes'] = $res['invalidTypes'];
$templates[$i]['ObjectTemplate']['invalidTypesMultiple'] = $res['invalidTypesMultiple'];
}
return array('templates' => $templates, 'types' => $attribute_types);
return array('templates' => $templates, 'types' => $attributeTypes);
}
public function groupAttributesIntoObject($user, $event_id, $object, $template, $selected_attribute_ids, $selected_object_relation_mapping, $hard_delete_attribute)
public function groupAttributesIntoObject(array $user, $event_id, array $object, $template, array $selected_attribute_ids, array $selected_object_relation_mapping, $hard_delete_attribute)
{
$saved_object_id = $this->saveObject($object, $event_id, $template, $user);
if (!is_numeric($saved_object_id)) {
return $saved_object_id;
}
$saved_object = $this->find('first', array(
'recursive' => -1,
'conditions' => array('Object.id' => $saved_object_id)
));
$existing_attributes = $this->Attribute->fetchAttributes($user, array('conditions' => array(
'Attribute.id' => $selected_attribute_ids,
'Attribute.event_id' => $event_id,
@ -1299,7 +1298,7 @@ class MispObject extends AppModel
$event = array('Event' => $existing_attributes[0]['Event']);
// Duplicate the attribute and its context, otherwise connected instances will drop the duplicated UUID
foreach ($existing_attributes as $i => $existing_attribute) {
foreach ($existing_attributes as $existing_attribute) {
if (isset($selected_object_relation_mapping[$existing_attribute['Attribute']['id']])) {
$sightings = $this->Event->Sighting->attachToEvent($event, $user, $existing_attribute['Attribute']['id']);
$object_relation = $selected_object_relation_mapping[$existing_attribute['Attribute']['id']];
@ -1308,20 +1307,18 @@ class MispObject extends AppModel
unset($created_attribute['id']);
unset($created_attribute['uuid']);
$created_attribute['object_relation'] = $object_relation;
$created_attribute['object_id'] = $saved_object['Object']['id'];
$created_attribute['object_id'] = $saved_object_id;
if (isset($existing_attribute['AttributeTag'])) {
$created_attribute['AttributeTag'] = $existing_attribute['AttributeTag'];
}
if (!empty($sightings)) {
$created_attribute['Sighting'] = $sightings;
}
$saved_object['Attribute'][$i] = $created_attribute;
$this->Attribute->captureAttribute($created_attribute, $event_id, $user, $saved_object['Object']['id']);
$this->Attribute->captureAttribute($created_attribute, $event_id, $user, $saved_object_id);
$this->Attribute->deleteAttribute($existing_attribute['Attribute']['id'], $user, $hard_delete_attribute);
}
}
return $saved_object['Object']['id'];
return $saved_object_id;
}
public function resolveUpdatedTemplate($template, $object, $update_template_available = false)
@ -1395,30 +1392,31 @@ class MispObject extends AppModel
}
$toReturn['updateable_attribute'] = $object['Attribute'];
$toReturn['not_updateable_attribute'] = array();
} else {
$toReturn['newer_template_version'] = false;
}
if (!empty($template_difference)) { // older template not completely embeded in newer
foreach ($template_difference as $temp_diff_element) {
foreach ($object['Attribute'] as $i => $attribute) {
if (
$attribute['object_relation'] == $temp_diff_element['object_relation']
&& $attribute['type'] == $temp_diff_element['type']
) { // This attribute cannot be merged automatically
$attribute['merge-possible'] = false;
$toReturn['not_updateable_attribute'][] = $attribute;
unset($toReturn['updateable_attribute'][$i]);
if (!empty($template_difference)) { // older template not completely embeded in newer
foreach ($template_difference as $temp_diff_element) {
foreach ($object['Attribute'] as $i => $attribute) {
if (
$attribute['object_relation'] == $temp_diff_element['object_relation']
&& $attribute['type'] == $temp_diff_element['type']
) { // This attribute cannot be merged automatically
$attribute['merge-possible'] = false;
$toReturn['not_updateable_attribute'][] = $attribute;
unset($toReturn['updateable_attribute'][$i]);
}
}
}
}
}
if ($update_template_available) { // template version bump requested
$toReturn['template'] = $newer_template; // bump the template version
}
return $toReturn;
}
public function reviseObject($revised_object, $object, $template) {
public function reviseObject($revised_object, $object, $template)
{
$revised_object_both = array('mergeable' => array(), 'notMergeable' => array());
// Loop through attributes to inject and perform the correct action

42
app/Model/ObjectTemplate.php
View File

@ -206,18 +206,27 @@ class ObjectTemplate extends AppModel
return true;
}
public function checkTemplateConformityBasedOnTypes($template, $attributes)
/**
* @param array $template
* @param array $attributes
* @return array
*/
public function checkTemplateConformityBasedOnTypes(array $template, array $attributes)
{
$to_return = array('valid' => true, 'missingTypes' => array());
if (!empty($template['ObjectTemplate']['requirements'])) {
// construct array containing ObjectTemplateElement with object_relation as key for faster search
$elementsByObjectRelationName = array_column($template['ObjectTemplateElement'], null, 'object_relation');
// check for all required attributes
if (!empty($template['ObjectTemplate']['requirements']['required'])) {
foreach ($template['ObjectTemplate']['requirements']['required'] as $requiredField) {
$requiredType = Hash::extract($template['ObjectTemplateElement'], sprintf('{n}[object_relation=%s].type', $requiredField))[0];
$requiredType = $elementsByObjectRelationName[$requiredField]['type'];
$found = false;
foreach ($attributes as $attribute) {
if ($attribute['Attribute']['type'] == $requiredType) {
if ($attribute['Attribute']['type'] === $requiredType) {
$found = true;
break;
}
}
if (!$found) {
@ -228,24 +237,24 @@ class ObjectTemplate extends AppModel
// check for all required one of attributes
if (!empty($template['ObjectTemplate']['requirements']['requiredOneOf'])) {
$found = false;
$all_required_type = array();
$allRequiredTypes = array();
foreach ($template['ObjectTemplate']['requirements']['requiredOneOf'] as $requiredField) {
$requiredType = Hash::extract($template['ObjectTemplateElement'], sprintf('{n}[object_relation=%s].type', $requiredField));
$requiredType = empty($requiredType) ? NULL : $requiredType[0];
$all_required_type[] = $requiredType;
$requiredType = $elementsByObjectRelationName[$requiredField]['type'] ?? null;
$allRequiredTypes[] = $requiredType;
foreach ($attributes as $attribute) {
if ($attribute['Attribute']['type'] == $requiredType) {
if ($attribute['Attribute']['type'] === $requiredType) {
$found = true;
break;
}
}
}
if (!$found) {
$to_return = array('valid' => false, 'missingTypes' => $all_required_type);
$to_return = array('valid' => false, 'missingTypes' => $allRequiredTypes);
}
}
}
// at this point, an object could created; checking if all attribute are valids
// at this point, an object could created; checking if all attribute are valid
$valid_types = array();
$to_return['invalidTypes'] = array();
$to_return['invalidTypesMultiple'] = array();
@ -266,8 +275,8 @@ class ObjectTemplate extends AppModel
$to_return['invalidTypes'][] = $attribute['Attribute']['type'];
}
}
$to_return['invalidTypes'] = array_unique($to_return['invalidTypes']);
$to_return['invalidTypesMultiple'] = array_unique($to_return['invalidTypesMultiple']);
$to_return['invalidTypes'] = array_unique($to_return['invalidTypes'], SORT_REGULAR);
$to_return['invalidTypesMultiple'] = array_unique($to_return['invalidTypesMultiple'], SORT_REGULAR);
if (!empty($to_return['invalidTypesMultiple'])) {
$to_return['valid'] = false;
}
@ -344,6 +353,10 @@ class ObjectTemplate extends AppModel
return FileAccessTool::readJsonFromFile($path);
}
/**
* @return Generator<array>
* @throws Exception
*/
private function readTemplatesFromDisk()
{
foreach ($this->getTemplateDirectoryPaths() as $dirpath) {
@ -355,8 +368,13 @@ class ObjectTemplate extends AppModel
}
}
/**
* @param bool $fullPath
* @return array
*/
private function getTemplateDirectoryPaths($fullPath=true)
{
App::uses('Folder', 'Utility');
$dir = new Folder(self::OBJECTS_DIR, false);
return $dir->read(true, false, $fullPath)[0];
}

29
app/Model/ObjectTemplateElement.php
View File

@ -1,30 +1,23 @@
<?php
App::uses('AppModel', 'Model');
class ObjectTemplateElement extends AppModel
{
public $actsAs = array(
'Containable'
);
public $belongsTo = array(
);
public $validate = array(
'Containable'
);
public function afterFind($results, $primary = false)
{
foreach ($results as $k => $result) {
if (isset($results[$k]['ObjectTemplateElement']['categories'])) {
$results[$k]['ObjectTemplateElement']['categories'] = json_decode($results[$k]['ObjectTemplateElement']['categories'], true);
foreach ($results as &$result) {
if (isset($result['ObjectTemplateElement']['categories'])) {
$result['ObjectTemplateElement']['categories'] = json_decode($result['ObjectTemplateElement']['categories'], true);
}
if (isset($results[$k]['ObjectTemplateElement']['values_list'])) {
$results[$k]['ObjectTemplateElement']['values_list'] = json_decode($results[$k]['ObjectTemplateElement']['values_list'], true);
if (isset($result['ObjectTemplateElement']['values_list'])) {
$result['ObjectTemplateElement']['values_list'] = json_decode($result['ObjectTemplateElement']['values_list'], true);
}
if (isset($results[$k]['ObjectTemplateElement']['sane_default'])) {
$results[$k]['ObjectTemplateElement']['sane_default'] = json_decode($results[$k]['ObjectTemplateElement']['sane_default'], true);
if (isset($result['ObjectTemplateElement']['sane_default'])) {
$result['ObjectTemplateElement']['sane_default'] = json_decode($result['ObjectTemplateElement']['sane_default'], true);
}
}
return $results;
@ -37,13 +30,13 @@ class ObjectTemplateElement extends AppModel
}
$json_fields = array('categories', 'values_list', 'sane_default');
foreach ($json_fields as $field) {
$this->data['ObjectTemplateElement'][$field] = empty($this->data['ObjectTemplateElement'][$field]) ? '[]' : json_encode($this->data['ObjectTemplateElement'][$field]);
$this->data['ObjectTemplateElement'][$field] = empty($this->data['ObjectTemplateElement'][$field]) ? '[]' : JsonTool::encode($this->data['ObjectTemplateElement'][$field]);
}
return true;
}
public function getAllAvailableTypes() {
public function getAllAvailableTypes()
{
$temp = $this->find('all', array(
'recursive' => -1,
'fields' => array('object_relation as type', 'description AS desc', 'categories'),

37
app/Model/OverCorrelatingValue.php
View File

@ -5,10 +5,6 @@ class OverCorrelatingValue extends AppModel
{
public $recursive = -1;
public $actsAs = array(
'Containable'
);
public function beforeValidate($options = array())
{
$this->data['OverCorrelatingValue']['value'] = self::truncate($this->data['OverCorrelatingValue']['value']);
@ -70,7 +66,7 @@ class OverCorrelatingValue extends AppModel
*/
public function getLimit()
{
return Configure::check('MISP.correlation_limit') ? Configure::read('MISP.correlation_limit') : 20;
return Configure::read('MISP.correlation_limit') ?: 20;
}
public function getOverCorrelations($query)
@ -87,19 +83,13 @@ class OverCorrelatingValue extends AppModel
return $data;
}
public function checkValue($value)
public function findOverCorrelatingValues(array $valuesToCheck): array
{
return $this->hasAny(['value' => self::truncate($value)]);
}
public function findOverCorrelatingValues(array $values_to_check): array
{
$values_to_check_truncated = array_unique(self::truncateValues($values_to_check));
$overCorrelatingValues = $this->find('column', [
'conditions' => ['value' => $values_to_check_truncated],
$valuesToCheck = array_unique(self::truncateValues($valuesToCheck), SORT_REGULAR);
return $this->find('column', [
'conditions' => ['value' => $valuesToCheck],
'fields' => ['value'],
]);
return $overCorrelatingValues;
}
public function generateOccurrencesRouter()
@ -139,14 +129,23 @@ class OverCorrelatingValue extends AppModel
]);
$this->Attribute = ClassRegistry::init('Attribute');
foreach ($overCorrelations as &$overCorrelation) {
$value = $overCorrelation['OverCorrelatingValue']['value'] . '%';
$count = $this->Attribute->find('count', [
'recursive' => -1,
'conditions' => [
'OR' => [
'Attribute.value1 LIKE' => $overCorrelation['OverCorrelatingValue']['value'] . '%',
'Attribute.value2 LIKE' => $overCorrelation['OverCorrelatingValue']['value'] . '%'
]
]
'Attribute.value1 LIKE' => $value,
'AND' => [
'Attribute.value2 LIKE' => $value,
'NOT' => ['Attribute.type' => Attribute::PRIMARY_ONLY_CORRELATING_TYPES]
],
],
'NOT' => ['Attribute.type' => Attribute::NON_CORRELATING_TYPES],
'Attribute.disable_correlation' => 0,
'Event.disable_correlation' => 0,
'Attribute.deleted' => 0,
],
'contain' => ['Event'],
]);
$overCorrelation['OverCorrelatingValue']['occurrence'] = $count;
}

48
app/Model/Server.php
View File

@ -659,7 +659,7 @@ class Server extends AppModel
}
$change = sprintf(
'%s events, %s proposals, %s sightings and %s galaxyClusters pulled or updated. %s events failed or didn\'t need an update.',
'%s events, %s proposals, %s sightings and %s galaxy clusters pulled or updated. %s events failed or didn\'t need an update.',
count($successes),
$pulledProposals,
$pulledSightings,
@ -796,6 +796,7 @@ class Server extends AppModel
* @return array
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
* @throws JsonException
*/
public function getEventIndexFromServer(ServerSyncTool $serverSync, $ignoreFilterRules = false)
{
@ -809,12 +810,37 @@ class Server extends AppModel
}
$filterRules['minimal'] = 1;
$filterRules['published'] = 1;
$eventIndex = $serverSync->eventIndex($filterRules)->json();
// Fetch event index from cache if exists and is not modified
$redis = RedisTool::init();
$indexFromCache = $redis->get("misp:event_index:{$serverSync->serverId()}");
if ($indexFromCache) {
list($etag, $eventIndex) = RedisTool::deserialize($indexFromCache);
} else {
$etag = '""'; // Provide empty ETag, so MISP will compute ETag for returned data
}
$response = $serverSync->eventIndex($filterRules, $etag);
if ($response->isNotModified() && $indexFromCache) {
return $eventIndex;
}
$eventIndex = $response->json();
// correct $eventArray if just one event, probably this response returns old MISP
if (isset($eventIndex['id'])) {
$eventIndex = [$eventIndex];
}
// Save to cache for 24 hours if ETag provided
if (isset($response->headers["ETag"])) {
$data = RedisTool::serialize([$response->headers["ETag"], $eventIndex]);
$redis->setex("misp:event_index:{$serverSync->serverId()}", 3600 * 24, $data);
} else if ($indexFromCache) {
$redis->del("misp:event_index:{$serverSync->serverId()}");
}
return $eventIndex;
}
@ -5708,7 +5734,7 @@ class Server extends AppModel
),
'custom_css' => array(
'level' => 2,
'description' => __('If you would like to customise the css, simply drop a css file in the /var/www/MISP/app/webroot/css directory and enter the name here.'),
'description' => __('If you would like to customise the CSS, simply drop a css file in the /var/www/MISP/app/webroot/css directory and enter the name here.'),
'value' => '',
'test' => 'testForStyleFile',
'type' => 'string',
@ -5776,6 +5802,22 @@ class Server extends AppModel
'type' => 'string',
'redacted' => true
),
'redis_serializer' => [
'level' => self::SETTING_OPTIONAL,
'description' => __('Redis serializer method. WARNING: Changing this setting will drop some cached data.'),
'value' => 'JSON',
'test' => null,
'type' => 'string',
'null' => true,
'afterHook' => function () {
$keysToDelete = ['taxonomies_cache:*', 'misp:warninglist_cache', 'misp:event_lock:*', 'misp:event_index:*'];
$redis = RedisTool::init();
foreach ($keysToDelete as $key) {
$redis->unlink($redis->keys($key));
}
return true;
},
],
'event_view_filter_fields' => array(
'level' => 2,
'description' => __('Specify which fields to filter on when you search on the event view. Default values are : "id, uuid, value, comment, type, category, Tag.name"'),

40
app/Model/Taxonomy.php
View File

@ -126,6 +126,9 @@ class Taxonomy extends AppModel
return (int)$result;
}
/**
* @throws Exception
*/
private function __updateVocab(array $vocab, array $current)
{
$enabled = 0;
@ -149,6 +152,9 @@ class Taxonomy extends AppModel
}
if (!empty($vocab['values'])) {
foreach ($vocab['values'] as $value) {
if (!isset($predicateLookup[$value['predicate']])) {
throw new Exception("Invalid taxonomy `{$vocab['namespace']}` provided. Predicate `{$value['predicate']}` is missing.");
}
$predicatePosition = $predicateLookup[$value['predicate']];
if (empty($taxonomy['Taxonomy']['TaxonomyPredicate'][$predicatePosition]['TaxonomyEntry'])) {
$taxonomy['Taxonomy']['TaxonomyPredicate'][$predicatePosition]['TaxonomyEntry'] = $value['entry'];
@ -194,30 +200,34 @@ class Taxonomy extends AppModel
foreach ($taxonomy['TaxonomyPredicate'] as $predicate) {
if (isset($predicate['TaxonomyEntry']) && !empty($predicate['TaxonomyEntry'])) {
foreach ($predicate['TaxonomyEntry'] as $entry) {
$temp = array('tag' => $taxonomy['Taxonomy']['namespace'] . ':' . $predicate['value'] . '="' . $entry['value'] . '"');
$temp['expanded'] = (!empty($predicate['expanded']) ? $predicate['expanded'] : $predicate['value']) . ': ' . (!empty($entry['expanded']) ? $entry['expanded'] : $entry['value']);
if (isset($entry['description']) && !empty($entry['description'])) {
$temp = [
'tag' => $taxonomy['Taxonomy']['namespace'] . ':' . $predicate['value'] . '="' . $entry['value'] . '"',
'expanded' => (!empty($predicate['expanded']) ? $predicate['expanded'] : $predicate['value']) . ': ' . (!empty($entry['expanded']) ? $entry['expanded'] : $entry['value']),
'exclusive_predicate' => $predicate['exclusive'],
];
if (!empty($entry['description'])) {
$temp['description'] = $entry['description'];
}
if (isset($entry['colour']) && !empty($entry['colour'])) {
if (!empty($entry['colour'])) {
$temp['colour'] = $entry['colour'];
}
if (isset($entry['numerical_value']) && $entry['numerical_value'] !== null) {
if (isset($entry['numerical_value'])) {
$temp['numerical_value'] = $entry['numerical_value'];
}
$temp['exclusive_predicate'] = $predicate['exclusive'];
$entries[] = $temp;
}
} else {
$temp = array('tag' => $taxonomy['Taxonomy']['namespace'] . ':' . $predicate['value']);
$temp['expanded'] = !empty($predicate['expanded']) ? $predicate['expanded'] : $predicate['value'];
if (isset($predicate['description']) && !empty($predicate['description'])) {
$temp = [
'tag' => $taxonomy['Taxonomy']['namespace'] . ':' . $predicate['value'],
'expanded' => !empty($predicate['expanded']) ? $predicate['expanded'] : $predicate['value']
];
if (!empty($predicate['description'])) {
$temp['description'] = $predicate['description'];
}
if (isset($predicate['colour']) && !empty($predicate['colour'])) {
if (!empty($predicate['colour'])) {
$temp['colour'] = $predicate['colour'];
}
if (isset($predicate['numerical_value']) && $predicate['numerical_value'] !== null) {
if (isset($predicate['numerical_value'])) {
$temp['numerical_value'] = $predicate['numerical_value'];
}
$entries[] = $temp;
@ -589,9 +599,9 @@ class Taxonomy extends AppModel
return false; // not taxonomy tag
}
$key = 'taxonomies_cache:tagName=' . $tagName . "&" . "metaOnly=$metaOnly" . "&" . "fullTaxonomy=$fullTaxonomy";
$key = "taxonomies_cache:tagName=$tagName&metaOnly=$metaOnly&fullTaxonomy=$fullTaxonomy";
$redis = $this->setupRedis();
$taxonomy = $redis ? json_decode($redis->get($key), true) : null;
$taxonomy = $redis ? RedisTool::deserialize($redis->get($key)) : null;
if (!$taxonomy) {
if (isset($splits['value'])) {
@ -634,7 +644,7 @@ class Taxonomy extends AppModel
}
if ($redis) {
$redis->setex($key, 1800, json_encode($taxonomy));
$redis->setex($key, 1800, RedisTool::serialize($taxonomy));
}
}
@ -753,7 +763,7 @@ class Taxonomy extends AppModel
/**
* @param string $tag
* @return array|null
* @return array|null Returns null if tag is not in taxonomy format
*/
public function splitTagToComponents($tag)
{

201
app/Model/User.php
View File

@ -493,14 +493,6 @@ class User extends AppModel
return $fails;
}
public function getOrgMemberCount($org)
{
return $this->find('count', array(
'conditions' => array(
'org =' => $org,
)));
}
/**
* 0 - true if key is valid
* 1 - User e-mail
@ -838,7 +830,7 @@ class User extends AppModel
* @param array $user
* @param SendEmailTemplate|string $body
* @param string|false $bodyNoEnc
* @param string $subject
* @param string|null $subject
* @param array|false $replyToUser
* @return bool
* @throws Crypt_GPG_BadPassphraseException
@ -1649,56 +1641,88 @@ class User extends AppModel
return substr(hash('sha256', "{$user['id']}|$salt"), 0, 8);
}
public function extractPeriodicSettingForUser($user, $decode=false): array
/**
* @param int $userId
* @param bool $decode
* @return array
* @throws JsonException
*/
public function fetchPeriodicSettingForUser($userId, $decode = false): array
{
$filter_names = ['orgc_id', 'distribution', 'sharing_group_id', 'event_info', 'tags', 'trending_for_tags'];
$filter_to_decode = ['tags', 'trending_for_tags', ];
if (is_numeric($user)) {
$user = $this->find('first', [
'recursive' => -1,
'conditions' => ['User.id' => $user],
'contain' => [
'UserSetting',
]
]);
$filterNames = ['orgc_id', 'distribution', 'sharing_group_id', 'event_info', 'tags', 'trending_for_tags', 'include_correlations', 'trending_period_amount'];
$filterToDecode = ['tags', 'trending_for_tags'];
$defaultPeriodicSettings = [
'orgc_id' => '',
'distribution' => -1,
'sharing_group_id' => '',
'event_info' => '',
'tags' => '[]',
'trending_for_tags' => '[]',
'include_correlations' => '',
'trending_period_amount' => 2,
];
$periodicSettings = $this->UserSetting->find('first', [
'recursive' => -1,
'conditions' => [
'user_id' => $userId,
'setting' => self::PERIODIC_USER_SETTING_KEY,
],
'fields' => ['value'],
]);
if (empty($periodicSettings)) {
$periodicSettings = $defaultPeriodicSettings;
} else {
$periodicSettings = $periodicSettings['UserSetting']['value'];
}
$periodic_settings = array_values(array_filter($user['UserSetting'], function ($userSetting) {
return $userSetting['setting'] == self::PERIODIC_USER_SETTING_KEY;
}));
if (empty($periodic_settings)) {
$periodic_settings = [['value' => [
'orgc_id' => '',
'distribution' => -1,
'sharing_group_id' => '',
'event_info' => '',
'tags' => '[]',
'trending_for_tags' => '[]'
]]];
$periodicSettingsIndexed = [];
foreach ($filterNames as $filterName) {
$periodicSettingsIndexed[$filterName] = $periodicSettings[$filterName] ?? $defaultPeriodicSettings[$filterName];
}
$periodic_settings_indexed = [];
if (!empty($periodic_settings)) {
foreach ($filter_names as $filter_name) {
$periodic_settings_indexed[$filter_name] = $periodic_settings[0]['value'][$filter_name];
if ($decode) {
foreach ($filterToDecode as $filter) {
if (!empty($periodicSettingsIndexed[$filter])) {
$periodicSettingsIndexed[$filter] = JsonTool::decode($periodicSettingsIndexed[$filter]);
}
}
}
foreach ($filter_to_decode as $filter) {
if (!empty($decode) && !empty($periodic_settings_indexed[$filter])) {
$periodic_settings_indexed[$filter] = JsonTool::decode($periodic_settings_indexed[$filter]);
}
}
return $periodic_settings_indexed;
return $periodicSettingsIndexed;
}
public function getUsablePeriodicSettingForUser(array $periodicSettings, $period='daily'): array
/**
* @param array $period_filters
* @param string $period
* @return array
*/
private function getUsablePeriodicSettingForUser(array $period_filters, $period='daily'): array
{
return $this->__getUsableFilters($periodicSettings, $period);
$filters = [
'last' => $this->__genTimerangeFilter($period),
'published' => true,
];
if (!empty($period_filters['orgc_id'])) {
$filters['orgc_id'] = $period_filters['orgc_id'];
}
if (isset($period_filters['distribution']) && $period_filters['distribution'] >= 0) {
$filters['distribution'] = intval($period_filters['distribution']);
}
if (!empty($period_filters['sharing_group_id'])) {
$filters['sharing_group_id'] = $period_filters['sharing_group_id'];
}
if (!empty($period_filters['event_info'])) {
$filters['event_info'] = $period_filters['event_info'];
}
if (!empty($period_filters['tags'])) {
$filters['tags'] = $period_filters['tags'];
}
return $filters;
}
public function saveNotificationSettings(int $user_id, array $data): bool
public function saveNotificationSettings(int $userId, array $data): bool
{
$existingUser = $this->find('first', [
'recursive' => -1,
'conditions' => ['User.id' => $user_id],
'conditions' => ['User.id' => $userId],
]);
if (empty($existingUser)) {
return false;
@ -1707,11 +1731,11 @@ class User extends AppModel
$existingUser['User'][$notification_period] = $data['User'][$notification_period];
}
$success = $this->save($existingUser, [
'fieldList' => self::PERIODIC_NOTIFICATIONS
'fieldList' => array_merge(self::PERIODIC_NOTIFICATIONS, ['date_modified']),
]);
if ($success) {
$periodic_settings = $data['periodic_settings'];
$param_to_decode = ['tags', 'trending_for_tags', ];
$param_to_decode = ['tags', 'trending_for_tags'];
foreach ($param_to_decode as $param) {
if (empty($periodic_settings[$param])) {
$periodic_settings[$param] = '[]';
@ -1729,6 +1753,8 @@ class User extends AppModel
'event_info' => $periodic_settings['event_info'] ?? '',
'tags' => $periodic_settings['tags'] ?? '[]',
'trending_for_tags' => $periodic_settings['trending_for_tags'] ?? '[]',
'include_correlations' => $periodic_settings['include_correlations'] ?? '',
'trending_period_amount' => $periodic_settings['trending_period_amount'] ?? 2,
];
$new_user_setting = [
'UserSetting' => [
@ -1746,40 +1772,45 @@ class User extends AppModel
{
return $this->find('all', [
'recursive' => -1,
'conditions' => ["notification_$period" => true],
'conditions' => [
"notification_$period" => true,
'disabled' => false,
],
]);
}
/**
* generatePeriodicSummary
*
* @param int $user_id
* @param string $period
* @param int $userId
* @param string $period Can be 'daily', 'weekly' or 'monthly'
* @param bool $rendered When false, instance of SendEmailTemplate will returned
* @return string|SendEmailTemplate
* @throws NotFoundException
* @throws InvalidArgumentException
*/
public function generatePeriodicSummary(int $user_id, string $period, $rendered=true)
public function generatePeriodicSummary(int $userId, string $period, $rendered=true)
{
$existingUser = $this->getUserById($user_id);
$user = $this->rearrangeToAuthForm($existingUser);
$allowed_periods = array_map(function($period) {
$allowedPeriods = array_map(function($period) {
return substr($period, strlen('notification_'));
}, self::PERIODIC_NOTIFICATIONS);
if (!in_array($period, $allowed_periods)) {
if (!in_array($period, $allowedPeriods, true)) {
throw new InvalidArgumentException(__('Invalid period. Must be one of %s', JsonTool::encode(self::PERIODIC_NOTIFICATIONS)));
}
$user = $this->getAuthUser($userId);
App::import('Tools', 'SendEmail');
$emailTemplate = $this->prepareEmailTemplate($period);
$periodicSettings = $this->extractPeriodicSettingForUser($user, true);
$periodicSettings = $this->fetchPeriodicSettingForUser($userId, true);
$filters = $this->getUsablePeriodicSettingForUser($periodicSettings, $period);
$filtersForRestSearch = $filters; // filters for restSearch are slightly different than fetchEvent
$filters['last'] = $this->resolveTimeDelta($filters['last']);
$filters['sgReferenceOnly'] = true;
$filters['includeEventCorrelations'] = false;
$filters['includeEventCorrelations'] = !empty($periodicSettings['include_correlations']);
$filters['includeGranularCorrelations'] = !empty($periodicSettings['include_correlations']);
$filters['noSightings'] = true;
$filters['includeGalaxy'] = false;
$events = $this->__getEventsForFilters($user, $filters);
$filters['fetchFullClusters'] = false;
$filters['includeScoresOnEvent'] = true;
$events = $this->Event->fetchEvent($user, $filters);
$elementCounter = 0;
$renderView = false;
@ -1792,11 +1823,10 @@ class User extends AppModel
unset($filtersForRestSearch['tags']);
}
$finalContext = $this->Event->restSearch($user, 'context', $filtersForRestSearch, false, false, $elementCounter, $renderView);
$finalContext = json_decode($finalContext->intoString(), true);
$finalContext = JsonTool::decode($finalContext->intoString());
$aggregated_context = $this->__renderAggregatedContext($finalContext);
$rollingWindows = 2;
$trendAnalysis = $this->Event->getTrendsForTagsFromEvents($events, $this->__periodToDays($period), $rollingWindows, $periodicSettings['trending_for_tags']);
$rollingWindows = $periodicSettings['trending_period_amount'] ?: 2;
$trendAnalysis = $this->Event->getTrendsForTagsFromEvents($events, $this->periodToDays($period), $rollingWindows, $periodicSettings['trending_for_tags']);
$tagFilterPrefixes = $periodicSettings['trending_for_tags'] ?: array_keys($trendAnalysis['all_tags']);
$trendData = [
'trendAnalysis' => $trendAnalysis,
@ -1804,17 +1834,20 @@ class User extends AppModel
];
$trending_summary = $this->__renderTrendingSummary($trendData);
$emailTemplate = $this->prepareEmailTemplate($period);
$emailTemplate->set('baseurl', $this->Event->__getAnnounceBaseurl());
$emailTemplate->set('events', $events);
$emailTemplate->set('filters', $filters);
$emailTemplate->set('periodicSettings', $periodicSettings);
$emailTemplate->set('period_days', $this->periodToDays($period));
$emailTemplate->set('period', $period);
$emailTemplate->set('aggregated_context', $aggregated_context);
$emailTemplate->set('trending_summary', $trending_summary);
$emailTemplate->set('analysisLevels', $this->Event->analysisLevels);
$emailTemplate->set('distributionLevels', $this->Event->distributionLevels);
if (!empty($rendered)) {
if ($rendered) {
$summary = $emailTemplate->render();
return $summary->format() == 'text' ? $summary->text : $summary->html;
return $summary->format() === 'text' ? $summary->text : $summary->html;
}
return $emailTemplate;
}
@ -1866,33 +1899,23 @@ class User extends AppModel
}
return $filters;
}
private function __genTimerangeFilter(string $period='daily'): string
{
$timerange = '1d';
if ($period == 'weekly') {
$timerange = '7d';
} else if ($period == 'monthly'){
$timerange = '31d';
return $this->periodToDays($period) . 'd';
}
private function periodToDays(string $period='daily'): int
{
if ($period === 'daily') {
return 1;
} else if ($period === 'weekly') {
return 7;
} else {
return 31;
}
return $timerange;
}
private function __periodToDays(string $period='daily'): int
{
return ($period == 'daily' ? 1 : (
$period == 'weekly' ? 7 : 31)
);
}
private function __getEventsForFilters(array $user, array $filters): array
{
$this->Event = ClassRegistry::init('Event');
$events = $this->Event->fetchEvent($user, $filters);
return $events;
}
public function prepareEmailTemplate(string $period='daily'): SendEmailTemplate
private function prepareEmailTemplate(string $period = 'daily'): SendEmailTemplate
{
$subject = sprintf('[%s MISP] %s %s', Configure::read('MISP.org'), Inflector::humanize($period), __('Notification - %s', (new DateTime())->format('Y-m-d')));
$template = new SendEmailTemplate("notification_$period");

100
app/Model/Warninglist.php
View File

@ -115,7 +115,7 @@ class Warninglist extends AppModel
}
try {
$redis = $this->setupRedisWithException();
$redis = RedisTool::init();
} catch (Exception $e) {
// fallback to default implementation when redis is not available
$eventWarnings = [];
@ -182,10 +182,10 @@ class Warninglist extends AppModel
}
$attributeKey = $keysToGet[$pos];
$saveToCache[$attributeKey] = empty($store) ? '' : json_encode($store);
$saveToCache[$attributeKey] = empty($store) ? '' : RedisTool::serialize($store);
} elseif (!empty($result)) { // skip empty string that means no warning list match
$matchedWarningList = json_decode($result, true);
$matchedWarningList = RedisTool::deserialize($result);
foreach ($matchedWarningList as $warninglistId => $matched) {
$attributes[$redisResultToAttributePos[$pos]]['warnings'][] = [
'value' => $matched[0],
@ -447,12 +447,7 @@ class Warninglist extends AppModel
$redis->del($redis->keys('misp:warninglist_entries_cache:*'));
}
$warninglists = $this->find('all', array(
'contain' => array('WarninglistType'),
'conditions' => array('enabled' => 1),
'fields' => ['id', 'name', 'type', 'category'],
));
$this->cacheWarninglists($warninglists);
$warninglists = $this->getEnabledAndCacheWarninglist();
foreach ($warninglists as $warninglist) {
if ($id && $warninglist['Warninglist']['id'] != $id) {
@ -467,17 +462,34 @@ class Warninglist extends AppModel
return true;
}
private function cacheWarninglists(array $warninglists)
/**
* Get enable warninglists and cache them.
* @return array
*/
private function getEnabledAndCacheWarninglist()
{
$redis = $this->setupRedis();
if ($redis !== false) {
$redis->del('misp:warninglist_cache');
foreach ($warninglists as $warninglist) {
$redis->sAdd('misp:warninglist_cache', json_encode($warninglist));
$warninglists = $this->find('all', [
'contain' => ['WarninglistType'],
'conditions' => ['enabled' => 1],
'fields' => ['id', 'name', 'type', 'category'],
]);
// Convert type to array
foreach ($warninglists as &$warninglist) {
$warninglist['types'] = [];
foreach ($warninglist['WarninglistType'] as $wt) {
$warninglist['types'][] = $wt['type'];
}
return true;
unset($warninglist['WarninglistType']);
}
return false;
try {
$redis = RedisTool::init();
$redis->set('misp:warninglist_cache', RedisTool::serialize($warninglists));
} catch (Exception $e) {
}
return $warninglists;
}
private function cacheWarninglistEntries(array $warninglistEntries, $id)
@ -500,6 +512,7 @@ class Warninglist extends AppModel
/**
* @return array
* @throws JsonException
*/
public function getEnabled()
{
@ -507,28 +520,18 @@ class Warninglist extends AppModel
return $this->enabledCache;
}
$redis = $this->setupRedis();
if ($redis !== false && $redis->exists('misp:warninglist_cache')) {
$warninglists = $redis->sMembers('misp:warninglist_cache');
foreach ($warninglists as $k => $v) {
$warninglists[$k] = json_decode($v, true);
}
} else {
$warninglists = $this->find('all', array(
'contain' => ['WarninglistType'],
'conditions' => ['enabled' => 1],
'fields' => ['id', 'name', 'type', 'category'],
));
$this->cacheWarninglists($warninglists);
try {
$redis = RedisTool::init();
$warninglists = RedisTool::deserialize($redis->get('misp:warninglist_cache'));
} catch (Exception $e) {
$warninglists = false;
}
foreach ($warninglists as &$warninglist) {
$warninglist['types'] = [];
foreach ($warninglist['WarninglistType'] as $wt) {
$warninglist['types'][] = $wt['type'];
}
unset($warninglist['WarninglistType']);
// $warninglists is false when nothing is cached
if ($warninglists === false) {
$warninglists = $this->getEnabledAndCacheWarninglist();
}
$this->enabledCache = $warninglists;
return $warninglists;
}
@ -539,17 +542,20 @@ class Warninglist extends AppModel
*/
private function getWarninglistEntries($id)
{
$redis = $this->setupRedis();
if ($redis !== false && $redis->exists('misp:warninglist_entries_cache:' . $id)) {
return $redis->sMembers('misp:warninglist_entries_cache:' . $id);
} else {
$entries = $this->WarninglistEntry->find('column', array(
'conditions' => array('warninglist_id' => $id),
'fields' => array('WarninglistEntry.value')
));
$this->cacheWarninglistEntries($entries, $id);
return $entries;
}
try {
$redis = RedisTool::init();
$entries = $redis->sMembers('misp:warninglist_entries_cache:' . $id);
if (!empty($entries)) {
return $entries;
}
} catch (Exception $e) {}
$entries = $this->WarninglistEntry->find('column', array(
'conditions' => array('warninglist_id' => $id),
'fields' => array('WarninglistEntry.value')
));
$this->cacheWarninglistEntries($entries, $id);
return $entries;
}
/**

2
app/Model/Workflow.php
View File

@ -723,7 +723,7 @@ class Workflow extends AppModel
* @return array
* @throws ModuleNotFoundException
*/
public function getModuleConfigByType($module_type, $id, $throwException=false): array
public function getModuleConfigByType($module_type, $id, $throwException=false): ?array
{
$this->loadAllWorkflowModules();
$moduleConfig = $this->loaded_modules[$module_type][$id] ?? null;

6
app/Test/AttributeValidationToolTest.php
View File

@ -135,6 +135,12 @@ class AttributeValidationToolTest extends TestCase
$this->assertEquals('xn--hkyrky-ptac70bc.cz|127.0.0.1', AttributeValidationTool::modifyBeforeValidation('domain|ip', 'HÁČKYČÁRKY.CZ|127.0.0.1'));
}
public function testSssdeep()
{
$this->shouldBeValid('ssdeep', ["768:+OFu8Q3w6QzfR5Jni6SQD7qSFDs6P93/q0XIc/UB5EPABWX:RFu8QAFzffJui79f13/AnB5EPAkX"]);
$this->shouldBeInvalid('ssdeep', ["768:+OFu8Q3w6QzfR5Jni6SQD7qSFDs6P93/q0XIc/UB5EPABWX\n\n:RFu8QAFzffJui79f13/AnB5EPAkX"]);
}
private function shouldBeValid($type, array $values)
{
foreach ($values as $value) {

159
app/View/Correlations/over_correlations.ctp
View File

@ -1,89 +1,92 @@
<?php
echo sprintf('<div%s>', empty($ajax) ? ' class="index"' : '');
echo $this->element('genericElements/IndexTable/index_table', [
'data' => [
'light_paginator' => 1,
'data' => $data,
'top_bar' => [
'children' => [
[
'children' => [
[
'type' => 'simple',
'url' => $baseurl . '/correlations/overCorrelations',
'text' => __('All')
],
[
'type' => 'simple',
'url' => $baseurl . '/correlations/overCorrelations/scope:over_correlating',
'text' => __('Over-correlating')
],
[
'type' => 'simple',
'url' => $baseurl . '/correlations/overCorrelations/scope:not_over_correlating',
'text' => __('Not over-correlating')
],
[
'type' => 'simple',
'url' => $baseurl . '/correlations/generateOccurrences',
'text' => __('Regenerate occurrence counts')
]
echo sprintf('<div%s>', empty($ajax) ? ' class="index"' : '');
echo $this->element('genericElements/IndexTable/index_table', [
'data' => [
'light_paginator' => 1,
'data' => $data,
'top_bar' => [
'children' => [
[
'children' => [
[
'type' => 'simple',
'url' => $baseurl . '/correlations/overCorrelations',
'text' => __('All'),
'active' => $scope === 'all',
],
[
'type' => 'simple',
'url' => $baseurl . '/correlations/overCorrelations/scope:over_correlating',
'text' => __('Over-correlating'),
'active' => $scope === 'over_correlating',
],
[
'type' => 'simple',
'url' => $baseurl . '/correlations/overCorrelations/scope:not_over_correlating',
'text' => __('Not over-correlating'),
'active' => $scope === 'not_over_correlating',
],
[
'type' => 'simple',
'url' => $baseurl . '/correlations/generateOccurrences',
'text' => __('Regenerate occurrence counts')
]
]
]
],
'fields' => [
[
'name' => 'Value',
'element' => 'postlink',
'data_path' => 'OverCorrelatingValue.value',
'url' => '/attributes/search/results',
'payload_paths' => [
'value' => 'OverCorrelatingValue.value'
]
],
[
'name' => 'Occurrences',
'data_path' => 'OverCorrelatingValue.occurrence',
'class' => 'shortish'
],
[
'name' => 'Blocked by Threshold',
'data_path' => 'OverCorrelatingValue.over_correlation',
'class' => 'shortish',
'element' => 'boolean'
],
[
'name' => 'Excluded by Exclusion List',
'data_path' => 'OverCorrelatingValue.excluded',
'class' => 'shortish',
'element' => 'boolean'
]
],
'fields' => [
[
'name' => 'Value',
'element' => 'postlink',
'data_path' => 'OverCorrelatingValue.value',
'url' => '/attributes/search/results',
'payload_paths' => [
'value' => 'OverCorrelatingValue.value'
]
],
'title' => empty($ajax) ? $title_for_layout : false,
'description' => empty($ajax) ? __('The values with the most correlation entries.') : false,
'pull' => 'right',
'actions' => [
[
'onclick' => sprintf(
'openGenericModal(\'%s/correlation_exclusions/add/redirect_controller:correlations/redirect:top/value:[onclick_params_data_path]\');',
$baseurl
),
'onclick_params_data_path' => 'OverCorrelatingValue.value',
'icon' => 'trash',
'title' => __('Add exclusion entry for value'),
'complex_requirement' => [
'function' => function (array $row) {
return !$row['OverCorrelatingValue']['excluded'];
}
]
[
'name' => 'Occurrences',
'data_path' => 'OverCorrelatingValue.occurrence',
'class' => 'shortish'
],
[
'name' => 'Blocked by Threshold',
'data_path' => 'OverCorrelatingValue.over_correlation',
'class' => 'shortish',
'element' => 'boolean'
],
[
'name' => 'Excluded by Exclusion List',
'data_path' => 'OverCorrelatingValue.excluded',
'class' => 'shortish',
'element' => 'boolean'
]
],
'title' => empty($ajax) ? $title_for_layout : false,
'description' => empty($ajax) ? __('The values with the most correlation entries.') : false,
'pull' => 'right',
'actions' => [
[
'onclick' => sprintf(
'openGenericModal(\'%s/correlation_exclusions/add/redirect_controller:correlations/redirect:top/value:[onclick_params_data_path]\');',
$baseurl
),
'onclick_params_data_path' => 'OverCorrelatingValue.value',
'icon' => 'trash',
'title' => __('Add exclusion entry for value'),
'complex_requirement' => [
'function' => function (array $row) {
return !$row['OverCorrelatingValue']['excluded'];
}
]
]
]
]);
echo '</div>';
if (empty($ajax)) {
echo $this->element('/genericElements/SideMenu/side_menu', $menuData);
}
]
]);
echo '</div>';
if (empty($ajax)) {
echo $this->element('/genericElements/SideMenu/side_menu', $menuData);
}

14
app/View/Elements/Events/View/attribute_correlations.ctp
View File

@ -1,7 +1,7 @@
<?php
if (!empty($event['Related' . $scope][$object['id']])) {
$i = 0;
$linkColour = ($scope == 'Attribute') ? 'red' : 'white';
$linkColour = $scope === 'Attribute' ? 'red' : 'white';
$withPivot = isset($withPivot) ? $withPivot : false;
// remove duplicates
$relatedEvents = array();
@ -15,16 +15,16 @@
$count = count($event['Related' . $scope][$object['id']]);
foreach ($event['Related' . $scope][$object['id']] as $relatedAttribute) {
if ($i == 4 && $count > 5) {
$expandButton = __('Show %s more...', $count - 4);
$expandButton = __('Show %s more…', $count - 4);
echo sprintf(
'<li class="no-side-padding correlation-expand-button useCursorPointer linkButton %s">%s</li>',
'<li class="no-side-padding correlation-expand-button useCursorPointer linkButton %s">%s</li> ',
$linkColour,
$expandButton
);
}
$relatedData = array(
'Orgc' => !empty($orgTable[$relatedAttribute['org_id']]) ? $orgTable[$relatedAttribute['org_id']] : 'N/A',
'Date' => isset($relatedAttribute['date']) ? $relatedAttribute['date'] : 'N/A',
'Orgc' => $orgTable[$relatedAttribute['org_id']] ?? 'N/A',
'Date' => $relatedAttribute['date'] ?? 'N/A',
'Event' => $relatedAttribute['info'],
'Correlating Value' => $relatedAttribute['value']
);
@ -59,13 +59,13 @@
}
if (!empty($object['correlation_exclusion'])) {
echo sprintf(
'<span class="bold red" title="%s">%s</span> ',
'<span class="red" title="%s">%s</span> ',
__('The attribute value matches a correlation exclusion rule defined by a site-administrator for this instance. Click the magnifying glass to search for all occurrences of the value.'),
__('Excluded.')
);
} else if (!empty($object['over_correlation'])) {
echo sprintf(
'<span class="bold red" title="%s">%s</span> ',
'<span class="red" title="%s">%s</span> ',
__('The instance threshold for the maximum number of correlations for the given attribute value has been exceeded. Click the magnifying glass to search for all occurrences of the value.'),
__('Too many correlations.')
);

50
app/View/Elements/Events/eventIndexTable.ctp
View File

@ -7,9 +7,7 @@
<?php else: ?>
<th style="padding-left:0;padding-right:0;">&nbsp;</th>
<?php endif;?>
<th class="filter">
<?php echo $this->Paginator->sort('published');?>
</th>
<th class="filter" title="<?= __('Published') ?>"><?= $this->Paginator->sort('published', '<i class="fa fa-upload"></i>', ['escape' => false]) ?></th>
<?php
if (Configure::read('MISP.showorgalternate') && Configure::read('MISP.showorg')):
?>
@ -37,7 +35,7 @@
<?php if (in_array('creator_user', $columns, true)): ?><th><?= $this->Paginator->sort('user_id', __('Creator user')) ?></th><?php endif; ?>
<th class="filter"><?= $this->Paginator->sort('date', null, array('direction' => 'desc'));?></th>
<?php if (in_array('timestamp', $columns, true)): ?><th title="<?= __('Last modified at') ?>"><?= $this->Paginator->sort('timestamp', __('Last modified at')) ?></th><?php endif; ?>
<?php if (in_array('publish_timestamp', $columns, true)): ?><th title="<?= __('Last modified at') ?>"><?= $this->Paginator->sort('publish_timestamp', __('Published at')) ?></th><?php endif; ?>
<?php if (in_array('publish_timestamp', $columns, true)): ?><th title="<?= __('Published at') ?>"><?= $this->Paginator->sort('publish_timestamp', __('Published at')) ?></th><?php endif; ?>
<th class="filter"><?= $this->Paginator->sort('info');?></th>
<th title="<?= $eventDescriptions['distribution']['desc'];?>"><?= $this->Paginator->sort('distribution');?></th>
<th class="actions"><?php echo __('Actions');?></th>
@ -45,13 +43,13 @@
<?php foreach ($events as $event): $eventId = (int)$event['Event']['id']; ?>
<tr id="event_<?= $eventId ?>">
<?php if ($isSiteAdmin || ($event['Event']['orgc_id'] == $me['org_id'])):?>
<td style="width:10px;">
<td style="width:10px">
<input class="select" type="checkbox" data-id="<?= $eventId ?>" data-uuid="<?= h($event['Event']['uuid']) ?>">
</td>
<?php else: ?>
<td style="padding-left:0;padding-right:0;"></td>
<?php endif; ?>
<td class="short dblclickElement">
<td class="dblclickElement" style="width:30px">
<a href="<?= "$baseurl/events/view/$eventId" ?>" title="<?= __('View') ?>" aria-label="<?= __('View') ?>">
<i class="fa <?= $event['Event']['published'] ? 'fa-check green' : 'fa-times grey' ?>"></i>
</a>
@ -98,53 +96,53 @@
<?php if (in_array('tags', $columns, true)): ?>
<td class="shortish">
<?= $this->element('ajaxTags', [
'event' => $event,
'tags' => $event['EventTag'],
'tagAccess' => false,
'localTagAccess' => false,
'missingTaxonomies' => false,
'columnised' => true,
'static_tags_only' => 1,
'tag_display_style' => Configure::check('MISP.full_tags_on_event_index') ? Configure::read('MISP.full_tags_on_event_index') : 1
]);
'event' => $event,
'tags' => $event['EventTag'],
'tagAccess' => false,
'localTagAccess' => false,
'missingTaxonomies' => false,
'columnised' => true,
'static_tags_only' => 1,
'tag_display_style' => Configure::check('MISP.full_tags_on_event_index') ? Configure::read('MISP.full_tags_on_event_index') : 1,
]);
?>
</td>
<?php endif; ?>
<?php if (in_array('attribute_count', $columns, true)): ?>
<td class="dblclickElement" style="width:30px;">
<td class="dblclickElement" style="width:30px">
<?= $event['Event']['attribute_count']; ?>
</td>
<?php endif; ?>
<?php if (in_array('correlations', $columns, true)): ?>
<td class="bold" style="width:30px;">
<td class="bold" style="width:30px">
<?php if (!empty($event['Event']['correlation_count'])): ?>
<a href="<?php echo $baseurl."/events/view/" . $eventId . '/correlation:1';?>" title="<?= __n('%s correlation', '%s correlations', $event['Event']['correlation_count'], $event['Event']['correlation_count']), '. ' . __('Show filtered event with correlation only.');?>">
<?php echo h($event['Event']['correlation_count']); ?>
<a href="<?= "$baseurl/events/view/$eventId/correlation:1" ?>" title="<?= __n('%s correlation', '%s correlations', $event['Event']['correlation_count'], $event['Event']['correlation_count']), '. ' . __('Show filtered event with correlation only.');?>">
<?= intval($event['Event']['correlation_count']); ?>
</a>
<?php endif; ?>
</td>
<?php endif; ?>
<?php if (in_array('report_count', $columns, true)): ?>
<td class="bold" style="width:30px;">
<td class="bold" style="width:30px">
<?= $event['Event']['report_count']; ?>
</td>
<?php endif; ?>
<?php if (in_array('sightings', $columns, true)): ?>
<td class="bold" style="width:30px;">
<td class="bold" style="width:30px">
<?php if (!empty($event['Event']['sightings_count'])): ?>
<a href="<?php echo $baseurl."/events/view/" . $eventId . '/sighting:1';?>" title="<?php echo (!empty($event['Event']['sightings_count']) ? h($event['Event']['sightings_count']) : '0') . ' sighting(s). Show filtered event with sighting(s) only.';?>">
<a href="<?= "$baseurl/events/view/$eventId/sighting:1" ?>" title="<?= __n("1 sighting. Show filtered event with sighting only.", "%s sightings. Show filtered event with sightings only.", $event['Event']['sightings_count'], intval($event['Event']['sightings_count'])) ?>">
<?= intval($event['Event']['sightings_count']) ?>
</a>
<?php endif; ?>
</td>
<?php endif; ?>
<?php if (in_array('proposals', $columns, true)): ?>
<td class="bold dblclickElement" style="width:30px;" title="<?= __n('%s proposal', '%s proposals', $event['Event']['proposals_count'], $event['Event']['proposals_count']) ?>">
<td class="bold dblclickElement" style="width:30px" title="<?= __n('%s proposal', '%s proposals', $event['Event']['proposals_count'], $event['Event']['proposals_count']) ?>">
<?= !empty($event['Event']['proposals_count']) ? intval($event['Event']['proposals_count']) : ''; ?>
</td>
<?php endif;?>
<?php if (in_array('discussion', $columns, true)): ?>
<td class="bold dblclickElement" style="width:30px;">
<td class="bold dblclickElement" style="width:30px">
<?php
if (!empty($event['Event']['post_count'])) {
$post_count = h($event['Event']['post_count']);
@ -179,7 +177,7 @@
<td class="dblclickElement">
<?= nl2br(h($event['Event']['info']), false) ?>
</td>
<td class="short dblclickElement<?php if ($event['Event']['distribution'] == 0) echo ' privateRedText';?>" title="<?php echo $event['Event']['distribution'] != 3 ? $distributionLevels[$event['Event']['distribution']] : __('All');?>">
<td class="short dblclickElement<?php if ($event['Event']['distribution'] == 0) echo ' privateRedText';?>" title="<?= $event['Event']['distribution'] != 3 ? $distributionLevels[$event['Event']['distribution']] : __('All');?>">
<?php if ($event['Event']['distribution'] == 4):?>
<a href="<?php echo $baseurl;?>/sharingGroups/view/<?= intval($event['SharingGroup']['id']); ?>"><?= h($event['SharingGroup']['name']) ?></a>
<?php else:
@ -191,7 +189,7 @@
'<it type="button" title="%s" class="%s" aria-hidden="true" style="font-size: x-small;" data-event-distribution="%s" data-event-distribution-name="%s" data-scope-id="%s"></it>',
__('Toggle advanced sharing network viewer'),
'fa fa-share-alt useCursorPointer distributionNetworkToggle',
h($event['Event']['distribution']),
intval($event['Event']['distribution']),
$event['Event']['distribution'] == 4 ? h($event['SharingGroup']['name']) : h($shortDist[$event['Event']['distribution']]),
$eventId
)

220
app/View/Elements/Events/trending_summary.ctp
View File

@ -5,13 +5,14 @@ $allTags = $trendAnalysis['all_tags'];
$allTimestamps = $trendAnalysis['all_timestamps'];
$currentPeriod = $allTimestamps[0];
$previousPeriod = $allTimestamps[1];
$previousPeriod2 = $allTimestamps[2];
$periods = [$previousPeriod2, $previousPeriod, $currentPeriod];
$periods = $allTimestamps;
$reversedPeriods = array_reverse($periods);
$periodCount = count($periods);
$allUniqueTagsPerPeriod = array_map(function ($tags) {
return array_keys($tags);
}, $clusteredTags);
$allUniqueTags = array_unique(array_merge($allUniqueTagsPerPeriod[$currentPeriod], $allUniqueTagsPerPeriod[$previousPeriod], $allUniqueTagsPerPeriod[$previousPeriod2]));
$allUniqueTags = array_values(array_unique(Hash::extract($allUniqueTagsPerPeriod, '{n}.{n}')));
App::uses('ColourPaletteTool', 'Tools');
$paletteTool = new ColourPaletteTool();
$COLOR_PALETTE = $paletteTool->createColourPalette(max(count($allUniqueTags), 1));
@ -29,34 +30,34 @@ $trendColorMapping = [
'?' => '#999999',
];
$now = new DateTime();
$currentPeriodDate = DateTime::createFromFormat('U', $currentPeriod);
$previousPeriodDate = DateTime::createFromFormat('U', $previousPeriod);
$previousPeriod2Date = DateTime::createFromFormat('U', $previousPeriod2);
$colorForTags = [];
$chartData = [];
$maxValue = 0;
foreach ($allUniqueTags as $i => $tag) {
if (
!empty($clusteredTags[$previousPeriod2][$tag]['occurence']) ||
!empty($clusteredTags[$previousPeriod][$tag]['occurence']) ||
!empty($clusteredTags[$currentPeriod][$tag]['occurence'])
) {
$sumForTags = array_reduce($clusteredTags, function ($carry, $clusteredTagsPerPeriod) use ($tag) {
return $carry + ($clusteredTagsPerPeriod[$tag]['occurrence'] ?? 0);
}, 0);
if ($sumForTags > 0) {
$colorForTags[$tag] = $COLOR_PALETTE[$i];
$chartData[$tag] = [
$clusteredTags[$previousPeriod2][$tag]['occurence'] ?? 0,
$clusteredTags[$previousPeriod][$tag]['occurence'] ?? 0,
$clusteredTags[$currentPeriod][$tag]['occurence'] ?? 0,
];
$chartData[$tag] = array_values(array_map(function ($clusteredTagsPerPeriod) use ($tag) {
return $clusteredTagsPerPeriod[$tag]['occurrence'] ?? 0;
}, $clusteredTags));
$chartData[$tag] = array_reverse($chartData[$tag]);
$maxValue = max($maxValue, max($chartData[$tag]));
}
$colorForTags[$tag] = $COLOR_PALETTE[$i];
}
$canvasWidth = 600;
$canvasHeight = 150;
foreach (array_keys($chartData) as $tag) {
$lastIndex = count($chartData[$tag]) - 1;
$canvasSubWidth = $lastIndex;
$chartData[$tag][0] = [0, $canvasHeight - ($chartData[$tag][0] / $maxValue) * $canvasHeight];
$chartData[$tag][1] = [$canvasWidth / 2, $canvasHeight - ($chartData[$tag][1] / $maxValue) * $canvasHeight];
$chartData[$tag][2] = [$canvasWidth, $canvasHeight - ($chartData[$tag][2] / $maxValue) * $canvasHeight];
for ($i = 1; $i < $lastIndex; $i++) {
$chartData[$tag][$i] = [$canvasWidth * ($i / $canvasSubWidth), $canvasHeight - ($chartData[$tag][$i] / $maxValue) * $canvasHeight];
}
$chartData[$tag][$lastIndex] = [$canvasWidth, $canvasHeight - ($chartData[$tag][$lastIndex] / $maxValue) * $canvasHeight];
}
if (!function_exists('reduceTag')) {
@ -93,6 +94,17 @@ if (!function_exists('computeLinePositions')) {
}
}
if (!function_exists('getColorFromYlOrBr')) {
function getColorFromYlOrBr(float $min, float $max, float $value): string
{
$YlOrBrPalette = ["#fff7bc", "#fee391", "#fec44f", "#fe9929", "#ec7014", "#cc4c02", "#993404", "#662506"];
$valuePercent = $value / ($max - $min);
$paletteRatio = $valuePercent * count($YlOrBrPalette);
$paletteIndex = max(round($paletteRatio - 1), 0);
return $YlOrBrPalette[$paletteIndex];
}
}
?>
<div style="display: flex; column-gap: 20px; justify-content: space-around; margin-bottom: 40px;">
@ -101,19 +113,19 @@ if (!function_exists('computeLinePositions')) {
<tbody>
<tr>
<td><?= __('Period duration') ?></td>
<td><?= __('%s days', $currentPeriodDate->diff($now)->format('%a')); ?></td>
<td><?= __('%s days', DateTime::createFromFormat('U', $currentPeriod)->diff($now)->format('%a')); ?></td>
</tr>
<tr>
<td><?= __('Period number') ?></td>
<td><?= count($periods) - 1 ?></td>
</tr>
<tr>
<td><?= __('Starting period') ?></td>
<td><?= sprintf('%s', $currentPeriodDate->format('M d, o. (\W\e\e\k W)')); ?></td>
<td><?= sprintf('%s', DateTime::createFromFormat('U', $currentPeriod)->format('M d, o. (\W\e\e\k W)')); ?></td>
</tr>
<tr>
<td><?= __('Previous period') ?></td>
<td><?= sprintf('%s', $previousPeriodDate->format('M d, o. (\W\e\e\k W)')); ?></td>
</tr>
<tr>
<td><?= __('Period-2') ?></td>
<td><?= sprintf('%s', $previousPeriod2Date->format('M d, o. (\W\e\e\k W)')); ?></td>
<td><?= __('Last period') ?></td>
<td><?= sprintf('%s', DateTime::createFromFormat('U', $periods[$periodCount - 1])->format('M d, o. (\W\e\e\k W)')); ?></td>
</tr>
</tbody>
</table>
@ -125,12 +137,12 @@ if (!function_exists('computeLinePositions')) {
<div>
<span class="y-axis-label" style="<?= sprintf('left: %spx; top: %spx; transform: translate(-100%%, %s%%)', 0, 0, -25) ?>"><?= h($maxValue) ?></span>
<span class="y-axis-label" style="<?= sprintf('left: %spx; top: %spx; transform: translate(-100%%, %s%%)', 0, ($canvasHeight - 20) / 2, 0) ?>"><?= h(round($maxValue / 2, 2)) ?></span>
<span class="y-axis-label" style="<?= sprintf('left: %spx; top: %spx; transform: translate(-100%%, %s%%)', 0, ($canvasHeight - 20), 25) ?>"><?= 0 ?></span>
<span class="y-axis-label" style="<?= sprintf('left: %spx; top: %spx; transform: translate(-100%%, %s%%)', 0, ($canvasHeight - 20), 25) ?>">0</span>
</div>
</div>
<div class="canvas">
<?php foreach ($chartData as $tag => $coords) : ?>
<?php for ($i = 0; $i < 3; $i++) : ?>
<?php for ($i = 0; $i < count($periods); $i++) : ?>
<?php
$coord = $coords[$i];
$previousCoord = isset($coords[$i - 1]) ? $coords[$i - 1] : false;
@ -154,9 +166,9 @@ if (!function_exists('computeLinePositions')) {
<?php endforeach ?>
</div>
<div class="x-axis-container">
<span class="x-axis-label" style="<?= sprintf('left: %spx; top: %spx;', 0, 0) ?>"><?= __('Period-2') ?></span>
<span class="x-axis-label" style="<?= sprintf('left: %spx; top: %spx;', $canvasWidth / 2, 0) ?>"><?= __('Previous period') ?></span>
<span class="x-axis-label" style="<?= sprintf('left: %spx; top: %spx;', $canvasWidth, 0) ?>"><?= __('Starting period') ?></span>
<?php foreach ($reversedPeriods as $i => $period) : ?>
<span class="x-axis-label" style="<?= sprintf('left: %spx; top: %spx;', $i * $canvasWidth / $canvasSubWidth, 0) ?>"><?= DateTime::createFromFormat('U', $period)->format('M. d, o') ?></span>
<?php endforeach; ?>
</div>
</div>
</div>
@ -170,54 +182,24 @@ if (!function_exists('computeLinePositions')) {
<thead>
<tr>
<th></th>
<th>
<span>
<div><?= __('Period-2') ?></div>
<div style="font-weight: normal;"><?= __('%s events', h($clusteredEvents[$previousPeriod2])) ?></div>
</span>
<table class="trending-table-data">
<thead style="font-size: small;">
<tr>
<td>#</td>
<td>⥮</td>
<td>%</td>
<td></td>
</tr>
</thead>
</table>
</th>
<th>
<span>
<div><?= __('Previous period') ?></div>
<div style="font-weight: normal;"><?= __('%s events', h($clusteredEvents[$previousPeriod])) ?></div>
</span>
<table class="trending-table-data">
<thead style="font-size: small;">
<tr>
<td>#</td>
<td>⥮</td>
<td>%</td>
<td></td>
</tr>
</thead>
</table>
</th>
<th>
<span>
<div><?= __('Starting period') ?></div>
<div style="font-weight: normal;"><?= __('%s events', h($clusteredEvents[$currentPeriod])) ?></div>
</span>
<table class="trending-table-data">
<thead style="font-size: small;">
<tr>
<td>#</td>
<td>⥮</td>
<td>%</td>
<td></td>
</tr>
</thead>
</table>
</th>
<?php foreach ($reversedPeriods as $i => $period) : ?>
<th>
<span>
<div><?= DateTime::createFromFormat('U', $period)->format('M. d, o') ?></div>
<div style="font-weight: normal;"><?= __('%s events', h($clusteredEvents[$period])) ?></div>
</span>
<table class="trending-table-data">
<thead style="font-size: small;">
<tr>
<td title="<?= __('Occurrence per events') ?>">#</td>
<td title="<?= __('Raw change') ?>">⥮</td>
<td title="<?= __('Percent change') ?>">%</td>
<td></td>
</tr>
</thead>
</table>
</th>
<?php endforeach; ?>
</tr>
</thead>
<?php foreach ($tagFilterPrefixes as $tagPrefix) : ?>
@ -238,59 +220,34 @@ if (!function_exists('computeLinePositions')) {
<span class="tag-legend" style="background-color: <?= $colorForTags[$tagName] ?>;"></span>
<code><?= h(reduceTag($tagName, count(explode(':', $tagPrefix)))) ?></code>
</td>
<td>
<table class="table-condensed no-border trending-table-data">
<tbody>
<tr>
<td><?= h($clusteredTags[$previousPeriod2][$tagName]['occurence'] ?? '-') ?></td>
<td><?= h($clusteredTags[$previousPeriod2][$tagName]['raw_change'] ?? '-') ?></td>
<td><?= h($clusteredTags[$previousPeriod2][$tagName]['percent_change'] ?? '-') ?>%</td>
<td style="font-size: large; color: <?= $trendColorMapping[$clusteredTags[$previousPeriod2][$tagName]['change_sign'] ?? '?'] ?>"><?= $trendIconMapping[$clusteredTags[$previousPeriod2][$tagName]['change_sign'] ?? '?'] ?></td>
</tr>
</tbody>
</table>
</td>
<td>
<table class="table-condensed no-border trending-table-data">
<tbody>
<tr>
<td><?= h($clusteredTags[$previousPeriod][$tagName]['occurence'] ?? '-') ?></td>
<td><?= h($clusteredTags[$previousPeriod][$tagName]['raw_change'] ?? '-') ?></td>
<td><?= h($clusteredTags[$previousPeriod][$tagName]['percent_change'] ?? '-') ?>%</td>
<td style="font-size: large; color: <?= $trendColorMapping[$clusteredTags[$previousPeriod][$tagName]['change_sign'] ?? '?'] ?>"><?= $trendIconMapping[$clusteredTags[$previousPeriod][$tagName]['change_sign'] ?? '?'] ?></td>
</tr>
</tbody>
</table>
</td>
<td>
<table class="table-condensed no-border trending-table-data">
<tbody>
<tr>
<td><?= h($clusteredTags[$currentPeriod][$tagName]['occurence'] ?? '-') ?></td>
<td><?= h($clusteredTags[$currentPeriod][$tagName]['raw_change'] ?? '-') ?></td>
<td><?= h($clusteredTags[$currentPeriod][$tagName]['percent_change'] ?? '-') ?>%</td>
<td style="font-size: large; color: <?= $trendColorMapping[$clusteredTags[$currentPeriod][$tagName]['change_sign'] ?? '?'] ?>"><?= $trendIconMapping[$clusteredTags[$currentPeriod][$tagName]['change_sign'] ?? '?'] ?></td>
</tr>
</tbody>
</table>
</td>
<?php foreach ($reversedPeriods as $i => $period) : ?>
<td>
<table class="table-condensed no-border trending-table-data">
<tbody>
<tr>
<td title="<?= __('Occurrence per events') ?>"><?= h($clusteredTags[$period][$tagName]['occurrence'] ?? '-') ?></td>
<td title="<?= __('Raw change') ?>"><?= h($clusteredTags[$period][$tagName]['raw_change'] ?? '-') ?></td>
<td title="<?= __('Percent change') ?>"><?= h($clusteredTags[$period][$tagName]['percent_change'] ?? '-') ?>%</td>
<?php if ($i > 0) : ?>
<td title="<?= __('Evolution') ?>" style="font-size: large; color: <?= $trendColorMapping[$clusteredTags[$period][$tagName]['change_sign'] ?? '?'] ?>"><?= $trendIconMapping[$clusteredTags[$period][$tagName]['change_sign'] ?? '?'] ?></td>
<?php endif; ?>
</tr>
</tbody>
</table>
</td>
<?php endforeach; ?>
</tr>
<td style="padding: 0;"></td>
<td colspan="3" style="padding: 0;">
<td colspan="<?= count($periods) ?>" style="padding: 0;">
<?php
$low = '#fee8c8';
$medium = '#f09c8f';
$high = '#bc2f1a';
$periodColorRatio = $clusteredTags[$currentPeriod][$tagName]['occurence'] / $maxValue;
$colorGradient = [];
foreach ($periods as $i => $period) {
$ratio = ($clusteredTags[$period][$tagName]['occurence'] ?? 0) / $maxValue;
$color = $ratio <= 0.33 ? $low : ($ratio >= 0.66 ? $high : $medium);
$length = 100 * $i / (count($periods) - 1);
$colorGradient[] = sprintf('%s %s%%', $color, $length);
}
$colorGradient = [];
foreach ($reversedPeriods as $i => $period) {
$color = getColorFromYlOrBr(0, $maxValue, $clusteredTags[$period][$tagName]['occurrence'] ?? 0);
$length = 100 * $i / (count($periods) - 1);
$length = $i > 0 ? $length - 5 : $length; // Small offset to better align colors on the table period header
$colorGradient[] = sprintf('%s %s%%', $color, $length);
}
?>
<div class="heatbar" style="background: <?= sprintf('linear-gradient(90deg, %s);', implode(', ', $colorGradient)) ?>;"></div>
</td>
<?php endforeach; ?>
@ -303,6 +260,7 @@ if (!function_exists('computeLinePositions')) {
table.trending-table table.trending-table-data {
width: 150px;
}
table.trending-table th:not(:first-child) {
width: 150px;
}
@ -369,7 +327,7 @@ if (!function_exists('computeLinePositions')) {
padding-left: inherit;
}
.y-axis-container > div {
.y-axis-container>div {
position: relative;
height: 100%;
}
@ -389,7 +347,7 @@ if (!function_exists('computeLinePositions')) {
}
.heatbar {
height: 3px;
width: calc(100% - 10px);
height: 3px;
width: calc(100% - 10px);
}
</style>

222
app/View/Emails/notification_common.ctp
View File

@ -9,6 +9,7 @@
* - `detailed-summary-type`
* - `detailed-summary-tags`
* - `detailed-summary-events`
* - `detailed-summary-correlations`
* - `aggregated-context`
*
* Additional variables:
@ -21,6 +22,8 @@ if (empty($this->__vars)) {
$default_vars = [
'event_table_include_basescore' => true,
'event_table_max_event_count' => 30,
'correlation_table_advanced_ui' => 10,
'correlation_table_max_count' => 50,
'additional_taxonomy_event_list' => [
'PAP' => 'PAP:'
],
@ -28,7 +31,7 @@ $default_vars = [
$vars = array_merge($default_vars, $this->__vars);
$now = new DateTime();
$start_date = new DateTime('7 days ago');
$start_date = new DateTime($period_days . ' days ago');
$event_number = count($events);
$attribute_number = 0;
$object_number = 0;
@ -49,6 +52,8 @@ $mitre_galaxy_tag_prefix = 'misp-galaxy:mitre-attack-pattern="';
$reportLink = sprintf('%s/users/viewPeriodicSummary/%s', $baseurl, $period);
$eventLink = sprintf('%s/events/index/searchpublished:1/searchPublishTimestamp:%s/searchPublishTimestamp:%s', $baseurl, h($start_date->format('Y-m-d H:i:s')), h($now->format('Y-m-d H:i:s')));
$processed_correlations = [];
$new_correlations = [];
foreach ($events as $event) {
$unique_tag_per_event = [];
$attribute_number += count($event['Attribute']);
@ -56,11 +61,10 @@ foreach ($events as $event) {
$event_report_number += count($event['EventReport']);
$proposal_number += count($event['ShadowAttribute']);
foreach ($event['EventTag'] as $event_tag) {
$tag = $event_tag['Tag'];
if (!empty($unique_tag_per_event[$tag['name']])) {
if (isset($unique_tag_per_event[$tag['name']])) {
continue; // Only one instance of tag per event
}
$unique_tag_per_event[$tag['name']] = true;
@ -69,24 +73,29 @@ foreach ($events as $event) {
$all_tag_amount[$tag['name']] = 0;
$tag_color_mapping[$tag['name']] = $tag['colour'];
}
$all_tag_amount[$tag['name']] += 1;
$all_tag_amount[$tag['name']]++;
if (!empty($tag['is_galaxy']) && substr($tag['name'], 0, strlen($mitre_galaxy_tag_prefix)) === $mitre_galaxy_tag_prefix) {
if ($tag['is_galaxy'] && substr($tag['name'], 0, strlen($mitre_galaxy_tag_prefix)) === $mitre_galaxy_tag_prefix) {
$technique = substr($tag['name'], strlen($mitre_galaxy_tag_prefix), strlen($tag['name']) - strlen($mitre_galaxy_tag_prefix) - 1);
$mitre_attack_techniques[$technique] = $event_tag;
}
}
$attribute_light_by_id = [];
foreach ($event['Attribute'] as $attribute) {
$attribute_light_by_id[$attribute['id']] = [
'timestamp' => $attribute['timestamp'],
'type' => $attribute['type'],
];
if (empty($attribute_types[$attribute['type']])) {
$attribute_types[$attribute['type']] = 0;
}
$attribute_types[$attribute['type']] += 1;
$attribute_types[$attribute['type']]++;
foreach ($attribute['AttributeTag'] as $attribute_tag) {
$tag = $attribute_tag['Tag'];
if (!empty($unique_tag_per_event[$tag['name']])) {
if (isset($unique_tag_per_event[$tag['name']])) {
continue; // Only one instance of tag per event
}
$unique_tag_per_event[$tag['name']] = true;
@ -95,9 +104,9 @@ foreach ($events as $event) {
$all_tag_amount[$tag['name']] = 0;
$tag_color_mapping[$tag['name']] = $tag['colour'];
}
$all_tag_amount[$tag['name']] += 1;
$all_tag_amount[$tag['name']]++;
if (!empty($tag['is_galaxy']) && substr($tag['name'], 0, strlen($mitre_galaxy_tag_prefix)) === $mitre_galaxy_tag_prefix) {
if ($tag['is_galaxy'] && substr($tag['name'], 0, strlen($mitre_galaxy_tag_prefix)) === $mitre_galaxy_tag_prefix) {
$technique = substr($tag['name'], strlen($mitre_galaxy_tag_prefix), strlen($tag['name']) - strlen($mitre_galaxy_tag_prefix) - 1);
$mitre_attack_techniques[$technique] = $attribute_tag;
}
@ -108,19 +117,23 @@ foreach ($events as $event) {
if (empty($object_types[$object['name']])) {
$object_types[$object['name']] = 0;
}
$object_types[$object['name']] += 1;
$object_types[$object['name']]++;
$attribute_number += count($object['Attribute']);
foreach ($object['Attribute'] as $attribute) {
$attribute_light_by_id[$attribute['id']] = [
'timestamp' => $attribute['timestamp'],
'type' => $attribute['type'],
];
if (empty($attribute_types[$attribute['type']])) {
$attribute_types[$attribute['type']] = 0;
}
$attribute_types[$attribute['type']] += 1;
$attribute_types[$attribute['type']]++;
foreach ($attribute['AttributeTag'] as $attribute_tag) {
$tag = $attribute_tag['Tag'];
if (!empty($unique_tag_per_event[$tag['name']])) {
if (isset($unique_tag_per_event[$tag['name']])) {
continue; // Only one instance of tag per event
}
$unique_tag_per_event[$tag['name']] = true;
@ -129,9 +142,9 @@ foreach ($events as $event) {
$all_tag_amount[$tag['name']] = 0;
$tag_color_mapping[$tag['name']] = $tag['colour'];
}
$all_tag_amount[$tag['name']] += 1;
$all_tag_amount[$tag['name']]++;
if (!empty($tag['is_galaxy']) && substr($tag['name'], 0, strlen($mitre_galaxy_tag_prefix)) === $mitre_galaxy_tag_prefix) {
if ($tag['is_galaxy'] && substr($tag['name'], 0, strlen($mitre_galaxy_tag_prefix)) === $mitre_galaxy_tag_prefix) {
$technique = substr($tag['name'], strlen($mitre_galaxy_tag_prefix), strlen($tag['name']) - strlen($mitre_galaxy_tag_prefix) - 1);
$mitre_attack_techniques[$technique] = $attribute_tag;
}
@ -147,6 +160,33 @@ foreach ($events as $event) {
'event_info' => $event['Event']['info'],
];
}
if (!empty($event['RelatedEvent'])) {
$related_event_by_id = [];
foreach ($event['RelatedEvent'] as $related_event) {
$related_event_by_id[$related_event['Event']['id']] = $related_event['Event'];
}
foreach ($event['RelatedAttribute'] as $attribute_id => $related_attributes) {
$has_attribute_been_modified_since_last_period = intval($attribute_light_by_id[$attribute_id]['timestamp']) >= intval($start_date->format('U'));
foreach ($related_attributes as $related_attribute) {
$correlation_id = sprintf('%s-%s', $related_attribute['attribute_id'], $attribute_id);
$reversed_correlation_id = sprintf('%s-%s', $attribute_id, $related_attribute['attribute_id']);
$has_correlation_been_processed = isset($processed_correlations[$correlation_id]); // We already added the correlation the other way around
if ($has_attribute_been_modified_since_last_period && !$has_correlation_been_processed) {
$source_event = $event['Event'];
$source_event['Orgc'] = $event['Orgc'];
$new_correlations[] = [
'source_event' => $source_event,
'target_event' => $related_event_by_id[$related_attribute['id']],
'attribute_value' => $related_attribute['value'],
'attribute_type' => $attribute_light_by_id[$attribute_id]['type'],
];
$processed_correlations[$reversed_correlation_id] = true;
}
}
}
}
}
if (!function_exists('findAndBuildTag')) {
@ -161,12 +201,14 @@ if (!function_exists('findAndBuildTag')) {
}
}
$unique_tag_number = count(array_keys($all_tag_amount));
$unique_tag_number = count($all_tag_amount);
arsort($attribute_types);
arsort($object_types);
arsort($all_tag_amount);
arsort($mitre_attack_techniques);
uasort($mitre_attack_techniques, function($tag1, $tag2) use ($all_tag_amount) {
return ($all_tag_amount[$tag1['Tag']['name']] < $all_tag_amount[$tag2['Tag']['name']]) ? 1 : -1;
});
array_splice($attribute_types, 10);
array_splice($object_types, 10);
@ -180,7 +222,7 @@ array_splice($mitre_attack_techniques, 10);
<?php if ($this->fetch('table-overview')) : ?>
<?= $this->fetch('table-overview'); ?>
<?php else : ?>
<?php else: ?>
<div class="panel">
<div class="panel-header">
<?= __('Data at a glance') ?>
@ -190,18 +232,16 @@ array_splice($mitre_attack_techniques, 10);
<tbody>
<tr>
<td><?= __('Summary period') ?></td>
<td><?= h($period) ?></td>
<td><?= h(ucfirst($period)) ?></td>
</tr>
<tr>
<td><?= __('Summary for dates') ?></td>
<td>
<?=
sprintf('<strong>%s</strong> (Week %s) ➞ <strong>%s</strong> (Week %s)',
<?= __('<strong>%s</strong> (Week %s) ➞ <strong>%s</strong> (Week %s)',
$start_date->format('M d, o'),
$start_date->format('W'),
$now->format('M d, o'),
$now->format('W'),
$start_date->format('M d, o')
$now->format('W')
)
?>
</td>
@ -211,7 +251,7 @@ array_splice($mitre_attack_techniques, 10);
<td><?= date("c"); ?></td>
</tr>
<tr>
<td><?= __('Events #') ?></td>
<td><?= __('Published Events #') ?></td>
<td><?= $event_number ?></td>
</tr>
<tr>
@ -234,6 +274,12 @@ array_splice($mitre_attack_techniques, 10);
<td><?= __('Unique tags #') ?></td>
<td><?= $unique_tag_number ?></td>
</tr>
<?php if (!empty($periodicSettings['include_correlations'])): ?>
<tr>
<td><?= __('New correlation #') ?></td>
<td><?= count($new_correlations) ?></td>
</tr>
<?php endif; ?>
</tbody>
</table>
⮞ <a href="<?= h($reportLink) ?>"><?= __('View this report in MISP') ?></a>
@ -243,7 +289,7 @@ array_splice($mitre_attack_techniques, 10);
<?php if ($this->fetch('detailed-summary-full')) : ?>
<?= $this->fetch('detailed-summary-full'); ?>
<?php else : ?>
<?php else: ?>
<div class="panel">
<div class="panel-header">
<?= __('Detailed summary') ?>
@ -251,9 +297,9 @@ array_splice($mitre_attack_techniques, 10);
<div class="panel-body">
<?php if ($this->fetch('detailed-summary-mitre-attack')) : ?>
<?= $this->fetch('detailed-summary-mitre-attack'); ?>
<?php else : ?>
<?php else: ?>
<?php if (!empty($mitre_attack_techniques)) : ?>
<h4><?= __('Top 10 Mitre Att&ck techniques') ?></h4>
<h4><?= __('Top 10 MITRE ATT&CK techniques') ?></h4>
<ul>
<?php foreach ($mitre_attack_techniques as $technique => $tag) : ?>
<li>
@ -272,7 +318,7 @@ array_splice($mitre_attack_techniques, 10);
<?php if ($this->fetch('detailed-summary-type')) : ?>
<?= $this->fetch('detailed-summary-type'); ?>
<?php else : ?>
<?php else: ?>
<?php if (!empty($attribute_types)) : ?>
<h4><?= __('Top 10 Attribute types') ?></h4>
<ul>
@ -307,10 +353,10 @@ array_splice($mitre_attack_techniques, 10);
<?php if ($this->fetch('detailed-summary-tags')) : ?>
<?= $this->fetch('detailed-summary-tags'); ?>
<?php else : ?>
<?php else: ?>
<h4><?= __('Top 10 Tags') ?></h4>
<ul>
<?php foreach ($all_tag_amount as $tag_name => $amount) : ?>
<?php array_splice($all_tag_amount, 10); foreach ($all_tag_amount as $tag_name => $amount) : ?>
<li>
<span class="tag" style="background-color: #999; color: #fff; border-radius: 9px; padding: 2px 8px;">
<?= $amount ?>
@ -323,7 +369,7 @@ array_splice($mitre_attack_techniques, 10);
<?php if ($this->fetch('detailed-summary-events')) : ?>
<?= $this->fetch('detailed-summary-events'); ?>
<?php else : ?>
<?php else: ?>
<?php if (!empty($events)) : ?>
<h4><?= __('Event list') ?> <small style="color: #999999;"><?= sprintf(' (%s)', count($events)) ?></small></h4>
<table class="table table-condensed">
@ -380,7 +426,7 @@ array_splice($mitre_attack_techniques, 10);
</tr>
<?php endforeach; ?>
</table>
<?php else : ?>
<?php else: ?>
&nbsp;
<?php endif; ?>
</td>
@ -390,7 +436,7 @@ array_splice($mitre_attack_techniques, 10);
<?php endforeach; ?>
</tbody>
</table>
<?php else : ?>
<?php else: ?>
<p><?= __('No events.') ?></p>
<?php endif; ?>
<?php if (count($events) > $vars['event_table_max_event_count']) : ?>
@ -405,14 +451,83 @@ array_splice($mitre_attack_techniques, 10);
<a href="<?= h($eventLink) ?>"><?= __('View all events in MISP') ?></a>
<?php endif; ?>
<?php endif; ?>
<?php if ($this->fetch('detailed-summary-correlations')) : ?>
<?php else: ?>
<?php if (!empty($new_correlations)) : ?>
<h4><?= __('New correlations') ?><small style="color: #999999;"><?= sprintf(' (%s)', count($new_correlations)) ?></small></h4>
<div>
<?php if (count($new_correlations) < $vars['correlation_table_advanced_ui']) : ?>
<?php foreach ($new_correlations as $correlation): ?>
<div style="display: flex; flex-wrap: nowrap; align-items: center; margin-top: 0.5em;">
<span>
<span class="correlating-event-container">
<span>
<a href="<?= sprintf('%s/events/view/%s', $baseurl, h($correlation['source_event']['id'])) ?>"><?= h($correlation['source_event']['info']) ?></a>
</span>
<span class="org-date">
<span><?= h($correlation['source_event']['date']) ?></span>
<span><?= h($correlation['source_event']['Orgc']['name']) ?></span>
</span>
</span>
</span>
<span class="correlating-attribute-container">
<span class="correlating-attribute">
<?= h($correlation['attribute_type']); ?> :: <b><?= h($correlation['attribute_value']) ?></b>
</span>
</span>
<span>
<span class="correlating-event-container">
<span>
<a href="<?= sprintf('%s/events/view/%s', $baseurl, h($correlation['target_event']['id'])) ?>"><?= h($correlation['target_event']['info']) ?></a>
</span>
<span class="org-date">
<span><?= h($correlation['target_event']['date']) ?></span>
<span><?= h($correlation['target_event']['Orgc']['name']) ?></span>
</span>
</span>
</span>
</div>
<?php endforeach; ?>
<?php else: ?>
<table class="table table-xcondensed">
<thead>
<tr>
<th><?= __('First event info') ?></th>
<th><?= __('Value') ?></th>
<th><?= __('Second event info') ?></th>
</tr>
</thead>
<tbody>
<?php foreach (array_slice($new_correlations, 0, $vars['correlation_table_max_count']) as $correlation): ?>
<tr>
<td><a href="<?= sprintf('%s/events/view/%s', $baseurl, h($correlation['source_event']['id'])) ?>"><?= h($correlation['source_event']['info']) ?></a></td>
<td><b><?= h($correlation['attribute_value']) ?></b></td>
<td><a href="<?= sprintf('%s/events/view/%s', $baseurl, h($correlation['target_event']['id'])) ?>"><?= h($correlation['target_event']['info']) ?></a></td>
</tr>
<?php endforeach; ?>
</tbody>
</table>
<?php
if (count($new_correlations) > $vars['correlation_table_max_count']) {
echo '⮞ ' . __n(
'%s correlation not displayed.',
'%s correlations not displayed.',
count($new_correlations) - $vars['correlation_table_max_count'],
sprintf('<strong>%s</strong>', count($new_correlations) - $vars['correlation_table_max_count'])
);
}
?>
<?php endif; ?>
</div>
<?php endif; ?>
<?php endif; ?>
</div>
</div>
<?php endif; // detailed-summary-full
?>
<?php if ($this->fetch('trending-summary')) : ?>
<?= $this->fetch('trending-summary'); ?>
<?php else : ?>
<?php else: ?>
<div class="panel">
<div class="panel-header">
<?= __('Tag trendings') ?>
@ -425,7 +540,7 @@ array_splice($mitre_attack_techniques, 10);
<?php if ($this->fetch('aggregated-context')) : ?>
<?= $this->fetch('aggregated-context'); ?>
<?php else : ?>
<?php else: ?>
<div class="panel">
<div class="panel-header">
<?= __('Context summary') ?>
@ -488,6 +603,41 @@ array_splice($mitre_attack_techniques, 10);
border-radius: 3px;
}
.correlating-attribute {
padding: 3px 5px;
border: 1px solid #ccc;
border-radius: 3px;
white-space: nowrap;
}
.correlating-attribute-container {
display: flex;
box-sizing: border-box;
margin: 0 0;
align-items: center;
min-width: 400px;
}
.correlating-attribute-container::before,
.correlating-attribute-container::after {
display: inline-block;
content: ' ';
height: 2px;
width: 100%;
background-color: #ccc;
}
.correlating-event-container {
display: flex;
flex-direction: column;
min-width: 180px;
border: 1px solid #ccc;
border-radius: 3px;
padding: 3px 5px;
}
.correlating-event-container > .org-date {
display: flex;
justify-content: space-between;
}
.no-overflow {
display: inline-block;
white-space: nowrap;

2
app/View/Events/index.ctp
View File

@ -44,7 +44,7 @@
'proposals' => __('Proposals'),
'discussion' => __('Posts'),
'report_count' => __('Report count'),
'timestamp' => __('Last change at'),
'timestamp' => __('Last modified at'),
'publish_timestamp' => __('Published at')
];

17
app/View/Events/module_views/context_view.ctp
View File

@ -48,29 +48,28 @@
<h2><?= __('Galaxy Clusters') ?></h2>
<div>
<?php
$htmlClusters = '';
foreach ($clusters as $tagname => $entries) {
$htmlClusters .= sprintf(
echo sprintf(
'<div><h4>%s %s</h4></div>',
sprintf('<i class="%s"></i>', $this->FontAwesome->getClass($entries[0]['Galaxy']['icon'])),
h($entries[0]['Galaxy']['name'])
);
if (!empty($entries[0]['Galaxy']['description'])) {
$htmlClusters .= sprintf('<div><i>%s</i></div>', h($entries[0]['Galaxy']['description']));
echo sprintf('<div><i>%s</i></div>', h($entries[0]['Galaxy']['description']));
}
$htmlClusters .= '<ul>';
echo '<ul>';
foreach ($entries as $cluster) {
$htmlClusters .= sprintf(
$description = $this->Markdown->toText($cluster['GalaxyCluster']['description']);
echo sprintf(
'<li><strong><a href="%s" target="_blank">%s</a></strong></li> %s',
$baseurl . '/galaxy_clusters/view/' . h($cluster['GalaxyCluster']['id']),
h($cluster['GalaxyCluster']['value']),
strlen(h($cluster['GalaxyCluster']['description'])) > 300 ?
(substr(h($cluster['GalaxyCluster']['description']), 0, 300) . '...') : h($cluster['GalaxyCluster']['description'])
strlen($description) > 300 ?
(h(mb_substr($description, 0, 300)) . '...') : h($description)
);
}
$htmlClusters .= '</ul>';
echo '</ul>';
}
echo $htmlClusters;
?>
</div>
<?php endif; ?>

8
app/View/ObjectReferences/ajax/add.ctp
View File

@ -13,7 +13,7 @@
'div' => false
));
?>
<div id="" class="hidden">
<div class="hidden">
<label for="ObjectReferenceRelationshipTypeSelect"><?php echo __('Relationship type');?></label>
<?php
echo $this->Form->input('relationship_type', array(
@ -44,7 +44,7 @@
'style' => 'width:320px;'
));
?>
<br />
<br>
<?php
$items = array();
@ -115,7 +115,6 @@
);
echo $this->element('generic_picker', array('items' => $items, 'options' => $options));
?>
</div>
<div class="span6">
<label for="selectedData"><?php echo __('Target Details');?></label>
@ -146,7 +145,7 @@
</div>
<script type="text/javascript">
var targetEvent = <?php echo json_encode($event); ?>;
$(document).ready(function() {
$(function() {
$('#ObjectReferenceReferencedUuid').on('input', function() {
objectReferenceInput();
});
@ -160,4 +159,3 @@
$('#ObjectReferenceRelationshipTypeSelect').chosen({ width: "100%" });
});
</script>
<?php echo $this->Js->writeBuffer(); // Write cached scripts

36
app/View/Objects/group_attributes_into_object.ctp
View File

@ -59,22 +59,22 @@ echo $this->Form->create('Object', array('url' => $baseurl . '/objects/groupAttr
<th><?php echo __('Distribution'); ?></th>
</tr>
</thead>
<tbody id='attributeMappingBody'>
<tbody id="attributeMappingBody">
<?php foreach ($attributes as $attribute): ?>
<tr>
<td id="isAttributeId"><?php echo h($attribute['Attribute']['id']); ?></td>
<td class="attributeId"><?= intval($attribute['Attribute']['id']); ?></td>
<td>
<span style="display: block;">
<select id="isAttributeMapping" style="margin-bottom: 5px;" onchange="updateObjectRelationDescription(this);">
<select class="attributeMapping" style="margin-bottom: 5px;">
<?php foreach ($object_relations[$attribute['Attribute']['type']] as $object_relation): ?>
<option value="<?php echo h($object_relation['object_relation']); ?>" title="<?php echo h($object_relation['description']); ?>"><?php echo h($object_relation['object_relation']); ?></option>
<?php endforeach; ?>
</select>
:: <?php echo h($attribute['Attribute']['type']); ?>
</span>
<i id="objectRelationDescription" class="apply_css_arrow"><?php echo h($object_relations[$attribute['Attribute']['type']][0]['description']); ?></i>
<i class="objectRelationDescription apply_css_arrow"><?php echo h($object_relations[$attribute['Attribute']['type']][0]['description']); ?></i>
</td>
<td style="min-width: 75px;"><?php echo h(date('Y-m-d', $attribute['Attribute']['timestamp'])); ?></td>
<td style="min-width: 75px;"><?= $this->Time->date($attribute['Attribute']['timestamp']); ?></td>
<td><?php echo h($attribute['Attribute']['category']); ?></td>
<td style="white-space: nowrap;"><?php echo h($attribute['Attribute']['value']); ?></td>
<td><?php echo h($distributionLevels[$attribute['Attribute']['distribution']]); ?></td>
@ -126,7 +126,7 @@ echo $this->Form->create('Object', array('url' => $baseurl . '/objects/groupAttr
<button class="btn btn-primary" onclick="submitMergeAttributeIntoObjectForm(this);"><?php echo __('Merge above Attributes into an Object'); ?></button>
</div>
<span class="red bold" data-original-title="" title="">
<?php echo sprintf(__('Selected Attributes will be %s deleted'), '<strong style="font-size: medium">' . ($hard_delete_attribute ? __('hard') : __('soft')) . '</strong>'); ?>
<?php echo __('Selected Attributes will be %s deleted', '<strong style="font-size: medium">' . ($hard_delete_attribute ? __('hard') : __('soft')) . '</strong>'); ?>
</span>
</div>
@ -135,25 +135,31 @@ $(".Object_distribution_select").change(function() {
checkAndEnable($(this).parent().find('.Object_sharing_group_id_select'), $(this).val() == 4);
});
$(".attributeMapping").change(function() {
var $select = $(this);
var text = $select.find(":selected").attr('title');
$select.parent().parent().find('.objectRelationDescription').text(text);
});
function submitMergeAttributeIntoObjectForm(btn) {
var $btn = $(btn);
var $form = $('#ObjectGroupAttributesIntoObjectForm');
var attribute_mapping = {};
$('#attributeMappingBody').find('tr').each(function() {
var $tr = $(this);
var attr_id = $tr.find('#isAttributeId').text();
var attr_mapping = $tr.find('#isAttributeMapping').val();
var attr_id = $tr.find('.attributeId').text();
var attr_mapping = $tr.find('.attributeMapping').val();
attribute_mapping[attr_id] = attr_mapping;
});
$('#ObjectSelectedObjectRelationMapping').val(JSON.stringify(attribute_mapping));
var btn_text_backup = '';
$.ajax({
data: $form.serialize(),
beforeSend: function (XMLHttpRequest) {
beforeSend: function () {
btn_text_backup = $btn.text();
$btn.html('<it class="fa fa-spinner fa-spin"></it>');
},
success:function (data, textStatus) {
success: function (data) {
if (data.errors !== undefined) {
showMessage('fail', responseArray.errors);
$btn.text(btn_text_backup);
@ -162,18 +168,12 @@ function submitMergeAttributeIntoObjectForm(btn) {
location.reload();
}
},
error:function() {
error: function() {
showMessage('fail', 'Could not merge Attributes into an Object.');
showObjectProposition();
},
type:"post",
type: "post",
url: $form.attr('action')
});
}
function updateObjectRelationDescription(changed) {
var $select = $(changed);
var text = $select.find(":selected").attr('title');
$select.parent().parent().find('#objectRelationDescription').text(text);
}
</script>

4
app/View/Objects/propose_objects_from_attributes.ctp
View File

@ -15,7 +15,7 @@
<th><?php echo __('Object name'); ?></th>
<th><?php echo __('Category'); ?></th>
<th><?php echo __('Description'); ?></th>
<th title="<?php echo __('Compatiblity or Attribute type missing from the selection'); ?>"><?php echo __('Compatiblity'); ?></th>
<th title="<?php echo __('Compatibility or Attribute type missing from the selection'); ?>"><?php echo __('Compatibility'); ?></th>
</tr>
</thead>
<tbody>
@ -28,7 +28,7 @@
<td><?php echo h($potential_template['ObjectTemplate']['meta-category']); ?></td>
<?php
$v = h($potential_template['ObjectTemplate']['description']);
$v = strlen($v) > 100 ? substr($v, 0, 100) . '...' : $v;
$v = strlen($v) > 100 ? mb_substr($v, 0, 100) . '&hellip;' : $v;
?>
<td style="max-width: 500px;" title="<?php echo h($potential_template['ObjectTemplate']['description']); ?>">
<?php echo $v; ?>

17
app/View/UserSettings/set_setting.ctp
View File

@ -10,7 +10,7 @@
array(
'div' => 'clear',
'class' => 'input input-xxlarge',
'options' => array($users),
'options' => $users,
'disabled' => count($users) === 1
)
),
@ -40,18 +40,16 @@
?>
<script type="text/javascript">
var validSettings = <?= json_encode($validSettings); ?>;
$(function() {
loadUserSettingValue();
changeUserSettingPlaceholder();
$('#UserSettingSetting').on('change', function() {
loadUserSettingValue();
changeUserSettingPlaceholder();
});
$('#UserSettingUserId').on('change', function() {
$('#UserSettingSetting, #UserSettingUserId').on('change', function() {
loadUserSettingValue();
changeUserSettingPlaceholder();
});
});
function loadUserSettingValue() {
var user_id = $('#UserSettingUserId').val();
var setting = $('#UserSettingSetting').val();
@ -66,6 +64,13 @@
data = JSON.stringify(data, undefined, 4);
}
$('#UserSettingValue').val(data);
},
error: function (xhr) {
if (xhr.status === 404) {
$('#UserSettingValue').val('');
} else {
xhrFailCallback(xhr);
}
}
});
}

3
app/View/Users/admin_add.ctp
View File

@ -38,7 +38,8 @@
$passwordPopover = '<span class="blue bold">' . __('Minimal length') . '</span>: ' . h($length) . '<br>';
$passwordPopover .= '<span class="blue bold">' . __('Complexity') . '</span>: ' . h($complexity);
echo $this->Form->input('password', array(
'label' => __('Password') . ' <span id="PasswordPopover" data-content="' . h($passwordPopover) . '" class="fas fa-info-circle"></span>'
'label' => __('Password') . ' <span id="PasswordPopover" data-content="' . h($passwordPopover) . '" class="fas fa-info-circle"></span>',
'autocomplete' => 'new-password'
));
echo $this->Form->input('confirm_password', array('type' => 'password', 'div' => array('class' => 'input password required')));
?>

3
app/View/Users/admin_edit.ctp
View File

@ -46,7 +46,8 @@
$passwordPopover = '<span class="blue bold">' . __('Length') .'</span>: ' . h($length) . '<br>';
$passwordPopover .= '<span class="blue bold">' . __('Complexity') .'</span>: ' . h($complexity);
echo $this->Form->input('password', array(
'label' => __('Password') . ' <span id="PasswordPopover" data-content="' . h($passwordPopover) .'" class="fas fa-info-circle"></span>'
'label' => __('Password') . ' <span id="PasswordPopover" data-content="' . h($passwordPopover) .'" class="fas fa-info-circle"></span>',
'autocomplete' => 'new-password'
));
echo $this->Form->input('confirm_password', array('type' => 'password', 'div' => array('class' => 'input password required')));
?>

12
app/View/Users/notification_settings.ctp
View File

@ -67,6 +67,18 @@ echo $this->element('genericElements/Form/genericForm', [
'type' => 'textarea',
'placeholder' => '["misp-galaxy:mitre-attack-pattern", "admiralty-scale"]',
],
[
'field' => 'periodic_settings.trending_period_amount',
'label' => __('Trending Period Amount'),
'class' => 'span6',
'type' => 'number',
],
[
'field' => 'periodic_settings.include_correlations',
'label' => __('Include events correlations'),
'default' => 0,
'type' => 'checkbox'
],
],
'submit' => [
'action' => $this->request->params['action'],

2
app/files/community-metadata/defaults.json
View File

@ -183,7 +183,7 @@
"uuid": "53a830e6-8d07-4dd8-98d5-18cc4e66fc8c",
"org_uuid": "",
"org_name": "",
"description": "The ultimate goal of the project is to develop a NATO capability, available to all NATO nations, through which nations commit to sharing their information. ",
"description": "The ultimate goal of the project is to develop a NATO capability, available to all NATO nations, through which nations commit to sharing their information. This community is only open for official government entities, sponsored by their nation representative in the NATO Multinational MISP Steering Board.",
"url": "https://misp.ncirc.nato.int",
"sector": "Governmental",
"nationality": "International",

2
app/files/misp-galaxy

@ -1 +1 @@
Subproject commit 1c8d82cfcc6ca14791d2c3311181170449de19dc
Subproject commit eacab6ca27e1d1996bb28b7c617943052a41e3fd

2
app/files/taxonomies

@ -1 +1 @@
Subproject commit 7d379245b7a62831cc4b5d32e73e1f0d923f2624
Subproject commit 57b125782cd372c8a762db7ce34f8a405752c6c1

2
app/files/warninglists

@ -1 +1 @@
Subproject commit 81d122e2df112888f93fd07f7a006e1ee432532b
Subproject commit 1b026ee5115e5a6c7fda4cb7a6032c01e6f69a9c

4
app/webroot/js/jquery.js vendored
View File

File diff suppressed because one or more lines are too long

98
app/webroot/js/misp.js
View File

@ -13,6 +13,10 @@ if (!String.prototype.startsWith) {
}
function escapeHtml(unsafe) {
if (typeof unsafe === "boolean" || typeof unsafe === "number") {
return unsafe;
}
return unsafe
.replace(/&/g, "&amp;")
.replace(/</g, "&lt;")
@ -52,7 +56,9 @@ function rgb2hex(rgb) {
}
function xhrFailCallback(xhr) {
if (xhr.status === 401) {
if (xhr.status === 0) {
showMessage('fail', 'Something went wrong – server is not responding.');
} if (xhr.status === 401) {
showMessage('fail', 'Unauthorized. Please reload page to log again.');
} else if (xhr.status === 403 || xhr.status === 405) {
showMessage('fail', 'Not allowed.');
@ -425,7 +431,7 @@ function eventUnpublish() {
$('.notPublished').show();
}
function updateIndex(id, context) {
function updateIndex(id, context, callback) {
var url, div;
if (context === 'event') {
if (typeof currentUri == 'undefined') {
@ -440,8 +446,12 @@ function updateIndex(id, context) {
}
xhr({
dataType: "html",
success:function (data) {
success: function (data) {
$(div).html(data);
if (typeof callback !== "undefined") {
callback("success");
}
if (typeof genericPopupCallback !== "undefined") {
genericPopupCallback("success");
} else {
@ -1275,10 +1285,8 @@ function submitPopoverForm(context_id, referer, update_context_id, modal, popove
$.get(baseurl + "/sightings/listSightings/" + id + "/attribute", function(data) {
$("#sightingsData").html(data);
}).fail(xhrFailCallback);
$('.sightingsToggle').removeClass('btn-primary');
$('.sightingsToggle').addClass('btn-inverse');
$('#sightingsListAllToggle').removeClass('btn-inverse');
$('#sightingsListAllToggle').addClass('btn-primary');
$('.sightingsToggle').removeClass('btn-primary').addClass('btn-inverse');
$('#sightingsListAllToggle').removeClass('btn-inverse').addClass('btn-primary');
}
if (referer === 'addEventReport' && typeof window.reloadEventReportTable === 'function') {
reloadEventReportTable()
@ -1286,8 +1294,8 @@ function submitPopoverForm(context_id, referer, update_context_id, modal, popove
}
if (
(
context == 'event' &&
(referer == 'add' || referer == 'massEdit' || referer == 'replaceAttributes' || referer == 'addObjectReference' || referer == 'quickAddAttributeForm')
context === 'event' &&
(referer === 'add' || referer === 'massEdit' || referer === 'replaceAttributes' || referer === 'addObjectReference' || referer === 'quickAddAttributeForm')
)
){
eventUnpublish();
@ -1340,10 +1348,15 @@ function handleAjaxModalResponse(response, context_id, url, referer, context, co
function handleAjaxPopoverResponse(response, context_id, url, referer, context, contextNamingConvention) {
responseArray = response;
var message = null;
var result = "fail";
if (responseArray.saved) {
updateIndex(context_id, context);
var callback = function() {
// Scroll to edited object after index is updated
if (referer === 'quickAddAttributeForm') {
scrollToElementIfNotVisible($("#Object_" + context_id + "_tr"));
}
}
updateIndex(context_id, context, callback);
if (responseArray.success) {
showMessage("success", responseArray.success);
result = "success";
@ -1354,8 +1367,8 @@ function handleAjaxPopoverResponse(response, context_id, url, referer, context,
} else {
var savedArray = saveValuesForPersistance();
$.ajax({
dataType:"html",
success:function (data, textStatus) {
dataType: "html",
success: function (data, textStatus) {
$("#popover_form").html(data);
openPopup("#popover_form");
var error_context = context.charAt(0).toUpperCase() + context.slice(1);
@ -1369,7 +1382,7 @@ function handleAjaxPopoverResponse(response, context_id, url, referer, context,
recoverValuesFromPersistance(savedArray);
$(".loading").hide();
},
url:url
url: url
});
}
return result;
@ -3666,6 +3679,24 @@ function pivotObjectReferences(url, uuid) {
fetchAttributes(currentUri, {"focus": uuid});
}
function scrollToElementIfNotVisible($el) {
var isInViewport = function($el) {
var elementTop = $el.offset().top;
var elementBottom = elementTop + $el.outerHeight();
var viewportTop = $(window).scrollTop();
var viewportBottom = viewportTop + $(window).height();
return elementBottom > viewportTop && elementTop < viewportBottom;
};
if ($el.length && !isInViewport($el)) {
$([document.documentElement, document.body]).animate({
scrollTop: $el.offset().top - 45, // 42px is #topBar size, so make little bit more space
});
}
}
// Attribute filtering
function filterAttributes(filter) {
var data;
@ -4500,38 +4531,43 @@ function add_basic_auth() {
}
function changeObjectReferenceSelectOption(selected, additionalData) {
var keys = {
"uuid": "UUID",
"category": "Category",
"type": "Type",
"value": "Value",
"to_ids": "To IDS",
"name": "Name",
"meta-category": "Meta category",
};
var uuid = selected;
var type = additionalData.itemOptions[uuid].type;
$('#ObjectReferenceReferencedUuid').val(uuid);
if (type == "Attribute") {
$('#targetData').html("");
var $targetData = $('#targetData');
if (type === "Attribute") {
$targetData.html("");
for (var k in targetEvent[type][uuid]) {
if ($.inArray(k, ['uuid', 'category', 'type', 'value', 'to_ids']) !== -1) {
$('#targetData').append('<div><span id="' + uuid + '_' + k + '_key" class="bold"></span>: <span id="' + uuid + '_' + k + '_data"></span></div>');
$('#' + uuid + '_' + k + '_key').text(k);
$('#' + uuid + '_' + k + '_data').text(targetEvent[type][uuid][k]);
$targetData.append('<div><span class="bold">' + keys[k] + '</span>: ' + escapeHtml(targetEvent[type][uuid][k]) + '</div>');
}
}
} else {
$('#targetData').html("");
$targetData.html("");
for (var k in targetEvent[type][uuid]) {
if (k == 'Attribute') {
$('#targetData').append('<br /><div><span id="header" class="bold">Attributes:</span>');
for (attribute in targetEvent[type][uuid]['Attribute']) {
for (k2 in targetEvent[type][uuid]['Attribute'][attribute]) {
if (k === 'Attribute') {
$targetData.append('<br><div><span id="header" class="bold">Attributes:</span>');
for (var attribute in targetEvent[type][uuid]['Attribute']) {
for (var k2 in targetEvent[type][uuid]['Attribute'][attribute]) {
if ($.inArray(k2, ['category', 'type', 'value', 'to_ids']) !== -1) {
$('#targetData').append('<div class="indent"><span id="' + targetEvent[type][uuid]['Attribute'][attribute]['uuid'] + '_' + k2 + '_key" class="bold"></span>: <span id="' + targetEvent[type][uuid]['Attribute'][attribute]['uuid'] + '_' + k2 + '_data"></span></div>');
$('#' + targetEvent[type][uuid]['Attribute'][attribute]['uuid'] + '_' + k2 + '_key').text(k2);
$('#' + targetEvent[type][uuid]['Attribute'][attribute]['uuid'] + '_' + k2 + '_data').text(targetEvent[type][uuid]['Attribute'][attribute][k2]);
$targetData.append('<div class="indent"><span class="bold">' + keys[k2] + '</span>: ' + escapeHtml(targetEvent[type][uuid]['Attribute'][attribute][k2]) + '</div>');
}
}
$('#targetData').append('<br />');
$targetData.append('<br>');
}
} else {
if ($.inArray(k, ['name', 'uuid', 'meta-category']) !== -1) {
$('#targetData').append('<div><span id="' + uuid + '_' + k + '_key" class="bold"></span>: <span id="' + uuid + '_' + k + '_data"></span></div>');
$('#' + uuid + '_' + k + '_key').text(k);
$('#' + uuid + '_' + k + '_data').text(targetEvent[type][uuid][k]);
$targetData.append('<div><span class="bold">' + keys[k] + '</span>: ' + escapeHtml(targetEvent[type][uuid][k]) + '</div>');
}
}
}

4
app/webroot/js/workflows-editor/workflows-editor.js
View File

@ -723,9 +723,9 @@ function loadWorkflow(workflow) {
Object.values(workflow.data).forEach(function (node) {
var module = all_modules_by_id[node.data.id] || all_triggers_by_id[node.data.id]
if (!module) {
console.error('Tried to add node for unknown module ' + node.data.module_data.id + ' (' + node.id + ')')
console.error('Tried to add node for unknown module ' + node.data.id + ' (' + node.id + ')')
var html = window['dotBlock_error']({
error: 'Invalid module id`' + node.data.module_data.id + '` (' + node.id + ')',
error: 'Invalid module id`' + node.data.id + '` (' + node.id + ')',
data: JSON.stringify(node.data.indexed_params, null, 2)
})
editor.addNode(

5
db_schema.json
View File

@ -8938,7 +8938,8 @@
"object_template_elements": {
"id": true,
"object_relation": false,
"type": false
"type": false,
"object_template_id": false
},
"organisations": {
"id": true,
@ -9149,5 +9150,5 @@
"uuid": false
}
},
"db_version": "97"
"db_version": "98"
}

4
tests/testlive_comprehensive_local.py
View File

@ -536,6 +536,10 @@ class TestComprehensive(unittest.TestCase):
for event in (first, second):
check_response(self.admin_misp_connector.delete_event(event))
def test_correlations_noacl(self):
with MISPSetting(self.admin_misp_connector, {"MISP.correlation_engine": "NoAcl"}):
self.test_correlations()
def test_advanced_correlations(self):
with MISPSetting(self.admin_misp_connector, {"MISP.enable_advanced_correlations": True}):
first = create_simple_event()