MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'develop' into 2.4

pull/9090/merge v2.4.190
iglocska 2024-04-18 15:05:04 +02:00
parent d2b18b0e8e 9f3e6ce20e
commit 471840ce33
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
60 changed files with 1188 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/main.yml vendored
View File

@ -269,13 +269,16 @@ jobs:
- name: Check requirements.txt
run: python tests/check_requirements.py
- name: Logs
- name: System logs
if: ${{ always() }}
# update logs_test.sh when adding more logsources here
run: |
tail -n +1 `pwd`/app/tmp/logs/*
tail -n +1 /var/log/apache2/*.log
- name: Application logs
if: ${{ always() }}
run: |
app/Console/cake Log export /tmp/logs.json.gz --without-changes
zcat /tmp/logs.json.gz

2
PyMISP

@ -1 +1 @@
Subproject commit 60aa6b9a0fce69507776429fcaaf3e0e3962a36c
Subproject commit 8b4f98ac4c2e6c8cc1dba064f937dac816b67d0f

2
VERSION.json
View File

@ -1 +1 @@
{"major":2, "minor":4, "hotfix":189}
{"major":2, "minor":4, "hotfix":190}

14
app/Console/Command/AppShell.php
View File

@ -18,6 +18,7 @@
App::uses('AppModel', 'Model');
App::uses('BackgroundJobsTool', 'Tools');
App::uses('BenchmarkTool', 'Tools');
require_once dirname(__DIR__) . '/../Model/Attribute.php'; // FIXME workaround bug where Vendor/symfony/polyfill-php80/Resources/stubs/Attribute.php is loaded instead
@ -38,7 +39,18 @@ abstract class AppShell extends Shell
{
$configLoad = $this->Tasks->load('ConfigLoad');
$configLoad->execute();
if (Configure::read('Plugin.Benchmarking_enable')) {
$Benchmark = new BenchmarkTool(ClassRegistry::init('User'));
$start_time = $Benchmark->startBenchmark();
register_shutdown_function(function () use ($start_time, $Benchmark) {
$Benchmark->stopBenchmark([
'user' => 0,
'controller' => 'Shell::' . $this->modelClass,
'action' => $this->command,
'start_time' => $start_time
]);
});
}
parent::initialize();
}

16
app/Console/Command/ServerShell.php
View File

@ -125,7 +125,6 @@ class ServerShell extends AppShell
if (empty($this->args[0]) || empty($this->args[1])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Pull'] . PHP_EOL);
}
$userId = $this->args[0];
$user = $this->getUser($userId);
$serverId = $this->args[1];
@ -145,6 +144,10 @@ class ServerShell extends AppShell
if (!empty($this->args[4]) && $this->args[4] === 'force') {
$force = true;
}
// Try to enable garbage collector as pulling events can use a lot of memory
gc_enable();
try {
$result = $this->Server->pull($user, $technique, $server, $jobId, $force);
if (is_array($result)) {
@ -166,7 +169,7 @@ class ServerShell extends AppShell
if (empty($this->args[0]) || empty($this->args[1])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Push'] . PHP_EOL);
}
$userId = $this->args[0];
$user = $this->getUser($userId);
$serverId = $this->args[1];
@ -370,7 +373,7 @@ class ServerShell extends AppShell
if (empty($this->args[0]) || empty($this->args[1])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Fetch feeds as local data'] . PHP_EOL);
}
$userId = $this->args[0];
$user = $this->getUser($userId);
$feedId = $this->args[1];
@ -426,7 +429,7 @@ class ServerShell extends AppShell
if (empty($this->args[0]) || empty($this->args[1])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Cache server'] . PHP_EOL);
}
$userId = $this->args[0];
$user = $this->getUser($userId);
$scope = $this->args[1];
@ -489,7 +492,7 @@ class ServerShell extends AppShell
if (empty($this->args[0]) || empty($this->args[1])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Cache feeds for quick lookups'] . PHP_EOL);
}
$userId = $this->args[0];
$user = $this->getUser($userId);
$scope = $this->args[1];
@ -735,6 +738,7 @@ class ServerShell extends AppShell
public function sendPeriodicSummaryToUsers()
{
$periods = $this->__getPeriodsForToday();
$start_time = time();
echo __n('Started periodic summary generation for the %s period', 'Started periodic summary generation for periods: %s', count($periods), implode(', ', $periods)) . PHP_EOL;
@ -800,7 +804,7 @@ class ServerShell extends AppShell
if (empty($this->args[0]) || empty($this->args[1])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Push Taxii'] . PHP_EOL);
}
$userId = $this->args[0];
$user = $this->getUser($userId);
$serverId = $this->args[1];

5
app/Controller/AnalystDataController.php
View File

@ -320,6 +320,11 @@ class AnalystDataController extends AppController
$this->AnalystData = $this->{$vt};
$this->modelClass = $vt;
$this->{$vt}->current_user = $this->Auth->user();
if (!empty($this->request->data)) {
if (!isset($this->request->data[$type])) {
$this->request->data = [$type => $this->request->data];
}
}
return $vt;
}
}

52
app/Controller/AppController.php
View File

@ -33,13 +33,19 @@ class AppController extends Controller
public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
private $__queryVersion = '159';
public $pyMispVersion = '2.4.188';
private $__queryVersion = '161';
public $pyMispVersion = '2.4.190';
public $phpmin = '7.2';
public $phprec = '7.4';
public $phptoonew = '8.0';
private $isApiAuthed = false;
/** @var redis */
private $redis = null;
/** @var benchmark_results */
private $benchmark_results = null;
public $baseurl = '';
public $restResponsePayload = null;
@ -57,9 +63,14 @@ class AppController extends Controller
/** @var ACLComponent */
public $ACL;
/** @var BenchmarkComponent */
public $Benchmark;
/** @var RestResponseComponent */
public $RestResponse;
public $start_time;
public function __construct($request = null, $response = null)
{
parent::__construct($request, $response);
@ -97,6 +108,12 @@ class AppController extends Controller
public function beforeFilter()
{
$this->User = ClassRegistry::init('User');
if (Configure::read('Plugin.Benchmarking_enable')) {
App::uses('BenchmarkTool', 'Tools');
$this->Benchmark = new BenchmarkTool($this->User);
$this->start_time = $this->Benchmark->startBenchmark();
}
$controller = $this->request->params['controller'];
$action = $this->request->params['action'];
@ -147,8 +164,6 @@ class AppController extends Controller
Configure::write('Config.language', 'eng');
}
$this->User = ClassRegistry::init('User');
if (!empty($this->request->params['named']['disable_background_processing'])) {
Configure::write('MISP.background_jobs', 0);
}
@ -863,6 +878,21 @@ class AppController extends Controller
public function afterFilter()
{
// benchmarking
if (Configure::read('Plugin.Benchmarking_enable')) {
$this->Benchmark->stopBenchmark([
'user' => $this->Auth->user('id'),
'controller' => $this->request->params['controller'],
'action' => $this->request->params['action'],
'start_time' => $this->start_time
]);
//if ($redis && !$redis->exists('misp:auth_fail_throttling:' . $key)) {
//$redis->setex('misp:auth_fail_throttling:' . $key, 3600, 1);
//return true;
//}
}
if ($this->isApiAuthed && $this->_isRest() && !Configure::read('Security.authkey_keep_session')) {
$this->Session->destroy();
}
@ -1033,7 +1063,19 @@ class AppController extends Controller
$data = array_merge($data, $temp);
} else {
foreach ($options['paramArray'] as $param) {
if (isset($temp[$param])) {
if (substr($param, -1) == '*') {
$root = substr($param, 0, strlen($param)-1);
foreach ($temp as $existingParamKey => $v) {
$leftover = substr($existingParamKey, strlen($param)-1);
if (
$root == substr($existingParamKey, 0, strlen($root)) &&
preg_match('/^[\w_-. ]+$/', $leftover) == 1
) {
$data[$existingParamKey] = $temp[$existingParamKey];
break;
}
}
} else if (isset($temp[$param])) {
$data[$param] = $temp[$param];
}
}

1
app/Controller/AttributesController.php
View File

@ -1580,6 +1580,7 @@ class AttributesController extends AppController
}
$this->paginate['conditions'] = $params['conditions'];
$this->paginate['ignoreIndexHint'] = 'deleted';
$attributes = $this->paginate();
$this->Attribute->attachTagsToAttributes($attributes, ['includeAllTags' => true]);

123
app/Controller/BenchmarksController.php Normal file
View File

@ -0,0 +1,123 @@
<?php
App::uses('AppController', 'Controller');
class BenchmarksController extends AppController
{
public $components = array('Session', 'RequestHandler');
public $paginate = [
'limit' => 60,
'maxLimit' => 9999,
];
public function beforeFilter()
{
parent::beforeFilter();
}
public function index()
{
$this->set('menuData', ['menuList' => 'admin', 'menuItem' => 'index']);
$this->loadModel('User');
App::uses('BenchmarkTool', 'Tools');
$this->Benchmark = new BenchmarkTool($this->User);
$passedArgs = $this->passedArgs;
$this->paginate['order'] = 'value';
$defaults = [
'days' => null,
'average' => false,
'aggregate' => false,
'scope' => null,
'field' => null,
'key' => null,
'quickFilter' => null
];
$filters = $this->IndexFilter->harvestParameters(array_keys($defaults));
foreach ($defaults as $key => $value) {
if (!isset($filters[$key])) {
$filters[$key] = $defaults[$key];
}
}
$temp = $this->Benchmark->getAllTopLists(
$filters['days'] ?? null,
$filters['limit'] ?? 100,
$filters['average'] ?? null,
$filters['aggregate'] ?? null
);
$settings = $this->Benchmark->getSettings();
$units = $this->Benchmark->getUnits();
$this->set('settings', $settings);
$data = [];
$userLookup = [];
foreach ($temp as $scope => $t) {
if (!empty($filters['scope']) && $filters['scope'] !== 'all' && $scope !== $filters['scope']) {
continue;
}
foreach ($t as $field => $t2) {
if (!empty($filters['field']) && $filters['field'] !== 'all' && $field !== $filters['field']) {
continue;
}
foreach ($t2 as $date => $t3) {
foreach ($t3 as $key => $value) {
if ($scope == 'user') {
if ($key === 'SYSTEM') {
$text = 'SYSTEM';
} else if (isset($userLookup[$key])) {
$text = $userLookup[$key];
} else {
$user = $this->User->find('first', [
'fields' => ['User.id', 'User.email'],
'recursive' => -1,
'conditions' => ['User.id' => $key]
]);
if (empty($user)) {
$text = '(' . $key . ') ' . __('Invalid user');
} else {
$text = '(' . $key . ') ' . $user['User']['email'];
}
$userLookup[$key] = $text;
}
} else {
$text = $key;
}
if (!empty($filters['quickFilter'])) {
$q = strtolower($filters['quickFilter']);
if (
strpos(strtolower($scope), $q) === false &&
strpos(strtolower($field), $q) === false &&
strpos(strtolower($key), $q) === false &&
strpos(strtolower($value), $q) === false &&
strpos(strtolower($date), $q) === false &&
strpos(strtolower($text), $q) === false
) {
continue;
}
}
if (empty($filters['key']) || $key == $filters['key']) {
$data[] = [
'scope' => $scope,
'field' => $field,
'date' => $date,
'key' => $key,
'text' => $text,
'value' => $value,
'unit' => $units[$field]
];
}
}
}
}
}
if ($this->_isRest()) {
return $this->RestResponse->viewData($data, $this->response->type());
}
App::uses('CustomPaginationTool', 'Tools');
$customPagination = new CustomPaginationTool();
$customPagination->truncateAndPaginate($data, $this->params, $this->modelClass, true);
$this->set('data', $data);
$this->set('passedArgs', json_encode($passedArgs));
$this->set('filters', $filters);
}
}

3
app/Controller/Component/ACLComponent.php
View File

@ -95,6 +95,9 @@ class ACLComponent extends Component
'index' => ['perm_auth'],
'view' => ['perm_auth'],
],
'benchmarks' => [
'index' => []
],
'cerebrates' => [
'add' => [],
'delete' => [],

6
app/Controller/Component/RestSearchComponent.php
View File

@ -144,7 +144,11 @@ class RestSearchComponent extends Component
'retry',
'expiry',
'minimum_ttl',
'ttl'
'ttl',
'org.sector',
'org.local',
'org.nationality',
'galaxy.*',
],
'Object' => [
'returnFormat',

15
app/Controller/EventReportsController.php
View File

@ -213,10 +213,13 @@ class EventReportsController extends AppController
public function extractAllFromReport($reportId)
{
if (!$this->request->is('ajax')) {
if (!$this->request->is('ajax') && !$this->_isRest()) {
throw new MethodNotAllowedException(__('This function can only be reached via AJAX.'));
}
if ($this->request->is('post')) {
if (!isset($this->data['EventReport'])) {
$this->data = ['EventReport' => $this->data];
}
$report = $this->EventReport->fetchIfAuthorized($this->Auth->user(), $reportId, 'edit', $throwErrors=true, $full=false);
$results = $this->EventReport->getComplexTypeToolResultWithReplacements($this->Auth->user(), $report);
$report['EventReport']['content'] = $results['replacementResult']['contentWithReplacements'];
@ -299,13 +302,16 @@ class EventReportsController extends AppController
public function importReportFromUrl($event_id)
{
if (!$this->request->is('ajax')) {
throw new MethodNotAllowedException(__('This function can only be reached via AJAX.'));
if (!$this->request->is('ajax') && !$this->_isRest()) {
throw new MethodNotAllowedException(__('This function can only be reached via AJAX and via the API.'));
}
$fetcherModule = $this->EventReport->isFetchURLModuleEnabled();
if ($this->request->is('post')) {
if (empty($this->data['EventReport'])) {
$this->data = ['EventReport' => $this->data];
}
if (empty($this->data['EventReport']['url'])) {
throw new MethodNotAllowedException(__('An URL must be provided'));
throw new MethodNotAllowedException(__('A URL must be provided'));
}
$url = $this->data['EventReport']['url'];
$format = 'html';
@ -316,7 +322,6 @@ class EventReportsController extends AppController
$format = $parsed_format;
}
}
$content = $this->EventReport->downloadMarkdownFromURL($event_id, $url, $format);
$errors = [];

22
app/Controller/EventsController.php
View File

@ -497,6 +497,11 @@ class EventsController extends AppController
continue 2;
}
$pieces = is_array($v) ? $v : explode('|', $v);
$isANDed = false;
if (count($pieces) == 1 && strpos($pieces[0], '&') !== -1) {
$pieces = explode('&', $v);
$isANDed = count($pieces) > 1;
}
$filterString = "";
$expectOR = false;
$tagRules = [];
@ -563,10 +568,19 @@ class EventsController extends AppController
}
if (!empty($tagRules['include'])) {
$include = $this->Event->EventTag->find('column', array(
'conditions' => array('EventTag.tag_id' => $tagRules['include']),
'fields' => ['EventTag.event_id'],
));
if ($isANDed) {
$include = $this->Event->EventTag->find('column', array(
'conditions' => ['EventTag.tag_id' => $tagRules['include']],
'fields' => ['EventTag.event_id'],
'group' => ['EventTag.event_id'],
'having' => ['COUNT(*) =' => count($tagRules['include'])],
));
} else {
$include = $this->Event->EventTag->find('column', array(
'conditions' => array('EventTag.tag_id' => $tagRules['include']),
'fields' => ['EventTag.event_id'],
));
}
if (!empty($include)) {
$this->paginate['conditions']['AND'][] = 'Event.id IN (' . implode(",", $include) . ')';
} else {

33
app/Controller/FeedsController.php
View File

@ -74,6 +74,8 @@ class FeedsController extends AppController
);
}
}
$loggedUser = $this->Auth->user();
$this->loadModel('TagCollection');
$this->CRUD->index([
'filters' => [
@ -92,7 +94,7 @@ class FeedsController extends AppController
'source_format'
],
'conditions' => $conditions,
'afterFind' => function (array $feeds) {
'afterFind' => function (array $feeds) use ($loggedUser) {
if ($this->_isSiteAdmin()) {
$feeds = $this->Feed->attachFeedCacheTimestamps($feeds);
}
@ -106,6 +108,19 @@ class FeedsController extends AppController
}
}
foreach ($feeds as &$feed) {
if (!empty($feed['Feed']['tag_collection_id'])) {
$tagCollection = $this->TagCollection->fetchTagCollection($loggedUser, [
'conditions' => [
'TagCollection.id' => $feed['Feed']['tag_collection_id'],
]
]);
if (!empty($tagCollection)) {
$feed['TagCollection'] = $tagCollection;
}
}
}
return $feeds;
}
]);
@ -294,6 +309,10 @@ class FeedsController extends AppController
}
$tags = $this->Event->EventTag->Tag->find('list', array('fields' => array('Tag.name'), 'order' => array('lower(Tag.name) asc')));
$tags[0] = 'None';
$this->loadModel('TagCollection');
$tagCollections = $this->TagCollection->fetchTagCollection($this->Auth->user());
$tagCollections = Hash::combine($tagCollections, '{n}.TagCollection.id', '{n}.TagCollection.name');
$tagCollections[0] = 'None';
$this->loadModel('Server');
$allTypes = $this->Server->getAllTypes();
@ -304,6 +323,7 @@ class FeedsController extends AppController
'order' => 'LOWER(name)'
)),
'tags' => $tags,
'tag_collections' => $tagCollections,
'feedTypes' => $this->Feed->getFeedTypesOptions(),
'sharingGroups' => $sharingGroups,
'distributionLevels' => $distributionLevels,
@ -340,6 +360,7 @@ class FeedsController extends AppController
'distribution',
'sharing_group_id',
'tag_id',
'tag_collection_id',
'event_id',
'publish',
'delta_merge',
@ -442,8 +463,17 @@ class FeedsController extends AppController
if (empty(Configure::read('Security.disable_local_feed_access'))) {
$inputSources['local'] = 'Local';
}
$tags = $this->Event->EventTag->Tag->find('all', [
'recursive' => -1,
'fields' => ['Tag.name', 'Tag.id'],
'order' => ['lower(Tag.name) asc']
]);
$tags = $this->Event->EventTag->Tag->find('list', array('fields' => array('Tag.name'), 'order' => array('lower(Tag.name) asc')));
$tags[0] = 'None';
$this->loadModel('TagCollection');
$tagCollections = $this->TagCollection->fetchTagCollection($this->Auth->user());
$tagCollections = Hash::combine($tagCollections, '{n}.TagCollection.id', '{n}.TagCollection.name');
$tagCollections[0] = 'None';
$this->loadModel('Server');
$allTypes = $this->Server->getAllTypes();
@ -457,6 +487,7 @@ class FeedsController extends AppController
'order' => 'LOWER(name)'
)),
'tags' => $tags,
'tag_collections' => $tagCollections,
'feedTypes' => $this->Feed->getFeedTypesOptions(),
'sharingGroups' => $sharingGroups,
'distributionLevels' => $distributionLevels,

2
app/Controller/TagCollectionsController.php
View File

@ -357,7 +357,7 @@ class TagCollectionsController extends AppController
if (!$tagCollection) {
throw new NotFoundException(__('Invalid tag collection.'));
}
if ($this->ACL->canModifyTagCollection($this->Auth->user(), $tagCollection)) {
if (!$this->ACL->canModifyTagCollection($this->Auth->user(), $tagCollection)) {
throw new ForbiddenException(__('You dont have a permission to do that'));
}
$tagCollectionTag = $this->TagCollection->TagCollectionTag->find('first', [

4
app/Controller/UsersController.php
View File

@ -2071,7 +2071,7 @@ class UsersController extends AppController
$stats['attribute_count'] = $this->User->Event->Attribute->find('count', array('conditions' => array('Attribute.deleted' => 0), 'recursive' => -1));
$stats['attribute_count_month'] = $this->User->Event->Attribute->find('count', array('conditions' => array('Attribute.timestamp >' => $this_month, 'Attribute.deleted' => 0), 'recursive' => -1));
$stats['attributes_per_event'] = round($stats['attribute_count'] / $stats['event_count']);
$stats['attributes_per_event'] = $stats['event_count'] != 0 ? round($stats['attribute_count'] / $stats['event_count']) : 0;
$stats['correlation_count'] = $this->User->Event->Attribute->Correlation->find('count', array('recursive' => -1));
@ -2082,7 +2082,7 @@ class UsersController extends AppController
$stats['org_count'] = count($orgs);
$stats['local_org_count'] = $local_orgs_count;
$stats['contributing_org_count'] = $this->User->Event->find('count', array('recursive' => -1, 'group' => array('Event.orgc_id')));
$stats['average_user_per_org'] = round($stats['user_count'] / $stats['local_org_count'], 1);
$stats['average_user_per_org'] = $stats['local_org_count'] != 0 ? round($stats['user_count'] / $stats['local_org_count'], 1) : 0;
$this->loadModel('Thread');
$stats['thread_count'] = $this->Thread->find('count', array('conditions' => array('Thread.post_count >' => 0), 'recursive' => -1));

115
app/Lib/Dashboard/BenchmarkTopListWidget.php Normal file
View File

@ -0,0 +1,115 @@
<?php
class BenchmarkTopListWidget
{
public $title = 'Benchmark top list';
public $render = 'MultiLineChart';
public $width = 3;
public $height = 3;
public $description = 'A graph showing the top list for a given scope and field in the captured metrics.';
public $cacheLifetime = false;
public $autoRefreshDelay = 30;
public $params = array(
'days' => 'Number of days to consider for the graph. There will be a data entry for each day (assuming the benchmarking has been enabled). Defaults to returning all data.',
'weeks' => 'Number of weeks to consider for the graph. There will be a data entry for each day (assuming the benchmarking has been enabled). Defaults to returning all data.',
'months' => 'Number of months to consider for the graph. There will be a data entry for each day (assuming the benchmarking has been enabled). Defaults to returning all data.',
'scope' => 'The scope of the benchmarking refers to what was being tracked. The following scopes are valid: user, endpoint, user_agent',
'field' => 'The individual metric to be queried from the benchmark results. Valid values are: time, sql_time, sql_queries, memory, endpoint',
'average' => 'If you wish to view the averages per scope/field, set this variable to true. It will divide the result by the number of executions recorded for the scope/field combination for the given day.'
);
public $Benchmark;
public $User;
public $placeholder =
'{
"days": "30",
"scope": "endpoints",
"field": "sql_time"
}';
public function handler($user, $options = array())
{
$this->User = ClassRegistry::init('User');
$currentTime = strtotime("now");
$endOfDay = strtotime("tomorrow", $currentTime) - 1;
if (!empty($options['days'])) {
$limit = (int)($options['days']);
$delta = 'day';
} else if (!empty($options['weeks'])) {
$limit = (int)($options['weeks']);
$delta = 'week';
} else if (!empty($options['months'])) {
$limit = (int)($options['months']);
$delta = 'month';
} else {
$limit = 30;
$delta = 'day';
}
$axis_info = [
'time' => 'Total time taken (ms)',
'sql_time' => 'SQL time taken (ms)',
'sql_queries' => 'Queries (#)',
'memory' => 'Memory (MB)',
'endpoint' => 'Queries to endpoint (#)'
];
$y_axis = $axis_info[isset($options['field']) ? $options['field'] : 'time'];
$data = ['y-axis' => $y_axis];
$data['data'] = array();
// Add total users data for all timestamps
for ($i = 0; $i < $limit; $i++) {
$itemTime = strtotime('- ' . $i . $delta, $endOfDay);
$item = array();
$date = strftime('%Y-%m-%d', $itemTime);
$item = $this->getData($date, $options);
if (!empty($item)) {
$item['date'] = $date;
$data['data'][] = $item;
}
}
$keys = [];
foreach ($data['data'] as $day_data) {
foreach ($day_data as $key => $temp) {
$keys[$key] = 1;
}
}
$keys = array_keys($keys);
foreach ($data['data'] as $k => $day_data) {
foreach ($keys as $key) {
if (!isset($day_data[$key])) {
$data['data'][$k][$key] = 0;
}
}
foreach ($day_data as $key => $temp) {
$keys[$key] = 1;
}
}
return $data;
}
private function getData($time, $options)
{
$dates = [$time];
$this->Benchmark = new BenchmarkTool($this->User);
$result = $this->Benchmark->getTopList(
isset($options['scope']) ? $options['scope'] : 'endpoint',
isset($options['field']) ? $options['field'] : 'memory',
$dates,
isset($options['limit']) ? $options['limit'] : 5,
isset($options['average']) ? $options['average'] : false,
);
if (!empty($result)) {
return $result[$time];
}
return false;
}
public function checkPermissions($user)
{
if (empty($user['Role']['perm_site_admin'])) {
return false;
}
return true;
}
}

5
app/Lib/Dashboard/EventEvolutionLineWidget.php
View File

@ -58,9 +58,11 @@ class EventEvolutionLineWidget
'recursive' => -1
];
$eparams = [];
$filteringOnOrg = false;
if (!empty($options['filter']) && is_array($options['filter'])) {
foreach ($this->validFilterKeys as $filterKey) {
if (!empty($options['filter'][$filterKey])) {
$filteringOnOrg = true;
if (!is_array($options['filter'][$filterKey])) {
$options['filter'][$filterKey] = [$options['filter'][$filterKey]];
}
@ -87,6 +89,9 @@ class EventEvolutionLineWidget
'conditions' => $oparams['conditions'],
'fields' => ['id']
]);
if ($filteringOnOrg) {
$eparams['conditions']['AND']['Event.orgc_id IN'] = !empty($org_ids) ? $org_ids : [-1];
}
$this->Event->virtualFields = [
'published_date' => null
];

181
app/Lib/Tools/BenchmarkTool.php Normal file
View File

@ -0,0 +1,181 @@
<?php
/**
* Get filter parameters from index searches
*/
class BenchmarkTool
{
/** @var Model */
public $Model;
/** @var redis */
public $redis;
/** @var retention */
private $retention = 0;
/** @var start_time */
public $start_timexxx;
const BENCHMARK_SCOPES = ['user', 'endpoint', 'user_agent'];
const BENCHMARK_FIELDS = ['time', 'sql_time', 'sql_queries', 'memory'];
const BENCHMARK_UNITS = [
'time' => 's',
'sql_time' => 'ms',
'sql_queries' => '',
'memory' => 'MB'
];
public $namespace = 'misp:benchmark:';
function __construct(Model $model) {
$this->Model = $model;
}
public function getSettings()
{
return [
'scope' => self::BENCHMARK_SCOPES,
'field' => self::BENCHMARK_FIELDS,
'average' => [0, 1],
'aggregate' => [0, 1]
];
}
public function getUnits()
{
return self::BENCHMARK_UNITS;
}
public function startBenchmark()
{
$start_time = microtime(true);
$this->redis = $this->Model->setupRedis();
$this->retention = Configure::check('Plugin.benchmark_retention') ? Configure::read('Plugin.benchmark_retention') : 0;
return $start_time;
}
public function stopBenchmark(array $options)
{
$start_time = $options['start_time'];
if (!empty($options['user'])) {
$sql = $this->Model->getDataSource()->getLog(false, false);
$benchmarkData = [
'user' => $options['user'],
'endpoint' => $options['controller'] . '/' . $options['action'],
'user_agent' => $_SERVER['HTTP_USER_AGENT'],
'sql_queries' => $sql['count'],
'sql_time' => $sql['time'],
'time' => (microtime(true) - $start_time),
'memory' => (int)(memory_get_peak_usage(true) / 1024 / 1024),
//'date' => date('Y-m-d', strtotime("-3 days"))
'date' => date('Y-m-d')
];
$this->pushBenchmarkDataToRedis($benchmarkData);
} else {
$sql = $this->Model->getDataSource()->getLog(false, false);
$benchmarkData = [
'user' => 'SYSTEM',
'endpoint' => $options['controller'] . '/' . $options['action'],
'user_agent' => 'CLI',
'sql_queries' => $sql['count'],
'sql_time' => $sql['time'],
'time' => (microtime(true) - $start_time),
'memory' => (int)(memory_get_peak_usage(true) / 1024 / 1024),
//'date' => date('Y-m-d', strtotime("-3 days"))
'date' => date('Y-m-d')
];
$this->pushBenchmarkDataToRedis($benchmarkData);
}
}
private function pushBenchmarkDataToRedis($benchmarkData)
{
$this->redis = $this->Model->setupRedis();
$this->redis->pipeline();
$this->redis->sAdd(
$this->namespace . 'days',
$benchmarkData['date']
);
foreach (self::BENCHMARK_SCOPES as $scope) {
$this->redis->sAdd(
$this->namespace . $scope . ':list',
$benchmarkData[$scope]
);
$this->redis->zIncrBy(
$this->namespace . $scope . ':count:' . $benchmarkData['date'],
1,
$benchmarkData[$scope]
);
foreach (self::BENCHMARK_FIELDS as $field) {
$this->redis->zIncrBy(
$this->namespace . $scope . ':' . $field . ':' . $benchmarkData['date'],
$benchmarkData[$field],
$benchmarkData[$scope]
);
}
$this->redis->zIncrBy(
$this->namespace . $scope . ':endpoint:' . $benchmarkData['date'] . ':' . $benchmarkData['user'],
1,
$benchmarkData['endpoint']
);
}
$this->redis->exec();
}
public function getTopList(string $scope, string $field, array $days = [], $limit = 10, $average = false, $aggregate = false)
{
if (empty($this->redis)) {
$this->redis = $this->Model->setupRedis();
}
$results = [];
if (is_string($days)) {
$days = [$days];
}
foreach ($days as $day) {
$temp = $this->redis->zrevrange($this->namespace . $scope . ':' . $field . ':' . $day, 0, $limit, true);
foreach ($temp as $k => $v) {
if ($average) {
$divisor = $this->redis->zscore($this->namespace . $scope . ':count:' . $day, $k);
if ($aggregate) {
$results['aggregate'][$k] = empty($results['aggregate'][$k]) ? ($v / $divisor) : ($results['aggregate'][$k] + ($v / $divisor));
} else {
$results[$day][$k] = (int)($v / $divisor);
}
} else {
if ($aggregate) {
$results['aggregate'][$k] = empty($results['aggregate'][$k]) ? $v : ($results['aggregate'][$k] + $v);
} else {
$results[$day][$k] = $v;
}
}
}
}
if ($aggregate && $average) {
$count_days = count($days);
foreach ($results['aggregate'] as $k => $result) {
$results['aggregate'][$k] = (int)($result / $count_days);
}
}
return $results;
}
public function getAllTopLists(array $days = null, $limit = 10, $average = false, $aggregate = false, $scope_filter = [])
{
if (empty($this->redis)) {
$this->redis = $this->Model->setupRedis();
}
if ($days === null) {
$days = $this->redis->smembers($this->namespace . 'days');
}
foreach (self::BENCHMARK_SCOPES as $scope) {
if (empty($scope_filter) || in_array($scope, $scope_filter)) {
foreach (self::BENCHMARK_FIELDS as $field) {
$results[$scope][$field] = $this->getTopList($scope, $field, $days, $limit, $average, $aggregate);
}
}
}
return $results;
}
}

19
app/Lib/Tools/CurlClient.php
View File

@ -6,8 +6,12 @@ class CurlClient extends HttpSocketExtended
/** @var resource */
private $ch;
/** @var int */
private $timeout = 10800;
/**
* Maximum time the transfer is allowed to complete in seconds
* 300 seconds is recommended timeout for MISP servers
* @var int
*/
private $timeout = 300;
/** @var string|null */
private $caFile;
@ -30,6 +34,9 @@ class CurlClient extends HttpSocketExtended
/** @var array */
private $proxy = [];
/** @var array */
private $defaultOptions;
/**
* @param array $params
* @noinspection PhpMissingParentConstructorInspection
@ -38,8 +45,6 @@ class CurlClient extends HttpSocketExtended
{
if (isset($params['timeout'])) {
$this->timeout = $params['timeout'];
} else {
$this->timeout = Configure::check('MISP.curl_request_timeout') ? Configure::read('MISP.curl_request_timeout') : 10800;
}
if (isset($params['ssl_cafile'])) {
$this->caFile = $params['ssl_cafile'];
@ -59,6 +64,7 @@ class CurlClient extends HttpSocketExtended
if (isset($params['ssl_verify_peer'])) {
$this->verifyPeer = $params['ssl_verify_peer'];
}
$this->defaultOptions = $this->generateDefaultOptions();
}
/**
@ -166,6 +172,7 @@ class CurlClient extends HttpSocketExtended
return;
}
$this->proxy = compact('host', 'port', 'method', 'user', 'pass');
$this->defaultOptions = $this->generateDefaultOptions(); // regenerate default options in case proxy setting is changed
}
/**
@ -196,7 +203,7 @@ class CurlClient extends HttpSocketExtended
$url .= '?' . http_build_query($query, '', '&', PHP_QUERY_RFC3986);
}
$options = $this->generateOptions();
$options = $this->defaultOptions; // this will copy default options
$options[CURLOPT_URL] = $url;
$options[CURLOPT_CUSTOMREQUEST] = $method;
@ -303,7 +310,7 @@ class CurlClient extends HttpSocketExtended
/**
* @return array
*/
private function generateOptions()
private function generateDefaultOptions()
{
$options = [
CURLOPT_FOLLOWLOCATION => true, // Allows to follow redirect

5
app/Lib/Tools/HttpSocketExtended.php
View File

@ -24,7 +24,7 @@ class HttpSocketHttpException extends Exception
$message .= " for URL $url";
}
if ($response->body) {
$message .= ': ' . substr($response->body, 0, 100);
$message .= ': ' . substr(ltrim($response->body), 0, 100);
}
parent::__construct($message, (int)$response->code);
@ -121,7 +121,8 @@ class HttpSocketResponseExtended extends HttpSocketResponse
try {
return JsonTool::decode($this->body);
} catch (Exception $e) {
throw new HttpSocketJsonException('Could not parse response as JSON.', $this, $e);
$contentType = $this->getHeader('content-type');
throw new HttpSocketJsonException("Could not parse HTTP response as JSON. Received Content-Type $contentType.", $this, $e);
}
}
}

28
app/Lib/Tools/ServerSyncTool.php
View File

@ -297,29 +297,24 @@ class ServerSyncTool
/**
* @param array $eventUuids
* @param array $blockedOrgs Blocked organisation UUIDs
* @return array
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
* @throws JsonException
*/
public function fetchSightingsForEvents(array $eventUuids)
public function fetchSightingsForEvents(array $eventUuids, array $blockedOrgs = [])
{
$SightingBlocklist = ClassRegistry::init('SightingBlocklist');
$blocked_sightings = $SightingBlocklist->find('column', [
'recursive' => -1,
'fields' => ['org_uuid']
]);
foreach ($blocked_sightings as $k => $uuid) {
$blocked_sightings[$k] = '!' . $uuid;
}
$postParams = [
'returnFormat' => 'json',
'last' => 0, // fetch all
'includeUuid' => true,
'uuid' => $eventUuids,
];
if (!empty($blocked_sightings)) {
$postParams['org_id'] = $blocked_sightings;
if (!empty($blockedOrgs)) {
$postParams['org_id'] = array_map(function ($uuid) {
return "!$uuid";
}, $blockedOrgs);
}
return $this->post('/sightings/restSearch/event', $postParams)->json()['response'];
}
@ -511,6 +506,16 @@ class ServerSyncTool
return $this->socket->getMetaData();
}
/**
* @param string $message
* @return void
*/
public function debug($message)
{
$memoryUsage = round(memory_get_usage() / 1024 / 1024, 2);
CakeLog::debug("[Server sync #{$this->serverId()}]: $message. Memory: $memoryUsage MB");
}
/**
* @params string $url Relative URL
* @return HttpSocketResponseExtended
@ -561,6 +566,7 @@ class ServerSyncTool
if ($etag) {
// Remove compression marks that adds Apache for compressed content
// This can be removed in future as this is already checked by MISP itself since 2024-03
$etagWithoutQuotes = trim($etag, '"');
$dashPos = strrpos($etagWithoutQuotes, '-');
if ($dashPos && in_array(substr($etagWithoutQuotes, $dashPos + 1), ['br', 'gzip'], true)) {

10
app/Lib/Tools/SyncTool.php
View File

@ -1,7 +1,6 @@
<?php
class SyncTool
{
const ALLOWED_CERT_FILE_EXTENSIONS = ['pem', 'crt'];
/**
@ -50,7 +49,7 @@ class SyncTool
* @return HttpSocketExtended
* @throws Exception
*/
public function createHttpSocket($params = array())
public function createHttpSocket(array $params = [])
{
// Use own CA PEM file
$caPath = Configure::read('MISP.ca_path');
@ -82,10 +81,11 @@ class SyncTool
}
$params['ssl_crypto_method'] = $version;
}
if (!isset($params['timeout'])) {
$params['timeout'] = Configure::check('MISP.curl_request_timeout') ? Configure::read('MISP.curl_request_timeout') : 10800;
}
if (function_exists('curl_init')) {
if (!isset($params['timeout']) && Configure::check('MISP.curl_request_timeout')) {
$params['timeout'] = (int)Configure::read('MISP.curl_request_timeout');
}
App::uses('CurlClient', 'Tools');
$HttpSocket = new CurlClient($params);
} else {

8
app/Model/AnalystData.php
View File

@ -43,7 +43,7 @@ class AnalystData extends AppModel
'distribution',
'sharing_group_id',
];
protected $EDITABLE_FIELDS = [];
public const EDITABLE_FIELDS = [];
/** @var object|null */
protected $Note;
@ -185,7 +185,7 @@ class AnalystData extends AppModel
public function getEditableFields(): array
{
return array_merge(self::BASE_EDITABLE_FIELDS, $this->EDITABLE_FIELDS);
return array_merge(static::BASE_EDITABLE_FIELDS, static::EDITABLE_FIELDS);
}
/**
@ -641,9 +641,8 @@ class AnalystData extends AppModel
return [];
}
$this->Server = ClassRegistry::init('Server');
$this->AnalystData = ClassRegistry::init('AnalystData');
$this->log("Starting Analyst Data sync with server #{$server['Server']['id']}", LOG_INFO);
$serverSync->debug("Starting Analyst Data sync");
$analystData = $this->collectDataForPush($serverSync->server());
$keyedAnalystData = [];
@ -1018,7 +1017,6 @@ class AnalystData extends AppModel
}
$this->Server = ClassRegistry::init('Server');
$this->AnalystData = ClassRegistry::init('AnalystData');
try {
$filterRules = $this->buildPullFilterRules($serverSync->server());
$remoteData = $serverSync->fetchIndexMinimal($filterRules)->json();

5
app/Model/AppModel.php
View File

@ -91,7 +91,7 @@ class AppModel extends Model
105 => false, 106 => false, 107 => false, 108 => false, 109 => false, 110 => false,
111 => false, 112 => false, 113 => true, 114 => false, 115 => false, 116 => false,
117 => false, 118 => false, 119 => false, 120 => false, 121 => false, 122 => false,
123 => false, 124 => false,
123 => false, 124 => false, 125 => false,
);
const ADVANCED_UPDATES_DESCRIPTION = array(
@ -2176,6 +2176,9 @@ class AppModel extends Model
INDEX `org_name` (`org_name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;';
break;
case 125:
$sqlArray[] = "ALTER TABLE `feeds` ADD COLUMN `tag_collection_id` INT(11) NOT NULL DEFAULT 0;";
break;
case 'fixNonEmptySharingGroupID':
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';

65
app/Model/Attribute.php
View File

@ -3163,6 +3163,7 @@ class Attribute extends AppModel
'includeFullModel' => !empty($filters['includeFullModel']) ? $filters['includeFullModel'] : 0,
'allow_proposal_blocking' => !empty($filters['allow_proposal_blocking']) ? $filters['allow_proposal_blocking'] : 0
);
if (!empty($filters['attackGalaxy'])) {
$params['attackGalaxy'] = $filters['attackGalaxy'];
}
@ -3388,20 +3389,60 @@ class Attribute extends AppModel
if (!empty($params['uuid'])) {
$params['uuid'] = $this->convert_filters($params['uuid']);
if (!empty($params['uuid']['OR'])) {
$conditions['AND'][] = array(
'OR' => array(
'Event.uuid' => $params['uuid']['OR'],
'Attribute.uuid' => $params['uuid']['OR']
)
);
if ($options['scope'] == 'Attribute') {
$subQuery = [
'conditions' => ['uuid' => $params['uuid']['OR']],
'fields' => ['id']
];
$pre_lookup = $this->Event->find('first', [
'conditions' => ['Event.uuid' => $params['uuid']['OR']],
'recursive' => -1,
'fields' => ['Event.id']
]);
if (empty($pre_lookup)) {
$conditions['AND'][] = array(
'OR' => array(
'Attribute.uuid' => $params['uuid']['OR']
)
);
} else {
$conditions['AND'][] = array(
'OR' => array(
$this->subQueryGenerator($this->Event, $subQuery, 'Attribute.event_id'),
'Attribute.uuid' => $params['uuid']['OR']
)
);
}
} else {
$conditions['AND'][] = array(
'OR' => array(
'Event.uuid' => $params['uuid']['OR'],
'Attribute.uuid' => $params['uuid']['OR']
)
);
}
}
if (!empty($params['uuid']['NOT'])) {
$conditions['AND'][] = array(
'NOT' => array(
'Event.uuid' => $params['uuid']['NOT'],
'Attribute.uuid' => $params['uuid']['NOT']
)
);
if ($options['scope'] == 'Attribute') {
$subQuery = [
'conditions' => ['uuid' => $params['uuid']['OR']],
'fields' => ['id']
];
$conditions['AND'][] = [
'NOT' => [
$this->subQueryGenerator($this->Event, $subQuery, 'Attribute.event_id'),
'Attribute.uuid' => $params['uuid']['NOT']
]
];
} else {
$conditions['AND'][] = array(
'NOT' => array(
'Event.uuid' => $params['uuid']['NOT'],
'Attribute.uuid' => $params['uuid']['NOT']
)
);
}
}
}
return $conditions;

6
app/Model/Benchmark.php Normal file
View File

@ -0,0 +1,6 @@
<?php
App::uses('AppModel', 'Model');
class Benchmark extends AppModel
{
}

2
app/Model/EventReport.php
View File

@ -710,7 +710,7 @@ class EventReport extends AppModel
'category' => $typeToCategoryMapping[$complexTypeToolEntry['default_type']][0],
'type' => $complexTypeToolEntry['default_type'],
'value' => $textToBeReplaced,
'to_ids' => $complexTypeToolEntry['to_ids'],
'to_ids' => $complexTypeToolEntry['to_ids'] ?? 0,
];
$replacedContent = str_replace($complexTypeToolEntry['original_value'], $textToInject, $replacedContent);
}

75
app/Model/Feed.php
View File

@ -1032,7 +1032,7 @@ class Feed extends AppModel
}
}
}
if ($feed['Feed']['tag_id']) {
if ($feed['Feed']['tag_id'] || $feed['Feed']['tag_collection_id']) {
if (empty($feed['Tag']['name'])) {
$feed_tag = $this->Tag->find('first', [
'conditions' => [
@ -1041,23 +1041,42 @@ class Feed extends AppModel
'recursive' => -1,
'fields' => ['Tag.name', 'Tag.colour', 'Tag.id']
]);
$feed['Tag'] = $feed_tag['Tag'];
if (!empty($feed_tag)) {
$feed['Tag'] = $feed_tag['Tag'];
}
}
if (!isset($event['Event']['Tag'])) {
$event['Event']['Tag'] = array();
}
$feedTag = $this->Tag->find('first', array('conditions' => array('Tag.id' => $feed['Feed']['tag_id']), 'recursive' => -1, 'fields' => array('Tag.name', 'Tag.colour', 'Tag.exportable')));
if (!empty($feedTag)) {
$found = false;
foreach ($event['Event']['Tag'] as $tag) {
if (strtolower($tag['name']) === strtolower($feedTag['Tag']['name'])) {
$found = true;
break;
}
if (!empty($feed['Feed']['tag_collection_id'])) {
$this->TagCollection = ClassRegistry::init('TagCollection');
$tagCollectionID = $feed['Feed']['tag_collection_id'];
$tagCollection = $this->TagCollection->find('first', [
'recursive' => -1,
'conditions' => [
'TagCollection.id' => $tagCollectionID,
],
'contain' => [
'TagCollectionTag' => ['Tag'],
]
]);
foreach ($tagCollection['TagCollectionTag'] as $collectionTag) {
$event['Event']['Tag'][] = $collectionTag['Tag'];
}
if (!$found) {
$event['Event']['Tag'][] = $feedTag['Tag'];
} else {
$feedTag = $this->Tag->find('first', array('conditions' => array('Tag.id' => $feed['Feed']['tag_id']), 'recursive' => -1, 'fields' => array('Tag.name', 'Tag.colour', 'Tag.exportable')));
if (!empty($feedTag)) {
$found = false;
foreach ($event['Event']['Tag'] as $tag) {
if (strtolower($tag['name']) === strtolower($feedTag['Tag']['name'])) {
$found = true;
break;
}
}
if (!$found) {
$event['Event']['Tag'][] = $feedTag['Tag'];
}
}
}
}
@ -1068,6 +1087,9 @@ class Feed extends AppModel
if (!empty($feed['Feed']['settings']['disable_correlation'])) {
$event['Event']['disable_correlation'] = (bool) $feed['Feed']['settings']['disable_correlation'];
}
if (!empty($feed['Feed']['settings']['unpublish_event'])) {
$event['Event']['published'] = (bool) $feed['Feed']['settings']['unpublish_event'];
}
}
return $event;
}
@ -1128,9 +1150,13 @@ class Feed extends AppModel
*/
private function __updateEventFromFeed(HttpSocket $HttpSocket = null, $feed, $uuid, $user, $filterRules)
{
$event = $this->downloadAndParseEventFromFeed($feed, $uuid, $HttpSocket);
$event = $this->downloadAndParseEventFromFeed($feed, $uuid, $HttpSocket);
$event = $this->__prepareEvent($event, $feed, $filterRules);
return $this->Event->_edit($event, $user, $uuid, $jobId = null);
if (is_array($event)) {
return $this->Event->_edit($event, $user, $uuid, $jobId = null);
} else {
return $event;
}
}
public function addDefaultFeeds($newFeeds)
@ -1374,8 +1400,25 @@ class Feed extends AppModel
if ($feed['Feed']['publish']) {
$this->Event->publishRouter($event['Event']['id'], null, $user);
}
if ($feed['Feed']['tag_id']) {
$this->Event->EventTag->attachTagToEvent($event['Event']['id'], ['id' => $feed['Feed']['tag_id']]);
if ($feed['Feed']['tag_id'] || $feed['Feed']['tag_collection_id']) {
if (!empty($feed['Feed']['tag_collection_id'])) {
$this->TagCollection = ClassRegistry::init('TagCollection');
$tagCollectionID = $feed['Feed']['tag_collection_id'];
$tagCollection = $this->TagCollection->find('first', [
'recursive' => -1,
'conditions' => [
'TagCollection.id' => $tagCollectionID,
],
'contain' => [
'TagCollectionTag',
]
]);
foreach ($tagCollection['TagCollectionTag'] as $collectionTag) {
$this->Event->EventTag->attachTagToEvent($event['Event']['id'], ['id' => $collectionTag['tag_id']]);
}
} else {
$this->Event->EventTag->attachTagToEvent($event['Event']['id'], ['id' => $feed['Feed']['tag_id']]);
}
}
return true;
}

3
app/Model/GalaxyCluster.php
View File

@ -1845,6 +1845,9 @@ class GalaxyCluster extends AppModel
if (!$compatible) {
return 0;
}
$serverSync->debug("Pulling galaxy clusters with technique $technique");
$clusterIds = $this->getClusterIdListBasedOnPullTechnique($user, $technique, $serverSync);
$successes = 0;
// now process the $clusterIds to pull each of the events sequentially

17
app/Model/Log.php
View File

@ -431,15 +431,14 @@ class Log extends AppModel
}
}
$entry = $data['Log']['action'];
if (!empty($data['Log']['title'])) {
$entry .= " -- {$data['Log']['title']}";
}
if (!empty($data['Log']['description'])) {
$entry .= " -- {$data['Log']['description']}";
} else if (!empty($data['Log']['change'])) {
$entry .= " -- " . JsonTool::encode($data['Log']['change']);
}
$entry = sprintf(
'%s -- %s -- %s',
$data['Log']['action'],
empty($data['Log']['title']) ? '' : $formatted_title = preg_replace('/\s+/', " ", $data['Log']['title']),
empty($data['Log']['description']) ?
(empty($data['Log']['change']) ? '' : preg_replace('/\s+/', " ", $data['Log']['change'])) :
preg_replace('/\s+/', " ", $data['Log']['description'])
);
$this->syslog->write($action, $entry);
}
}

61
app/Model/Server.php
View File

@ -604,6 +604,7 @@ class Server extends AppModel
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
* @throws JsonException
* @throws Exception
*/
public function pull(array $user, $technique, array $server, $jobId = false, $force = false)
{
@ -619,7 +620,7 @@ class Server extends AppModel
try {
$server['Server']['version'] = $serverSync->info()['version'];
} catch (Exception $e) {
$this->logException("Could not get remote server `{$server['Server']['name']}` version.", $e);
$this->logException("Could not get remote server `{$serverSync->serverName()}` version.", $e);
if ($e instanceof HttpSocketHttpException && $e->getCode() === 403) {
$message = __('Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server. Another reason could be an incorrect sync server setting.');
} else {
@ -648,6 +649,8 @@ class Server extends AppModel
}
}
$serverSync->debug("Pulling event list with technique $technique");
try {
$eventIds = $this->__getEventIdListBasedOnPullTechnique($technique, $serverSync, $force);
} catch (Exception $e) {
@ -673,26 +676,29 @@ class Server extends AppModel
$job->saveProgress($jobId, __n('Pulling %s event.', 'Pulling %s events.', count($eventIds), count($eventIds)));
}
foreach ($eventIds as $k => $eventId) {
$serverSync->debug("Pulling event $eventId");
$this->__pullEvent($eventId, $successes, $fails, $eventModel, $serverSync, $user, $jobId, $force);
if ($jobId && $k % 10 === 0) {
$job->saveProgress($jobId, null, 10 + 40 * (($k + 1) / count($eventIds)));
}
}
foreach ($fails as $eventid => $message) {
$this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['Server']['id'], "Failed to pull event #$eventid.", 'Reason: ' . $message);
$this->loadLog()->createLogEntry($user, 'pull', 'Server', $serverSync->serverId(), "Failed to pull event #$eventid.", 'Reason: ' . $message);
}
}
if ($jobId) {
$job->saveProgress($jobId, 'Pulling proposals.', 50);
}
$pulledProposals = $pulledSightings = 0;
$pulledProposals = $pulledSightings = $pulledAnalystData = 0;
if ($technique === 'full' || $technique === 'update') {
$pulledProposals = $eventModel->ShadowAttribute->pullProposals($user, $serverSync);
if ($jobId) {
$job->saveProgress($jobId, 'Pulling sightings.', 75);
}
$pulledSightings = $eventModel->Sighting->pullSightings($user, $serverSync);
$this->AnalystData = ClassRegistry::init('AnalystData');
$pulledAnalystData = $this->AnalystData->pull($user, $serverSync);
}
@ -819,7 +825,7 @@ class Server extends AppModel
*/
public function getElligibleClusterIdsFromServerForPull(ServerSyncTool $serverSync, $onlyUpdateLocalCluster=true, array $eligibleClusters=array(), array $conditions=array())
{
$this->log("Fetching eligible clusters from server #{$serverSync->serverId()} for pull: " . JsonTool::encode($conditions), LOG_INFO);
$serverSync->debug("Fetching eligible clusters for pull: " . JsonTool::encode($conditions));
if ($onlyUpdateLocalCluster && empty($eligibleClusters)) {
return []; // no clusters for update
@ -875,7 +881,7 @@ class Server extends AppModel
*/
private function getElligibleClusterIdsFromServerForPush(ServerSyncTool $serverSync, array $localClusters=array(), array $conditions=array())
{
$this->log("Fetching eligible clusters from server #{$serverSync->serverId()} for push: " . JsonTool::encode($conditions), LOG_INFO);
$serverSync->debug("Fetching eligible clusters for push: " . JsonTool::encode($conditions));
$clusterArray = $this->fetchCustomClusterIdsFromServer($serverSync, $conditions=$conditions);
$keyedClusterArray = Hash::combine($clusterArray, '{n}.GalaxyCluster.uuid', '{n}.GalaxyCluster.version');
if (!empty($localClusters)) {
@ -915,9 +921,14 @@ class Server extends AppModel
// Fetch event index from cache if exists and is not modified
$redis = RedisTool::init();
$indexFromCache = $redis->get("misp:event_index:{$serverSync->serverId()}");
$indexFromCache = $redis->get("misp:event_index_cache:{$serverSync->serverId()}");
if ($indexFromCache) {
list($etag, $eventIndex) = RedisTool::deserialize(RedisTool::decompress($indexFromCache));
$etagPos = strpos($indexFromCache, "\n");
if ($etagPos === false) {
throw new RuntimeException("Could not find etag in cache fro server {$serverSync->serverId()}");
}
$etag = substr($indexFromCache, 0, $etagPos);
$serverSync->debug("Event index loaded from Redis cache with etag $etag containing");
} else {
$etag = '""'; // Provide empty ETag, so MISP will compute ETag for returned data
}
@ -925,9 +936,21 @@ class Server extends AppModel
$response = $serverSync->eventIndex($filterRules, $etag);
if ($response->isNotModified() && $indexFromCache) {
return $eventIndex;
return JsonTool::decode(RedisTool::decompress(substr($indexFromCache, $etagPos + 1)));
}
// Save to cache for 24 hours if ETag provided
$etag = $response->getHeader('etag');
if ($etag) {
$serverSync->debug("Event index from remote server has different etag $etag, saving to cache");
$data = "$etag\n" . RedisTool::compress($response->body);
$redis->setex("misp:event_index_cache:{$serverSync->serverId()}", 3600 * 24, $data);
} elseif ($indexFromCache) {
RedisTool::unlink($redis, "misp:event_index_cache:{$serverSync->serverId()}");
}
unset($indexFromCache); // clean up memory
$eventIndex = $response->json();
// correct $eventArray if just one event, probably this response returns old MISP
@ -935,15 +958,6 @@ class Server extends AppModel
$eventIndex = [$eventIndex];
}
// Save to cache for 24 hours if ETag provided
$etag = $response->getHeader('etag');
if ($etag) {
$data = RedisTool::compress(RedisTool::serialize([$etag, $eventIndex]));
$redis->setex("misp:event_index:{$serverSync->serverId()}", 3600 * 24, $data);
} elseif ($indexFromCache) {
RedisTool::unlink($redis, "misp:event_index:{$serverSync->serverId()}");
}
return $eventIndex;
}
@ -1372,7 +1386,7 @@ class Server extends AppModel
return []; // pushing clusters is not enabled
}
$this->log("Starting $technique clusters sync with server #{$serverSync->serverId()}", LOG_INFO);
$serverSync->debug("Starting $technique clusters sync");
$this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
$this->Event = ClassRegistry::init('Event');
@ -5125,8 +5139,8 @@ class Server extends AppModel
),
'curl_request_timeout' => [
'level' => 1,
'description' => __('Control the timeout of curl requests issued by MISP (during synchronisation, feed fetching, etc.'),
'value' => 10800,
'description' => __('Control the default timeout in seconds of curl HTTP requests issued by MISP (during synchronisation, feed fetching, etc.)'),
'value' => 300,
'test' => 'testForNumeric',
'type' => 'numeric',
'null' => true
@ -7536,6 +7550,13 @@ class Server extends AppModel
'test' => 'testBool',
'type' => 'boolean'
),
'Benchmarking_enable' => [
'level' => 2,
'description' => __('Enable the benchmarking functionalities to capture information about execution times, SQL query loads and more per user and per endpoint.'),
'value' => false,
'test' => 'testBool',
'type' => 'boolean'
],
'Enrichment_services_enable' => array(
'level' => 0,
'description' => __('Enable/disable the enrichment services'),

2
app/Model/ShadowAttribute.php
View File

@ -706,6 +706,8 @@ class ShadowAttribute extends AppModel
return 0;
}
$serverSync->debug("Pulling proposals");
$i = 1;
$fetchedCount = 0;
$chunkSize = 1000;

17
app/Model/Sighting.php
View File

@ -1418,11 +1418,13 @@ class Sighting extends AppModel
*/
public function pullSightings(array $user, ServerSyncTool $serverSync)
{
$serverSync->debug("Fetching event index for pulling sightings");
$this->Server = ClassRegistry::init('Server');
try {
$remoteEvents = $this->Server->getEventIndexFromServer($serverSync);
} catch (Exception $e) {
$this->logException("Could not fetch event IDs from server {$serverSync->server()['Server']['name']}", $e);
$this->logException("Could not fetch event IDs from server {$serverSync->serverName()}", $e);
return 0;
}
// Remove events from list that do not have published sightings.
@ -1452,6 +1454,8 @@ class Sighting extends AppModel
return 0;
}
$serverSync->debug("Pulling sightings for " . count($eventUuids) . " events");
if ($serverSync->isSupported(ServerSyncTool::FEATURE_SIGHTING_REST_SEARCH)) {
return $this->pullSightingNewWay($user, $eventUuids, $serverSync);
} else {
@ -1470,12 +1474,19 @@ class Sighting extends AppModel
*/
private function pullSightingNewWay(array $user, array $eventUuids, ServerSyncTool $serverSync)
{
$SightingBlocklist = ClassRegistry::init('SightingBlocklist');
$blockedSightingsOrgs = $SightingBlocklist->find('column', [
'recursive' => -1,
'fields' => ['org_uuid']
]);
$uuids = array_keys($eventUuids);
shuffle($uuids); // shuffle array to avoid keeping events with a lof ot sightings in same batch all the time
$saved = 0;
$savedEventUuids = [];
foreach (array_chunk($uuids, 100) as $chunk) {
foreach (array_chunk($uuids, 20) as $chunk) {
try {
$sightings = $serverSync->fetchSightingsForEvents($chunk);
$sightings = $serverSync->fetchSightingsForEvents($chunk, $blockedSightingsOrgs);
} catch (Exception $e) {
$this->logException("Failed to download sightings from remote server {$serverSync->server()['Server']['name']}.", $e);
continue;

2
app/Model/WorkflowModules/WorkflowBaseModule.php
View File

@ -213,6 +213,8 @@ class WorkflowBaseModule
if ($operator == 'in_or') {
return !empty($matching);
} elseif ($operator == 'in_and') {
sort($matching);
sort($value);
return array_values($matching) == array_values($value);
} elseif ($operator == 'not_in_or') {
return empty($matching);

16
app/Model/WorkflowModules/action/Module_stop_execution.php
View File

@ -6,6 +6,7 @@ class Module_stop_execution extends WorkflowBaseActionModule
public $blocking = true;
public $id = 'stop-execution';
public $name = 'Stop execution';
public $version = '0.2';
public $description = 'Essentially stops the execution for blocking workflows. Do nothing for non-blocking ones';
public $icon = 'ban';
public $inputs = 1;
@ -15,12 +16,25 @@ class Module_stop_execution extends WorkflowBaseActionModule
public function __construct()
{
parent::__construct();
$this->params = [
[
'id' => 'message',
'label' => 'Stop message',
'type' => 'input',
'default' => __('Execution stopped'),
'placeholder' => __('Execution stopped'),
'jinja_supported' => true,
],
];
}
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
{
parent::exec($node, $roamingData, $errors);
$errors[] = __('Execution stopped');
$rData = $roamingData->getData();
$params = $this->getParamsWithValues($node, $rData);
$errors[] = empty($params['message']['value']) ? $params['message']['default'] : $params['message']['value'];
return false;
}
}

19
app/Model/WorkflowModules/logic/Module_distribution_if.php
View File

@ -5,7 +5,7 @@ class Module_distribution_if extends WorkflowBaseLogicModule
{
public $id = 'distribution-if';
public $name = 'IF :: Distribution';
public $version = '0.2';
public $version = '0.3';
public $description = 'Distribution IF / ELSE condition block. The `then` output will be used if the encoded conditions is satisfied, otherwise the `else` output will be used.';
public $icon = 'code-branch';
public $inputs = 1;
@ -103,12 +103,15 @@ class Module_distribution_if extends WorkflowBaseLogicModule
$final_sharing_group = $this->__extractSharingGroupIDs(
$data['Event'],
$data['Event']['Attribute'][0]['Object'] ?? [],
$data['Event']['Attribute'][0]
$data['Event']['Attribute'][0],
$scope
);
if ($operator == 'equals') {
return !array_diff($final_sharing_group, $selected_sharing_groups); // All sharing groups are in the selection
return empty($selected_sharing_groups) ? !empty($final_sharing_group) :
!array_diff($final_sharing_group, $selected_sharing_groups); // All sharing groups are in the selection
} else if ($operator == 'not_equals') {
return count(array_diff($final_sharing_group, $selected_sharing_groups)) == count($final_sharing_group); // All sharing groups are in the selection
return empty($selected_sharing_groups) ? empty($final_sharing_group) :
count(array_diff($final_sharing_group, $selected_sharing_groups)) == count($final_sharing_group); // All sharing groups are in the selection
}
$errors[] = __('Condition operator not supported for that distribution level');
return false;
@ -159,9 +162,15 @@ class Module_distribution_if extends WorkflowBaseLogicModule
return min($distri1, $distri2);
}
private function __extractSharingGroupIDs(array $event, array $object=[], array $attribute=[]): array
private function __extractSharingGroupIDs(array $event, array $object=[], array $attribute=[], $scope='event'): array
{
$sgIDs = [];
if ($scope == 'event') {
if (!empty($event) && $event['distribution'] == 4) {
$sgIDs[] = $event['sharing_group_id'];
}
return $sgIDs;
}
if (!empty($event) && $event['distribution'] == 4) {
$sgIDs[] = $event['sharing_group_id'];
}

50
app/View/AnalystData/view.ctp
View File

@ -120,15 +120,57 @@ echo $this->element(
);
$object_uuid = Hash::get($data, $modelSelection . '.uuid');
$notes = $data[$modelSelection]['Note'] ?? [];
$opinions = $data[$modelSelection]['Opinion'] ?? [];
$relationships_outbound = $data[$modelSelection]['Relationship'] ?? [];
$relationships_inbound = $data[$modelSelection]['RelationshipInbound'] ?? [];
$notesOpinions = array_merge($notes, $opinions);
if(!function_exists("countNotes")) {
function countNotes($notesOpinions) {
$notesTotalCount = count($notesOpinions);
$notesCount = 0;
$relationsCount = 0;
foreach ($notesOpinions as $notesOpinion) {
if ($notesOpinion['note_type'] == 2) { // relationship
$relationsCount += 1;
} else {
$notesCount += 1;
}
if (!empty($notesOpinion['Note'])) {
$nestedCounts = countNotes($notesOpinion['Note']);
$notesTotalCount += $nestedCounts['total'];
$notesCount += $nestedCounts['notesOpinions'];
$relationsCount += $nestedCounts['relations'];
}
if (!empty($notesOpinion['Opinion'])) {
$nestedCounts = countNotes($notesOpinion['Opinion']);
$notesTotalCount += $nestedCounts['total'];
$notesCount += $nestedCounts['notesOpinions'];
$relationsCount += $nestedCounts['relations'];
}
}
return ['total' => $notesTotalCount, 'notesOpinions' => $notesCount, 'relations' => $relationsCount];
}
}
$counts = countNotes($notesOpinions);
$notesOpinionCount = $counts['notesOpinions'];
$allCounts = [
'notesOpinions' => $counts['notesOpinions'],
'relationships_outbound' => count($relationships_outbound),
'relationships_inbound' => count($relationships_inbound),
];
$options = [
'container_id' => 'analyst_data_thread',
'object_type' => $modelSelection,
'object_uuid' => $object_uuid,
'shortDist' => $shortDist,
'notes' => $data[$modelSelection]['Note'] ?? [],
'opinions' => $data[$modelSelection]['Opinion'] ?? [],
'relationships_outbound' => $data[$modelSelection]['Relationship'] ?? [],
'relationships_inbound' => $data[$modelSelection]['RelationshipInbound'] ?? [],
'notes' => $notes,
'opinions' => $opinions,
'relationships_outbound' => $relationships_outbound,
'relationships_inbound' => $relationships_inbound,
'allCounts' => $allCounts,
];
echo $this->element('genericElements/assetLoader', [

110
app/View/Benchmarks/index.ctp Normal file
View File

@ -0,0 +1,110 @@
<?php
$passedArgsArray = json_decode($passedArgs, true);
$fields = [
[
'name' => __('Date'),
'sort' => 'date',
'data_path' => 'date'
],
[
'name' => __('scope'),
'sort' => 'scope',
'data_path' => 'scope'
],
[
'name' => __('Key'),
'sort' => 'key',
'data_path' => 'text'
],
[
'name' => __('field'),
'sort' => 'field',
'data_path' => 'field'
],
[
'name' => __('Value'),
'element' => 'custom',
'function' => function($row) {
return empty($row['unit']) ? h($row['value']) : h($row['value'] . ' ' . $row['unit']);
},
'sort' => 'value'
]
];
$quick_filters = [];
foreach ($settings as $key => $setting_data) {
$temp = $filters;
$url = $baseurl . '/benchmarks/index';
foreach ($temp as $s => $v) {
if ($v && $s != $key) {
if (is_array($v)) {
foreach ($v as $multi_v) {
$url .= '/' . $s . '[]:' . $multi_v;
}
} else {
$url .= '/' . $s . ':' . $v;
}
}
}
if ($key != 'average' && $key != 'aggregate') {
$quick_filters[$key]['all'] = [
'url' => h($url),
'text' => __('All'),
'active' => !$filters[$key],
'style' => 'display:inline;'
];
}
foreach ($setting_data as $setting_element) {
$text = $setting_element;
if ($key == 'average') {
$text = $setting_element ? 'average / request' : 'total';
}
if ($key == 'aggregate') {
$text = $setting_element ? 'aggregate' : 'daily';
}
$quick_filters[$key][] = [
'url' => h($url . '/' . $key . ':' . $setting_element),
'text' => $text,
'active' => $filters[$key] == $setting_element,
'style' => 'display:inline;'
];
}
}
echo $this->element('genericElements/IndexTable/scaffold', [
'scaffold_data' => [
'passedArgsArray' => $passedArgsArray,
'data' => [
'persistUrlParams' => array_keys($settings),
'data' => $data,
'top_bar' => [
'pull' => 'right',
'children' => [
[
'children' => $quick_filters['scope']
],
[
'children' => $quick_filters['field']
],
[
'children' => $quick_filters['average']
],
[
'children' => $quick_filters['aggregate']
],
[
'type' => 'search',
'button' => __('Filter'),
'placeholder' => __('Enter value to search'),
'data' => '',
'searchKey' => 'quickFilter'
]
]
],
'fields' => $fields,
'title' => empty($ajax) ? __('Benchmark results') : false,
'description' => empty($ajax) ? __('Results of the collected benchmarks. You can filter it further by passing the limit, scope, field parameters.') : false,
]
]
]);
?>

2
app/View/Dashboards/index.ctp
View File

@ -23,7 +23,7 @@ $(function () {
saveDashboardState();
});
grid.on('added', function(event, items) {
resetDashboardGrid(grid);
resetDashboardGrid(grid, false);
});
grid.on('gsresizestop', function(event, element) {
$(element).find('.widgetContentInner').trigger('widget-resized')

3
app/View/Elements/dashboard/Widgets/MultiLineChart.ctp
View File

@ -6,6 +6,7 @@
h($data['formula'])
);
}
$y_axis = $data['y-axis'] ?? 'Count';
?>
<div id="chartContainer-<?= $seed ?>" style="flex-grow: 1; position:relative;"></div>
<script>
@ -50,7 +51,7 @@ function init<?= $seed ?>() { // variables and functions have their own scope (n
show_legend: true,
style: {
xlabel: "Date",
ylabel: "Count",
ylabel: "<?= h($y_axis) ?>",
hideXAxis: false,
hideYAxis: false,
},

14
app/View/Elements/genericElements/Analyst_data/generic.ctp
View File

@ -65,20 +65,8 @@ $allCounts = [
$(document).ready(function() {
$('.node-opener-<?= $seed ?>').click(function() {
openNotes(this)
openNotes<?= $seed ?>(this)
})
function adjustPopoverPosition() {
var $popover = $('.popover:last');
$popover.css('top', Math.max($popover.position().top, 50) + 'px')
}
function openNotes(clicked) {
openPopover(clicked, renderedNotes<?= $seed ?>, undefined, undefined, function() {
adjustPopoverPosition()
$(clicked).removeClass('have-a-popover') // avoid closing the popover if a confirm popover (like the delete one) is called
})
}
})
</script>

75
app/View/Elements/genericElements/Analyst_data/thread.ctp
View File

@ -4,6 +4,7 @@
$URL_DELETE = '/analystData/delete/';
$seed = isset($seed) ? $seed : mt_rand();
$injectInPage = !empty($container_id) ? true : false;
$notes = !empty($notes) ? $notes : [];
$opinions = !empty($opinions) ? $opinions : [];
@ -41,7 +42,28 @@
if (!window.shortDist) {
var shortDist = <?= json_encode($shortDist) ?>;
}
var renderedNotes<?= $seed ?> = null
var container_id = false
<?php if (isset($container_id)): ?>
container_id = '<?= h($container_id) ?>'
<?php endif; ?>
function adjustPopoverPosition() {
var $popover = $('.popover:last');
$popover.css('top', Math.max($popover.position().top, 50) + 'px')
}
function openNotes<?= $seed ?>(clicked) {
var notes = <?= json_encode($notesOpinions) ?>;
var relationships = <?= json_encode($relationshipsOutbound) ?>;
var relationships_inbound = <?= json_encode($relationshipsInbound) ?>;
var relationship_related_object = <?= json_encode($related_objects) ?>;
var renderedNotes = renderAllNotesWithForm<?= $seed ?>(notes, relationships, relationships_inbound, relationship_related_object)
openPopover(clicked, renderedNotes, undefined, undefined, function() {
adjustPopoverPosition()
$(clicked).removeClass('have-a-popover') // avoid closing the popover if a confirm popover (like the delete one) is called
})
}
function renderNotes(notes, relationship_related_object, emptyMessage='<?= __('Empty') ?>', isInbound=false) {
var renderedNotesArray = []
@ -406,18 +428,7 @@ function fetchMoreNotes(clicked, noteType, uuid) {
}
(function() {
var notes = <?= json_encode($notesOpinions) ?>;
var relationships = <?= json_encode($relationshipsOutbound) ?>;
var relationships_inbound = <?= json_encode($relationshipsInbound) ?>;
var relationship_related_object = <?= json_encode($related_objects) ?>;
var container_id = false
<?php if (isset($container_id)): ?>
container_id = '<?= h($container_id) ?>'
<?php endif; ?>
var nodeContainerTemplate = doT.template('\
var nodeContainerTemplate<?= $seed ?> = doT.template('\
<div> \
<ul class="nav nav-tabs" style="margin-bottom: 10px;"> \
<li class="active"><a href="#notes-<?= $seed ?>" data-toggle="tab"><i class="<?= $this->FontAwesome->getClass('sticky-note') ?>"></i> <?= __('Notes & Opinions') ?> <span class="label label-secondary"><?= $allCounts['notesOpinions'] ?></span></a></li> \
@ -439,18 +450,6 @@ function fetchMoreNotes(clicked, noteType, uuid) {
</div> \
')
function renderAllNotesWithForm(relationship_related_object) {
var buttonContainer = '<div id="add-button-container" style="margin-top: 0.5rem;">' + addNoteButton + addOpinionButton + '</div>'
renderedNotes<?= $seed ?> = nodeContainerTemplate({
content_notes: renderNotes(notes.filter(function(note) { return note.note_type != 2}), relationship_related_object, '<?= __('No notes for this UUID.') ?>') + buttonContainer,
content_relationships_outbound: renderNotes(relationships, relationship_related_object, '<?= __('No relationship from this UUID') ?>') + addRelationshipButton,
content_relationships_inbound: renderNotes(relationships_inbound, relationship_related_object, '<?= __('No element are referencing this UUID') ?>', true),
})
if (container_id) {
$('#' + container_id).html(renderedNotes<?= $seed ?>)
}
}
var addNoteButton = '<button class="btn btn-small btn-block btn-primary" type="button" onclick="createNewNote(this, \'<?= $object_type ?>\', \'<?= $object_uuid ?>\')"> \
<i class="<?= $this->FontAwesome->getClass('plus') ?>"></i> <?= __('Add a note') ?> \
</button>'
@ -461,10 +460,15 @@ function fetchMoreNotes(clicked, noteType, uuid) {
<i class="<?= $this->FontAwesome->getClass('plus') ?>"></i> <?= __('Add a relationship') ?> \
</button>'
$(document).ready(function() {
renderAllNotesWithForm(relationship_related_object)
})
})()
function renderAllNotesWithForm<?= $seed ?>(notes, relationships, relationships_inbound, relationship_related_object) {
var buttonContainer = '<div id="add-button-container" style="margin-top: 0.5rem;">' + addNoteButton + addOpinionButton + '</div>'
var renderedNotes = nodeContainerTemplate<?= $seed ?>({
content_notes: renderNotes(notes.filter(function(note) { return note.note_type != 2}), relationship_related_object, '<?= __('No notes for this UUID.') ?>') + buttonContainer,
content_relationships_outbound: renderNotes(relationships, relationship_related_object, '<?= __('No relationship from this UUID') ?>') + addRelationshipButton,
content_relationships_inbound: renderNotes(relationships_inbound, relationship_related_object, '<?= __('No element are referencing this UUID') ?>', true),
})
return renderedNotes
}
function createNewNote(clicked, object_type, object_uuid) {
note_type = 'Note';
@ -516,6 +520,19 @@ function fetchMoreNotes(clicked, noteType, uuid) {
}
}
<?php if(!empty($injectInPage)): ?>
$(document).ready(function() {
var notes = <?= json_encode($notesOpinions) ?>;
var relationships = <?= json_encode($relationshipsOutbound) ?>;
var relationships_inbound = <?= json_encode($relationshipsInbound) ?>;
var relationship_related_object = <?= json_encode($related_objects) ?>;
var renderedNotes = renderAllNotesWithForm<?= $seed ?>(notes, relationships, relationships_inbound, relationship_related_object)
if (container_id) {
$('#' + container_id).html(renderedNotes)
}
})
<?php endif; ?>
</script>
<style>

24
app/View/Elements/genericElements/IndexTable/Fields/tags.ctp
View File

@ -25,5 +25,29 @@
'hide_global_scope' => isset($field['hide_global_scope']) ? $field['hide_global_scope'] : false
]
);
} else if (!empty($field['includeTagCollection']) && empty($tags)) {
if (!empty($row['TagCollection'])) {
echo sprintf('<a class="badge" style="background-color: #fff; color: #000; border: 1px solid #000;" title="%s" href="%s">%s :: %s</a>',
__('Tag Collection'),
'/tag_collections/view/' . h($row['TagCollection'][0]['TagCollection']['id']),
__('Tag Collection'),
h($row['TagCollection'][0]['TagCollection']['name'])
);
echo '<div>';
echo $this->element(
'ajaxTags',
[
'scope' => '',
'attributeId' => 0,
'tags' => Hash::extract($row['TagCollection'][0]['TagCollectionTag'], '{n}.Tag'),
'tagAccess' => false,
'localTagAccess' => false,
'static_tags_only' => 1,
'scope' => isset($field['scope']) ? $field['scope'] : 'event',
'hide_global_scope' => isset($field['hide_global_scope']) ? $field['hide_global_scope'] : false
]
);
echo '</div>';
}
}
?>

2
app/View/Elements/genericElements/IndexTable/index_table.ctp
View File

@ -35,7 +35,7 @@
if (!empty($data['persistUrlParams'])) {
foreach ($data['persistUrlParams'] as $persistedParam) {
if (!empty($passedArgsArray[$persistedParam])) {
$data['paginatorOptions']['url'][] = $passedArgsArray[$persistedParam];
$data['paginatorOptions']['url'][$persistedParam] = $passedArgsArray[$persistedParam];
}
}
}

7
app/View/Elements/genericElements/SideMenu/side_menu.ctp
View File

@ -1100,6 +1100,13 @@ $divider = '<li class="divider"></li>';
'url' => $baseurl . '/servers/updateProgress',
'text' => __('Update Progress')
));
if (Configure::read('Plugin.Benchmarking_enable')) {
echo $divider;
echo $this->element('/genericElements/SideMenu/side_menu_link', array(
'url' => $baseurl . '/benchmarks/index',
'text' => __('Benchmarks')
));
}
echo $divider;
if (Configure::read('MISP.background_jobs')) {
echo $this->element('/genericElements/SideMenu/side_menu_link', array(

5
app/View/Elements/global_menu.ctp
View File

@ -408,6 +408,11 @@ if (!empty($me)) {
'url' => $baseurl . '/servers/serverSettings',
'requirement' => $isSiteAdmin
),
[
'text' => __('Benchmarking'),
'url' => $baseurl . '/benchmarks/index',
'requirement' => $isSiteAdmin && Configure::read('Plugin.Benchmarking_enable')
],
array(
'type' => 'separator',
'requirement' => $isSiteAdmin

13
app/View/Feeds/add.ctp
View File

@ -26,6 +26,11 @@ echo $this->element('genericElements/Form/genericForm', [
'label' => __('Disable correlation'),
'type' => 'checkbox'
],
[
'field' => 'Feed.settings.unpublish_event',
'label' => __('Unpublish events'),
'type' => 'checkbox'
],
[
'field' => 'name',
'label' => __('Name'),
@ -161,6 +166,14 @@ echo $this->element('genericElements/Form/genericForm', [
'type' => 'dropdown',
'searchable' => 1
],
[
'field' => 'tag_collection_id',
'label' => __('Default Tag Collection'),
'options' => $dropdownData['tag_collections'],
'selected' => isset($entity['Feed']['tag_collection_id']) ? $entity['Feed']['tag_collection_id'] : '0',
'type' => 'dropdown',
'searchable' => 1
],
[
'field' => 'rules',
'label' => __('Filter rules'),

3
app/View/Feeds/index.ctp
View File

@ -193,7 +193,8 @@
'class' => 'short',
'data_path' => 'Tag',
'element' => 'tags',
'scope' => 'feeds'
'scope' => 'feeds',
'includeTagCollection' => true,
),
array(
'name' => __('Visible'),

50
app/View/GalaxyClusters/view.ctp
View File

@ -170,15 +170,57 @@ $md.html(md.render($md.text()));
<?php
$object_uuid = $cluster['GalaxyCluster']['uuid'];
$notes = $cluster['GalaxyCluster']['Note'] ?? [];
$opinions = $cluster['GalaxyCluster']['Opinion'] ?? [];
$relationships_outbound = $cluster['GalaxyCluster']['Relationship'] ?? [];
$relationships_inbound = $cluster['GalaxyCluster']['RelationshipInbound'] ?? [];
$notesOpinions = array_merge($notes, $opinions);
if(!function_exists("countNotes")) {
function countNotes($notesOpinions) {
$notesTotalCount = count($notesOpinions);
$notesCount = 0;
$relationsCount = 0;
foreach ($notesOpinions as $notesOpinion) {
if ($notesOpinion['note_type'] == 2) { // relationship
$relationsCount += 1;
} else {
$notesCount += 1;
}
if (!empty($notesOpinion['Note'])) {
$nestedCounts = countNotes($notesOpinion['Note']);
$notesTotalCount += $nestedCounts['total'];
$notesCount += $nestedCounts['notesOpinions'];
$relationsCount += $nestedCounts['relations'];
}
if (!empty($notesOpinion['Opinion'])) {
$nestedCounts = countNotes($notesOpinion['Opinion']);
$notesTotalCount += $nestedCounts['total'];
$notesCount += $nestedCounts['notesOpinions'];
$relationsCount += $nestedCounts['relations'];
}
}
return ['total' => $notesTotalCount, 'notesOpinions' => $notesCount, 'relations' => $relationsCount];
}
}
$counts = countNotes($notesOpinions);
$notesOpinionCount = $counts['notesOpinions'];
$allCounts = [
'notesOpinions' => $counts['notesOpinions'],
'relationships_outbound' => count($relationships_outbound),
'relationships_inbound' => count($relationships_inbound),
];
$options = [
'container_id' => 'analyst_data_thread',
'object_type' => 'GalaxyCluster',
'object_uuid' => $object_uuid,
'shortDist' => $shortDist,
'notes' => $cluster['GalaxyCluster']['Note'] ?? [],
'opinions' => $cluster['GalaxyCluster']['Opinion'] ?? [],
'relationships_outbound' => $cluster['GalaxyCluster']['Relationship'] ?? [],
'relationships_inbound' => $cluster['GalaxyCluster']['RelationshipInbound'] ?? [],
'notes' => $notes,
'opinions' => $opinions,
'relationships_outbound' => $relationships_outbound,
'relationships_inbound' => $relationships_inbound,
'allCounts' => $allCounts,
];
echo $this->element('genericElements/Analyst_data/thread', $options);

11
app/View/Users/view.ctp
View File

@ -177,7 +177,7 @@ if ($isAdmin && $isTotp) {
'js' => array('vis', 'jquery-ui.min', 'network-distribution-graph')
));
echo sprintf(
'<div class="users view"><div class="row-fluid"><div class="span8" style="margin:0px;">%s</div></div>%s%s%s<div style="margin-top:20px;">%s%s</div></div>',
'<div class="users view"><div class="row-fluid"><div class="span8" style="margin:0px;">%s</div></div>%s%s%s<div style="margin-top:20px;">%s%s%s</div></div>',
sprintf(
'<h2>%s</h2>%s',
__('User %s', h($user['User']['email'])),
@ -210,6 +210,15 @@ if ($isAdmin && $isTotp) {
__('Review user logins')
),
$me['Role']['perm_auth'] ? $this->element('/genericElements/accordion', array('title' => __('Auth keys'), 'url' => '/auth_keys/index/' . h($user['User']['id']))) : '',
$me['Role']['perm_site_admin'] ?
$this->element(
'/genericElements/accordion',
[
'title' => __('Benchmarks'),
'url' => '/benchmarks/index/scope:user/average:1/aggregate:1/key:' . h($user['User']['id'])
]
) :
'',
$this->element('/genericElements/accordion', array('title' => 'Events', 'url' => '/events/index/searchemail:' . urlencode(h($user['User']['email']))))
);
$current_menu = [

2
app/files/misp-galaxy

@ -1 +1 @@
Subproject commit c953d8ee5d1ec43bf8a418f957ad6f920fe1c0e8
Subproject commit 8e8c3fa93de383c86b27c00830bfb6314d2c8087

2
app/files/misp-objects

@ -1 +1 @@
Subproject commit dc52c10844cbed9e2f39f0429665b4f9b1caef3e
Subproject commit 96492b9c932a4b307216550abeadddc727e17cec

2
app/files/taxonomies

@ -1 +1 @@
Subproject commit 55e0f57d5d2953417e4084fe311a80bae99270f5
Subproject commit 272192fe3223a2a5f382b1f24540e49cf6a945ff

2
app/files/warninglists

@ -1 +1 @@
Subproject commit 7c69f4f987ee753901e952071556d5ad0fafd115
Subproject commit c282baa102adf07c10dec8ca94488a1ffd933e32

2
app/webroot/js/attack_matrix.js
View File

@ -17,6 +17,8 @@
});
adapt_position_from_viewport();
var firstTabId = $('#attack-matrix-tabscontroller span[data-toggle="tab"]:first').attr('href');
resizeHeader(firstTabId);
$('.ajax_popover_form .btn-matrix-submit').click(function() {
makeTagging(pickedGalaxies);

19
app/webroot/js/misp.js
View File

@ -5385,6 +5385,18 @@ function submitDashboardAddWidget() {
var height = $('#DashboardHeight').val();
var el = null;
var k = $('#last-element-counter').data('element-counter');
if (config === '') {
config = '[]'
}
try {
config = JSON.parse(config);
} catch (error) {
showMessage('fail', error.message)
return
}
config = JSON.stringify(config);
$.ajax({
url: baseurl + '/dashboards/getEmptyWidget/' + widget + '/' + (k+1),
type: 'GET',
@ -5398,14 +5410,7 @@ function submitDashboardAddWidget() {
"autoposition": 1
}
);
if (config !== '') {
config = JSON.parse(config);
config = JSON.stringify(config);
} else {
config = '[]';
}
$('#widget_' + (k+1)).attr('config', config);
saveDashboardState();
$('#last-element-counter').data('element-counter', (k+1));
},
complete: function(data) {

7
app/webroot/js/workflows-editor/workflows-editor.js
View File

@ -1836,10 +1836,9 @@ function genPicker(options, forNode = true) {
var $container = genSelect(options)
var $select = $container.find('select')
$select.addClass('start-chosen')
if (options.picker_options) {
// $select.data('chosen_options', options.picker_options)
$select.attr('data-chosen_options', JSON.stringify(options.picker_options))
}
var pickerOptions = options.picker_options ?? {}
pickerOptions['max_shown_results'] = 100
$select.attr('data-chosen_options', JSON.stringify(pickerOptions))
return $container
}

13
db_schema.json
View File

@ -3206,6 +3206,17 @@
"column_type": "int(11)",
"column_default": "0",
"extra": ""
},
{
"column_name": "tag_collection_id",
"is_nullable": "NO",
"data_type": "int",
"character_maximum_length": null,
"numeric_precision": "10",
"collation_name": null,
"column_type": "int(11)",
"column_default": "0",
"extra": ""
}
],
"fuzzy_correlate_ssdeep": [
@ -10603,5 +10614,5 @@
"uuid": false
}
},
"db_version": "124"
"db_version": "125"
}

2
requirements.txt
View File

@ -6,7 +6,7 @@ misp-lib-stix2>=3.0.1.1
mixbox>=1.0.5
plyara>=2.1.1
pydeep2>=0.5.1
pymisp==2.4.188
pymisp==2.4.190
python-magic>=0.4.27
pyzmq>=25.1.1
redis>=5.0.1