diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d60373e3a..f9d766b50 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -269,13 +269,16 @@ jobs: - name: Check requirements.txt run: python tests/check_requirements.py - - name: Logs + - name: System logs if: ${{ always() }} # update logs_test.sh when adding more logsources here run: | tail -n +1 `pwd`/app/tmp/logs/* tail -n +1 /var/log/apache2/*.log + - name: Application logs + if: ${{ always() }} + run: | app/Console/cake Log export /tmp/logs.json.gz --without-changes zcat /tmp/logs.json.gz diff --git a/PyMISP b/PyMISP index 60aa6b9a0..8b4f98ac4 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit 60aa6b9a0fce69507776429fcaaf3e0e3962a36c +Subproject commit 8b4f98ac4c2e6c8cc1dba064f937dac816b67d0f diff --git a/VERSION.json b/VERSION.json index e7997ba86..01970df01 100644 --- a/VERSION.json +++ b/VERSION.json @@ -1 +1 @@ -{"major":2, "minor":4, "hotfix":189} +{"major":2, "minor":4, "hotfix":190} diff --git a/app/Console/Command/AppShell.php b/app/Console/Command/AppShell.php index 6d63a94f4..869ae313d 100644 --- a/app/Console/Command/AppShell.php +++ b/app/Console/Command/AppShell.php @@ -18,6 +18,7 @@ App::uses('AppModel', 'Model'); App::uses('BackgroundJobsTool', 'Tools'); +App::uses('BenchmarkTool', 'Tools'); require_once dirname(__DIR__) . '/../Model/Attribute.php'; // FIXME workaround bug where Vendor/symfony/polyfill-php80/Resources/stubs/Attribute.php is loaded instead @@ -38,7 +39,18 @@ abstract class AppShell extends Shell { $configLoad = $this->Tasks->load('ConfigLoad'); $configLoad->execute(); - + if (Configure::read('Plugin.Benchmarking_enable')) { + $Benchmark = new BenchmarkTool(ClassRegistry::init('User')); + $start_time = $Benchmark->startBenchmark(); + register_shutdown_function(function () use ($start_time, $Benchmark) { + $Benchmark->stopBenchmark([ + 'user' => 0, + 'controller' => 'Shell::' . $this->modelClass, + 'action' => $this->command, + 'start_time' => $start_time + ]); + }); + } parent::initialize(); } diff --git a/app/Console/Command/ServerShell.php b/app/Console/Command/ServerShell.php index ad2d8df72..f6961516b 100644 --- a/app/Console/Command/ServerShell.php +++ b/app/Console/Command/ServerShell.php @@ -125,7 +125,6 @@ class ServerShell extends AppShell if (empty($this->args[0]) || empty($this->args[1])) { die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Pull'] . PHP_EOL); } - $userId = $this->args[0]; $user = $this->getUser($userId); $serverId = $this->args[1]; @@ -145,6 +144,10 @@ class ServerShell extends AppShell if (!empty($this->args[4]) && $this->args[4] === 'force') { $force = true; } + + // Try to enable garbage collector as pulling events can use a lot of memory + gc_enable(); + try { $result = $this->Server->pull($user, $technique, $server, $jobId, $force); if (is_array($result)) { @@ -166,7 +169,7 @@ class ServerShell extends AppShell if (empty($this->args[0]) || empty($this->args[1])) { die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Push'] . PHP_EOL); } - + $userId = $this->args[0]; $user = $this->getUser($userId); $serverId = $this->args[1]; @@ -370,7 +373,7 @@ class ServerShell extends AppShell if (empty($this->args[0]) || empty($this->args[1])) { die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Fetch feeds as local data'] . PHP_EOL); } - + $userId = $this->args[0]; $user = $this->getUser($userId); $feedId = $this->args[1]; @@ -426,7 +429,7 @@ class ServerShell extends AppShell if (empty($this->args[0]) || empty($this->args[1])) { die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Cache server'] . PHP_EOL); } - + $userId = $this->args[0]; $user = $this->getUser($userId); $scope = $this->args[1]; @@ -489,7 +492,7 @@ class ServerShell extends AppShell if (empty($this->args[0]) || empty($this->args[1])) { die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Cache feeds for quick lookups'] . PHP_EOL); } - + $userId = $this->args[0]; $user = $this->getUser($userId); $scope = $this->args[1]; @@ -735,6 +738,7 @@ class ServerShell extends AppShell public function sendPeriodicSummaryToUsers() { + $periods = $this->__getPeriodsForToday(); $start_time = time(); echo __n('Started periodic summary generation for the %s period', 'Started periodic summary generation for periods: %s', count($periods), implode(', ', $periods)) . PHP_EOL; @@ -800,7 +804,7 @@ class ServerShell extends AppShell if (empty($this->args[0]) || empty($this->args[1])) { die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Push Taxii'] . PHP_EOL); } - + $userId = $this->args[0]; $user = $this->getUser($userId); $serverId = $this->args[1]; diff --git a/app/Controller/AnalystDataController.php b/app/Controller/AnalystDataController.php index 9659555d8..c54800aae 100644 --- a/app/Controller/AnalystDataController.php +++ b/app/Controller/AnalystDataController.php @@ -320,6 +320,11 @@ class AnalystDataController extends AppController $this->AnalystData = $this->{$vt}; $this->modelClass = $vt; $this->{$vt}->current_user = $this->Auth->user(); + if (!empty($this->request->data)) { + if (!isset($this->request->data[$type])) { + $this->request->data = [$type => $this->request->data]; + } + } return $vt; } } diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index 56486d44f..97660006d 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -33,13 +33,19 @@ class AppController extends Controller public $helpers = array('OrgImg', 'FontAwesome', 'UserName'); - private $__queryVersion = '159'; - public $pyMispVersion = '2.4.188'; + private $__queryVersion = '161'; + public $pyMispVersion = '2.4.190'; public $phpmin = '7.2'; public $phprec = '7.4'; public $phptoonew = '8.0'; private $isApiAuthed = false; + /** @var redis */ + private $redis = null; + + /** @var benchmark_results */ + private $benchmark_results = null; + public $baseurl = ''; public $restResponsePayload = null; @@ -57,9 +63,14 @@ class AppController extends Controller /** @var ACLComponent */ public $ACL; + /** @var BenchmarkComponent */ + public $Benchmark; + /** @var RestResponseComponent */ public $RestResponse; + public $start_time; + public function __construct($request = null, $response = null) { parent::__construct($request, $response); @@ -97,6 +108,12 @@ class AppController extends Controller public function beforeFilter() { + $this->User = ClassRegistry::init('User'); + if (Configure::read('Plugin.Benchmarking_enable')) { + App::uses('BenchmarkTool', 'Tools'); + $this->Benchmark = new BenchmarkTool($this->User); + $this->start_time = $this->Benchmark->startBenchmark(); + } $controller = $this->request->params['controller']; $action = $this->request->params['action']; @@ -147,8 +164,6 @@ class AppController extends Controller Configure::write('Config.language', 'eng'); } - $this->User = ClassRegistry::init('User'); - if (!empty($this->request->params['named']['disable_background_processing'])) { Configure::write('MISP.background_jobs', 0); } @@ -863,6 +878,21 @@ class AppController extends Controller public function afterFilter() { + // benchmarking + if (Configure::read('Plugin.Benchmarking_enable')) { + $this->Benchmark->stopBenchmark([ + 'user' => $this->Auth->user('id'), + 'controller' => $this->request->params['controller'], + 'action' => $this->request->params['action'], + 'start_time' => $this->start_time + ]); + + //if ($redis && !$redis->exists('misp:auth_fail_throttling:' . $key)) { + //$redis->setex('misp:auth_fail_throttling:' . $key, 3600, 1); + //return true; + //} + + } if ($this->isApiAuthed && $this->_isRest() && !Configure::read('Security.authkey_keep_session')) { $this->Session->destroy(); } @@ -1033,7 +1063,19 @@ class AppController extends Controller $data = array_merge($data, $temp); } else { foreach ($options['paramArray'] as $param) { - if (isset($temp[$param])) { + if (substr($param, -1) == '*') { + $root = substr($param, 0, strlen($param)-1); + foreach ($temp as $existingParamKey => $v) { + $leftover = substr($existingParamKey, strlen($param)-1); + if ( + $root == substr($existingParamKey, 0, strlen($root)) && + preg_match('/^[\w_-. ]+$/', $leftover) == 1 + ) { + $data[$existingParamKey] = $temp[$existingParamKey]; + break; + } + } + } else if (isset($temp[$param])) { $data[$param] = $temp[$param]; } } diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 9694cfac8..1ef438956 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -1580,6 +1580,7 @@ class AttributesController extends AppController } $this->paginate['conditions'] = $params['conditions']; + $this->paginate['ignoreIndexHint'] = 'deleted'; $attributes = $this->paginate(); $this->Attribute->attachTagsToAttributes($attributes, ['includeAllTags' => true]); diff --git a/app/Controller/BenchmarksController.php b/app/Controller/BenchmarksController.php new file mode 100644 index 000000000..6fe4ce773 --- /dev/null +++ b/app/Controller/BenchmarksController.php @@ -0,0 +1,123 @@ + 60, + 'maxLimit' => 9999, + + ]; + + public function beforeFilter() + { + parent::beforeFilter(); + } + + public function index() + { + $this->set('menuData', ['menuList' => 'admin', 'menuItem' => 'index']); + $this->loadModel('User'); + App::uses('BenchmarkTool', 'Tools'); + $this->Benchmark = new BenchmarkTool($this->User); + $passedArgs = $this->passedArgs; + $this->paginate['order'] = 'value'; + $defaults = [ + 'days' => null, + 'average' => false, + 'aggregate' => false, + 'scope' => null, + 'field' => null, + 'key' => null, + 'quickFilter' => null + ]; + $filters = $this->IndexFilter->harvestParameters(array_keys($defaults)); + foreach ($defaults as $key => $value) { + if (!isset($filters[$key])) { + $filters[$key] = $defaults[$key]; + } + } + $temp = $this->Benchmark->getAllTopLists( + $filters['days'] ?? null, + $filters['limit'] ?? 100, + $filters['average'] ?? null, + $filters['aggregate'] ?? null + ); + $settings = $this->Benchmark->getSettings(); + $units = $this->Benchmark->getUnits(); + $this->set('settings', $settings); + $data = []; + $userLookup = []; + foreach ($temp as $scope => $t) { + if (!empty($filters['scope']) && $filters['scope'] !== 'all' && $scope !== $filters['scope']) { + continue; + } + foreach ($t as $field => $t2) { + if (!empty($filters['field']) && $filters['field'] !== 'all' && $field !== $filters['field']) { + continue; + } + foreach ($t2 as $date => $t3) { + foreach ($t3 as $key => $value) { + if ($scope == 'user') { + if ($key === 'SYSTEM') { + $text = 'SYSTEM'; + } else if (isset($userLookup[$key])) { + $text = $userLookup[$key]; + } else { + $user = $this->User->find('first', [ + 'fields' => ['User.id', 'User.email'], + 'recursive' => -1, + 'conditions' => ['User.id' => $key] + ]); + if (empty($user)) { + $text = '(' . $key . ') ' . __('Invalid user'); + } else { + $text = '(' . $key . ') ' . $user['User']['email']; + } + $userLookup[$key] = $text; + } + } else { + $text = $key; + } + if (!empty($filters['quickFilter'])) { + $q = strtolower($filters['quickFilter']); + if ( + strpos(strtolower($scope), $q) === false && + strpos(strtolower($field), $q) === false && + strpos(strtolower($key), $q) === false && + strpos(strtolower($value), $q) === false && + strpos(strtolower($date), $q) === false && + strpos(strtolower($text), $q) === false + ) { + continue; + } + } + if (empty($filters['key']) || $key == $filters['key']) { + $data[] = [ + 'scope' => $scope, + 'field' => $field, + 'date' => $date, + 'key' => $key, + 'text' => $text, + 'value' => $value, + 'unit' => $units[$field] + ]; + } + } + } + } + } + if ($this->_isRest()) { + return $this->RestResponse->viewData($data, $this->response->type()); + } + App::uses('CustomPaginationTool', 'Tools'); + $customPagination = new CustomPaginationTool(); + $customPagination->truncateAndPaginate($data, $this->params, $this->modelClass, true); + $this->set('data', $data); + $this->set('passedArgs', json_encode($passedArgs)); + $this->set('filters', $filters); + } + +} diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php index cfe7147a9..e121f5c5e 100644 --- a/app/Controller/Component/ACLComponent.php +++ b/app/Controller/Component/ACLComponent.php @@ -95,6 +95,9 @@ class ACLComponent extends Component 'index' => ['perm_auth'], 'view' => ['perm_auth'], ], + 'benchmarks' => [ + 'index' => [] + ], 'cerebrates' => [ 'add' => [], 'delete' => [], diff --git a/app/Controller/Component/RestSearchComponent.php b/app/Controller/Component/RestSearchComponent.php index ae8a4a34e..9e8ab024f 100644 --- a/app/Controller/Component/RestSearchComponent.php +++ b/app/Controller/Component/RestSearchComponent.php @@ -144,7 +144,11 @@ class RestSearchComponent extends Component 'retry', 'expiry', 'minimum_ttl', - 'ttl' + 'ttl', + 'org.sector', + 'org.local', + 'org.nationality', + 'galaxy.*', ], 'Object' => [ 'returnFormat', diff --git a/app/Controller/EventReportsController.php b/app/Controller/EventReportsController.php index 04199f2b5..efeb9ddd4 100644 --- a/app/Controller/EventReportsController.php +++ b/app/Controller/EventReportsController.php @@ -213,10 +213,13 @@ class EventReportsController extends AppController public function extractAllFromReport($reportId) { - if (!$this->request->is('ajax')) { + if (!$this->request->is('ajax') && !$this->_isRest()) { throw new MethodNotAllowedException(__('This function can only be reached via AJAX.')); } if ($this->request->is('post')) { + if (!isset($this->data['EventReport'])) { + $this->data = ['EventReport' => $this->data]; + } $report = $this->EventReport->fetchIfAuthorized($this->Auth->user(), $reportId, 'edit', $throwErrors=true, $full=false); $results = $this->EventReport->getComplexTypeToolResultWithReplacements($this->Auth->user(), $report); $report['EventReport']['content'] = $results['replacementResult']['contentWithReplacements']; @@ -299,13 +302,16 @@ class EventReportsController extends AppController public function importReportFromUrl($event_id) { - if (!$this->request->is('ajax')) { - throw new MethodNotAllowedException(__('This function can only be reached via AJAX.')); + if (!$this->request->is('ajax') && !$this->_isRest()) { + throw new MethodNotAllowedException(__('This function can only be reached via AJAX and via the API.')); } $fetcherModule = $this->EventReport->isFetchURLModuleEnabled(); if ($this->request->is('post')) { + if (empty($this->data['EventReport'])) { + $this->data = ['EventReport' => $this->data]; + } if (empty($this->data['EventReport']['url'])) { - throw new MethodNotAllowedException(__('An URL must be provided')); + throw new MethodNotAllowedException(__('A URL must be provided')); } $url = $this->data['EventReport']['url']; $format = 'html'; @@ -316,7 +322,6 @@ class EventReportsController extends AppController $format = $parsed_format; } } - $content = $this->EventReport->downloadMarkdownFromURL($event_id, $url, $format); $errors = []; diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 4e695ac67..418a40cce 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -497,6 +497,11 @@ class EventsController extends AppController continue 2; } $pieces = is_array($v) ? $v : explode('|', $v); + $isANDed = false; + if (count($pieces) == 1 && strpos($pieces[0], '&') !== -1) { + $pieces = explode('&', $v); + $isANDed = count($pieces) > 1; + } $filterString = ""; $expectOR = false; $tagRules = []; @@ -563,10 +568,19 @@ class EventsController extends AppController } if (!empty($tagRules['include'])) { - $include = $this->Event->EventTag->find('column', array( - 'conditions' => array('EventTag.tag_id' => $tagRules['include']), - 'fields' => ['EventTag.event_id'], - )); + if ($isANDed) { + $include = $this->Event->EventTag->find('column', array( + 'conditions' => ['EventTag.tag_id' => $tagRules['include']], + 'fields' => ['EventTag.event_id'], + 'group' => ['EventTag.event_id'], + 'having' => ['COUNT(*) =' => count($tagRules['include'])], + )); + } else { + $include = $this->Event->EventTag->find('column', array( + 'conditions' => array('EventTag.tag_id' => $tagRules['include']), + 'fields' => ['EventTag.event_id'], + )); + } if (!empty($include)) { $this->paginate['conditions']['AND'][] = 'Event.id IN (' . implode(",", $include) . ')'; } else { diff --git a/app/Controller/FeedsController.php b/app/Controller/FeedsController.php index 5eafbf435..954bf95f6 100644 --- a/app/Controller/FeedsController.php +++ b/app/Controller/FeedsController.php @@ -74,6 +74,8 @@ class FeedsController extends AppController ); } } + $loggedUser = $this->Auth->user(); + $this->loadModel('TagCollection'); $this->CRUD->index([ 'filters' => [ @@ -92,7 +94,7 @@ class FeedsController extends AppController 'source_format' ], 'conditions' => $conditions, - 'afterFind' => function (array $feeds) { + 'afterFind' => function (array $feeds) use ($loggedUser) { if ($this->_isSiteAdmin()) { $feeds = $this->Feed->attachFeedCacheTimestamps($feeds); } @@ -106,6 +108,19 @@ class FeedsController extends AppController } } + foreach ($feeds as &$feed) { + if (!empty($feed['Feed']['tag_collection_id'])) { + $tagCollection = $this->TagCollection->fetchTagCollection($loggedUser, [ + 'conditions' => [ + 'TagCollection.id' => $feed['Feed']['tag_collection_id'], + ] + ]); + if (!empty($tagCollection)) { + $feed['TagCollection'] = $tagCollection; + } + } + } + return $feeds; } ]); @@ -294,6 +309,10 @@ class FeedsController extends AppController } $tags = $this->Event->EventTag->Tag->find('list', array('fields' => array('Tag.name'), 'order' => array('lower(Tag.name) asc'))); $tags[0] = 'None'; + $this->loadModel('TagCollection'); + $tagCollections = $this->TagCollection->fetchTagCollection($this->Auth->user()); + $tagCollections = Hash::combine($tagCollections, '{n}.TagCollection.id', '{n}.TagCollection.name'); + $tagCollections[0] = 'None'; $this->loadModel('Server'); $allTypes = $this->Server->getAllTypes(); @@ -304,6 +323,7 @@ class FeedsController extends AppController 'order' => 'LOWER(name)' )), 'tags' => $tags, + 'tag_collections' => $tagCollections, 'feedTypes' => $this->Feed->getFeedTypesOptions(), 'sharingGroups' => $sharingGroups, 'distributionLevels' => $distributionLevels, @@ -340,6 +360,7 @@ class FeedsController extends AppController 'distribution', 'sharing_group_id', 'tag_id', + 'tag_collection_id', 'event_id', 'publish', 'delta_merge', @@ -442,8 +463,17 @@ class FeedsController extends AppController if (empty(Configure::read('Security.disable_local_feed_access'))) { $inputSources['local'] = 'Local'; } + $tags = $this->Event->EventTag->Tag->find('all', [ + 'recursive' => -1, + 'fields' => ['Tag.name', 'Tag.id'], + 'order' => ['lower(Tag.name) asc'] + ]); $tags = $this->Event->EventTag->Tag->find('list', array('fields' => array('Tag.name'), 'order' => array('lower(Tag.name) asc'))); $tags[0] = 'None'; + $this->loadModel('TagCollection'); + $tagCollections = $this->TagCollection->fetchTagCollection($this->Auth->user()); + $tagCollections = Hash::combine($tagCollections, '{n}.TagCollection.id', '{n}.TagCollection.name'); + $tagCollections[0] = 'None'; $this->loadModel('Server'); $allTypes = $this->Server->getAllTypes(); @@ -457,6 +487,7 @@ class FeedsController extends AppController 'order' => 'LOWER(name)' )), 'tags' => $tags, + 'tag_collections' => $tagCollections, 'feedTypes' => $this->Feed->getFeedTypesOptions(), 'sharingGroups' => $sharingGroups, 'distributionLevels' => $distributionLevels, diff --git a/app/Controller/TagCollectionsController.php b/app/Controller/TagCollectionsController.php index 0131790ab..0f204e261 100644 --- a/app/Controller/TagCollectionsController.php +++ b/app/Controller/TagCollectionsController.php @@ -357,7 +357,7 @@ class TagCollectionsController extends AppController if (!$tagCollection) { throw new NotFoundException(__('Invalid tag collection.')); } - if ($this->ACL->canModifyTagCollection($this->Auth->user(), $tagCollection)) { + if (!$this->ACL->canModifyTagCollection($this->Auth->user(), $tagCollection)) { throw new ForbiddenException(__('You dont have a permission to do that')); } $tagCollectionTag = $this->TagCollection->TagCollectionTag->find('first', [ diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php index 09bc73493..4baa2378e 100644 --- a/app/Controller/UsersController.php +++ b/app/Controller/UsersController.php @@ -2071,7 +2071,7 @@ class UsersController extends AppController $stats['attribute_count'] = $this->User->Event->Attribute->find('count', array('conditions' => array('Attribute.deleted' => 0), 'recursive' => -1)); $stats['attribute_count_month'] = $this->User->Event->Attribute->find('count', array('conditions' => array('Attribute.timestamp >' => $this_month, 'Attribute.deleted' => 0), 'recursive' => -1)); - $stats['attributes_per_event'] = round($stats['attribute_count'] / $stats['event_count']); + $stats['attributes_per_event'] = $stats['event_count'] != 0 ? round($stats['attribute_count'] / $stats['event_count']) : 0; $stats['correlation_count'] = $this->User->Event->Attribute->Correlation->find('count', array('recursive' => -1)); @@ -2082,7 +2082,7 @@ class UsersController extends AppController $stats['org_count'] = count($orgs); $stats['local_org_count'] = $local_orgs_count; $stats['contributing_org_count'] = $this->User->Event->find('count', array('recursive' => -1, 'group' => array('Event.orgc_id'))); - $stats['average_user_per_org'] = round($stats['user_count'] / $stats['local_org_count'], 1); + $stats['average_user_per_org'] = $stats['local_org_count'] != 0 ? round($stats['user_count'] / $stats['local_org_count'], 1) : 0; $this->loadModel('Thread'); $stats['thread_count'] = $this->Thread->find('count', array('conditions' => array('Thread.post_count >' => 0), 'recursive' => -1)); diff --git a/app/Lib/Dashboard/BenchmarkTopListWidget.php b/app/Lib/Dashboard/BenchmarkTopListWidget.php new file mode 100644 index 000000000..1fd36ec1b --- /dev/null +++ b/app/Lib/Dashboard/BenchmarkTopListWidget.php @@ -0,0 +1,115 @@ + 'Number of days to consider for the graph. There will be a data entry for each day (assuming the benchmarking has been enabled). Defaults to returning all data.', + 'weeks' => 'Number of weeks to consider for the graph. There will be a data entry for each day (assuming the benchmarking has been enabled). Defaults to returning all data.', + 'months' => 'Number of months to consider for the graph. There will be a data entry for each day (assuming the benchmarking has been enabled). Defaults to returning all data.', + 'scope' => 'The scope of the benchmarking refers to what was being tracked. The following scopes are valid: user, endpoint, user_agent', + 'field' => 'The individual metric to be queried from the benchmark results. Valid values are: time, sql_time, sql_queries, memory, endpoint', + 'average' => 'If you wish to view the averages per scope/field, set this variable to true. It will divide the result by the number of executions recorded for the scope/field combination for the given day.' + ); + public $Benchmark; + public $User; + + public $placeholder = + '{ + "days": "30", + "scope": "endpoints", + "field": "sql_time" +}'; + + public function handler($user, $options = array()) + { + $this->User = ClassRegistry::init('User'); + $currentTime = strtotime("now"); + $endOfDay = strtotime("tomorrow", $currentTime) - 1; + if (!empty($options['days'])) { + $limit = (int)($options['days']); + $delta = 'day'; + } else if (!empty($options['weeks'])) { + $limit = (int)($options['weeks']); + $delta = 'week'; + } else if (!empty($options['months'])) { + $limit = (int)($options['months']); + $delta = 'month'; + } else { + $limit = 30; + $delta = 'day'; + } + $axis_info = [ + 'time' => 'Total time taken (ms)', + 'sql_time' => 'SQL time taken (ms)', + 'sql_queries' => 'Queries (#)', + 'memory' => 'Memory (MB)', + 'endpoint' => 'Queries to endpoint (#)' + ]; + $y_axis = $axis_info[isset($options['field']) ? $options['field'] : 'time']; + $data = ['y-axis' => $y_axis]; + $data['data'] = array(); + // Add total users data for all timestamps + + for ($i = 0; $i < $limit; $i++) { + $itemTime = strtotime('- ' . $i . $delta, $endOfDay); + $item = array(); + $date = strftime('%Y-%m-%d', $itemTime); + $item = $this->getData($date, $options); + if (!empty($item)) { + $item['date'] = $date; + $data['data'][] = $item; + } + } + $keys = []; + foreach ($data['data'] as $day_data) { + foreach ($day_data as $key => $temp) { + $keys[$key] = 1; + } + } + $keys = array_keys($keys); + foreach ($data['data'] as $k => $day_data) { + foreach ($keys as $key) { + if (!isset($day_data[$key])) { + $data['data'][$k][$key] = 0; + } + } + foreach ($day_data as $key => $temp) { + $keys[$key] = 1; + } + } + return $data; + } + + private function getData($time, $options) + { + $dates = [$time]; + $this->Benchmark = new BenchmarkTool($this->User); + $result = $this->Benchmark->getTopList( + isset($options['scope']) ? $options['scope'] : 'endpoint', + isset($options['field']) ? $options['field'] : 'memory', + $dates, + isset($options['limit']) ? $options['limit'] : 5, + isset($options['average']) ? $options['average'] : false, + ); + if (!empty($result)) { + return $result[$time]; + } + return false; + } + + public function checkPermissions($user) + { + if (empty($user['Role']['perm_site_admin'])) { + return false; + } + return true; + } +} diff --git a/app/Lib/Dashboard/EventEvolutionLineWidget.php b/app/Lib/Dashboard/EventEvolutionLineWidget.php index a4c06e007..680e17355 100644 --- a/app/Lib/Dashboard/EventEvolutionLineWidget.php +++ b/app/Lib/Dashboard/EventEvolutionLineWidget.php @@ -58,9 +58,11 @@ class EventEvolutionLineWidget 'recursive' => -1 ]; $eparams = []; + $filteringOnOrg = false; if (!empty($options['filter']) && is_array($options['filter'])) { foreach ($this->validFilterKeys as $filterKey) { if (!empty($options['filter'][$filterKey])) { + $filteringOnOrg = true; if (!is_array($options['filter'][$filterKey])) { $options['filter'][$filterKey] = [$options['filter'][$filterKey]]; } @@ -87,6 +89,9 @@ class EventEvolutionLineWidget 'conditions' => $oparams['conditions'], 'fields' => ['id'] ]); + if ($filteringOnOrg) { + $eparams['conditions']['AND']['Event.orgc_id IN'] = !empty($org_ids) ? $org_ids : [-1]; + } $this->Event->virtualFields = [ 'published_date' => null ]; diff --git a/app/Lib/Tools/BenchmarkTool.php b/app/Lib/Tools/BenchmarkTool.php new file mode 100644 index 000000000..8f741b854 --- /dev/null +++ b/app/Lib/Tools/BenchmarkTool.php @@ -0,0 +1,181 @@ + 's', + 'sql_time' => 'ms', + 'sql_queries' => '', + 'memory' => 'MB' + ]; + + public $namespace = 'misp:benchmark:'; + + function __construct(Model $model) { + $this->Model = $model; + } + + public function getSettings() + { + return [ + 'scope' => self::BENCHMARK_SCOPES, + 'field' => self::BENCHMARK_FIELDS, + 'average' => [0, 1], + 'aggregate' => [0, 1] + ]; + } + + public function getUnits() + { + return self::BENCHMARK_UNITS; + } + + public function startBenchmark() + { + $start_time = microtime(true); + $this->redis = $this->Model->setupRedis(); + $this->retention = Configure::check('Plugin.benchmark_retention') ? Configure::read('Plugin.benchmark_retention') : 0; + return $start_time; + } + + public function stopBenchmark(array $options) + { + $start_time = $options['start_time']; + if (!empty($options['user'])) { + $sql = $this->Model->getDataSource()->getLog(false, false); + $benchmarkData = [ + 'user' => $options['user'], + 'endpoint' => $options['controller'] . '/' . $options['action'], + 'user_agent' => $_SERVER['HTTP_USER_AGENT'], + 'sql_queries' => $sql['count'], + 'sql_time' => $sql['time'], + 'time' => (microtime(true) - $start_time), + 'memory' => (int)(memory_get_peak_usage(true) / 1024 / 1024), + //'date' => date('Y-m-d', strtotime("-3 days")) + 'date' => date('Y-m-d') + ]; + $this->pushBenchmarkDataToRedis($benchmarkData); + } else { + $sql = $this->Model->getDataSource()->getLog(false, false); + $benchmarkData = [ + 'user' => 'SYSTEM', + 'endpoint' => $options['controller'] . '/' . $options['action'], + 'user_agent' => 'CLI', + 'sql_queries' => $sql['count'], + 'sql_time' => $sql['time'], + 'time' => (microtime(true) - $start_time), + 'memory' => (int)(memory_get_peak_usage(true) / 1024 / 1024), + //'date' => date('Y-m-d', strtotime("-3 days")) + 'date' => date('Y-m-d') + ]; + $this->pushBenchmarkDataToRedis($benchmarkData); + } + } + + private function pushBenchmarkDataToRedis($benchmarkData) + { + $this->redis = $this->Model->setupRedis(); + $this->redis->pipeline(); + $this->redis->sAdd( + $this->namespace . 'days', + $benchmarkData['date'] + ); + foreach (self::BENCHMARK_SCOPES as $scope) { + $this->redis->sAdd( + $this->namespace . $scope . ':list', + $benchmarkData[$scope] + ); + $this->redis->zIncrBy( + $this->namespace . $scope . ':count:' . $benchmarkData['date'], + 1, + $benchmarkData[$scope] + ); + foreach (self::BENCHMARK_FIELDS as $field) { + $this->redis->zIncrBy( + $this->namespace . $scope . ':' . $field . ':' . $benchmarkData['date'], + $benchmarkData[$field], + $benchmarkData[$scope] + ); + } + $this->redis->zIncrBy( + $this->namespace . $scope . ':endpoint:' . $benchmarkData['date'] . ':' . $benchmarkData['user'], + 1, + $benchmarkData['endpoint'] + ); + } + $this->redis->exec(); + } + + public function getTopList(string $scope, string $field, array $days = [], $limit = 10, $average = false, $aggregate = false) + { + if (empty($this->redis)) { + $this->redis = $this->Model->setupRedis(); + } + $results = []; + if (is_string($days)) { + $days = [$days]; + } + foreach ($days as $day) { + $temp = $this->redis->zrevrange($this->namespace . $scope . ':' . $field . ':' . $day, 0, $limit, true); + foreach ($temp as $k => $v) { + if ($average) { + $divisor = $this->redis->zscore($this->namespace . $scope . ':count:' . $day, $k); + if ($aggregate) { + $results['aggregate'][$k] = empty($results['aggregate'][$k]) ? ($v / $divisor) : ($results['aggregate'][$k] + ($v / $divisor)); + } else { + $results[$day][$k] = (int)($v / $divisor); + } + } else { + if ($aggregate) { + $results['aggregate'][$k] = empty($results['aggregate'][$k]) ? $v : ($results['aggregate'][$k] + $v); + } else { + $results[$day][$k] = $v; + } + } + } + } + if ($aggregate && $average) { + $count_days = count($days); + foreach ($results['aggregate'] as $k => $result) { + $results['aggregate'][$k] = (int)($result / $count_days); + } + } + return $results; + } + + public function getAllTopLists(array $days = null, $limit = 10, $average = false, $aggregate = false, $scope_filter = []) + { + if (empty($this->redis)) { + $this->redis = $this->Model->setupRedis(); + } + if ($days === null) { + $days = $this->redis->smembers($this->namespace . 'days'); + } + foreach (self::BENCHMARK_SCOPES as $scope) { + if (empty($scope_filter) || in_array($scope, $scope_filter)) { + foreach (self::BENCHMARK_FIELDS as $field) { + $results[$scope][$field] = $this->getTopList($scope, $field, $days, $limit, $average, $aggregate); + } + } + } + return $results; + } +} diff --git a/app/Lib/Tools/CurlClient.php b/app/Lib/Tools/CurlClient.php index 006b58f3f..4e19b8462 100644 --- a/app/Lib/Tools/CurlClient.php +++ b/app/Lib/Tools/CurlClient.php @@ -6,8 +6,12 @@ class CurlClient extends HttpSocketExtended /** @var resource */ private $ch; - /** @var int */ - private $timeout = 10800; + /** + * Maximum time the transfer is allowed to complete in seconds + * 300 seconds is recommended timeout for MISP servers + * @var int + */ + private $timeout = 300; /** @var string|null */ private $caFile; @@ -30,6 +34,9 @@ class CurlClient extends HttpSocketExtended /** @var array */ private $proxy = []; + /** @var array */ + private $defaultOptions; + /** * @param array $params * @noinspection PhpMissingParentConstructorInspection @@ -38,8 +45,6 @@ class CurlClient extends HttpSocketExtended { if (isset($params['timeout'])) { $this->timeout = $params['timeout']; - } else { - $this->timeout = Configure::check('MISP.curl_request_timeout') ? Configure::read('MISP.curl_request_timeout') : 10800; } if (isset($params['ssl_cafile'])) { $this->caFile = $params['ssl_cafile']; @@ -59,6 +64,7 @@ class CurlClient extends HttpSocketExtended if (isset($params['ssl_verify_peer'])) { $this->verifyPeer = $params['ssl_verify_peer']; } + $this->defaultOptions = $this->generateDefaultOptions(); } /** @@ -166,6 +172,7 @@ class CurlClient extends HttpSocketExtended return; } $this->proxy = compact('host', 'port', 'method', 'user', 'pass'); + $this->defaultOptions = $this->generateDefaultOptions(); // regenerate default options in case proxy setting is changed } /** @@ -196,7 +203,7 @@ class CurlClient extends HttpSocketExtended $url .= '?' . http_build_query($query, '', '&', PHP_QUERY_RFC3986); } - $options = $this->generateOptions(); + $options = $this->defaultOptions; // this will copy default options $options[CURLOPT_URL] = $url; $options[CURLOPT_CUSTOMREQUEST] = $method; @@ -303,7 +310,7 @@ class CurlClient extends HttpSocketExtended /** * @return array */ - private function generateOptions() + private function generateDefaultOptions() { $options = [ CURLOPT_FOLLOWLOCATION => true, // Allows to follow redirect diff --git a/app/Lib/Tools/HttpSocketExtended.php b/app/Lib/Tools/HttpSocketExtended.php index cfe097c28..375144f5b 100644 --- a/app/Lib/Tools/HttpSocketExtended.php +++ b/app/Lib/Tools/HttpSocketExtended.php @@ -24,7 +24,7 @@ class HttpSocketHttpException extends Exception $message .= " for URL $url"; } if ($response->body) { - $message .= ': ' . substr($response->body, 0, 100); + $message .= ': ' . substr(ltrim($response->body), 0, 100); } parent::__construct($message, (int)$response->code); @@ -121,7 +121,8 @@ class HttpSocketResponseExtended extends HttpSocketResponse try { return JsonTool::decode($this->body); } catch (Exception $e) { - throw new HttpSocketJsonException('Could not parse response as JSON.', $this, $e); + $contentType = $this->getHeader('content-type'); + throw new HttpSocketJsonException("Could not parse HTTP response as JSON. Received Content-Type $contentType.", $this, $e); } } } diff --git a/app/Lib/Tools/ServerSyncTool.php b/app/Lib/Tools/ServerSyncTool.php index 40e555d4a..6f433bcba 100644 --- a/app/Lib/Tools/ServerSyncTool.php +++ b/app/Lib/Tools/ServerSyncTool.php @@ -297,29 +297,24 @@ class ServerSyncTool /** * @param array $eventUuids + * @param array $blockedOrgs Blocked organisation UUIDs * @return array * @throws HttpSocketHttpException * @throws HttpSocketJsonException * @throws JsonException */ - public function fetchSightingsForEvents(array $eventUuids) + public function fetchSightingsForEvents(array $eventUuids, array $blockedOrgs = []) { - $SightingBlocklist = ClassRegistry::init('SightingBlocklist'); - $blocked_sightings = $SightingBlocklist->find('column', [ - 'recursive' => -1, - 'fields' => ['org_uuid'] - ]); - foreach ($blocked_sightings as $k => $uuid) { - $blocked_sightings[$k] = '!' . $uuid; - } $postParams = [ 'returnFormat' => 'json', 'last' => 0, // fetch all 'includeUuid' => true, 'uuid' => $eventUuids, ]; - if (!empty($blocked_sightings)) { - $postParams['org_id'] = $blocked_sightings; + if (!empty($blockedOrgs)) { + $postParams['org_id'] = array_map(function ($uuid) { + return "!$uuid"; + }, $blockedOrgs); } return $this->post('/sightings/restSearch/event', $postParams)->json()['response']; } @@ -511,6 +506,16 @@ class ServerSyncTool return $this->socket->getMetaData(); } + /** + * @param string $message + * @return void + */ + public function debug($message) + { + $memoryUsage = round(memory_get_usage() / 1024 / 1024, 2); + CakeLog::debug("[Server sync #{$this->serverId()}]: $message. Memory: $memoryUsage MB"); + } + /** * @params string $url Relative URL * @return HttpSocketResponseExtended @@ -561,6 +566,7 @@ class ServerSyncTool if ($etag) { // Remove compression marks that adds Apache for compressed content + // This can be removed in future as this is already checked by MISP itself since 2024-03 $etagWithoutQuotes = trim($etag, '"'); $dashPos = strrpos($etagWithoutQuotes, '-'); if ($dashPos && in_array(substr($etagWithoutQuotes, $dashPos + 1), ['br', 'gzip'], true)) { diff --git a/app/Lib/Tools/SyncTool.php b/app/Lib/Tools/SyncTool.php index b0ef40a41..ee8cdb3bf 100644 --- a/app/Lib/Tools/SyncTool.php +++ b/app/Lib/Tools/SyncTool.php @@ -1,7 +1,6 @@ EDITABLE_FIELDS); + return array_merge(static::BASE_EDITABLE_FIELDS, static::EDITABLE_FIELDS); } /** @@ -641,9 +641,8 @@ class AnalystData extends AppModel return []; } $this->Server = ClassRegistry::init('Server'); - $this->AnalystData = ClassRegistry::init('AnalystData'); - $this->log("Starting Analyst Data sync with server #{$server['Server']['id']}", LOG_INFO); + $serverSync->debug("Starting Analyst Data sync"); $analystData = $this->collectDataForPush($serverSync->server()); $keyedAnalystData = []; @@ -1018,7 +1017,6 @@ class AnalystData extends AppModel } $this->Server = ClassRegistry::init('Server'); - $this->AnalystData = ClassRegistry::init('AnalystData'); try { $filterRules = $this->buildPullFilterRules($serverSync->server()); $remoteData = $serverSync->fetchIndexMinimal($filterRules)->json(); diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 738c4d3d8..cb8232d2e 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -91,7 +91,7 @@ class AppModel extends Model 105 => false, 106 => false, 107 => false, 108 => false, 109 => false, 110 => false, 111 => false, 112 => false, 113 => true, 114 => false, 115 => false, 116 => false, 117 => false, 118 => false, 119 => false, 120 => false, 121 => false, 122 => false, - 123 => false, 124 => false, + 123 => false, 124 => false, 125 => false, ); const ADVANCED_UPDATES_DESCRIPTION = array( @@ -2176,6 +2176,9 @@ class AppModel extends Model INDEX `org_name` (`org_name`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;'; break; + case 125: + $sqlArray[] = "ALTER TABLE `feeds` ADD COLUMN `tag_collection_id` INT(11) NOT NULL DEFAULT 0;"; + break; case 'fixNonEmptySharingGroupID': $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 3981d0e21..a5d05dd74 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -3163,6 +3163,7 @@ class Attribute extends AppModel 'includeFullModel' => !empty($filters['includeFullModel']) ? $filters['includeFullModel'] : 0, 'allow_proposal_blocking' => !empty($filters['allow_proposal_blocking']) ? $filters['allow_proposal_blocking'] : 0 ); + if (!empty($filters['attackGalaxy'])) { $params['attackGalaxy'] = $filters['attackGalaxy']; } @@ -3388,20 +3389,60 @@ class Attribute extends AppModel if (!empty($params['uuid'])) { $params['uuid'] = $this->convert_filters($params['uuid']); if (!empty($params['uuid']['OR'])) { - $conditions['AND'][] = array( - 'OR' => array( - 'Event.uuid' => $params['uuid']['OR'], - 'Attribute.uuid' => $params['uuid']['OR'] - ) - ); + if ($options['scope'] == 'Attribute') { + $subQuery = [ + 'conditions' => ['uuid' => $params['uuid']['OR']], + 'fields' => ['id'] + ]; + $pre_lookup = $this->Event->find('first', [ + 'conditions' => ['Event.uuid' => $params['uuid']['OR']], + 'recursive' => -1, + 'fields' => ['Event.id'] + ]); + if (empty($pre_lookup)) { + $conditions['AND'][] = array( + 'OR' => array( + 'Attribute.uuid' => $params['uuid']['OR'] + ) + ); + } else { + $conditions['AND'][] = array( + 'OR' => array( + $this->subQueryGenerator($this->Event, $subQuery, 'Attribute.event_id'), + 'Attribute.uuid' => $params['uuid']['OR'] + ) + ); + } + + } else { + $conditions['AND'][] = array( + 'OR' => array( + 'Event.uuid' => $params['uuid']['OR'], + 'Attribute.uuid' => $params['uuid']['OR'] + ) + ); + } } if (!empty($params['uuid']['NOT'])) { - $conditions['AND'][] = array( - 'NOT' => array( - 'Event.uuid' => $params['uuid']['NOT'], - 'Attribute.uuid' => $params['uuid']['NOT'] - ) - ); + if ($options['scope'] == 'Attribute') { + $subQuery = [ + 'conditions' => ['uuid' => $params['uuid']['OR']], + 'fields' => ['id'] + ]; + $conditions['AND'][] = [ + 'NOT' => [ + $this->subQueryGenerator($this->Event, $subQuery, 'Attribute.event_id'), + 'Attribute.uuid' => $params['uuid']['NOT'] + ] + ]; + } else { + $conditions['AND'][] = array( + 'NOT' => array( + 'Event.uuid' => $params['uuid']['NOT'], + 'Attribute.uuid' => $params['uuid']['NOT'] + ) + ); + } } } return $conditions; diff --git a/app/Model/Benchmark.php b/app/Model/Benchmark.php new file mode 100644 index 000000000..c350c6398 --- /dev/null +++ b/app/Model/Benchmark.php @@ -0,0 +1,6 @@ + $typeToCategoryMapping[$complexTypeToolEntry['default_type']][0], 'type' => $complexTypeToolEntry['default_type'], 'value' => $textToBeReplaced, - 'to_ids' => $complexTypeToolEntry['to_ids'], + 'to_ids' => $complexTypeToolEntry['to_ids'] ?? 0, ]; $replacedContent = str_replace($complexTypeToolEntry['original_value'], $textToInject, $replacedContent); } diff --git a/app/Model/Feed.php b/app/Model/Feed.php index ef448a536..e17fa4b45 100644 --- a/app/Model/Feed.php +++ b/app/Model/Feed.php @@ -1032,7 +1032,7 @@ class Feed extends AppModel } } } - if ($feed['Feed']['tag_id']) { + if ($feed['Feed']['tag_id'] || $feed['Feed']['tag_collection_id']) { if (empty($feed['Tag']['name'])) { $feed_tag = $this->Tag->find('first', [ 'conditions' => [ @@ -1041,23 +1041,42 @@ class Feed extends AppModel 'recursive' => -1, 'fields' => ['Tag.name', 'Tag.colour', 'Tag.id'] ]); - $feed['Tag'] = $feed_tag['Tag']; + if (!empty($feed_tag)) { + $feed['Tag'] = $feed_tag['Tag']; + } } if (!isset($event['Event']['Tag'])) { $event['Event']['Tag'] = array(); } - $feedTag = $this->Tag->find('first', array('conditions' => array('Tag.id' => $feed['Feed']['tag_id']), 'recursive' => -1, 'fields' => array('Tag.name', 'Tag.colour', 'Tag.exportable'))); - if (!empty($feedTag)) { - $found = false; - foreach ($event['Event']['Tag'] as $tag) { - if (strtolower($tag['name']) === strtolower($feedTag['Tag']['name'])) { - $found = true; - break; - } + if (!empty($feed['Feed']['tag_collection_id'])) { + $this->TagCollection = ClassRegistry::init('TagCollection'); + $tagCollectionID = $feed['Feed']['tag_collection_id']; + $tagCollection = $this->TagCollection->find('first', [ + 'recursive' => -1, + 'conditions' => [ + 'TagCollection.id' => $tagCollectionID, + ], + 'contain' => [ + 'TagCollectionTag' => ['Tag'], + ] + ]); + foreach ($tagCollection['TagCollectionTag'] as $collectionTag) { + $event['Event']['Tag'][] = $collectionTag['Tag']; } - if (!$found) { - $event['Event']['Tag'][] = $feedTag['Tag']; + } else { + $feedTag = $this->Tag->find('first', array('conditions' => array('Tag.id' => $feed['Feed']['tag_id']), 'recursive' => -1, 'fields' => array('Tag.name', 'Tag.colour', 'Tag.exportable'))); + if (!empty($feedTag)) { + $found = false; + foreach ($event['Event']['Tag'] as $tag) { + if (strtolower($tag['name']) === strtolower($feedTag['Tag']['name'])) { + $found = true; + break; + } + } + if (!$found) { + $event['Event']['Tag'][] = $feedTag['Tag']; + } } } } @@ -1068,6 +1087,9 @@ class Feed extends AppModel if (!empty($feed['Feed']['settings']['disable_correlation'])) { $event['Event']['disable_correlation'] = (bool) $feed['Feed']['settings']['disable_correlation']; } + if (!empty($feed['Feed']['settings']['unpublish_event'])) { + $event['Event']['published'] = (bool) $feed['Feed']['settings']['unpublish_event']; + } } return $event; } @@ -1128,9 +1150,13 @@ class Feed extends AppModel */ private function __updateEventFromFeed(HttpSocket $HttpSocket = null, $feed, $uuid, $user, $filterRules) { - $event = $this->downloadAndParseEventFromFeed($feed, $uuid, $HttpSocket); + $event = $this->downloadAndParseEventFromFeed($feed, $uuid, $HttpSocket); $event = $this->__prepareEvent($event, $feed, $filterRules); - return $this->Event->_edit($event, $user, $uuid, $jobId = null); + if (is_array($event)) { + return $this->Event->_edit($event, $user, $uuid, $jobId = null); + } else { + return $event; + } } public function addDefaultFeeds($newFeeds) @@ -1374,8 +1400,25 @@ class Feed extends AppModel if ($feed['Feed']['publish']) { $this->Event->publishRouter($event['Event']['id'], null, $user); } - if ($feed['Feed']['tag_id']) { - $this->Event->EventTag->attachTagToEvent($event['Event']['id'], ['id' => $feed['Feed']['tag_id']]); + if ($feed['Feed']['tag_id'] || $feed['Feed']['tag_collection_id']) { + if (!empty($feed['Feed']['tag_collection_id'])) { + $this->TagCollection = ClassRegistry::init('TagCollection'); + $tagCollectionID = $feed['Feed']['tag_collection_id']; + $tagCollection = $this->TagCollection->find('first', [ + 'recursive' => -1, + 'conditions' => [ + 'TagCollection.id' => $tagCollectionID, + ], + 'contain' => [ + 'TagCollectionTag', + ] + ]); + foreach ($tagCollection['TagCollectionTag'] as $collectionTag) { + $this->Event->EventTag->attachTagToEvent($event['Event']['id'], ['id' => $collectionTag['tag_id']]); + } + } else { + $this->Event->EventTag->attachTagToEvent($event['Event']['id'], ['id' => $feed['Feed']['tag_id']]); + } } return true; } diff --git a/app/Model/GalaxyCluster.php b/app/Model/GalaxyCluster.php index b02cead26..885d959ec 100644 --- a/app/Model/GalaxyCluster.php +++ b/app/Model/GalaxyCluster.php @@ -1845,6 +1845,9 @@ class GalaxyCluster extends AppModel if (!$compatible) { return 0; } + + $serverSync->debug("Pulling galaxy clusters with technique $technique"); + $clusterIds = $this->getClusterIdListBasedOnPullTechnique($user, $technique, $serverSync); $successes = 0; // now process the $clusterIds to pull each of the events sequentially diff --git a/app/Model/Log.php b/app/Model/Log.php index 2155a2e99..6c810db1d 100644 --- a/app/Model/Log.php +++ b/app/Model/Log.php @@ -431,15 +431,14 @@ class Log extends AppModel } } - $entry = $data['Log']['action']; - if (!empty($data['Log']['title'])) { - $entry .= " -- {$data['Log']['title']}"; - } - if (!empty($data['Log']['description'])) { - $entry .= " -- {$data['Log']['description']}"; - } else if (!empty($data['Log']['change'])) { - $entry .= " -- " . JsonTool::encode($data['Log']['change']); - } + $entry = sprintf( + '%s -- %s -- %s', + $data['Log']['action'], + empty($data['Log']['title']) ? '' : $formatted_title = preg_replace('/\s+/', " ", $data['Log']['title']), + empty($data['Log']['description']) ? + (empty($data['Log']['change']) ? '' : preg_replace('/\s+/', " ", $data['Log']['change'])) : + preg_replace('/\s+/', " ", $data['Log']['description']) + ); $this->syslog->write($action, $entry); } } diff --git a/app/Model/Server.php b/app/Model/Server.php index 933fadf31..aea2e9b1e 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -604,6 +604,7 @@ class Server extends AppModel * @throws HttpSocketHttpException * @throws HttpSocketJsonException * @throws JsonException + * @throws Exception */ public function pull(array $user, $technique, array $server, $jobId = false, $force = false) { @@ -619,7 +620,7 @@ class Server extends AppModel try { $server['Server']['version'] = $serverSync->info()['version']; } catch (Exception $e) { - $this->logException("Could not get remote server `{$server['Server']['name']}` version.", $e); + $this->logException("Could not get remote server `{$serverSync->serverName()}` version.", $e); if ($e instanceof HttpSocketHttpException && $e->getCode() === 403) { $message = __('Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server. Another reason could be an incorrect sync server setting.'); } else { @@ -648,6 +649,8 @@ class Server extends AppModel } } + $serverSync->debug("Pulling event list with technique $technique"); + try { $eventIds = $this->__getEventIdListBasedOnPullTechnique($technique, $serverSync, $force); } catch (Exception $e) { @@ -673,26 +676,29 @@ class Server extends AppModel $job->saveProgress($jobId, __n('Pulling %s event.', 'Pulling %s events.', count($eventIds), count($eventIds))); } foreach ($eventIds as $k => $eventId) { + $serverSync->debug("Pulling event $eventId"); $this->__pullEvent($eventId, $successes, $fails, $eventModel, $serverSync, $user, $jobId, $force); if ($jobId && $k % 10 === 0) { $job->saveProgress($jobId, null, 10 + 40 * (($k + 1) / count($eventIds))); } } foreach ($fails as $eventid => $message) { - $this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['Server']['id'], "Failed to pull event #$eventid.", 'Reason: ' . $message); + $this->loadLog()->createLogEntry($user, 'pull', 'Server', $serverSync->serverId(), "Failed to pull event #$eventid.", 'Reason: ' . $message); } } if ($jobId) { $job->saveProgress($jobId, 'Pulling proposals.', 50); } - $pulledProposals = $pulledSightings = 0; + $pulledProposals = $pulledSightings = $pulledAnalystData = 0; if ($technique === 'full' || $technique === 'update') { $pulledProposals = $eventModel->ShadowAttribute->pullProposals($user, $serverSync); if ($jobId) { $job->saveProgress($jobId, 'Pulling sightings.', 75); } + $pulledSightings = $eventModel->Sighting->pullSightings($user, $serverSync); + $this->AnalystData = ClassRegistry::init('AnalystData'); $pulledAnalystData = $this->AnalystData->pull($user, $serverSync); } @@ -819,7 +825,7 @@ class Server extends AppModel */ public function getElligibleClusterIdsFromServerForPull(ServerSyncTool $serverSync, $onlyUpdateLocalCluster=true, array $eligibleClusters=array(), array $conditions=array()) { - $this->log("Fetching eligible clusters from server #{$serverSync->serverId()} for pull: " . JsonTool::encode($conditions), LOG_INFO); + $serverSync->debug("Fetching eligible clusters for pull: " . JsonTool::encode($conditions)); if ($onlyUpdateLocalCluster && empty($eligibleClusters)) { return []; // no clusters for update @@ -875,7 +881,7 @@ class Server extends AppModel */ private function getElligibleClusterIdsFromServerForPush(ServerSyncTool $serverSync, array $localClusters=array(), array $conditions=array()) { - $this->log("Fetching eligible clusters from server #{$serverSync->serverId()} for push: " . JsonTool::encode($conditions), LOG_INFO); + $serverSync->debug("Fetching eligible clusters for push: " . JsonTool::encode($conditions)); $clusterArray = $this->fetchCustomClusterIdsFromServer($serverSync, $conditions=$conditions); $keyedClusterArray = Hash::combine($clusterArray, '{n}.GalaxyCluster.uuid', '{n}.GalaxyCluster.version'); if (!empty($localClusters)) { @@ -915,9 +921,14 @@ class Server extends AppModel // Fetch event index from cache if exists and is not modified $redis = RedisTool::init(); - $indexFromCache = $redis->get("misp:event_index:{$serverSync->serverId()}"); + $indexFromCache = $redis->get("misp:event_index_cache:{$serverSync->serverId()}"); if ($indexFromCache) { - list($etag, $eventIndex) = RedisTool::deserialize(RedisTool::decompress($indexFromCache)); + $etagPos = strpos($indexFromCache, "\n"); + if ($etagPos === false) { + throw new RuntimeException("Could not find etag in cache fro server {$serverSync->serverId()}"); + } + $etag = substr($indexFromCache, 0, $etagPos); + $serverSync->debug("Event index loaded from Redis cache with etag $etag containing"); } else { $etag = '""'; // Provide empty ETag, so MISP will compute ETag for returned data } @@ -925,9 +936,21 @@ class Server extends AppModel $response = $serverSync->eventIndex($filterRules, $etag); if ($response->isNotModified() && $indexFromCache) { - return $eventIndex; + return JsonTool::decode(RedisTool::decompress(substr($indexFromCache, $etagPos + 1))); } + // Save to cache for 24 hours if ETag provided + $etag = $response->getHeader('etag'); + if ($etag) { + $serverSync->debug("Event index from remote server has different etag $etag, saving to cache"); + $data = "$etag\n" . RedisTool::compress($response->body); + $redis->setex("misp:event_index_cache:{$serverSync->serverId()}", 3600 * 24, $data); + } elseif ($indexFromCache) { + RedisTool::unlink($redis, "misp:event_index_cache:{$serverSync->serverId()}"); + } + + unset($indexFromCache); // clean up memory + $eventIndex = $response->json(); // correct $eventArray if just one event, probably this response returns old MISP @@ -935,15 +958,6 @@ class Server extends AppModel $eventIndex = [$eventIndex]; } - // Save to cache for 24 hours if ETag provided - $etag = $response->getHeader('etag'); - if ($etag) { - $data = RedisTool::compress(RedisTool::serialize([$etag, $eventIndex])); - $redis->setex("misp:event_index:{$serverSync->serverId()}", 3600 * 24, $data); - } elseif ($indexFromCache) { - RedisTool::unlink($redis, "misp:event_index:{$serverSync->serverId()}"); - } - return $eventIndex; } @@ -1372,7 +1386,7 @@ class Server extends AppModel return []; // pushing clusters is not enabled } - $this->log("Starting $technique clusters sync with server #{$serverSync->serverId()}", LOG_INFO); + $serverSync->debug("Starting $technique clusters sync"); $this->GalaxyCluster = ClassRegistry::init('GalaxyCluster'); $this->Event = ClassRegistry::init('Event'); @@ -5125,8 +5139,8 @@ class Server extends AppModel ), 'curl_request_timeout' => [ 'level' => 1, - 'description' => __('Control the timeout of curl requests issued by MISP (during synchronisation, feed fetching, etc.'), - 'value' => 10800, + 'description' => __('Control the default timeout in seconds of curl HTTP requests issued by MISP (during synchronisation, feed fetching, etc.)'), + 'value' => 300, 'test' => 'testForNumeric', 'type' => 'numeric', 'null' => true @@ -7536,6 +7550,13 @@ class Server extends AppModel 'test' => 'testBool', 'type' => 'boolean' ), + 'Benchmarking_enable' => [ + 'level' => 2, + 'description' => __('Enable the benchmarking functionalities to capture information about execution times, SQL query loads and more per user and per endpoint.'), + 'value' => false, + 'test' => 'testBool', + 'type' => 'boolean' + ], 'Enrichment_services_enable' => array( 'level' => 0, 'description' => __('Enable/disable the enrichment services'), diff --git a/app/Model/ShadowAttribute.php b/app/Model/ShadowAttribute.php index 645da2dec..a1e6b002b 100644 --- a/app/Model/ShadowAttribute.php +++ b/app/Model/ShadowAttribute.php @@ -706,6 +706,8 @@ class ShadowAttribute extends AppModel return 0; } + $serverSync->debug("Pulling proposals"); + $i = 1; $fetchedCount = 0; $chunkSize = 1000; diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php index df09b80cc..fc1b1096e 100644 --- a/app/Model/Sighting.php +++ b/app/Model/Sighting.php @@ -1418,11 +1418,13 @@ class Sighting extends AppModel */ public function pullSightings(array $user, ServerSyncTool $serverSync) { + $serverSync->debug("Fetching event index for pulling sightings"); + $this->Server = ClassRegistry::init('Server'); try { $remoteEvents = $this->Server->getEventIndexFromServer($serverSync); } catch (Exception $e) { - $this->logException("Could not fetch event IDs from server {$serverSync->server()['Server']['name']}", $e); + $this->logException("Could not fetch event IDs from server {$serverSync->serverName()}", $e); return 0; } // Remove events from list that do not have published sightings. @@ -1452,6 +1454,8 @@ class Sighting extends AppModel return 0; } + $serverSync->debug("Pulling sightings for " . count($eventUuids) . " events"); + if ($serverSync->isSupported(ServerSyncTool::FEATURE_SIGHTING_REST_SEARCH)) { return $this->pullSightingNewWay($user, $eventUuids, $serverSync); } else { @@ -1470,12 +1474,19 @@ class Sighting extends AppModel */ private function pullSightingNewWay(array $user, array $eventUuids, ServerSyncTool $serverSync) { + $SightingBlocklist = ClassRegistry::init('SightingBlocklist'); + $blockedSightingsOrgs = $SightingBlocklist->find('column', [ + 'recursive' => -1, + 'fields' => ['org_uuid'] + ]); + $uuids = array_keys($eventUuids); + shuffle($uuids); // shuffle array to avoid keeping events with a lof ot sightings in same batch all the time $saved = 0; $savedEventUuids = []; - foreach (array_chunk($uuids, 100) as $chunk) { + foreach (array_chunk($uuids, 20) as $chunk) { try { - $sightings = $serverSync->fetchSightingsForEvents($chunk); + $sightings = $serverSync->fetchSightingsForEvents($chunk, $blockedSightingsOrgs); } catch (Exception $e) { $this->logException("Failed to download sightings from remote server {$serverSync->server()['Server']['name']}.", $e); continue; diff --git a/app/Model/WorkflowModules/WorkflowBaseModule.php b/app/Model/WorkflowModules/WorkflowBaseModule.php index 66defb1bf..8a4e2f668 100644 --- a/app/Model/WorkflowModules/WorkflowBaseModule.php +++ b/app/Model/WorkflowModules/WorkflowBaseModule.php @@ -213,6 +213,8 @@ class WorkflowBaseModule if ($operator == 'in_or') { return !empty($matching); } elseif ($operator == 'in_and') { + sort($matching); + sort($value); return array_values($matching) == array_values($value); } elseif ($operator == 'not_in_or') { return empty($matching); diff --git a/app/Model/WorkflowModules/action/Module_stop_execution.php b/app/Model/WorkflowModules/action/Module_stop_execution.php index 5266a1557..b02bce689 100644 --- a/app/Model/WorkflowModules/action/Module_stop_execution.php +++ b/app/Model/WorkflowModules/action/Module_stop_execution.php @@ -6,6 +6,7 @@ class Module_stop_execution extends WorkflowBaseActionModule public $blocking = true; public $id = 'stop-execution'; public $name = 'Stop execution'; + public $version = '0.2'; public $description = 'Essentially stops the execution for blocking workflows. Do nothing for non-blocking ones'; public $icon = 'ban'; public $inputs = 1; @@ -15,12 +16,25 @@ class Module_stop_execution extends WorkflowBaseActionModule public function __construct() { parent::__construct(); + + $this->params = [ + [ + 'id' => 'message', + 'label' => 'Stop message', + 'type' => 'input', + 'default' => __('Execution stopped'), + 'placeholder' => __('Execution stopped'), + 'jinja_supported' => true, + ], + ]; } public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool { parent::exec($node, $roamingData, $errors); - $errors[] = __('Execution stopped'); + $rData = $roamingData->getData(); + $params = $this->getParamsWithValues($node, $rData); + $errors[] = empty($params['message']['value']) ? $params['message']['default'] : $params['message']['value']; return false; } } diff --git a/app/Model/WorkflowModules/logic/Module_distribution_if.php b/app/Model/WorkflowModules/logic/Module_distribution_if.php index 6135385f2..a4f98cecd 100644 --- a/app/Model/WorkflowModules/logic/Module_distribution_if.php +++ b/app/Model/WorkflowModules/logic/Module_distribution_if.php @@ -5,7 +5,7 @@ class Module_distribution_if extends WorkflowBaseLogicModule { public $id = 'distribution-if'; public $name = 'IF :: Distribution'; - public $version = '0.2'; + public $version = '0.3'; public $description = 'Distribution IF / ELSE condition block. The `then` output will be used if the encoded conditions is satisfied, otherwise the `else` output will be used.'; public $icon = 'code-branch'; public $inputs = 1; @@ -103,12 +103,15 @@ class Module_distribution_if extends WorkflowBaseLogicModule $final_sharing_group = $this->__extractSharingGroupIDs( $data['Event'], $data['Event']['Attribute'][0]['Object'] ?? [], - $data['Event']['Attribute'][0] + $data['Event']['Attribute'][0], + $scope ); if ($operator == 'equals') { - return !array_diff($final_sharing_group, $selected_sharing_groups); // All sharing groups are in the selection + return empty($selected_sharing_groups) ? !empty($final_sharing_group) : + !array_diff($final_sharing_group, $selected_sharing_groups); // All sharing groups are in the selection } else if ($operator == 'not_equals') { - return count(array_diff($final_sharing_group, $selected_sharing_groups)) == count($final_sharing_group); // All sharing groups are in the selection + return empty($selected_sharing_groups) ? empty($final_sharing_group) : + count(array_diff($final_sharing_group, $selected_sharing_groups)) == count($final_sharing_group); // All sharing groups are in the selection } $errors[] = __('Condition operator not supported for that distribution level'); return false; @@ -159,9 +162,15 @@ class Module_distribution_if extends WorkflowBaseLogicModule return min($distri1, $distri2); } - private function __extractSharingGroupIDs(array $event, array $object=[], array $attribute=[]): array + private function __extractSharingGroupIDs(array $event, array $object=[], array $attribute=[], $scope='event'): array { $sgIDs = []; + if ($scope == 'event') { + if (!empty($event) && $event['distribution'] == 4) { + $sgIDs[] = $event['sharing_group_id']; + } + return $sgIDs; + } if (!empty($event) && $event['distribution'] == 4) { $sgIDs[] = $event['sharing_group_id']; } diff --git a/app/View/AnalystData/view.ctp b/app/View/AnalystData/view.ctp index f56460e7a..c9715d63f 100644 --- a/app/View/AnalystData/view.ctp +++ b/app/View/AnalystData/view.ctp @@ -120,15 +120,57 @@ echo $this->element( ); $object_uuid = Hash::get($data, $modelSelection . '.uuid'); + +$notes = $data[$modelSelection]['Note'] ?? []; +$opinions = $data[$modelSelection]['Opinion'] ?? []; +$relationships_outbound = $data[$modelSelection]['Relationship'] ?? []; +$relationships_inbound = $data[$modelSelection]['RelationshipInbound'] ?? []; +$notesOpinions = array_merge($notes, $opinions); +if(!function_exists("countNotes")) { + function countNotes($notesOpinions) { + $notesTotalCount = count($notesOpinions); + $notesCount = 0; + $relationsCount = 0; + foreach ($notesOpinions as $notesOpinion) { + if ($notesOpinion['note_type'] == 2) { // relationship + $relationsCount += 1; + } else { + $notesCount += 1; + } + if (!empty($notesOpinion['Note'])) { + $nestedCounts = countNotes($notesOpinion['Note']); + $notesTotalCount += $nestedCounts['total']; + $notesCount += $nestedCounts['notesOpinions']; + $relationsCount += $nestedCounts['relations']; + } + if (!empty($notesOpinion['Opinion'])) { + $nestedCounts = countNotes($notesOpinion['Opinion']); + $notesTotalCount += $nestedCounts['total']; + $notesCount += $nestedCounts['notesOpinions']; + $relationsCount += $nestedCounts['relations']; + } + } + return ['total' => $notesTotalCount, 'notesOpinions' => $notesCount, 'relations' => $relationsCount]; + } +} +$counts = countNotes($notesOpinions); +$notesOpinionCount = $counts['notesOpinions']; +$allCounts = [ + 'notesOpinions' => $counts['notesOpinions'], + 'relationships_outbound' => count($relationships_outbound), + 'relationships_inbound' => count($relationships_inbound), +]; + $options = [ 'container_id' => 'analyst_data_thread', 'object_type' => $modelSelection, 'object_uuid' => $object_uuid, 'shortDist' => $shortDist, - 'notes' => $data[$modelSelection]['Note'] ?? [], - 'opinions' => $data[$modelSelection]['Opinion'] ?? [], - 'relationships_outbound' => $data[$modelSelection]['Relationship'] ?? [], - 'relationships_inbound' => $data[$modelSelection]['RelationshipInbound'] ?? [], + 'notes' => $notes, + 'opinions' => $opinions, + 'relationships_outbound' => $relationships_outbound, + 'relationships_inbound' => $relationships_inbound, + 'allCounts' => $allCounts, ]; echo $this->element('genericElements/assetLoader', [ diff --git a/app/View/Benchmarks/index.ctp b/app/View/Benchmarks/index.ctp new file mode 100644 index 000000000..df2c9e282 --- /dev/null +++ b/app/View/Benchmarks/index.ctp @@ -0,0 +1,110 @@ + __('Date'), + 'sort' => 'date', + 'data_path' => 'date' + ], + [ + 'name' => __('scope'), + 'sort' => 'scope', + 'data_path' => 'scope' + ], + [ + 'name' => __('Key'), + 'sort' => 'key', + 'data_path' => 'text' + ], + [ + 'name' => __('field'), + 'sort' => 'field', + 'data_path' => 'field' + ], + [ + 'name' => __('Value'), + 'element' => 'custom', + 'function' => function($row) { + return empty($row['unit']) ? h($row['value']) : h($row['value'] . ' ' . $row['unit']); + }, + 'sort' => 'value' + ] + ]; + $quick_filters = []; + foreach ($settings as $key => $setting_data) { + $temp = $filters; + $url = $baseurl . '/benchmarks/index'; + foreach ($temp as $s => $v) { + if ($v && $s != $key) { + if (is_array($v)) { + foreach ($v as $multi_v) { + $url .= '/' . $s . '[]:' . $multi_v; + } + } else { + $url .= '/' . $s . ':' . $v; + } + + } + } + if ($key != 'average' && $key != 'aggregate') { + $quick_filters[$key]['all'] = [ + 'url' => h($url), + 'text' => __('All'), + 'active' => !$filters[$key], + 'style' => 'display:inline;' + ]; + } + foreach ($setting_data as $setting_element) { + $text = $setting_element; + if ($key == 'average') { + $text = $setting_element ? 'average / request' : 'total'; + } + if ($key == 'aggregate') { + $text = $setting_element ? 'aggregate' : 'daily'; + } + $quick_filters[$key][] = [ + 'url' => h($url . '/' . $key . ':' . $setting_element), + 'text' => $text, + 'active' => $filters[$key] == $setting_element, + 'style' => 'display:inline;' + ]; + } + } + echo $this->element('genericElements/IndexTable/scaffold', [ + 'scaffold_data' => [ + 'passedArgsArray' => $passedArgsArray, + 'data' => [ + 'persistUrlParams' => array_keys($settings), + 'data' => $data, + 'top_bar' => [ + 'pull' => 'right', + 'children' => [ + [ + 'children' => $quick_filters['scope'] + ], + [ + 'children' => $quick_filters['field'] + ], + [ + 'children' => $quick_filters['average'] + ], + [ + 'children' => $quick_filters['aggregate'] + ], + [ + 'type' => 'search', + 'button' => __('Filter'), + 'placeholder' => __('Enter value to search'), + 'data' => '', + 'searchKey' => 'quickFilter' + ] + ] + ], + 'fields' => $fields, + 'title' => empty($ajax) ? __('Benchmark results') : false, + 'description' => empty($ajax) ? __('Results of the collected benchmarks. You can filter it further by passing the limit, scope, field parameters.') : false, + ] + ] + ]); + +?> diff --git a/app/View/Dashboards/index.ctp b/app/View/Dashboards/index.ctp index 278f588f3..4e0b31775 100644 --- a/app/View/Dashboards/index.ctp +++ b/app/View/Dashboards/index.ctp @@ -23,7 +23,7 @@ $(function () { saveDashboardState(); }); grid.on('added', function(event, items) { - resetDashboardGrid(grid); + resetDashboardGrid(grid, false); }); grid.on('gsresizestop', function(event, element) { $(element).find('.widgetContentInner').trigger('widget-resized') diff --git a/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp b/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp index a6057be90..10115e48d 100644 --- a/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp +++ b/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp @@ -6,6 +6,7 @@ h($data['formula']) ); } + $y_axis = $data['y-axis'] ?? 'Count'; ?>
diff --git a/app/View/Elements/genericElements/Analyst_data/thread.ctp b/app/View/Elements/genericElements/Analyst_data/thread.ctp index 5930f6c5d..bad0a3b8d 100644 --- a/app/View/Elements/genericElements/Analyst_data/thread.ctp +++ b/app/View/Elements/genericElements/Analyst_data/thread.ctp @@ -4,6 +4,7 @@ $URL_DELETE = '/analystData/delete/'; $seed = isset($seed) ? $seed : mt_rand(); + $injectInPage = !empty($container_id) ? true : false; $notes = !empty($notes) ? $notes : []; $opinions = !empty($opinions) ? $opinions : []; @@ -41,7 +42,28 @@ if (!window.shortDist) { var shortDist = ; } -var renderedNotes = null + +var container_id = false + + container_id = '' + + +function adjustPopoverPosition() { + var $popover = $('.popover:last'); + $popover.css('top', Math.max($popover.position().top, 50) + 'px') +} + +function openNotes(clicked) { + var notes = ; + var relationships = ; + var relationships_inbound = ; + var relationship_related_object = ; + var renderedNotes = renderAllNotesWithForm(notes, relationships, relationships_inbound, relationship_related_object) + openPopover(clicked, renderedNotes, undefined, undefined, function() { + adjustPopoverPosition() + $(clicked).removeClass('have-a-popover') // avoid closing the popover if a confirm popover (like the delete one) is called + }) +} function renderNotes(notes, relationship_related_object, emptyMessage='', isInbound=false) { var renderedNotesArray = [] @@ -406,18 +428,7 @@ function fetchMoreNotes(clicked, noteType, uuid) { } - -(function() { - var notes = ; - var relationships = ; - var relationships_inbound = ; - var relationship_related_object = ; - var container_id = false - - container_id = '' - - - var nodeContainerTemplate = doT.template('\ + var nodeContainerTemplate = doT.template('\
\
\ ') - function renderAllNotesWithForm(relationship_related_object) { - var buttonContainer = '
' + addNoteButton + addOpinionButton + '
' - renderedNotes = nodeContainerTemplate({ - content_notes: renderNotes(notes.filter(function(note) { return note.note_type != 2}), relationship_related_object, '') + buttonContainer, - content_relationships_outbound: renderNotes(relationships, relationship_related_object, '') + addRelationshipButton, - content_relationships_inbound: renderNotes(relationships_inbound, relationship_related_object, '', true), - }) - if (container_id) { - $('#' + container_id).html(renderedNotes) - } - } - var addNoteButton = '' @@ -461,10 +460,15 @@ function fetchMoreNotes(clicked, noteType, uuid) { \ ' - $(document).ready(function() { - renderAllNotesWithForm(relationship_related_object) - }) -})() + function renderAllNotesWithForm(notes, relationships, relationships_inbound, relationship_related_object) { + var buttonContainer = '
' + addNoteButton + addOpinionButton + '
' + var renderedNotes = nodeContainerTemplate({ + content_notes: renderNotes(notes.filter(function(note) { return note.note_type != 2}), relationship_related_object, '') + buttonContainer, + content_relationships_outbound: renderNotes(relationships, relationship_related_object, '') + addRelationshipButton, + content_relationships_inbound: renderNotes(relationships_inbound, relationship_related_object, '', true), + }) + return renderedNotes + } function createNewNote(clicked, object_type, object_uuid) { note_type = 'Note'; @@ -516,6 +520,19 @@ function fetchMoreNotes(clicked, noteType, uuid) { } } + + $(document).ready(function() { + var notes = ; + var relationships = ; + var relationships_inbound = ; + var relationship_related_object = ; + var renderedNotes = renderAllNotesWithForm(notes, relationships, relationships_inbound, relationship_related_object) + if (container_id) { + $('#' + container_id).html(renderedNotes) + } + }) + +