diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d60373e3a..f9d766b50 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -269,13 +269,16 @@ jobs: - name: Check requirements.txt run: python tests/check_requirements.py - - name: Logs + - name: System logs if: ${{ always() }} # update logs_test.sh when adding more logsources here run: | tail -n +1 `pwd`/app/tmp/logs/* tail -n +1 /var/log/apache2/*.log + - name: Application logs + if: ${{ always() }} + run: | app/Console/cake Log export /tmp/logs.json.gz --without-changes zcat /tmp/logs.json.gz diff --git a/README.md b/README.md index b98ff74a8..0732a6faf 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,8 @@ The objective of MISP is to foster the sharing of structured information within +[![CLA FREE initiative](https://raw.githubusercontent.com/ossbase-org/ossbase.org/main/logos/cla-free-small.png)](https://ossbase.org/initiatives/cla-free/) + Core functions ------------------ - An **efficient IOC and indicators** database, allowing to store technical and non-technical information about malware samples, incidents, attackers and intelligence. diff --git a/app/Console/Command/ServerShell.php b/app/Console/Command/ServerShell.php index 6b29266f4..f6961516b 100644 --- a/app/Console/Command/ServerShell.php +++ b/app/Console/Command/ServerShell.php @@ -144,6 +144,10 @@ class ServerShell extends AppShell if (!empty($this->args[4]) && $this->args[4] === 'force') { $force = true; } + + // Try to enable garbage collector as pulling events can use a lot of memory + gc_enable(); + try { $result = $this->Server->pull($user, $technique, $server, $jobId, $force); if (is_array($result)) { diff --git a/app/Controller/AnalystDataController.php b/app/Controller/AnalystDataController.php index 9659555d8..c54800aae 100644 --- a/app/Controller/AnalystDataController.php +++ b/app/Controller/AnalystDataController.php @@ -320,6 +320,11 @@ class AnalystDataController extends AppController $this->AnalystData = $this->{$vt}; $this->modelClass = $vt; $this->{$vt}->current_user = $this->Auth->user(); + if (!empty($this->request->data)) { + if (!isset($this->request->data[$type])) { + $this->request->data = [$type => $this->request->data]; + } + } return $vt; } } diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index 010af593b..38e8df162 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -33,7 +33,7 @@ class AppController extends Controller public $helpers = array('OrgImg', 'FontAwesome', 'UserName'); - private $__queryVersion = '159'; + private $__queryVersion = '161'; public $pyMispVersion = '2.4.188'; public $phpmin = '7.2'; public $phprec = '7.4'; @@ -1063,7 +1063,19 @@ class AppController extends Controller $data = array_merge($data, $temp); } else { foreach ($options['paramArray'] as $param) { - if (isset($temp[$param])) { + if (substr($param, -1) == '*') { + $root = substr($param, 0, strlen($param)-1); + foreach ($temp as $existingParamKey => $v) { + $leftover = substr($existingParamKey, strlen($param)-1); + if ( + $root == substr($existingParamKey, 0, strlen($root)) && + preg_match('/^[\w_-. ]+$/', $leftover) == 1 + ) { + $data[$existingParamKey] = $temp[$existingParamKey]; + break; + } + } + } else if (isset($temp[$param])) { $data[$param] = $temp[$param]; } } diff --git a/app/Controller/Component/RestSearchComponent.php b/app/Controller/Component/RestSearchComponent.php index ae8a4a34e..9e8ab024f 100644 --- a/app/Controller/Component/RestSearchComponent.php +++ b/app/Controller/Component/RestSearchComponent.php @@ -144,7 +144,11 @@ class RestSearchComponent extends Component 'retry', 'expiry', 'minimum_ttl', - 'ttl' + 'ttl', + 'org.sector', + 'org.local', + 'org.nationality', + 'galaxy.*', ], 'Object' => [ 'returnFormat', diff --git a/app/Controller/EventReportsController.php b/app/Controller/EventReportsController.php index 04199f2b5..efeb9ddd4 100644 --- a/app/Controller/EventReportsController.php +++ b/app/Controller/EventReportsController.php @@ -213,10 +213,13 @@ class EventReportsController extends AppController public function extractAllFromReport($reportId) { - if (!$this->request->is('ajax')) { + if (!$this->request->is('ajax') && !$this->_isRest()) { throw new MethodNotAllowedException(__('This function can only be reached via AJAX.')); } if ($this->request->is('post')) { + if (!isset($this->data['EventReport'])) { + $this->data = ['EventReport' => $this->data]; + } $report = $this->EventReport->fetchIfAuthorized($this->Auth->user(), $reportId, 'edit', $throwErrors=true, $full=false); $results = $this->EventReport->getComplexTypeToolResultWithReplacements($this->Auth->user(), $report); $report['EventReport']['content'] = $results['replacementResult']['contentWithReplacements']; @@ -299,13 +302,16 @@ class EventReportsController extends AppController public function importReportFromUrl($event_id) { - if (!$this->request->is('ajax')) { - throw new MethodNotAllowedException(__('This function can only be reached via AJAX.')); + if (!$this->request->is('ajax') && !$this->_isRest()) { + throw new MethodNotAllowedException(__('This function can only be reached via AJAX and via the API.')); } $fetcherModule = $this->EventReport->isFetchURLModuleEnabled(); if ($this->request->is('post')) { + if (empty($this->data['EventReport'])) { + $this->data = ['EventReport' => $this->data]; + } if (empty($this->data['EventReport']['url'])) { - throw new MethodNotAllowedException(__('An URL must be provided')); + throw new MethodNotAllowedException(__('A URL must be provided')); } $url = $this->data['EventReport']['url']; $format = 'html'; @@ -316,7 +322,6 @@ class EventReportsController extends AppController $format = $parsed_format; } } - $content = $this->EventReport->downloadMarkdownFromURL($event_id, $url, $format); $errors = []; diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 4e695ac67..418a40cce 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -497,6 +497,11 @@ class EventsController extends AppController continue 2; } $pieces = is_array($v) ? $v : explode('|', $v); + $isANDed = false; + if (count($pieces) == 1 && strpos($pieces[0], '&') !== -1) { + $pieces = explode('&', $v); + $isANDed = count($pieces) > 1; + } $filterString = ""; $expectOR = false; $tagRules = []; @@ -563,10 +568,19 @@ class EventsController extends AppController } if (!empty($tagRules['include'])) { - $include = $this->Event->EventTag->find('column', array( - 'conditions' => array('EventTag.tag_id' => $tagRules['include']), - 'fields' => ['EventTag.event_id'], - )); + if ($isANDed) { + $include = $this->Event->EventTag->find('column', array( + 'conditions' => ['EventTag.tag_id' => $tagRules['include']], + 'fields' => ['EventTag.event_id'], + 'group' => ['EventTag.event_id'], + 'having' => ['COUNT(*) =' => count($tagRules['include'])], + )); + } else { + $include = $this->Event->EventTag->find('column', array( + 'conditions' => array('EventTag.tag_id' => $tagRules['include']), + 'fields' => ['EventTag.event_id'], + )); + } if (!empty($include)) { $this->paginate['conditions']['AND'][] = 'Event.id IN (' . implode(",", $include) . ')'; } else { diff --git a/app/Controller/TagCollectionsController.php b/app/Controller/TagCollectionsController.php index 0131790ab..0f204e261 100644 --- a/app/Controller/TagCollectionsController.php +++ b/app/Controller/TagCollectionsController.php @@ -357,7 +357,7 @@ class TagCollectionsController extends AppController if (!$tagCollection) { throw new NotFoundException(__('Invalid tag collection.')); } - if ($this->ACL->canModifyTagCollection($this->Auth->user(), $tagCollection)) { + if (!$this->ACL->canModifyTagCollection($this->Auth->user(), $tagCollection)) { throw new ForbiddenException(__('You dont have a permission to do that')); } $tagCollectionTag = $this->TagCollection->TagCollectionTag->find('first', [ diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php index 09bc73493..4baa2378e 100644 --- a/app/Controller/UsersController.php +++ b/app/Controller/UsersController.php @@ -2071,7 +2071,7 @@ class UsersController extends AppController $stats['attribute_count'] = $this->User->Event->Attribute->find('count', array('conditions' => array('Attribute.deleted' => 0), 'recursive' => -1)); $stats['attribute_count_month'] = $this->User->Event->Attribute->find('count', array('conditions' => array('Attribute.timestamp >' => $this_month, 'Attribute.deleted' => 0), 'recursive' => -1)); - $stats['attributes_per_event'] = round($stats['attribute_count'] / $stats['event_count']); + $stats['attributes_per_event'] = $stats['event_count'] != 0 ? round($stats['attribute_count'] / $stats['event_count']) : 0; $stats['correlation_count'] = $this->User->Event->Attribute->Correlation->find('count', array('recursive' => -1)); @@ -2082,7 +2082,7 @@ class UsersController extends AppController $stats['org_count'] = count($orgs); $stats['local_org_count'] = $local_orgs_count; $stats['contributing_org_count'] = $this->User->Event->find('count', array('recursive' => -1, 'group' => array('Event.orgc_id'))); - $stats['average_user_per_org'] = round($stats['user_count'] / $stats['local_org_count'], 1); + $stats['average_user_per_org'] = $stats['local_org_count'] != 0 ? round($stats['user_count'] / $stats['local_org_count'], 1) : 0; $this->loadModel('Thread'); $stats['thread_count'] = $this->Thread->find('count', array('conditions' => array('Thread.post_count >' => 0), 'recursive' => -1)); diff --git a/app/Lib/Dashboard/EventEvolutionLineWidget.php b/app/Lib/Dashboard/EventEvolutionLineWidget.php index a4c06e007..680e17355 100644 --- a/app/Lib/Dashboard/EventEvolutionLineWidget.php +++ b/app/Lib/Dashboard/EventEvolutionLineWidget.php @@ -58,9 +58,11 @@ class EventEvolutionLineWidget 'recursive' => -1 ]; $eparams = []; + $filteringOnOrg = false; if (!empty($options['filter']) && is_array($options['filter'])) { foreach ($this->validFilterKeys as $filterKey) { if (!empty($options['filter'][$filterKey])) { + $filteringOnOrg = true; if (!is_array($options['filter'][$filterKey])) { $options['filter'][$filterKey] = [$options['filter'][$filterKey]]; } @@ -87,6 +89,9 @@ class EventEvolutionLineWidget 'conditions' => $oparams['conditions'], 'fields' => ['id'] ]); + if ($filteringOnOrg) { + $eparams['conditions']['AND']['Event.orgc_id IN'] = !empty($org_ids) ? $org_ids : [-1]; + } $this->Event->virtualFields = [ 'published_date' => null ]; diff --git a/app/Lib/Tools/CurlClient.php b/app/Lib/Tools/CurlClient.php index 006b58f3f..4e19b8462 100644 --- a/app/Lib/Tools/CurlClient.php +++ b/app/Lib/Tools/CurlClient.php @@ -6,8 +6,12 @@ class CurlClient extends HttpSocketExtended /** @var resource */ private $ch; - /** @var int */ - private $timeout = 10800; + /** + * Maximum time the transfer is allowed to complete in seconds + * 300 seconds is recommended timeout for MISP servers + * @var int + */ + private $timeout = 300; /** @var string|null */ private $caFile; @@ -30,6 +34,9 @@ class CurlClient extends HttpSocketExtended /** @var array */ private $proxy = []; + /** @var array */ + private $defaultOptions; + /** * @param array $params * @noinspection PhpMissingParentConstructorInspection @@ -38,8 +45,6 @@ class CurlClient extends HttpSocketExtended { if (isset($params['timeout'])) { $this->timeout = $params['timeout']; - } else { - $this->timeout = Configure::check('MISP.curl_request_timeout') ? Configure::read('MISP.curl_request_timeout') : 10800; } if (isset($params['ssl_cafile'])) { $this->caFile = $params['ssl_cafile']; @@ -59,6 +64,7 @@ class CurlClient extends HttpSocketExtended if (isset($params['ssl_verify_peer'])) { $this->verifyPeer = $params['ssl_verify_peer']; } + $this->defaultOptions = $this->generateDefaultOptions(); } /** @@ -166,6 +172,7 @@ class CurlClient extends HttpSocketExtended return; } $this->proxy = compact('host', 'port', 'method', 'user', 'pass'); + $this->defaultOptions = $this->generateDefaultOptions(); // regenerate default options in case proxy setting is changed } /** @@ -196,7 +203,7 @@ class CurlClient extends HttpSocketExtended $url .= '?' . http_build_query($query, '', '&', PHP_QUERY_RFC3986); } - $options = $this->generateOptions(); + $options = $this->defaultOptions; // this will copy default options $options[CURLOPT_URL] = $url; $options[CURLOPT_CUSTOMREQUEST] = $method; @@ -303,7 +310,7 @@ class CurlClient extends HttpSocketExtended /** * @return array */ - private function generateOptions() + private function generateDefaultOptions() { $options = [ CURLOPT_FOLLOWLOCATION => true, // Allows to follow redirect diff --git a/app/Lib/Tools/HttpSocketExtended.php b/app/Lib/Tools/HttpSocketExtended.php index cfe097c28..375144f5b 100644 --- a/app/Lib/Tools/HttpSocketExtended.php +++ b/app/Lib/Tools/HttpSocketExtended.php @@ -24,7 +24,7 @@ class HttpSocketHttpException extends Exception $message .= " for URL $url"; } if ($response->body) { - $message .= ': ' . substr($response->body, 0, 100); + $message .= ': ' . substr(ltrim($response->body), 0, 100); } parent::__construct($message, (int)$response->code); @@ -121,7 +121,8 @@ class HttpSocketResponseExtended extends HttpSocketResponse try { return JsonTool::decode($this->body); } catch (Exception $e) { - throw new HttpSocketJsonException('Could not parse response as JSON.', $this, $e); + $contentType = $this->getHeader('content-type'); + throw new HttpSocketJsonException("Could not parse HTTP response as JSON. Received Content-Type $contentType.", $this, $e); } } } diff --git a/app/Lib/Tools/ServerSyncTool.php b/app/Lib/Tools/ServerSyncTool.php index 11c1c9d61..6f433bcba 100644 --- a/app/Lib/Tools/ServerSyncTool.php +++ b/app/Lib/Tools/ServerSyncTool.php @@ -506,6 +506,16 @@ class ServerSyncTool return $this->socket->getMetaData(); } + /** + * @param string $message + * @return void + */ + public function debug($message) + { + $memoryUsage = round(memory_get_usage() / 1024 / 1024, 2); + CakeLog::debug("[Server sync #{$this->serverId()}]: $message. Memory: $memoryUsage MB"); + } + /** * @params string $url Relative URL * @return HttpSocketResponseExtended @@ -556,6 +566,7 @@ class ServerSyncTool if ($etag) { // Remove compression marks that adds Apache for compressed content + // This can be removed in future as this is already checked by MISP itself since 2024-03 $etagWithoutQuotes = trim($etag, '"'); $dashPos = strrpos($etagWithoutQuotes, '-'); if ($dashPos && in_array(substr($etagWithoutQuotes, $dashPos + 1), ['br', 'gzip'], true)) { diff --git a/app/Lib/Tools/SyncTool.php b/app/Lib/Tools/SyncTool.php index b0ef40a41..ee8cdb3bf 100644 --- a/app/Lib/Tools/SyncTool.php +++ b/app/Lib/Tools/SyncTool.php @@ -1,7 +1,6 @@ EDITABLE_FIELDS); + return array_merge(static::BASE_EDITABLE_FIELDS, static::EDITABLE_FIELDS); } /** @@ -641,9 +641,8 @@ class AnalystData extends AppModel return []; } $this->Server = ClassRegistry::init('Server'); - $this->AnalystData = ClassRegistry::init('AnalystData'); - $this->log("Starting Analyst Data sync with server #{$server['Server']['id']}", LOG_INFO); + $serverSync->debug("Starting Analyst Data sync"); $analystData = $this->collectDataForPush($serverSync->server()); $keyedAnalystData = []; @@ -1018,7 +1017,6 @@ class AnalystData extends AppModel } $this->Server = ClassRegistry::init('Server'); - $this->AnalystData = ClassRegistry::init('AnalystData'); try { $filterRules = $this->buildPullFilterRules($serverSync->server()); $remoteData = $serverSync->fetchIndexMinimal($filterRules)->json(); diff --git a/app/Model/EventReport.php b/app/Model/EventReport.php index cbf552612..2e5a592fd 100644 --- a/app/Model/EventReport.php +++ b/app/Model/EventReport.php @@ -710,7 +710,7 @@ class EventReport extends AppModel 'category' => $typeToCategoryMapping[$complexTypeToolEntry['default_type']][0], 'type' => $complexTypeToolEntry['default_type'], 'value' => $textToBeReplaced, - 'to_ids' => $complexTypeToolEntry['to_ids'], + 'to_ids' => $complexTypeToolEntry['to_ids'] ?? 0, ]; $replacedContent = str_replace($complexTypeToolEntry['original_value'], $textToInject, $replacedContent); } diff --git a/app/Model/Feed.php b/app/Model/Feed.php index ef448a536..74acee2ae 100644 --- a/app/Model/Feed.php +++ b/app/Model/Feed.php @@ -1068,6 +1068,9 @@ class Feed extends AppModel if (!empty($feed['Feed']['settings']['disable_correlation'])) { $event['Event']['disable_correlation'] = (bool) $feed['Feed']['settings']['disable_correlation']; } + if (!empty($feed['Feed']['settings']['unpublish_event'])) { + $event['Event']['published'] = (bool) $feed['Feed']['settings']['unpublish_event']; + } } return $event; } diff --git a/app/Model/GalaxyCluster.php b/app/Model/GalaxyCluster.php index b02cead26..885d959ec 100644 --- a/app/Model/GalaxyCluster.php +++ b/app/Model/GalaxyCluster.php @@ -1845,6 +1845,9 @@ class GalaxyCluster extends AppModel if (!$compatible) { return 0; } + + $serverSync->debug("Pulling galaxy clusters with technique $technique"); + $clusterIds = $this->getClusterIdListBasedOnPullTechnique($user, $technique, $serverSync); $successes = 0; // now process the $clusterIds to pull each of the events sequentially diff --git a/app/Model/Server.php b/app/Model/Server.php index 8354e2683..aea2e9b1e 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -604,6 +604,7 @@ class Server extends AppModel * @throws HttpSocketHttpException * @throws HttpSocketJsonException * @throws JsonException + * @throws Exception */ public function pull(array $user, $technique, array $server, $jobId = false, $force = false) { @@ -619,7 +620,7 @@ class Server extends AppModel try { $server['Server']['version'] = $serverSync->info()['version']; } catch (Exception $e) { - $this->logException("Could not get remote server `{$server['Server']['name']}` version.", $e); + $this->logException("Could not get remote server `{$serverSync->serverName()}` version.", $e); if ($e instanceof HttpSocketHttpException && $e->getCode() === 403) { $message = __('Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server. Another reason could be an incorrect sync server setting.'); } else { @@ -648,6 +649,8 @@ class Server extends AppModel } } + $serverSync->debug("Pulling event list with technique $technique"); + try { $eventIds = $this->__getEventIdListBasedOnPullTechnique($technique, $serverSync, $force); } catch (Exception $e) { @@ -673,26 +676,29 @@ class Server extends AppModel $job->saveProgress($jobId, __n('Pulling %s event.', 'Pulling %s events.', count($eventIds), count($eventIds))); } foreach ($eventIds as $k => $eventId) { + $serverSync->debug("Pulling event $eventId"); $this->__pullEvent($eventId, $successes, $fails, $eventModel, $serverSync, $user, $jobId, $force); if ($jobId && $k % 10 === 0) { $job->saveProgress($jobId, null, 10 + 40 * (($k + 1) / count($eventIds))); } } foreach ($fails as $eventid => $message) { - $this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['Server']['id'], "Failed to pull event #$eventid.", 'Reason: ' . $message); + $this->loadLog()->createLogEntry($user, 'pull', 'Server', $serverSync->serverId(), "Failed to pull event #$eventid.", 'Reason: ' . $message); } } if ($jobId) { $job->saveProgress($jobId, 'Pulling proposals.', 50); } - $pulledProposals = $pulledSightings = 0; + $pulledProposals = $pulledSightings = $pulledAnalystData = 0; if ($technique === 'full' || $technique === 'update') { $pulledProposals = $eventModel->ShadowAttribute->pullProposals($user, $serverSync); if ($jobId) { $job->saveProgress($jobId, 'Pulling sightings.', 75); } + $pulledSightings = $eventModel->Sighting->pullSightings($user, $serverSync); + $this->AnalystData = ClassRegistry::init('AnalystData'); $pulledAnalystData = $this->AnalystData->pull($user, $serverSync); } @@ -819,7 +825,7 @@ class Server extends AppModel */ public function getElligibleClusterIdsFromServerForPull(ServerSyncTool $serverSync, $onlyUpdateLocalCluster=true, array $eligibleClusters=array(), array $conditions=array()) { - $this->log("Fetching eligible clusters from server #{$serverSync->serverId()} for pull: " . JsonTool::encode($conditions), LOG_INFO); + $serverSync->debug("Fetching eligible clusters for pull: " . JsonTool::encode($conditions)); if ($onlyUpdateLocalCluster && empty($eligibleClusters)) { return []; // no clusters for update @@ -875,7 +881,7 @@ class Server extends AppModel */ private function getElligibleClusterIdsFromServerForPush(ServerSyncTool $serverSync, array $localClusters=array(), array $conditions=array()) { - $this->log("Fetching eligible clusters from server #{$serverSync->serverId()} for push: " . JsonTool::encode($conditions), LOG_INFO); + $serverSync->debug("Fetching eligible clusters for push: " . JsonTool::encode($conditions)); $clusterArray = $this->fetchCustomClusterIdsFromServer($serverSync, $conditions=$conditions); $keyedClusterArray = Hash::combine($clusterArray, '{n}.GalaxyCluster.uuid', '{n}.GalaxyCluster.version'); if (!empty($localClusters)) { @@ -915,9 +921,14 @@ class Server extends AppModel // Fetch event index from cache if exists and is not modified $redis = RedisTool::init(); - $indexFromCache = $redis->get("misp:event_index:{$serverSync->serverId()}"); + $indexFromCache = $redis->get("misp:event_index_cache:{$serverSync->serverId()}"); if ($indexFromCache) { - list($etag, $eventIndex) = RedisTool::deserialize(RedisTool::decompress($indexFromCache)); + $etagPos = strpos($indexFromCache, "\n"); + if ($etagPos === false) { + throw new RuntimeException("Could not find etag in cache fro server {$serverSync->serverId()}"); + } + $etag = substr($indexFromCache, 0, $etagPos); + $serverSync->debug("Event index loaded from Redis cache with etag $etag containing"); } else { $etag = '""'; // Provide empty ETag, so MISP will compute ETag for returned data } @@ -925,9 +936,21 @@ class Server extends AppModel $response = $serverSync->eventIndex($filterRules, $etag); if ($response->isNotModified() && $indexFromCache) { - return $eventIndex; + return JsonTool::decode(RedisTool::decompress(substr($indexFromCache, $etagPos + 1))); } + // Save to cache for 24 hours if ETag provided + $etag = $response->getHeader('etag'); + if ($etag) { + $serverSync->debug("Event index from remote server has different etag $etag, saving to cache"); + $data = "$etag\n" . RedisTool::compress($response->body); + $redis->setex("misp:event_index_cache:{$serverSync->serverId()}", 3600 * 24, $data); + } elseif ($indexFromCache) { + RedisTool::unlink($redis, "misp:event_index_cache:{$serverSync->serverId()}"); + } + + unset($indexFromCache); // clean up memory + $eventIndex = $response->json(); // correct $eventArray if just one event, probably this response returns old MISP @@ -935,15 +958,6 @@ class Server extends AppModel $eventIndex = [$eventIndex]; } - // Save to cache for 24 hours if ETag provided - $etag = $response->getHeader('etag'); - if ($etag) { - $data = RedisTool::compress(RedisTool::serialize([$etag, $eventIndex])); - $redis->setex("misp:event_index:{$serverSync->serverId()}", 3600 * 24, $data); - } elseif ($indexFromCache) { - RedisTool::unlink($redis, "misp:event_index:{$serverSync->serverId()}"); - } - return $eventIndex; } @@ -1372,7 +1386,7 @@ class Server extends AppModel return []; // pushing clusters is not enabled } - $this->log("Starting $technique clusters sync with server #{$serverSync->serverId()}", LOG_INFO); + $serverSync->debug("Starting $technique clusters sync"); $this->GalaxyCluster = ClassRegistry::init('GalaxyCluster'); $this->Event = ClassRegistry::init('Event'); @@ -5125,8 +5139,8 @@ class Server extends AppModel ), 'curl_request_timeout' => [ 'level' => 1, - 'description' => __('Control the timeout of curl requests issued by MISP (during synchronisation, feed fetching, etc.'), - 'value' => 10800, + 'description' => __('Control the default timeout in seconds of curl HTTP requests issued by MISP (during synchronisation, feed fetching, etc.)'), + 'value' => 300, 'test' => 'testForNumeric', 'type' => 'numeric', 'null' => true diff --git a/app/Model/ShadowAttribute.php b/app/Model/ShadowAttribute.php index 645da2dec..a1e6b002b 100644 --- a/app/Model/ShadowAttribute.php +++ b/app/Model/ShadowAttribute.php @@ -706,6 +706,8 @@ class ShadowAttribute extends AppModel return 0; } + $serverSync->debug("Pulling proposals"); + $i = 1; $fetchedCount = 0; $chunkSize = 1000; diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php index 0a33a87ca..fc1b1096e 100644 --- a/app/Model/Sighting.php +++ b/app/Model/Sighting.php @@ -1418,11 +1418,13 @@ class Sighting extends AppModel */ public function pullSightings(array $user, ServerSyncTool $serverSync) { + $serverSync->debug("Fetching event index for pulling sightings"); + $this->Server = ClassRegistry::init('Server'); try { $remoteEvents = $this->Server->getEventIndexFromServer($serverSync); } catch (Exception $e) { - $this->logException("Could not fetch event IDs from server {$serverSync->server()['Server']['name']}", $e); + $this->logException("Could not fetch event IDs from server {$serverSync->serverName()}", $e); return 0; } // Remove events from list that do not have published sightings. @@ -1452,6 +1454,8 @@ class Sighting extends AppModel return 0; } + $serverSync->debug("Pulling sightings for " . count($eventUuids) . " events"); + if ($serverSync->isSupported(ServerSyncTool::FEATURE_SIGHTING_REST_SEARCH)) { return $this->pullSightingNewWay($user, $eventUuids, $serverSync); } else { diff --git a/app/Model/WorkflowModules/WorkflowBaseModule.php b/app/Model/WorkflowModules/WorkflowBaseModule.php index 66defb1bf..8a4e2f668 100644 --- a/app/Model/WorkflowModules/WorkflowBaseModule.php +++ b/app/Model/WorkflowModules/WorkflowBaseModule.php @@ -213,6 +213,8 @@ class WorkflowBaseModule if ($operator == 'in_or') { return !empty($matching); } elseif ($operator == 'in_and') { + sort($matching); + sort($value); return array_values($matching) == array_values($value); } elseif ($operator == 'not_in_or') { return empty($matching); diff --git a/app/Model/WorkflowModules/action/Module_stop_execution.php b/app/Model/WorkflowModules/action/Module_stop_execution.php index 5266a1557..b02bce689 100644 --- a/app/Model/WorkflowModules/action/Module_stop_execution.php +++ b/app/Model/WorkflowModules/action/Module_stop_execution.php @@ -6,6 +6,7 @@ class Module_stop_execution extends WorkflowBaseActionModule public $blocking = true; public $id = 'stop-execution'; public $name = 'Stop execution'; + public $version = '0.2'; public $description = 'Essentially stops the execution for blocking workflows. Do nothing for non-blocking ones'; public $icon = 'ban'; public $inputs = 1; @@ -15,12 +16,25 @@ class Module_stop_execution extends WorkflowBaseActionModule public function __construct() { parent::__construct(); + + $this->params = [ + [ + 'id' => 'message', + 'label' => 'Stop message', + 'type' => 'input', + 'default' => __('Execution stopped'), + 'placeholder' => __('Execution stopped'), + 'jinja_supported' => true, + ], + ]; } public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool { parent::exec($node, $roamingData, $errors); - $errors[] = __('Execution stopped'); + $rData = $roamingData->getData(); + $params = $this->getParamsWithValues($node, $rData); + $errors[] = empty($params['message']['value']) ? $params['message']['default'] : $params['message']['value']; return false; } } diff --git a/app/Model/WorkflowModules/logic/Module_distribution_if.php b/app/Model/WorkflowModules/logic/Module_distribution_if.php index 6135385f2..a4f98cecd 100644 --- a/app/Model/WorkflowModules/logic/Module_distribution_if.php +++ b/app/Model/WorkflowModules/logic/Module_distribution_if.php @@ -5,7 +5,7 @@ class Module_distribution_if extends WorkflowBaseLogicModule { public $id = 'distribution-if'; public $name = 'IF :: Distribution'; - public $version = '0.2'; + public $version = '0.3'; public $description = 'Distribution IF / ELSE condition block. The `then` output will be used if the encoded conditions is satisfied, otherwise the `else` output will be used.'; public $icon = 'code-branch'; public $inputs = 1; @@ -103,12 +103,15 @@ class Module_distribution_if extends WorkflowBaseLogicModule $final_sharing_group = $this->__extractSharingGroupIDs( $data['Event'], $data['Event']['Attribute'][0]['Object'] ?? [], - $data['Event']['Attribute'][0] + $data['Event']['Attribute'][0], + $scope ); if ($operator == 'equals') { - return !array_diff($final_sharing_group, $selected_sharing_groups); // All sharing groups are in the selection + return empty($selected_sharing_groups) ? !empty($final_sharing_group) : + !array_diff($final_sharing_group, $selected_sharing_groups); // All sharing groups are in the selection } else if ($operator == 'not_equals') { - return count(array_diff($final_sharing_group, $selected_sharing_groups)) == count($final_sharing_group); // All sharing groups are in the selection + return empty($selected_sharing_groups) ? empty($final_sharing_group) : + count(array_diff($final_sharing_group, $selected_sharing_groups)) == count($final_sharing_group); // All sharing groups are in the selection } $errors[] = __('Condition operator not supported for that distribution level'); return false; @@ -159,9 +162,15 @@ class Module_distribution_if extends WorkflowBaseLogicModule return min($distri1, $distri2); } - private function __extractSharingGroupIDs(array $event, array $object=[], array $attribute=[]): array + private function __extractSharingGroupIDs(array $event, array $object=[], array $attribute=[], $scope='event'): array { $sgIDs = []; + if ($scope == 'event') { + if (!empty($event) && $event['distribution'] == 4) { + $sgIDs[] = $event['sharing_group_id']; + } + return $sgIDs; + } if (!empty($event) && $event['distribution'] == 4) { $sgIDs[] = $event['sharing_group_id']; } diff --git a/app/View/AnalystData/view.ctp b/app/View/AnalystData/view.ctp index f56460e7a..c9715d63f 100644 --- a/app/View/AnalystData/view.ctp +++ b/app/View/AnalystData/view.ctp @@ -120,15 +120,57 @@ echo $this->element( ); $object_uuid = Hash::get($data, $modelSelection . '.uuid'); + +$notes = $data[$modelSelection]['Note'] ?? []; +$opinions = $data[$modelSelection]['Opinion'] ?? []; +$relationships_outbound = $data[$modelSelection]['Relationship'] ?? []; +$relationships_inbound = $data[$modelSelection]['RelationshipInbound'] ?? []; +$notesOpinions = array_merge($notes, $opinions); +if(!function_exists("countNotes")) { + function countNotes($notesOpinions) { + $notesTotalCount = count($notesOpinions); + $notesCount = 0; + $relationsCount = 0; + foreach ($notesOpinions as $notesOpinion) { + if ($notesOpinion['note_type'] == 2) { // relationship + $relationsCount += 1; + } else { + $notesCount += 1; + } + if (!empty($notesOpinion['Note'])) { + $nestedCounts = countNotes($notesOpinion['Note']); + $notesTotalCount += $nestedCounts['total']; + $notesCount += $nestedCounts['notesOpinions']; + $relationsCount += $nestedCounts['relations']; + } + if (!empty($notesOpinion['Opinion'])) { + $nestedCounts = countNotes($notesOpinion['Opinion']); + $notesTotalCount += $nestedCounts['total']; + $notesCount += $nestedCounts['notesOpinions']; + $relationsCount += $nestedCounts['relations']; + } + } + return ['total' => $notesTotalCount, 'notesOpinions' => $notesCount, 'relations' => $relationsCount]; + } +} +$counts = countNotes($notesOpinions); +$notesOpinionCount = $counts['notesOpinions']; +$allCounts = [ + 'notesOpinions' => $counts['notesOpinions'], + 'relationships_outbound' => count($relationships_outbound), + 'relationships_inbound' => count($relationships_inbound), +]; + $options = [ 'container_id' => 'analyst_data_thread', 'object_type' => $modelSelection, 'object_uuid' => $object_uuid, 'shortDist' => $shortDist, - 'notes' => $data[$modelSelection]['Note'] ?? [], - 'opinions' => $data[$modelSelection]['Opinion'] ?? [], - 'relationships_outbound' => $data[$modelSelection]['Relationship'] ?? [], - 'relationships_inbound' => $data[$modelSelection]['RelationshipInbound'] ?? [], + 'notes' => $notes, + 'opinions' => $opinions, + 'relationships_outbound' => $relationships_outbound, + 'relationships_inbound' => $relationships_inbound, + 'allCounts' => $allCounts, ]; echo $this->element('genericElements/assetLoader', [ diff --git a/app/View/Dashboards/index.ctp b/app/View/Dashboards/index.ctp index 278f588f3..4e0b31775 100644 --- a/app/View/Dashboards/index.ctp +++ b/app/View/Dashboards/index.ctp @@ -23,7 +23,7 @@ $(function () { saveDashboardState(); }); grid.on('added', function(event, items) { - resetDashboardGrid(grid); + resetDashboardGrid(grid, false); }); grid.on('gsresizestop', function(event, element) { $(element).find('.widgetContentInner').trigger('widget-resized') diff --git a/app/View/Elements/Events/View/row_object.ctp b/app/View/Elements/Events/View/row_object.ctp index 1022b609c..9889612ee 100644 --- a/app/View/Elements/Events/View/row_object.ctp +++ b/app/View/Elements/Events/View/row_object.ctp @@ -36,6 +36,7 @@ $objectId = intval($object['id']); $notes = !empty($object['Note']) ? $object['Note'] : []; $opinions = !empty($object['Opinion']) ? $object['Opinion'] : []; $relationships = !empty($object['Relationship']) ? $object['Relationship'] : []; + $relationshipsInbound = !empty($object['RelationshipInbound']) ? $object['RelationshipInbound'] : []; echo $this->element('genericElements/Analyst_data/generic', [ 'analyst_data' => ['notes' => $notes, 'opinions' => $opinions, 'relationships_outbound' => $relationships, 'relationships_inbound' => $relationshipsInbound], 'object_uuid' => $object['uuid'], diff --git a/app/View/Elements/genericElements/Analyst_data/generic.ctp b/app/View/Elements/genericElements/Analyst_data/generic.ctp index 2ccdc4eb0..acbfae91a 100644 --- a/app/View/Elements/genericElements/Analyst_data/generic.ctp +++ b/app/View/Elements/genericElements/Analyst_data/generic.ctp @@ -65,20 +65,8 @@ $allCounts = [ $(document).ready(function() { $('.node-opener-').click(function() { - openNotes(this) + openNotes(this) }) - - function adjustPopoverPosition() { - var $popover = $('.popover:last'); - $popover.css('top', Math.max($popover.position().top, 50) + 'px') - } - - function openNotes(clicked) { - openPopover(clicked, renderedNotes, undefined, undefined, function() { - adjustPopoverPosition() - $(clicked).removeClass('have-a-popover') // avoid closing the popover if a confirm popover (like the delete one) is called - }) - } }) diff --git a/app/View/Elements/genericElements/Analyst_data/thread.ctp b/app/View/Elements/genericElements/Analyst_data/thread.ctp index 5930f6c5d..bad0a3b8d 100644 --- a/app/View/Elements/genericElements/Analyst_data/thread.ctp +++ b/app/View/Elements/genericElements/Analyst_data/thread.ctp @@ -4,6 +4,7 @@ $URL_DELETE = '/analystData/delete/'; $seed = isset($seed) ? $seed : mt_rand(); + $injectInPage = !empty($container_id) ? true : false; $notes = !empty($notes) ? $notes : []; $opinions = !empty($opinions) ? $opinions : []; @@ -41,7 +42,28 @@ if (!window.shortDist) { var shortDist = ; } -var renderedNotes = null + +var container_id = false + + container_id = '' + + +function adjustPopoverPosition() { + var $popover = $('.popover:last'); + $popover.css('top', Math.max($popover.position().top, 50) + 'px') +} + +function openNotes(clicked) { + var notes = ; + var relationships = ; + var relationships_inbound = ; + var relationship_related_object = ; + var renderedNotes = renderAllNotesWithForm(notes, relationships, relationships_inbound, relationship_related_object) + openPopover(clicked, renderedNotes, undefined, undefined, function() { + adjustPopoverPosition() + $(clicked).removeClass('have-a-popover') // avoid closing the popover if a confirm popover (like the delete one) is called + }) +} function renderNotes(notes, relationship_related_object, emptyMessage='', isInbound=false) { var renderedNotesArray = [] @@ -406,18 +428,7 @@ function fetchMoreNotes(clicked, noteType, uuid) { } - -(function() { - var notes = ; - var relationships = ; - var relationships_inbound = ; - var relationship_related_object = ; - var container_id = false - - container_id = '' - - - var nodeContainerTemplate = doT.template('\ + var nodeContainerTemplate = doT.template('\
\
\ ') - function renderAllNotesWithForm(relationship_related_object) { - var buttonContainer = '
' + addNoteButton + addOpinionButton + '
' - renderedNotes = nodeContainerTemplate({ - content_notes: renderNotes(notes.filter(function(note) { return note.note_type != 2}), relationship_related_object, '') + buttonContainer, - content_relationships_outbound: renderNotes(relationships, relationship_related_object, '') + addRelationshipButton, - content_relationships_inbound: renderNotes(relationships_inbound, relationship_related_object, '', true), - }) - if (container_id) { - $('#' + container_id).html(renderedNotes) - } - } - var addNoteButton = '' @@ -461,10 +460,15 @@ function fetchMoreNotes(clicked, noteType, uuid) { \ ' - $(document).ready(function() { - renderAllNotesWithForm(relationship_related_object) - }) -})() + function renderAllNotesWithForm(notes, relationships, relationships_inbound, relationship_related_object) { + var buttonContainer = '
' + addNoteButton + addOpinionButton + '
' + var renderedNotes = nodeContainerTemplate({ + content_notes: renderNotes(notes.filter(function(note) { return note.note_type != 2}), relationship_related_object, '') + buttonContainer, + content_relationships_outbound: renderNotes(relationships, relationship_related_object, '') + addRelationshipButton, + content_relationships_inbound: renderNotes(relationships_inbound, relationship_related_object, '', true), + }) + return renderedNotes + } function createNewNote(clicked, object_type, object_uuid) { note_type = 'Note'; @@ -516,6 +520,19 @@ function fetchMoreNotes(clicked, noteType, uuid) { } } + + $(document).ready(function() { + var notes = ; + var relationships = ; + var relationships_inbound = ; + var relationship_related_object = ; + var renderedNotes = renderAllNotesWithForm(notes, relationships, relationships_inbound, relationship_related_object) + if (container_id) { + $('#' + container_id).html(renderedNotes) + } + }) + +