MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'develop' of github.com:MISP/MISP into develop

feature/better-logic-for-merge-attribute-into-object
iglocska 2024-04-17 15:10:30 +02:00
parent 4544ef2516 254b6d7646
commit a55a19cd09
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
36 changed files with 350 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/main.yml vendored
View File

@ -269,13 +269,16 @@ jobs:
- name: Check requirements.txt
run: python tests/check_requirements.py
- name: Logs
- name: System logs
if: ${{ always() }}
# update logs_test.sh when adding more logsources here
run: |
tail -n +1 `pwd`/app/tmp/logs/*
tail -n +1 /var/log/apache2/*.log
- name: Application logs
if: ${{ always() }}
run: |
app/Console/cake Log export /tmp/logs.json.gz --without-changes
zcat /tmp/logs.json.gz

2
README.md
View File

@ -46,6 +46,8 @@ The objective of MISP is to foster the sharing of structured information within
</tr>
</table>
[![CLA FREE initiative](https://raw.githubusercontent.com/ossbase-org/ossbase.org/main/logos/cla-free-small.png)](https://ossbase.org/initiatives/cla-free/)
Core functions
------------------
- An **efficient IOC and indicators** database, allowing to store technical and non-technical information about malware samples, incidents, attackers and intelligence.

4
app/Console/Command/ServerShell.php
View File

@ -144,6 +144,10 @@ class ServerShell extends AppShell
if (!empty($this->args[4]) && $this->args[4] === 'force') {
$force = true;
}
// Try to enable garbage collector as pulling events can use a lot of memory
gc_enable();
try {
$result = $this->Server->pull($user, $technique, $server, $jobId, $force);
if (is_array($result)) {

5
app/Controller/AnalystDataController.php
View File

@ -320,6 +320,11 @@ class AnalystDataController extends AppController
$this->AnalystData = $this->{$vt};
$this->modelClass = $vt;
$this->{$vt}->current_user = $this->Auth->user();
if (!empty($this->request->data)) {
if (!isset($this->request->data[$type])) {
$this->request->data = [$type => $this->request->data];
}
}
return $vt;
}
}

16
app/Controller/AppController.php
View File

@ -33,7 +33,7 @@ class AppController extends Controller
public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
private $__queryVersion = '159';
private $__queryVersion = '161';
public $pyMispVersion = '2.4.188';
public $phpmin = '7.2';
public $phprec = '7.4';
@ -1063,7 +1063,19 @@ class AppController extends Controller
$data = array_merge($data, $temp);
} else {
foreach ($options['paramArray'] as $param) {
if (isset($temp[$param])) {
if (substr($param, -1) == '*') {
$root = substr($param, 0, strlen($param)-1);
foreach ($temp as $existingParamKey => $v) {
$leftover = substr($existingParamKey, strlen($param)-1);
if (
$root == substr($existingParamKey, 0, strlen($root)) &&
preg_match('/^[\w_-. ]+$/', $leftover) == 1
) {
$data[$existingParamKey] = $temp[$existingParamKey];
break;
}
}
} else if (isset($temp[$param])) {
$data[$param] = $temp[$param];
}
}

6
app/Controller/Component/RestSearchComponent.php
View File

@ -144,7 +144,11 @@ class RestSearchComponent extends Component
'retry',
'expiry',
'minimum_ttl',
'ttl'
'ttl',
'org.sector',
'org.local',
'org.nationality',
'galaxy.*',
],
'Object' => [
'returnFormat',

15
app/Controller/EventReportsController.php
View File

@ -213,10 +213,13 @@ class EventReportsController extends AppController
public function extractAllFromReport($reportId)
{
if (!$this->request->is('ajax')) {
if (!$this->request->is('ajax') && !$this->_isRest()) {
throw new MethodNotAllowedException(__('This function can only be reached via AJAX.'));
}
if ($this->request->is('post')) {
if (!isset($this->data['EventReport'])) {
$this->data = ['EventReport' => $this->data];
}
$report = $this->EventReport->fetchIfAuthorized($this->Auth->user(), $reportId, 'edit', $throwErrors=true, $full=false);
$results = $this->EventReport->getComplexTypeToolResultWithReplacements($this->Auth->user(), $report);
$report['EventReport']['content'] = $results['replacementResult']['contentWithReplacements'];
@ -299,13 +302,16 @@ class EventReportsController extends AppController
public function importReportFromUrl($event_id)
{
if (!$this->request->is('ajax')) {
throw new MethodNotAllowedException(__('This function can only be reached via AJAX.'));
if (!$this->request->is('ajax') && !$this->_isRest()) {
throw new MethodNotAllowedException(__('This function can only be reached via AJAX and via the API.'));
}
$fetcherModule = $this->EventReport->isFetchURLModuleEnabled();
if ($this->request->is('post')) {
if (empty($this->data['EventReport'])) {
$this->data = ['EventReport' => $this->data];
}
if (empty($this->data['EventReport']['url'])) {
throw new MethodNotAllowedException(__('An URL must be provided'));
throw new MethodNotAllowedException(__('A URL must be provided'));
}
$url = $this->data['EventReport']['url'];
$format = 'html';
@ -316,7 +322,6 @@ class EventReportsController extends AppController
$format = $parsed_format;
}
}
$content = $this->EventReport->downloadMarkdownFromURL($event_id, $url, $format);
$errors = [];

22
app/Controller/EventsController.php
View File

@ -497,6 +497,11 @@ class EventsController extends AppController
continue 2;
}
$pieces = is_array($v) ? $v : explode('|', $v);
$isANDed = false;
if (count($pieces) == 1 && strpos($pieces[0], '&') !== -1) {
$pieces = explode('&', $v);
$isANDed = count($pieces) > 1;
}
$filterString = "";
$expectOR = false;
$tagRules = [];
@ -563,10 +568,19 @@ class EventsController extends AppController
}
if (!empty($tagRules['include'])) {
$include = $this->Event->EventTag->find('column', array(
'conditions' => array('EventTag.tag_id' => $tagRules['include']),
'fields' => ['EventTag.event_id'],
));
if ($isANDed) {
$include = $this->Event->EventTag->find('column', array(
'conditions' => ['EventTag.tag_id' => $tagRules['include']],
'fields' => ['EventTag.event_id'],
'group' => ['EventTag.event_id'],
'having' => ['COUNT(*) =' => count($tagRules['include'])],
));
} else {
$include = $this->Event->EventTag->find('column', array(
'conditions' => array('EventTag.tag_id' => $tagRules['include']),
'fields' => ['EventTag.event_id'],
));
}
if (!empty($include)) {
$this->paginate['conditions']['AND'][] = 'Event.id IN (' . implode(",", $include) . ')';
} else {

2
app/Controller/TagCollectionsController.php
View File

@ -357,7 +357,7 @@ class TagCollectionsController extends AppController
if (!$tagCollection) {
throw new NotFoundException(__('Invalid tag collection.'));
}
if ($this->ACL->canModifyTagCollection($this->Auth->user(), $tagCollection)) {
if (!$this->ACL->canModifyTagCollection($this->Auth->user(), $tagCollection)) {
throw new ForbiddenException(__('You dont have a permission to do that'));
}
$tagCollectionTag = $this->TagCollection->TagCollectionTag->find('first', [

4
app/Controller/UsersController.php
View File

@ -2071,7 +2071,7 @@ class UsersController extends AppController
$stats['attribute_count'] = $this->User->Event->Attribute->find('count', array('conditions' => array('Attribute.deleted' => 0), 'recursive' => -1));
$stats['attribute_count_month'] = $this->User->Event->Attribute->find('count', array('conditions' => array('Attribute.timestamp >' => $this_month, 'Attribute.deleted' => 0), 'recursive' => -1));
$stats['attributes_per_event'] = round($stats['attribute_count'] / $stats['event_count']);
$stats['attributes_per_event'] = $stats['event_count'] != 0 ? round($stats['attribute_count'] / $stats['event_count']) : 0;
$stats['correlation_count'] = $this->User->Event->Attribute->Correlation->find('count', array('recursive' => -1));
@ -2082,7 +2082,7 @@ class UsersController extends AppController
$stats['org_count'] = count($orgs);
$stats['local_org_count'] = $local_orgs_count;
$stats['contributing_org_count'] = $this->User->Event->find('count', array('recursive' => -1, 'group' => array('Event.orgc_id')));
$stats['average_user_per_org'] = round($stats['user_count'] / $stats['local_org_count'], 1);
$stats['average_user_per_org'] = $stats['local_org_count'] != 0 ? round($stats['user_count'] / $stats['local_org_count'], 1) : 0;
$this->loadModel('Thread');
$stats['thread_count'] = $this->Thread->find('count', array('conditions' => array('Thread.post_count >' => 0), 'recursive' => -1));

5
app/Lib/Dashboard/EventEvolutionLineWidget.php
View File

@ -58,9 +58,11 @@ class EventEvolutionLineWidget
'recursive' => -1
];
$eparams = [];
$filteringOnOrg = false;
if (!empty($options['filter']) && is_array($options['filter'])) {
foreach ($this->validFilterKeys as $filterKey) {
if (!empty($options['filter'][$filterKey])) {
$filteringOnOrg = true;
if (!is_array($options['filter'][$filterKey])) {
$options['filter'][$filterKey] = [$options['filter'][$filterKey]];
}
@ -87,6 +89,9 @@ class EventEvolutionLineWidget
'conditions' => $oparams['conditions'],
'fields' => ['id']
]);
if ($filteringOnOrg) {
$eparams['conditions']['AND']['Event.orgc_id IN'] = !empty($org_ids) ? $org_ids : [-1];
}
$this->Event->virtualFields = [
'published_date' => null
];

19
app/Lib/Tools/CurlClient.php
View File

@ -6,8 +6,12 @@ class CurlClient extends HttpSocketExtended
/** @var resource */
private $ch;
/** @var int */
private $timeout = 10800;
/**
* Maximum time the transfer is allowed to complete in seconds
* 300 seconds is recommended timeout for MISP servers
* @var int
*/
private $timeout = 300;
/** @var string|null */
private $caFile;
@ -30,6 +34,9 @@ class CurlClient extends HttpSocketExtended
/** @var array */
private $proxy = [];
/** @var array */
private $defaultOptions;
/**
* @param array $params
* @noinspection PhpMissingParentConstructorInspection
@ -38,8 +45,6 @@ class CurlClient extends HttpSocketExtended
{
if (isset($params['timeout'])) {
$this->timeout = $params['timeout'];
} else {
$this->timeout = Configure::check('MISP.curl_request_timeout') ? Configure::read('MISP.curl_request_timeout') : 10800;
}
if (isset($params['ssl_cafile'])) {
$this->caFile = $params['ssl_cafile'];
@ -59,6 +64,7 @@ class CurlClient extends HttpSocketExtended
if (isset($params['ssl_verify_peer'])) {
$this->verifyPeer = $params['ssl_verify_peer'];
}
$this->defaultOptions = $this->generateDefaultOptions();
}
/**
@ -166,6 +172,7 @@ class CurlClient extends HttpSocketExtended
return;
}
$this->proxy = compact('host', 'port', 'method', 'user', 'pass');
$this->defaultOptions = $this->generateDefaultOptions(); // regenerate default options in case proxy setting is changed
}
/**
@ -196,7 +203,7 @@ class CurlClient extends HttpSocketExtended
$url .= '?' . http_build_query($query, '', '&', PHP_QUERY_RFC3986);
}
$options = $this->generateOptions();
$options = $this->defaultOptions; // this will copy default options
$options[CURLOPT_URL] = $url;
$options[CURLOPT_CUSTOMREQUEST] = $method;
@ -303,7 +310,7 @@ class CurlClient extends HttpSocketExtended
/**
* @return array
*/
private function generateOptions()
private function generateDefaultOptions()
{
$options = [
CURLOPT_FOLLOWLOCATION => true, // Allows to follow redirect

5
app/Lib/Tools/HttpSocketExtended.php
View File

@ -24,7 +24,7 @@ class HttpSocketHttpException extends Exception
$message .= " for URL $url";
}
if ($response->body) {
$message .= ': ' . substr($response->body, 0, 100);
$message .= ': ' . substr(ltrim($response->body), 0, 100);
}
parent::__construct($message, (int)$response->code);
@ -121,7 +121,8 @@ class HttpSocketResponseExtended extends HttpSocketResponse
try {
return JsonTool::decode($this->body);
} catch (Exception $e) {
throw new HttpSocketJsonException('Could not parse response as JSON.', $this, $e);
$contentType = $this->getHeader('content-type');
throw new HttpSocketJsonException("Could not parse HTTP response as JSON. Received Content-Type $contentType.", $this, $e);
}
}
}

11
app/Lib/Tools/ServerSyncTool.php
View File

@ -506,6 +506,16 @@ class ServerSyncTool
return $this->socket->getMetaData();
}
/**
* @param string $message
* @return void
*/
public function debug($message)
{
$memoryUsage = round(memory_get_usage() / 1024 / 1024, 2);
CakeLog::debug("[Server sync #{$this->serverId()}]: $message. Memory: $memoryUsage MB");
}
/**
* @params string $url Relative URL
* @return HttpSocketResponseExtended
@ -556,6 +566,7 @@ class ServerSyncTool
if ($etag) {
// Remove compression marks that adds Apache for compressed content
// This can be removed in future as this is already checked by MISP itself since 2024-03
$etagWithoutQuotes = trim($etag, '"');
$dashPos = strrpos($etagWithoutQuotes, '-');
if ($dashPos && in_array(substr($etagWithoutQuotes, $dashPos + 1), ['br', 'gzip'], true)) {

10
app/Lib/Tools/SyncTool.php
View File

@ -1,7 +1,6 @@
<?php
class SyncTool
{
const ALLOWED_CERT_FILE_EXTENSIONS = ['pem', 'crt'];
/**
@ -50,7 +49,7 @@ class SyncTool
* @return HttpSocketExtended
* @throws Exception
*/
public function createHttpSocket($params = array())
public function createHttpSocket(array $params = [])
{
// Use own CA PEM file
$caPath = Configure::read('MISP.ca_path');
@ -82,10 +81,11 @@ class SyncTool
}
$params['ssl_crypto_method'] = $version;
}
if (!isset($params['timeout'])) {
$params['timeout'] = Configure::check('MISP.curl_request_timeout') ? Configure::read('MISP.curl_request_timeout') : 10800;
}
if (function_exists('curl_init')) {
if (!isset($params['timeout']) && Configure::check('MISP.curl_request_timeout')) {
$params['timeout'] = (int)Configure::read('MISP.curl_request_timeout');
}
App::uses('CurlClient', 'Tools');
$HttpSocket = new CurlClient($params);
} else {

8
app/Model/AnalystData.php
View File

@ -43,7 +43,7 @@ class AnalystData extends AppModel
'distribution',
'sharing_group_id',
];
protected $EDITABLE_FIELDS = [];
public const EDITABLE_FIELDS = [];
/** @var object|null */
protected $Note;
@ -185,7 +185,7 @@ class AnalystData extends AppModel
public function getEditableFields(): array
{
return array_merge(self::BASE_EDITABLE_FIELDS, $this->EDITABLE_FIELDS);
return array_merge(static::BASE_EDITABLE_FIELDS, static::EDITABLE_FIELDS);
}
/**
@ -641,9 +641,8 @@ class AnalystData extends AppModel
return [];
}
$this->Server = ClassRegistry::init('Server');
$this->AnalystData = ClassRegistry::init('AnalystData');
$this->log("Starting Analyst Data sync with server #{$server['Server']['id']}", LOG_INFO);
$serverSync->debug("Starting Analyst Data sync");
$analystData = $this->collectDataForPush($serverSync->server());
$keyedAnalystData = [];
@ -1018,7 +1017,6 @@ class AnalystData extends AppModel
}
$this->Server = ClassRegistry::init('Server');
$this->AnalystData = ClassRegistry::init('AnalystData');
try {
$filterRules = $this->buildPullFilterRules($serverSync->server());
$remoteData = $serverSync->fetchIndexMinimal($filterRules)->json();

2
app/Model/EventReport.php
View File

@ -710,7 +710,7 @@ class EventReport extends AppModel
'category' => $typeToCategoryMapping[$complexTypeToolEntry['default_type']][0],
'type' => $complexTypeToolEntry['default_type'],
'value' => $textToBeReplaced,
'to_ids' => $complexTypeToolEntry['to_ids'],
'to_ids' => $complexTypeToolEntry['to_ids'] ?? 0,
];
$replacedContent = str_replace($complexTypeToolEntry['original_value'], $textToInject, $replacedContent);
}

3
app/Model/Feed.php
View File

@ -1068,6 +1068,9 @@ class Feed extends AppModel
if (!empty($feed['Feed']['settings']['disable_correlation'])) {
$event['Event']['disable_correlation'] = (bool) $feed['Feed']['settings']['disable_correlation'];
}
if (!empty($feed['Feed']['settings']['unpublish_event'])) {
$event['Event']['published'] = (bool) $feed['Feed']['settings']['unpublish_event'];
}
}
return $event;
}

3
app/Model/GalaxyCluster.php
View File

@ -1845,6 +1845,9 @@ class GalaxyCluster extends AppModel
if (!$compatible) {
return 0;
}
$serverSync->debug("Pulling galaxy clusters with technique $technique");
$clusterIds = $this->getClusterIdListBasedOnPullTechnique($user, $technique, $serverSync);
$successes = 0;
// now process the $clusterIds to pull each of the events sequentially

54
app/Model/Server.php
View File

@ -604,6 +604,7 @@ class Server extends AppModel
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
* @throws JsonException
* @throws Exception
*/
public function pull(array $user, $technique, array $server, $jobId = false, $force = false)
{
@ -619,7 +620,7 @@ class Server extends AppModel
try {
$server['Server']['version'] = $serverSync->info()['version'];
} catch (Exception $e) {
$this->logException("Could not get remote server `{$server['Server']['name']}` version.", $e);
$this->logException("Could not get remote server `{$serverSync->serverName()}` version.", $e);
if ($e instanceof HttpSocketHttpException && $e->getCode() === 403) {
$message = __('Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server. Another reason could be an incorrect sync server setting.');
} else {
@ -648,6 +649,8 @@ class Server extends AppModel
}
}
$serverSync->debug("Pulling event list with technique $technique");
try {
$eventIds = $this->__getEventIdListBasedOnPullTechnique($technique, $serverSync, $force);
} catch (Exception $e) {
@ -673,26 +676,29 @@ class Server extends AppModel
$job->saveProgress($jobId, __n('Pulling %s event.', 'Pulling %s events.', count($eventIds), count($eventIds)));
}
foreach ($eventIds as $k => $eventId) {
$serverSync->debug("Pulling event $eventId");
$this->__pullEvent($eventId, $successes, $fails, $eventModel, $serverSync, $user, $jobId, $force);
if ($jobId && $k % 10 === 0) {
$job->saveProgress($jobId, null, 10 + 40 * (($k + 1) / count($eventIds)));
}
}
foreach ($fails as $eventid => $message) {
$this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['Server']['id'], "Failed to pull event #$eventid.", 'Reason: ' . $message);
$this->loadLog()->createLogEntry($user, 'pull', 'Server', $serverSync->serverId(), "Failed to pull event #$eventid.", 'Reason: ' . $message);
}
}
if ($jobId) {
$job->saveProgress($jobId, 'Pulling proposals.', 50);
}
$pulledProposals = $pulledSightings = 0;
$pulledProposals = $pulledSightings = $pulledAnalystData = 0;
if ($technique === 'full' || $technique === 'update') {
$pulledProposals = $eventModel->ShadowAttribute->pullProposals($user, $serverSync);
if ($jobId) {
$job->saveProgress($jobId, 'Pulling sightings.', 75);
}
$pulledSightings = $eventModel->Sighting->pullSightings($user, $serverSync);
$this->AnalystData = ClassRegistry::init('AnalystData');
$pulledAnalystData = $this->AnalystData->pull($user, $serverSync);
}
@ -819,7 +825,7 @@ class Server extends AppModel
*/
public function getElligibleClusterIdsFromServerForPull(ServerSyncTool $serverSync, $onlyUpdateLocalCluster=true, array $eligibleClusters=array(), array $conditions=array())
{
$this->log("Fetching eligible clusters from server #{$serverSync->serverId()} for pull: " . JsonTool::encode($conditions), LOG_INFO);
$serverSync->debug("Fetching eligible clusters for pull: " . JsonTool::encode($conditions));
if ($onlyUpdateLocalCluster && empty($eligibleClusters)) {
return []; // no clusters for update
@ -875,7 +881,7 @@ class Server extends AppModel
*/
private function getElligibleClusterIdsFromServerForPush(ServerSyncTool $serverSync, array $localClusters=array(), array $conditions=array())
{
$this->log("Fetching eligible clusters from server #{$serverSync->serverId()} for push: " . JsonTool::encode($conditions), LOG_INFO);
$serverSync->debug("Fetching eligible clusters for push: " . JsonTool::encode($conditions));
$clusterArray = $this->fetchCustomClusterIdsFromServer($serverSync, $conditions=$conditions);
$keyedClusterArray = Hash::combine($clusterArray, '{n}.GalaxyCluster.uuid', '{n}.GalaxyCluster.version');
if (!empty($localClusters)) {
@ -915,9 +921,14 @@ class Server extends AppModel
// Fetch event index from cache if exists and is not modified
$redis = RedisTool::init();
$indexFromCache = $redis->get("misp:event_index:{$serverSync->serverId()}");
$indexFromCache = $redis->get("misp:event_index_cache:{$serverSync->serverId()}");
if ($indexFromCache) {
list($etag, $eventIndex) = RedisTool::deserialize(RedisTool::decompress($indexFromCache));
$etagPos = strpos($indexFromCache, "\n");
if ($etagPos === false) {
throw new RuntimeException("Could not find etag in cache fro server {$serverSync->serverId()}");
}
$etag = substr($indexFromCache, 0, $etagPos);
$serverSync->debug("Event index loaded from Redis cache with etag $etag containing");
} else {
$etag = '""'; // Provide empty ETag, so MISP will compute ETag for returned data
}
@ -925,9 +936,21 @@ class Server extends AppModel
$response = $serverSync->eventIndex($filterRules, $etag);
if ($response->isNotModified() && $indexFromCache) {
return $eventIndex;
return JsonTool::decode(RedisTool::decompress(substr($indexFromCache, $etagPos + 1)));
}
// Save to cache for 24 hours if ETag provided
$etag = $response->getHeader('etag');
if ($etag) {
$serverSync->debug("Event index from remote server has different etag $etag, saving to cache");
$data = "$etag\n" . RedisTool::compress($response->body);
$redis->setex("misp:event_index_cache:{$serverSync->serverId()}", 3600 * 24, $data);
} elseif ($indexFromCache) {
RedisTool::unlink($redis, "misp:event_index_cache:{$serverSync->serverId()}");
}
unset($indexFromCache); // clean up memory
$eventIndex = $response->json();
// correct $eventArray if just one event, probably this response returns old MISP
@ -935,15 +958,6 @@ class Server extends AppModel
$eventIndex = [$eventIndex];
}
// Save to cache for 24 hours if ETag provided
$etag = $response->getHeader('etag');
if ($etag) {
$data = RedisTool::compress(RedisTool::serialize([$etag, $eventIndex]));
$redis->setex("misp:event_index:{$serverSync->serverId()}", 3600 * 24, $data);
} elseif ($indexFromCache) {
RedisTool::unlink($redis, "misp:event_index:{$serverSync->serverId()}");
}
return $eventIndex;
}
@ -1372,7 +1386,7 @@ class Server extends AppModel
return []; // pushing clusters is not enabled
}
$this->log("Starting $technique clusters sync with server #{$serverSync->serverId()}", LOG_INFO);
$serverSync->debug("Starting $technique clusters sync");
$this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
$this->Event = ClassRegistry::init('Event');
@ -5125,8 +5139,8 @@ class Server extends AppModel
),
'curl_request_timeout' => [
'level' => 1,
'description' => __('Control the timeout of curl requests issued by MISP (during synchronisation, feed fetching, etc.'),
'value' => 10800,
'description' => __('Control the default timeout in seconds of curl HTTP requests issued by MISP (during synchronisation, feed fetching, etc.)'),
'value' => 300,
'test' => 'testForNumeric',
'type' => 'numeric',
'null' => true

2
app/Model/ShadowAttribute.php
View File

@ -706,6 +706,8 @@ class ShadowAttribute extends AppModel
return 0;
}
$serverSync->debug("Pulling proposals");
$i = 1;
$fetchedCount = 0;
$chunkSize = 1000;

6
app/Model/Sighting.php
View File

@ -1418,11 +1418,13 @@ class Sighting extends AppModel
*/
public function pullSightings(array $user, ServerSyncTool $serverSync)
{
$serverSync->debug("Fetching event index for pulling sightings");
$this->Server = ClassRegistry::init('Server');
try {
$remoteEvents = $this->Server->getEventIndexFromServer($serverSync);
} catch (Exception $e) {
$this->logException("Could not fetch event IDs from server {$serverSync->server()['Server']['name']}", $e);
$this->logException("Could not fetch event IDs from server {$serverSync->serverName()}", $e);
return 0;
}
// Remove events from list that do not have published sightings.
@ -1452,6 +1454,8 @@ class Sighting extends AppModel
return 0;
}
$serverSync->debug("Pulling sightings for " . count($eventUuids) . " events");
if ($serverSync->isSupported(ServerSyncTool::FEATURE_SIGHTING_REST_SEARCH)) {
return $this->pullSightingNewWay($user, $eventUuids, $serverSync);
} else {

2
app/Model/WorkflowModules/WorkflowBaseModule.php
View File

@ -213,6 +213,8 @@ class WorkflowBaseModule
if ($operator == 'in_or') {
return !empty($matching);
} elseif ($operator == 'in_and') {
sort($matching);
sort($value);
return array_values($matching) == array_values($value);
} elseif ($operator == 'not_in_or') {
return empty($matching);

16
app/Model/WorkflowModules/action/Module_stop_execution.php
View File

@ -6,6 +6,7 @@ class Module_stop_execution extends WorkflowBaseActionModule
public $blocking = true;
public $id = 'stop-execution';
public $name = 'Stop execution';
public $version = '0.2';
public $description = 'Essentially stops the execution for blocking workflows. Do nothing for non-blocking ones';
public $icon = 'ban';
public $inputs = 1;
@ -15,12 +16,25 @@ class Module_stop_execution extends WorkflowBaseActionModule
public function __construct()
{
parent::__construct();
$this->params = [
[
'id' => 'message',
'label' => 'Stop message',
'type' => 'input',
'default' => __('Execution stopped'),
'placeholder' => __('Execution stopped'),
'jinja_supported' => true,
],
];
}
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
{
parent::exec($node, $roamingData, $errors);
$errors[] = __('Execution stopped');
$rData = $roamingData->getData();
$params = $this->getParamsWithValues($node, $rData);
$errors[] = empty($params['message']['value']) ? $params['message']['default'] : $params['message']['value'];
return false;
}
}

19
app/Model/WorkflowModules/logic/Module_distribution_if.php
View File

@ -5,7 +5,7 @@ class Module_distribution_if extends WorkflowBaseLogicModule
{
public $id = 'distribution-if';
public $name = 'IF :: Distribution';
public $version = '0.2';
public $version = '0.3';
public $description = 'Distribution IF / ELSE condition block. The `then` output will be used if the encoded conditions is satisfied, otherwise the `else` output will be used.';
public $icon = 'code-branch';
public $inputs = 1;
@ -103,12 +103,15 @@ class Module_distribution_if extends WorkflowBaseLogicModule
$final_sharing_group = $this->__extractSharingGroupIDs(
$data['Event'],
$data['Event']['Attribute'][0]['Object'] ?? [],
$data['Event']['Attribute'][0]
$data['Event']['Attribute'][0],
$scope
);
if ($operator == 'equals') {
return !array_diff($final_sharing_group, $selected_sharing_groups); // All sharing groups are in the selection
return empty($selected_sharing_groups) ? !empty($final_sharing_group) :
!array_diff($final_sharing_group, $selected_sharing_groups); // All sharing groups are in the selection
} else if ($operator == 'not_equals') {
return count(array_diff($final_sharing_group, $selected_sharing_groups)) == count($final_sharing_group); // All sharing groups are in the selection
return empty($selected_sharing_groups) ? empty($final_sharing_group) :
count(array_diff($final_sharing_group, $selected_sharing_groups)) == count($final_sharing_group); // All sharing groups are in the selection
}
$errors[] = __('Condition operator not supported for that distribution level');
return false;
@ -159,9 +162,15 @@ class Module_distribution_if extends WorkflowBaseLogicModule
return min($distri1, $distri2);
}
private function __extractSharingGroupIDs(array $event, array $object=[], array $attribute=[]): array
private function __extractSharingGroupIDs(array $event, array $object=[], array $attribute=[], $scope='event'): array
{
$sgIDs = [];
if ($scope == 'event') {
if (!empty($event) && $event['distribution'] == 4) {
$sgIDs[] = $event['sharing_group_id'];
}
return $sgIDs;
}
if (!empty($event) && $event['distribution'] == 4) {
$sgIDs[] = $event['sharing_group_id'];
}

50
app/View/AnalystData/view.ctp
View File

@ -120,15 +120,57 @@ echo $this->element(
);
$object_uuid = Hash::get($data, $modelSelection . '.uuid');
$notes = $data[$modelSelection]['Note'] ?? [];
$opinions = $data[$modelSelection]['Opinion'] ?? [];
$relationships_outbound = $data[$modelSelection]['Relationship'] ?? [];
$relationships_inbound = $data[$modelSelection]['RelationshipInbound'] ?? [];
$notesOpinions = array_merge($notes, $opinions);
if(!function_exists("countNotes")) {
function countNotes($notesOpinions) {
$notesTotalCount = count($notesOpinions);
$notesCount = 0;
$relationsCount = 0;
foreach ($notesOpinions as $notesOpinion) {
if ($notesOpinion['note_type'] == 2) { // relationship
$relationsCount += 1;
} else {
$notesCount += 1;
}
if (!empty($notesOpinion['Note'])) {
$nestedCounts = countNotes($notesOpinion['Note']);
$notesTotalCount += $nestedCounts['total'];
$notesCount += $nestedCounts['notesOpinions'];
$relationsCount += $nestedCounts['relations'];
}
if (!empty($notesOpinion['Opinion'])) {
$nestedCounts = countNotes($notesOpinion['Opinion']);
$notesTotalCount += $nestedCounts['total'];
$notesCount += $nestedCounts['notesOpinions'];
$relationsCount += $nestedCounts['relations'];
}
}
return ['total' => $notesTotalCount, 'notesOpinions' => $notesCount, 'relations' => $relationsCount];
}
}
$counts = countNotes($notesOpinions);
$notesOpinionCount = $counts['notesOpinions'];
$allCounts = [
'notesOpinions' => $counts['notesOpinions'],
'relationships_outbound' => count($relationships_outbound),
'relationships_inbound' => count($relationships_inbound),
];
$options = [
'container_id' => 'analyst_data_thread',
'object_type' => $modelSelection,
'object_uuid' => $object_uuid,
'shortDist' => $shortDist,
'notes' => $data[$modelSelection]['Note'] ?? [],
'opinions' => $data[$modelSelection]['Opinion'] ?? [],
'relationships_outbound' => $data[$modelSelection]['Relationship'] ?? [],
'relationships_inbound' => $data[$modelSelection]['RelationshipInbound'] ?? [],
'notes' => $notes,
'opinions' => $opinions,
'relationships_outbound' => $relationships_outbound,
'relationships_inbound' => $relationships_inbound,
'allCounts' => $allCounts,
];
echo $this->element('genericElements/assetLoader', [

2
app/View/Dashboards/index.ctp
View File

@ -23,7 +23,7 @@ $(function () {
saveDashboardState();
});
grid.on('added', function(event, items) {
resetDashboardGrid(grid);
resetDashboardGrid(grid, false);
});
grid.on('gsresizestop', function(event, element) {
$(element).find('.widgetContentInner').trigger('widget-resized')

1
app/View/Elements/Events/View/row_object.ctp
View File

@ -36,6 +36,7 @@ $objectId = intval($object['id']);
$notes = !empty($object['Note']) ? $object['Note'] : [];
$opinions = !empty($object['Opinion']) ? $object['Opinion'] : [];
$relationships = !empty($object['Relationship']) ? $object['Relationship'] : [];
$relationshipsInbound = !empty($object['RelationshipInbound']) ? $object['RelationshipInbound'] : [];
echo $this->element('genericElements/Analyst_data/generic', [
'analyst_data' => ['notes' => $notes, 'opinions' => $opinions, 'relationships_outbound' => $relationships, 'relationships_inbound' => $relationshipsInbound],
'object_uuid' => $object['uuid'],

14
app/View/Elements/genericElements/Analyst_data/generic.ctp
View File

@ -65,20 +65,8 @@ $allCounts = [
$(document).ready(function() {
$('.node-opener-<?= $seed ?>').click(function() {
openNotes(this)
openNotes<?= $seed ?>(this)
})
function adjustPopoverPosition() {
var $popover = $('.popover:last');
$popover.css('top', Math.max($popover.position().top, 50) + 'px')
}
function openNotes(clicked) {
openPopover(clicked, renderedNotes<?= $seed ?>, undefined, undefined, function() {
adjustPopoverPosition()
$(clicked).removeClass('have-a-popover') // avoid closing the popover if a confirm popover (like the delete one) is called
})
}
})
</script>

75
app/View/Elements/genericElements/Analyst_data/thread.ctp
View File

@ -4,6 +4,7 @@
$URL_DELETE = '/analystData/delete/';
$seed = isset($seed) ? $seed : mt_rand();
$injectInPage = !empty($container_id) ? true : false;
$notes = !empty($notes) ? $notes : [];
$opinions = !empty($opinions) ? $opinions : [];
@ -41,7 +42,28 @@
if (!window.shortDist) {
var shortDist = <?= json_encode($shortDist) ?>;
}
var renderedNotes<?= $seed ?> = null
var container_id = false
<?php if (isset($container_id)): ?>
container_id = '<?= h($container_id) ?>'
<?php endif; ?>
function adjustPopoverPosition() {
var $popover = $('.popover:last');
$popover.css('top', Math.max($popover.position().top, 50) + 'px')
}
function openNotes<?= $seed ?>(clicked) {
var notes = <?= json_encode($notesOpinions) ?>;
var relationships = <?= json_encode($relationshipsOutbound) ?>;
var relationships_inbound = <?= json_encode($relationshipsInbound) ?>;
var relationship_related_object = <?= json_encode($related_objects) ?>;
var renderedNotes = renderAllNotesWithForm<?= $seed ?>(notes, relationships, relationships_inbound, relationship_related_object)
openPopover(clicked, renderedNotes, undefined, undefined, function() {
adjustPopoverPosition()
$(clicked).removeClass('have-a-popover') // avoid closing the popover if a confirm popover (like the delete one) is called
})
}
function renderNotes(notes, relationship_related_object, emptyMessage='<?= __('Empty') ?>', isInbound=false) {
var renderedNotesArray = []
@ -406,18 +428,7 @@ function fetchMoreNotes(clicked, noteType, uuid) {
}
(function() {
var notes = <?= json_encode($notesOpinions) ?>;
var relationships = <?= json_encode($relationshipsOutbound) ?>;
var relationships_inbound = <?= json_encode($relationshipsInbound) ?>;
var relationship_related_object = <?= json_encode($related_objects) ?>;
var container_id = false
<?php if (isset($container_id)): ?>
container_id = '<?= h($container_id) ?>'
<?php endif; ?>
var nodeContainerTemplate = doT.template('\
var nodeContainerTemplate<?= $seed ?> = doT.template('\
<div> \
<ul class="nav nav-tabs" style="margin-bottom: 10px;"> \
<li class="active"><a href="#notes-<?= $seed ?>" data-toggle="tab"><i class="<?= $this->FontAwesome->getClass('sticky-note') ?>"></i> <?= __('Notes & Opinions') ?> <span class="label label-secondary"><?= $allCounts['notesOpinions'] ?></span></a></li> \
@ -439,18 +450,6 @@ function fetchMoreNotes(clicked, noteType, uuid) {
</div> \
')
function renderAllNotesWithForm(relationship_related_object) {
var buttonContainer = '<div id="add-button-container" style="margin-top: 0.5rem;">' + addNoteButton + addOpinionButton + '</div>'
renderedNotes<?= $seed ?> = nodeContainerTemplate({
content_notes: renderNotes(notes.filter(function(note) { return note.note_type != 2}), relationship_related_object, '<?= __('No notes for this UUID.') ?>') + buttonContainer,
content_relationships_outbound: renderNotes(relationships, relationship_related_object, '<?= __('No relationship from this UUID') ?>') + addRelationshipButton,
content_relationships_inbound: renderNotes(relationships_inbound, relationship_related_object, '<?= __('No element are referencing this UUID') ?>', true),
})
if (container_id) {
$('#' + container_id).html(renderedNotes<?= $seed ?>)
}
}
var addNoteButton = '<button class="btn btn-small btn-block btn-primary" type="button" onclick="createNewNote(this, \'<?= $object_type ?>\', \'<?= $object_uuid ?>\')"> \
<i class="<?= $this->FontAwesome->getClass('plus') ?>"></i> <?= __('Add a note') ?> \
</button>'
@ -461,10 +460,15 @@ function fetchMoreNotes(clicked, noteType, uuid) {
<i class="<?= $this->FontAwesome->getClass('plus') ?>"></i> <?= __('Add a relationship') ?> \
</button>'
$(document).ready(function() {
renderAllNotesWithForm(relationship_related_object)
})
})()
function renderAllNotesWithForm<?= $seed ?>(notes, relationships, relationships_inbound, relationship_related_object) {
var buttonContainer = '<div id="add-button-container" style="margin-top: 0.5rem;">' + addNoteButton + addOpinionButton + '</div>'
var renderedNotes = nodeContainerTemplate<?= $seed ?>({
content_notes: renderNotes(notes.filter(function(note) { return note.note_type != 2}), relationship_related_object, '<?= __('No notes for this UUID.') ?>') + buttonContainer,
content_relationships_outbound: renderNotes(relationships, relationship_related_object, '<?= __('No relationship from this UUID') ?>') + addRelationshipButton,
content_relationships_inbound: renderNotes(relationships_inbound, relationship_related_object, '<?= __('No element are referencing this UUID') ?>', true),
})
return renderedNotes
}
function createNewNote(clicked, object_type, object_uuid) {
note_type = 'Note';
@ -516,6 +520,19 @@ function fetchMoreNotes(clicked, noteType, uuid) {
}
}
<?php if(!empty($injectInPage)): ?>
$(document).ready(function() {
var notes = <?= json_encode($notesOpinions) ?>;
var relationships = <?= json_encode($relationshipsOutbound) ?>;
var relationships_inbound = <?= json_encode($relationshipsInbound) ?>;
var relationship_related_object = <?= json_encode($related_objects) ?>;
var renderedNotes = renderAllNotesWithForm<?= $seed ?>(notes, relationships, relationships_inbound, relationship_related_object)
if (container_id) {
$('#' + container_id).html(renderedNotes)
}
})
<?php endif; ?>
</script>
<style>

5
app/View/Feeds/add.ctp
View File

@ -26,6 +26,11 @@ echo $this->element('genericElements/Form/genericForm', [
'label' => __('Disable correlation'),
'type' => 'checkbox'
],
[
'field' => 'Feed.settings.unpublish_event',
'label' => __('Unpublish events'),
'type' => 'checkbox'
],
[
'field' => 'name',
'label' => __('Name'),

50
app/View/GalaxyClusters/view.ctp
View File

@ -170,15 +170,57 @@ $md.html(md.render($md.text()));
<?php
$object_uuid = $cluster['GalaxyCluster']['uuid'];
$notes = $cluster['GalaxyCluster']['Note'] ?? [];
$opinions = $cluster['GalaxyCluster']['Opinion'] ?? [];
$relationships_outbound = $cluster['GalaxyCluster']['Relationship'] ?? [];
$relationships_inbound = $cluster['GalaxyCluster']['RelationshipInbound'] ?? [];
$notesOpinions = array_merge($notes, $opinions);
if(!function_exists("countNotes")) {
function countNotes($notesOpinions) {
$notesTotalCount = count($notesOpinions);
$notesCount = 0;
$relationsCount = 0;
foreach ($notesOpinions as $notesOpinion) {
if ($notesOpinion['note_type'] == 2) { // relationship
$relationsCount += 1;
} else {
$notesCount += 1;
}
if (!empty($notesOpinion['Note'])) {
$nestedCounts = countNotes($notesOpinion['Note']);
$notesTotalCount += $nestedCounts['total'];
$notesCount += $nestedCounts['notesOpinions'];
$relationsCount += $nestedCounts['relations'];
}
if (!empty($notesOpinion['Opinion'])) {
$nestedCounts = countNotes($notesOpinion['Opinion']);
$notesTotalCount += $nestedCounts['total'];
$notesCount += $nestedCounts['notesOpinions'];
$relationsCount += $nestedCounts['relations'];
}
}
return ['total' => $notesTotalCount, 'notesOpinions' => $notesCount, 'relations' => $relationsCount];
}
}
$counts = countNotes($notesOpinions);
$notesOpinionCount = $counts['notesOpinions'];
$allCounts = [
'notesOpinions' => $counts['notesOpinions'],
'relationships_outbound' => count($relationships_outbound),
'relationships_inbound' => count($relationships_inbound),
];
$options = [
'container_id' => 'analyst_data_thread',
'object_type' => 'GalaxyCluster',
'object_uuid' => $object_uuid,
'shortDist' => $shortDist,
'notes' => $cluster['GalaxyCluster']['Note'] ?? [],
'opinions' => $cluster['GalaxyCluster']['Opinion'] ?? [],
'relationships_outbound' => $cluster['GalaxyCluster']['Relationship'] ?? [],
'relationships_inbound' => $cluster['GalaxyCluster']['RelationshipInbound'] ?? [],
'notes' => $notes,
'opinions' => $opinions,
'relationships_outbound' => $relationships_outbound,
'relationships_inbound' => $relationships_inbound,
'allCounts' => $allCounts,
];
echo $this->element('genericElements/Analyst_data/thread', $options);

3
app/webroot/doc/openapi.yaml
View File

@ -6162,6 +6162,9 @@ components:
- xml
- csv
- text
- stix
- stix2
- stix-json
- hashes
- cache
- count

2
app/webroot/js/attack_matrix.js
View File

@ -17,6 +17,8 @@
});
adapt_position_from_viewport();
var firstTabId = $('#attack-matrix-tabscontroller span[data-toggle="tab"]:first').attr('href');
resizeHeader(firstTabId);
$('.ajax_popover_form .btn-matrix-submit').click(function() {
makeTagging(pickedGalaxies);

19
app/webroot/js/misp.js
View File

@ -5385,6 +5385,18 @@ function submitDashboardAddWidget() {
var height = $('#DashboardHeight').val();
var el = null;
var k = $('#last-element-counter').data('element-counter');
if (config === '') {
config = '[]'
}
try {
config = JSON.parse(config);
} catch (error) {
showMessage('fail', error.message)
return
}
config = JSON.stringify(config);
$.ajax({
url: baseurl + '/dashboards/getEmptyWidget/' + widget + '/' + (k+1),
type: 'GET',
@ -5398,14 +5410,7 @@ function submitDashboardAddWidget() {
"autoposition": 1
}
);
if (config !== '') {
config = JSON.parse(config);
config = JSON.stringify(config);
} else {
config = '[]';
}
$('#widget_' + (k+1)).attr('config', config);
saveDashboardState();
$('#last-element-counter').data('element-counter', (k+1));
},
complete: function(data) {

7
app/webroot/js/workflows-editor/workflows-editor.js
View File

@ -1836,10 +1836,9 @@ function genPicker(options, forNode = true) {
var $container = genSelect(options)
var $select = $container.find('select')
$select.addClass('start-chosen')
if (options.picker_options) {
// $select.data('chosen_options', options.picker_options)
$select.attr('data-chosen_options', JSON.stringify(options.picker_options))
}
var pickerOptions = options.picker_options ?? {}
pickerOptions['max_shown_results'] = 100
$select.attr('data-chosen_options', JSON.stringify(pickerOptions))
return $container
}