Merge branch 'develop' into 2.4

2024-03-22 16:00:38 +01:00 · 2024-03-22 16:00:38 +01:00 · 8ac96cc104
parent 0fb58cff44 d39d9b4714
commit 8ac96cc104
30 changed files with 4409 additions and 3594 deletions
--- a/2
+++ b/2
@ -1 +1 @@
-Subproject commit 601e534778817d19bdda5227df983c1766ad10cd
+Subproject commit 8a2e52ac7ee2318623eb9f817cf999cc9734afb1
--- a/VERSION.json
+++ b/VERSION.json
@ -1 +1 @@
-{"major":2, "minor":4, "hotfix":187}
+{"major":2, "minor":4, "hotfix":188}
--- a/app/Console/Command/AdminShell.php
+++ b/app/Console/Command/AdminShell.php
@ -616,9 +616,9 @@ class AdminShell extends AppShell
        try {
            $redis = RedisTool::init();
            for ($i = 0; $i < 10; $i++) {
-                $persistence = $redis->info('persistence');
-                if (isset($persistence['loading']) && $persistence['loading']) {
-                    $this->out('Redis is still loading...');
+                $pong = $redis->ping();
+                if ($pong !== true) {
+                    $this->out('Redis is still loading... ' . $pong);
                    sleep(1);
                } else {
                    break;
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@ -34,7 +34,7 @@ class AppController extends Controller
    public $helpers = array('OrgImg', 'FontAwesome', 'UserName');

    private $__queryVersion = '159';
-    public $pyMispVersion = '2.4.187';
+    public $pyMispVersion = '2.4.188';
    public $phpmin = '7.2';
    public $phprec = '7.4';
    public $phptoonew = '8.0';
--- a/app/Controller/Component/CompressedRequestHandlerComponent.php
+++ b/app/Controller/Component/CompressedRequestHandlerComponent.php
@ -37,14 +37,17 @@ class CompressedRequestHandlerComponent extends Component
    private function decodeGzipEncodedContent(Controller $controller)
    {
        if (function_exists('gzdecode')) {
-            $decoded = gzdecode($controller->request->input());
+            $input = $controller->request->input();
+            if (empty($input)) {
+                throw new BadRequestException('Request data should be gzip encoded, but request is empty.');
+            }
+            $decoded = gzdecode($input);
            if ($decoded === false) {
                throw new BadRequestException('Invalid compressed data.');
            }
            return $decoded;
-        } else {
-            throw new BadRequestException("This server doesn't support GZIP compressed requests.");
        }
+        throw new BadRequestException("This server doesn't support GZIP compressed requests.");
    }

    /**
--- a/app/Controller/EventsController.php
+++ b/app/Controller/EventsController.php
@ -3112,9 +3112,9 @@ class EventsController extends AppController
                        $errors['Module'] = 'Module failure.';
                    }
                } else {
+                    $errors['failed_servers'] = $result;
                    $lastResult = array_pop($result);
                    $resultString = (count($result) > 0) ? implode(', ', $result) . ' and ' . $lastResult : $lastResult;
-                    $errors['failed_servers'] = $result;
                    $message = __('Event published but not pushed to %s, re-try later. If the issue persists, make sure that the correct sync user credentials are used for the server link and that the sync user on the remote server has authentication privileges.', $resultString);
                }
            } else {
--- a/app/Lib/Tools/CurlClient.php
+++ b/app/Lib/Tools/CurlClient.php
@ -202,6 +202,10 @@ class CurlClient extends HttpSocketExtended
            $options[CURLOPT_POSTFIELDS] = $query;
        }

+        if ($method === 'HEAD') {
+            $options[CURLOPT_NOBODY] = true;
+        }
+
        if (!empty($request['header'])) {
            $headers = [];
            foreach ($request['header'] as $key => $value) {
@ -231,7 +235,6 @@ class CurlClient extends HttpSocketExtended
            }
            return $len;
        };
-
        if (!curl_setopt_array($this->ch, $options)) {
            throw new \RuntimeException('curl error: Could not set options');
        }
--- a/app/Model/AttachmentScan.php
+++ b/app/Model/AttachmentScan.php
@ -258,7 +258,7 @@ class AttachmentScan extends AppModel
                    $scanned++;
                }
            } catch (Exception $e) {
-                $this->logException("Could not scan attachment for $type {$attribute['Attribute']['id']}", $e, LOG_WARNING);
+                $this->logException("Could not scan attachment for $type {$attribute[$type]['id']}", $e, LOG_WARNING);
                $fails++;
            }

--- a/app/Model/Attribute.php
+++ b/app/Model/Attribute.php
--- a/app/Model/Datasource/Database/MysqlExtended.php
+++ b/app/Model/Datasource/Database/MysqlExtended.php
@ -58,6 +58,7 @@ class MysqlExtended extends Mysql
            'having' => $this->having($query['having'], true, $Model),
            'lock' => $this->getLockingHint($query['lock']),
            'indexHint' => $this->__buildIndexHint($query['forceIndexHint'] ?? null),
+            'ignoreIndexHint' => $this->__buildIgnoreIndexHint($query['ignoreIndexHint'] ?? null)
        ));
    }

@ -90,6 +91,7 @@ class MysqlExtended extends Mysql
                'having' => $queryData['having'],
                'lock' => $queryData['lock'],
                'forceIndexHint' => $queryData['forceIndexHint'] ?? null,
+                'ignoreIndexHint' => $queryData['ignoreIndexHint'] ?? null,
            ),
            $Model
        );
@ -117,14 +119,25 @@ class MysqlExtended extends Mysql
    }

    /**
-     * Builds the index hint for the query
+     * Builds the force index hint for the query
     * 
-     * @param string|null $forceIndexHint FORCE INDEX hint
+     * @param string|null $forceIndexHint INDEX hint
     * @return string
     */
    private function __buildIndexHint($forceIndexHint = null): ?string
    {
-        return isset($forceIndexHint) ? ('FORCE INDEX ' . $forceIndexHint) : null;
+        return isset($forceIndexHint) ? ('FORCE INDEX (' . $forceIndexHint . ')') : null;
+    }
+
+        /**
+     * Builds the ignore index hint for the query
+     * 
+     * @param string|null $ignoreIndexHint INDEX hint
+     * @return string
+     */
+    private function __buildIgnoreIndexHint($ignoreIndexHint = null): ?string
+    {
+        return isset($ignoreIndexHint) ? ('IGNORE INDEX (' . $ignoreIndexHint . ')') : null;
    }

    /**
--- a/app/Model/Datasource/Database/MysqlObserverExtended.php
+++ b/app/Model/Datasource/Database/MysqlObserverExtended.php
@ -50,6 +50,7 @@ class MysqlObserverExtended extends Mysql
            'having' => $this->having($query['having'], true, $Model),
            'lock' => $this->getLockingHint($query['lock']),
            'indexHint' => $this->__buildIndexHint($query['forceIndexHint'] ?? null),
+            'ignoreIndexHint' => $this->__buildIgnoreIndexHint($query['ignoreIndexHint'] ?? null),
        ));
    }

@ -82,6 +83,7 @@ class MysqlObserverExtended extends Mysql
                'having' => $queryData['having'],
                'lock' => $queryData['lock'],
                'forceIndexHint' => $queryData['forceIndexHint'] ?? null,
+                'ignoreIndexHint' => $queryData['ignoreIndexHint'] ?? null,
            ),
            $Model
        );
@ -103,20 +105,31 @@ class MysqlObserverExtended extends Mysql
            extract($data);
            $having = !empty($having) ? " $having" : '';
            $lock = !empty($lock) ? " $lock" : '';
-            return rtrim("SELECT {$fields} FROM {$table} {$alias} {$indexHint} {$joins} {$conditions} {$group}{$having} {$order} {$limit}{$lock}");
+            return rtrim("SELECT {$fields} FROM {$table} {$alias} {$indexHint} {$ignoreIndexHint} {$joins} {$conditions} {$group}{$having} {$order} {$limit}{$lock}");
        }
        return parent::renderStatement($type, $data);
    }

    /**
-     * Builds the index hint for the query
+     * Builds the force index hint for the query
     * 
-     * @param string|null $forceIndexHint FORCE INDEX hint
+     * @param string|null $forceIndexHint INDEX hint
     * @return string
     */
    private function __buildIndexHint($forceIndexHint = null): ?string
    {
-        return isset($forceIndexHint) ? ('FORCE INDEX ' . $forceIndexHint) : null;
+        return isset($forceIndexHint) ? ('FORCE INDEX (' . $forceIndexHint . ')') : null;
+    }
+
+        /**
+     * Builds the ignore index hint for the query
+     * 
+     * @param string|null $ignoreIndexHint INDEX hint
+     * @return string
+     */
+    private function __buildIgnoreIndexHint($ignoreIndexHint = null): ?string
+    {
+        return isset($ignoreIndexHint) ? ('IGNORE INDEX (' . $ignoreIndexHint . ')') : null;
    }

    /**
--- a/app/Model/DecayingModel.php
+++ b/app/Model/DecayingModel.php
@ -16,6 +16,9 @@ class DecayingModel extends AppModel
        )
    );

+    private $modelCache = [];
+    private $modelCacheForType = [];
+
    private $__registered_model_classes = array(); // Proxy for already instantiated classes
    public $allowed_overrides = array('threshold' => 1, 'lifetime' => 1, 'decay_speed' => 1);

@ -263,6 +266,10 @@ class DecayingModel extends AppModel
    // - full attach Attribute types associated to the requested model
    public function fetchModel($user, $id, $full=true, $conditions=array(), $attach_editable=0)
    {
+        $cacheKey = sprintf('%s', $id);
+        if (isset($this->modelCache[$cacheKey])) {
+            return $this->modelCache[$cacheKey];
+        }
        $conditions['id'] = $id;
        $searchOptions = array(
            'conditions' => $conditions,
@ -290,6 +297,7 @@ class DecayingModel extends AppModel
            $decayingModel['DecayingModel']['attribute_types'] = $this->DecayingModelMapping->getAssociatedTypes($user, $decayingModel);
        }
        $decayingModel = $this->attachIsEditableByCurrentUser($user, $decayingModel);
+        $this->modelCache[$cacheKey] = $decayingModel;
        return $decayingModel;
    }

@ -612,11 +620,15 @@ class DecayingModel extends AppModel
        if ($model_id === false) { // fetch all allowed and associated models
            $associated_model_ids = $this->DecayingModelMapping->getAssociatedModels($user, $attribute['type'], true);
            $associated_model_ids = isset($associated_model_ids[$attribute['type']]) ? array_values($associated_model_ids[$attribute['type']]) : array();
-            if (!empty($associated_model_ids)) {
+            if (isset($this->modelCacheForType[$attribute['type']])) {
+                $models = $this->modelCacheForType[$attribute['type']];
+            } else if (!empty($associated_model_ids)) {
                $models = $this->fetchModels($user, $associated_model_ids, false, array('enabled' => true));
+                $this->modelCacheForType[$attribute['type']] = $models;
            }
        } else {
            $models = $this->fetchModels($user, $model_id, false, array());
+            $this->modelCacheForType[$attribute['type']] = $models;
        }
        foreach ($models as $model) {
            if (!empty($model_overrides)) {
--- a/app/Model/DecayingModelMapping.php
+++ b/app/Model/DecayingModelMapping.php
@ -25,6 +25,8 @@ class DecayingModelMapping extends AppModel
        )
    );

+    private $modelCache = [];
+
    public function resetMappingForModel($new_model, $user) {
        if (empty($new_model['model_id'])) {
            throw new NotFoundException(__('No Decaying Model with the provided ID exists'));
@ -76,6 +78,10 @@ class DecayingModelMapping extends AppModel
    }

    public function getAssociatedModels($user, $attribute_type = false) {
+        $cacheKey = sprintf('%s', $attribute_type);
+        if (isset($this->modelCache[$cacheKey])) {
+            return $this->modelCache[$cacheKey];
+        }
        $conditions = array(
            'OR' => array(
                'DecayingModel.org_id' => $user['org_id'],
@ -111,6 +117,7 @@ class DecayingModelMapping extends AppModel
        }
        $associated_models = Hash::combine($associated_models, '{n}.DecayingModelMapping.model_id', '{n}.DecayingModelMapping.model_id', '{n}.DecayingModelMapping.attribute_type');
        $models = array_merge_recursive($associated_default_models, $associated_models);
+        $this->modelCache[$cacheKey] = $models;
        return $models;
    }

--- a/app/Model/DecayingModelsFormulas/Base.php
+++ b/app/Model/DecayingModelsFormulas/Base.php
@ -133,9 +133,9 @@ abstract class DecayingModelBase
        }
        if ($last_sighting_timestamp === false) {
            $this->Sighting = ClassRegistry::init('Sighting');
-            $all_sightings = $this->Sighting->listSightings($user, $attribute['id'], 'attribute', false, 0, true);
-            if (!empty($all_sightings)) {
-                $last_sighting_timestamp = $all_sightings[0]['Sighting']['date_sighting'];
+            $last_sighting = $this->Sighting->getLastSightingForAttribute($user, $attribute['id']);
+            if (!empty($last_sighting)) {
+                $last_sighting_timestamp = $last_sighting['Sighting']['date_sighting'];
            } elseif (!is_null($attribute['last_seen'])) {
                $last_sighting_timestamp = (new DateTime($attribute['last_seen']))->format('U');
            } else {
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@ -1058,7 +1058,11 @@ class Event extends AppModel
        // prepare attribute for sync
        if (!empty($data['Attribute'])) {
            foreach ($data['Attribute'] as $key => $attribute) {
-                if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && in_array($attribute['type'], $pushRules['type_attributes']['NOT'])) {
+                if (
+                    !empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) &&
+                    !empty($pushRules['type_attributes']['NOT']) &&
+                    in_array($attribute['type'], $pushRules['type_attributes']['NOT'])
+                ) {
                    unset($data['Attribute'][$key]);
                    continue;
                }
@ -3280,6 +3284,7 @@ class Event extends AppModel
        $template->set('distributionLevels', $this->distributionLevels);
        $template->set('analysisLevels', $this->analysisLevels);
        $template->set('tlp', $subjMarkingString);
+        $template->set('title', Configure::read('MISP.title_text'));
        $template->subject($subject);
        $template->referenceId("event-alert|{$event['Event']['id']}");

--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@ -5408,6 +5408,13 @@ class Server extends AppModel
                    'test' => 'testForEmpty',
                    'type' => 'string',
                ],
+                'email_reply_to' => [
+                    'level' => 2,
+                    'description' => __('Reply to e-mail address for e-mails send from MISP instance.'),
+                    'value' => '',
+                    'test' => 'testForEmpty',
+                    'type' => 'string',
+                ],
                'taxii_sync' => array(
                    'level' => 3,
                    'description' => __('This setting is deprecated and can be safely removed.'),
--- a/app/Model/ShadowAttribute.php
+++ b/app/Model/ShadowAttribute.php
@ -196,6 +196,24 @@ class ShadowAttribute extends AppModel

        // convert into utc and micro sec
        $this->data = $this->Attribute->ISODatetimeToUTC($this->data, $this->alias);
+
+        $trigger_id = 'shadow-attribute-before-save';
+        $isTriggerCallable = $this->isTriggerCallable($trigger_id);
+        if ($isTriggerCallable) {
+            $triggerData = $this->data;
+            $shadowAttribute_id = $triggerData['ShadowAttribute']['id'] ?? 0;
+            $workflowErrors = [];
+            $logging = [
+                'model' => 'ShadowAttribute',
+                'action' => 'add',
+                'id' => $shadowAttribute_id,
+                'message' => __('The workflow `%s` prevented the saving of this proposal.', $trigger_id)
+            ];
+            $workflowSuccess = $this->executeTrigger($trigger_id, $triggerData, $workflowErrors, $logging);
+            if (!$workflowSuccess) {
+                return false;
+            }
+        }
        return true;
    }

--- a/app/Model/Sighting.php
+++ b/app/Model/Sighting.php
@ -1010,6 +1010,33 @@ class Sighting extends AppModel
        return $sightings;
    }

+     /**
+     * @param int $id
+     * @return array
+     */
+    public function getLastSightingForAttribute(array $user, $id): array
+    {
+        $conditions = [
+            'Sighting.attribute_id' => $id,
+            'Sighting.type' => 0,
+        ];
+
+        $sightingsPolicy = $this->sightingsPolicy();
+        if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER || $sightingsPolicy === self::SIGHTING_POLICY_HOST_ORG) {
+            $conditions['Sighting.org_id'] = [$user['org_id'], Configure::read('MISP.host_org_id')];
+        } else if ($sightingsPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) {
+            $all_sightings = $this->listSightings($user, [$id], 'attribute', false, 0, true);
+            $sighting = $all_sightings[0]['Sighting']['date_sighting'];
+            return $sighting;
+        }
+        $sighting = $this->find('first', [
+            'conditions' => $conditions,
+            'recursive' => -1,
+            'order' => ['Sighting.date_sighting DESC']
+        ]);
+        return empty($sighting) ? [] : $sighting;
+    }
+
    /**
     * @param array $user
     * @param string $returnFormat
--- a/app/Model/WorkflowModules/trigger/Module_shadow_attribute_before_save.php
+++ b/app/Model/WorkflowModules/trigger/Module_shadow_attribute_before_save.php
@ -0,0 +1,45 @@
+<?php
+include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
+
+class Module_shadow_attribute_before_save extends WorkflowBaseTriggerModule
+{
+    public $id = 'shadow-attribute-before-save';
+    public $scope = 'shadow-attribute';
+    public $name = 'Shadow Attribute Before Save';
+    public $description = 'This trigger is called just before a Shadow Attribute is saved in the database';
+    public $icon = 'comment';
+    public $inputs = 0;
+    public $outputs = 1;
+    public $blocking = true;
+    public $misp_core_format = true;
+    public $trigger_overhead = self::OVERHEAD_MEDIUM;
+
+    public function __construct()
+    {
+        parent::__construct();
+        $this->trigger_overhead_message = __('This trigger is called each time a Shadow Attribute is about to be saved. This means that when a large quantity of Shadow Attributes are being saved (e.g. Feed pulling or synchronisation), the workflow will be run for as many time as there are Shadow Attributes.');  
+    }
+
+    public function normalizeData(array $data)
+    {
+        $this->Event = ClassRegistry::init('Event');
+        $this->Attribute = ClassRegistry::init('Attribute');
+
+        if (empty($data['ShadowAttribute'])) {
+            return false;
+        }
+        
+        // If we're dealing with a proposed edit, we retrieve the data about the attribute 
+        if ($data['ShadowAttribute']['old_id']) {
+            $event = $this->Attribute->fetchAttribute($data['ShadowAttribute']['old_id']);
+            $event['Attribute']['ShadowAttribute'] = array($data['ShadowAttribute']);
+        } else {
+            // If it is a proposal to add a new attribute, we retrieve only the data about the event
+            $event = $this->Event->quickFetchEvent($data['ShadowAttribute']['event_id']);
+            $event['Event']['ShadowAttribute'] = [$data['ShadowAttribute']];
+        }
+
+        $event = parent::normalizeData($event);
+        return $event;
+    }
+}
--- a/app/Plugin/EcsLog/Lib/Log/Engine/EcsLog.php
+++ b/app/Plugin/EcsLog/Lib/Log/Engine/EcsLog.php
@ -182,10 +182,10 @@ class EcsLog implements CakeLogInterface
    }

    /**
-     * @param Exception $exception
+     * @param Throwable $exception
     * @return void
     */
-    public static function handleException(Exception $exception)
+    public static function handleException(Throwable $exception)
    {
        $code = $exception->getCode();
        $code = ($code && is_int($code)) ? $code : 1;
--- a/app/Plugin/OidcAuth/Controller/Component/Auth/OidcAuthenticate.php
+++ b/app/Plugin/OidcAuth/Controller/Component/Auth/OidcAuthenticate.php
@ -13,7 +13,7 @@ App::uses('Oidc', 'OidcAuth.Lib');
 *  - OidcAuth.organisation_property (default: `organization`)
 *  - OidcAuth.organisation_uuid_property (default: `organization_uuid`)
 *  - OidcAuth.roles_property (default: `roles`)
- *  - OidcAuth.default_org
+ *  - OidcAuth.default_org - organisation ID, UUID or name if organsation is not provided by OIDC
 *  - OidcAuth.unblock (boolean, default: false)
 *  - OidcAuth.offline_access (boolean, default: false)
 *  - OidcAuth.check_user_validity (integer, default `0`)
--- a/app/Plugin/OidcAuth/Lib/Oidc.php
+++ b/app/Plugin/OidcAuth/Lib/Oidc.php
@ -49,17 +49,22 @@ class Oidc
        }

        $organisationProperty = $this->getConfig('organisation_property', 'organization');
-        $organisationName = $claims->{$organisationProperty} ?? $this->getConfig('default_org');
+        $organisationName = $claims->{$organisationProperty} ?? null;

        $organisationUuidProperty = $this->getConfig('organisation_uuid_property', 'organization_uuid');
        $organisationUuid = $claims->{$organisationUuidProperty} ?? null;

        $organisationId = $this->checkOrganization($organisationName, $organisationUuid, $mispUsername);
        if (!$organisationId) {
-            if ($user) {
-                $this->block($user);
+            $defaultOrganisationId = $this->defaultOrganisationId();
+            if ($defaultOrganisationId) {
+                $organisationId = $defaultOrganisationId;
+            } else {
+                if ($user) {
+                    $this->block($user);
+                }
+                return false;
            }
-            return false;
        }

        $roleProperty = $this->getConfig('roles_property', 'roles');
@ -123,7 +128,7 @@ class Oidc
            return $user;
        }

-        $this->log($mispUsername, 'User not found in database.');
+        $this->log($mispUsername, 'User not found in database, creating new one.');

        $time = time();
        $userData = [
@ -320,6 +325,8 @@ class Oidc
    }

    /**
+     * Fetch organisation ID from database by provided name and UUID. If organisation is not found, it is created. If
+     * organisation with given UUID has different name, then is renamed.
     * @param string $orgName Organisation name or UUID
     * @param string|null $orgUuid Organisation UUID
     * @param string $mispUsername
@ -376,6 +383,41 @@ class Oidc
        return $orgId;
    }

+    /**
+     * @return false|int Organisation ID or false if org not found
+     */
+    private function defaultOrganisationId()
+    {
+        $defaultOrgName = $this->getConfig('default_org');
+        if (empty($defaultOrgName)) {
+            return false;
+        }
+
+        if (is_numeric($defaultOrgName)) {
+            $conditions = ['id' => $defaultOrgName];
+        } else if (Validation::uuid($defaultOrgName)) {
+            $conditions = ['uuid' => strtolower($defaultOrgName)];
+        } else {
+            $conditions = ['name' => $defaultOrgName];
+        }
+        $orgAux = $this->User->Organisation->find('first', [
+            'fields' => ['Organisation.id'],
+            'conditions' => $conditions,
+        ]);
+        if (empty($orgAux)) {
+            if (is_numeric($defaultOrgName)) {
+                $this->log(null, "Could not find default organisation with ID `$defaultOrgName`.");
+            } else if (Validation::uuid($defaultOrgName)) {
+                $this->log(null, "Could not find default organisation with UUID `$defaultOrgName`.");
+            } else {
+                $this->log(null, "Could not find default organisation with name `$defaultOrgName`.");
+            }
+            return false;
+        }
+
+        return $orgAux['Organisation']['id'];
+    }
+
    /**
     * @param int $orgId
     * @param string $newName
--- a/app/files/misp-galaxy
+++ b/app/files/misp-galaxy
@ -1 +1 @@
-Subproject commit 2eca8cb04715be2f743bb8b73a83a4fb3a2a1798
+Subproject commit e18e5c16c6451c9a15ba94f9f0d50c97c13dc808
--- a/app/files/misp-objects
+++ b/app/files/misp-objects
@ -1 +1 @@
-Subproject commit 4bf694a8463b7d24bfea00b98e941ddffcb7c7a0
+Subproject commit 8ccd583d217624c322a6927bcbdb7fe412a2e855
--- a/app/files/scripts/misp-stix
+++ b/app/files/scripts/misp-stix
@ -1 +1 @@
-Subproject commit 0428b4a43a67feed33837107ece1b144042c619e
+Subproject commit f531a2cdce0e1ff2bcc879d57d2872f6318fb5cf
--- a/app/files/warninglists
+++ b/app/files/warninglists
@ -1 +1 @@
-Subproject commit 42ecbaf75fbf9c6e130b9d1a66252f51b467ae52
+Subproject commit d5c586c1bdbc84f3b7b6d0d1a4185c3b23f61bbb
--- a/app/webroot/doc/openapi.yaml
+++ b/app/webroot/doc/openapi.yaml
@ -58,6 +58,8 @@ info:
 servers:
  - url: https://misp.local
 tags:
+  - name: Analyst Data
+    description: "Analyst Data allows analysts to share and add their own analysis to any MISP data having an UUID. They can be one of these three type: Analyst Note, Analyst Opionion or Analyst Relationship."
  - name: Attributes
    description: "Attributes in MISP can be network indicators (e.g. IP address), system indicators (e.g. a string in memory) or even bank account details."
    externalDocs:
@ -128,8 +130,104 @@ tags:
    description: "MISP Taxonomies is a set of common classification libraries to tag, classify and organise information. Taxonomy allows to express the same vocabulary among a distributed set of users and organisations."
    externalDocs:
      url: https://www.circl.lu/doc/misp/taxonomy/#taxonomies
+  - name: Event Report
+    description: "Event reports can contain text under the markdown format and are stored inside an event. They can be used to describe events or processes in a readable way."
+

 paths:
+  /analystData/add/{analystType}/{objectUUID}/{ObjectType}:
+    post:
+      summary: "Add analyst data"
+      operationId: addAnalystData
+      tags:
+        - Analyst Data
+      parameters:
+        - $ref: "#/components/parameters/analystDataTypeParameter"
+        - $ref: "#/components/parameters/analystDataObjectUUIDParameter"
+        - $ref: "#/components/parameters/analystDataObjectTypeParameter"
+      requestBody:
+        $ref: "#/components/requestBodies/AddAnalystDataRequest"
+      responses:
+        "200":
+          $ref: "#/components/responses/AnalystDataResponse"
+        "403":
+          $ref: "#/components/responses/UnauthorizedApiErrorResponse"
+        default:
+          $ref: "#/components/responses/ApiErrorResponse"
+
+  /analystData/edit/{analystType}/{analystDataID}:
+    post:
+      summary: "Edit analyst data"
+      operationId: editAnalystData
+      tags:
+        - Analyst Data
+      parameters:
+        - $ref: "#/components/parameters/analystDataTypeParameter"
+        - $ref: "#/components/parameters/analystDataIdParameter"
+      requestBody:
+        $ref: "#/components/requestBodies/AddAnalystDataRequest"
+      responses:
+        "200":
+          $ref: "#/components/responses/AnalystDataResponse"
+        "403":
+          $ref: "#/components/responses/UnauthorizedApiErrorResponse"
+        default:
+          $ref: "#/components/responses/ApiErrorResponse"
+
+  /analystData/delete/{analystType}/{analystDataID}:
+    delete:
+      summary: "Delete Analyst data"
+      operationId: deleteAnalystData
+      tags:
+        - Analyst Data
+      parameters:
+        - $ref: "#/components/parameters/analystDataTypeParameter"
+        - $ref: "#/components/parameters/analystDataIdParameter"
+      responses:
+        "200":
+          $ref: "#/components/responses/DeleteAnalystDataResponse"
+        "403":
+          $ref: "#/components/responses/UnauthorizedApiErrorResponse"
+        "404":
+          $ref: "#/components/responses/NotFoundApiErrorResponse"
+        default:
+          $ref: "#/components/responses/ApiErrorResponse"
+
+  /analystData/index/{analystType}:
+    get:
+      summary: "List Analyst data"
+      operationId: indexAnalystData
+      tags:
+        - Analyst Data
+      parameters:
+        - $ref: "#/components/parameters/analystDataTypeParameter"
+      responses:
+        "200":
+          $ref: "#/components/responses/AnalystDataListResponse"
+        "403":
+          $ref: "#/components/responses/UnauthorizedApiErrorResponse"
+        "404":
+          $ref: "#/components/responses/NotFoundApiErrorResponse"
+        default:
+          $ref: "#/components/responses/ApiErrorResponse"
+
+  /analystData/view/{analystType}/{analystDataID}:
+    get:
+      summary: "Get Analyst Data by ID"
+      operationId: getEventById
+      tags:
+        - Analyst Data
+      parameters:
+        - $ref: "#/components/parameters/analystDataTypeParameter"
+        - $ref: "#/components/parameters/analystDataIdParameter"
+      responses:
+        "200":
+          $ref: "#/components/responses/AnalystDataResponse"
+        "403":
+          $ref: "#/components/responses/UnauthorizedApiErrorResponse"
+        default:
+          $ref: "#/components/responses/ApiErrorResponse"
+
  /attributes/restSearch:
    post:
      summary: "[restSearch] Get a filtered and paginated list of attributes"
@ -322,6 +420,121 @@ paths:
        default:
          $ref: "#/components/responses/ApiErrorResponse"

+  /eventReports/view/{eventReportId}:
+    get:
+      summary: "Get event report by ID"
+      operationId: viewEventReport
+      tags:
+        - Event Report
+      parameters:
+        - $ref: "#/components/parameters/eventReportIdParameter"
+      responses:
+        "200":
+          $ref: "#/components/responses/EventReportResponse"
+        "403":
+          $ref: "#/components/responses/UnauthorizedApiErrorResponse"
+        default:
+          $ref: "#/components/responses/ApiErrorResponse"
+
+  /eventReports/add/{eventId}:
+    post:
+      summary: "Add Event Report"
+      operationId: addEventReport
+      tags:
+        - Event Report
+      parameters:
+        - $ref: "#/components/parameters/eventIdParameter"
+      requestBody:
+        $ref: "#/components/requestBodies/AddEventReportRequest"
+      responses:
+        "200":
+          $ref: "#/components/responses/EventReportResponse"
+        "403":
+          $ref: "#/components/responses/UnauthorizedApiErrorResponse"
+        default:
+          $ref: "#/components/responses/ApiErrorResponse"
+
+  /eventReports/edit/{eventReportId}:
+    post:
+      summary: "Edit Event Report"
+      operationId: editEventReport
+      tags:
+        - Event Report
+      parameters:
+        - $ref: "#/components/parameters/eventReportIdParameter"
+      responses:
+        "200":
+          $ref: "#/components/responses/EventReportResponse"
+        "403":
+          $ref: "#/components/responses/UnauthorizedApiErrorResponse"
+        default:
+          $ref: "#/components/responses/ApiErrorResponse"
+
+  /eventReports/delete/{eventReportId}/{hardDelete}:
+    post:
+      summary: "Delete Event Report"
+      operationId: deleteEventReport
+      tags:
+        - Event Report
+      parameters:
+        - $ref: "#/components/parameters/eventReportIdParameter"
+        - $ref: "#/components/parameters/hardDeleteParameterNotRequired"
+      responses:
+        "200":
+          $ref: "#/components/responses/DeleteEventReportResponse"
+        "403":
+          $ref: "#/components/responses/UnauthorizedApiErrorResponse"
+        default:
+          $ref: "#/components/responses/ApiErrorResponse"
+
+  /eventReports/restore/{eventReportId}:
+    post:
+      summary: "Restore Event Report"
+      operationId: restoreEventReport
+      tags:
+        - Event Report
+      parameters:
+        - $ref: "#/components/parameters/eventReportIdParameter"
+      responses:
+        "200":
+          $ref: "#/components/responses/RestoreEventReportResponse"
+        "403":
+          $ref: "#/components/responses/UnauthorizedApiErrorResponse"
+        default:
+          $ref: "#/components/responses/ApiErrorResponse"
+
+  /eventReports/index:
+    get:
+      summary: "Get event reports"
+      operationId: indexEventReport
+      tags:
+        - Event Report
+      responses:
+        "200":
+          $ref: "#/components/responses/EventReportListResponse"
+        "403":
+          $ref: "#/components/responses/UnauthorizedApiErrorResponse"
+        default:
+          $ref: "#/components/responses/ApiErrorResponse"
+
+  /eventReports/importReportFromUrl/{eventId}:
+    post:
+      summary: "Import Report From URL"
+      operationId: importFromURLEventReport
+      tags:
+        - Event Report
+      parameters:
+        - $ref: "#/components/parameters/eventIdParameter"
+      requestBody:
+        $ref: "#/components/requestBodies/ImportFromURLEventReportRequest"
+      responses:
+        "200":
+          $ref: "#/components/responses/EventReportImportFromUrlResponse"
+        "403":
+          $ref: "#/components/responses/UnauthorizedApiErrorResponse"
+        default:
+          $ref: "#/components/responses/ApiErrorResponse"
+
  /events/restSearch:
    post:
      summary: "[restSearch] Get a filtered and paginated list of events"
@ -2502,6 +2715,129 @@ paths:

 components:
  schemas:
+    # Analyst Data
+    AnalystDataID:
+      type: string
+      pattern: '^\d+$'
+      maxLength: 10
+      example: "12345"
+
+    AnalystDataType:
+      type: string
+      enum:
+        - "Note"
+        - "Opinion"
+        - "Relationship"
+
+    AnalystObjectType:
+      type: string
+      enum:
+        - "Attribute"
+        - "Event"
+        - "EventReport"
+        - "GalaxyCluster"
+        - "Galaxy"
+        - "Object"
+        - "Note"
+        - "Opinion"
+        - "Relationship"
+        - "Organisation"
+        - "SharingGroup"
+
+    AnalystDataAuthors:
+      type: string
+      example: "john.doe@admin.test"
+
+    AnalystNoteNote:
+      type: string
+      example: "Provide more context"
+
+    AnalystNoteLanguage:
+      type: string
+      description: "RFC5646 Language code"
+      example: "fr-BE"
+
+    AnalystNoteOpinion:
+      type: number
+      description: "The opinion expressed on a scale from 0 to 100 where values < 50 are negatives, 50 is neutral and values > 50 are positives"
+      format: integer
+      minimum: 0
+      maximum: 100
+      example: 70
+
+    AnalystData:
+      type: object
+      properties:
+        uuid:
+          $ref: "#/components/schemas/UUID"
+        object_uuid:
+          $ref: "#/components/schemas/UUID"
+        object_type:
+          $ref: "#/components/schemas/AnalystObjectType"
+        authors:
+          $ref: "#/components/schemas/AnalystDataAuthors"
+        org_uuid:
+          $ref: "#/components/schemas/UUID"
+        orgc_uuid:
+          $ref: "#/components/schemas/UUID"
+        created:
+          type: string
+          example: "2024-03-19 11:10:24"
+        modified:
+          type: string
+          example: "2024-03-19 11:10:24"
+        distribution:
+          $ref: "#/components/schemas/DistributionLevelId"
+        sharing_group_id:
+          $ref: "#/components/schemas/SharingGroupId"
+        locked:
+          $ref: "#/components/schemas/IsLocked"
+
+    AnalystNote:
+      allOf:
+        - type: object
+          properties:
+            note:
+              $ref: "#/components/schemas/AnalystNoteNote"
+            language:
+              $ref: "#/components/schemas/AnalystNoteLanguage"
+            note_type_name:
+              type: string
+              enum:
+                - "Note"
+        - $ref: "#/components/schemas/AnalystData"
+
+    AnalystOpinion:
+      allOf:
+        - type: object
+          properties:
+            comment:
+              $ref: "#/components/schemas/AnalystNoteNote"
+            opinion:
+              $ref: "#/components/schemas/AnalystNoteOpinion"
+            note_type_name:
+              type: string
+              enum:
+                - "Opinion"
+        - $ref: "#/components/schemas/AnalystData"
+
+    AnalystRelationship:
+      allOf:
+        - type: object
+          properties:
+            related_object_uuid:
+              $ref: "#/components/schemas/UUID"
+            related_object_type:
+              $ref: "#/components/schemas/AnalystObjectType"
+            relationship_type:
+              type: string
+              example: "related-to"
+            note_type_name:
+              type: string
+              enum:
+                - "Relationship"
+        - $ref: "#/components/schemas/AnalystData"
+
    # Attributes
    AttributeId:
      type: string
@ -3128,8 +3464,47 @@ components:
      items:
        $ref: "#/components/schemas/EventTag"

-    EventReport: # TODO: describe
+    EventReportId:
+      type: string
+      pattern: '^\d+$'
+      maxLength: 10
+      example: "12345"
+
+    EventReportName:
+      type: string
+      maxLength: 65535
+      example: "Report of the incident"
+
+    EventReportValue:
+      type: string
+
+    EventReport:
+      allOf:
+        - type: object
+          properties:
+            id:
+              $ref: "#/components/schemas/EventReportId"
+        - $ref: "#/components/schemas/EventReportNoId"
+
+    EventReportNoId:
      type: object
+      properties:
+        uuid:
+          $ref: "#/components/schemas/UUID"
+        event_id:
+          $ref: "#/components/schemas/EventId"
+        name:
+          $ref: "#/components/schemas/EventReportName"
+        content:
+          $ref: "#/components/schemas/EventReportValue"
+        distribution:
+          $ref: "#/components/schemas/DistributionLevelId"
+        sharing_group_id:
+          $ref: "#/components/schemas/SharingGroupId"
+        timestamp:
+          $ref: "#/components/schemas/NullableTimestamp"
+        deleted:
+          $ref: "#/components/schemas/SoftDeletedFlag"

    EventNoId:
      type: object
@ -6209,6 +6584,17 @@ components:
          - "0"
          - "1"

+    hardDeleteParameterNotRequired:
+      name: hardDelete
+      in: path
+      description: "`1` for hard delete the entity, `0` for soft deletion."
+      required: false
+      schema:
+        type: string
+        enum:
+          - "0"
+          - "1"
+
    objectTemplateIdParameter:
      name: objectTemplateId
      in: path
@ -6253,6 +6639,50 @@ components:
      schema:
        $ref: "#/components/schemas/TaxonomyId"

+    eventReportIdParameter:
+      name: eventReportId
+      in: path
+      description: "UUID or numeric ID of the event report"
+      required: true
+      schema:
+        oneOf:
+          - $ref: "#/components/schemas/EventReportId"
+          - $ref: "#/components/schemas/UUID"
+
+    analystDataIdParameter:
+      name: analystID
+      in: path
+      description: "UUID or numeric ID of the Analyst data"
+      required: true
+      schema:
+        oneOf:
+          - $ref: "#/components/schemas/AnalystDataID"
+          - $ref: "#/components/schemas/UUID"
+
+    analystDataTypeParameter:
+      name: analystType
+      in: path
+      description: "Type of the analyst data."
+      required: true
+      schema:
+        $ref: "#/components/schemas/AnalystDataType"
+
+    analystDataObjectUUIDParameter:
+      name: analystObjectUUID
+      in: path
+      description: "Object UUID that has an analyst data"
+      required: true
+      schema:
+        $ref: "#/components/schemas/UUID"
+
+    analystDataObjectTypeParameter:
+      name: analystObjectType
+      in: path
+      description: "Object type that has an analyst data"
+      required: true
+      schema:
+        $ref: "#/components/schemas/analystObjectType"
+
  securitySchemes:
    ApiKeyAuth:
      type: apiKey
@ -7052,7 +7482,58 @@ components:
              - $ref: "#/components/schemas/TagNumbericalValueOverrideUserSetting"
              - $ref: "#/components/schemas/EventIndexHideColumnsUserSetting"

+    AddEventReportRequest:
+      required: true
+      content:
+        application/json:
+          schema:
+            $ref: "#/components/schemas/EventReportNoId"
+
+    ImportFromURLEventReportRequest:
+      required: true
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              url:
+                type: string
+                example: "https://domain.example/blogpost/123.pdf"
+
  responses:
+    AnalystDataResponse:
+      description: "An analyst data. Could be a Note, Opinion or Relationship depending on the `analystType` parameter"
+      content:
+        application/json:
+          schema:
+            oneOf:
+              - $ref: "#/components/schemas/AnalystNote"
+              - $ref: "#/components/schemas/AnalystOpinion"
+              - $ref: "#/components/schemas/AnalystRelationship"
+
+    DeleteAnalystDataResponse:
+      description: "Delete analyst data response"
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              message:
+                type: string
+                example: Analyst Note deleted.
+
+    AnalystDataListResponse:
+      description: "A list of Analyst Data"
+      content:
+        application/json:
+          schema:
+            type: array
+            items:
+              oneOf:
+                - $ref: "#/components/schemas/AnalystNote"
+                - $ref: "#/components/schemas/AnalystOpinion"
+                - $ref: "#/components/schemas/AnalystRelationship"
+
    AttributeResponse:
      description: "An attribute"
      content:
@ -7206,6 +7687,93 @@ components:
                nullable: true
                example: "Event was not deleted."

+    DeleteEventReportResponse:
+      description: "Delete event report response"
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              saved:
+                description: "`true` if the event was succesfully deleted, `false` if it failed"
+                type: boolean
+              success:
+                description: "`true` if the event was succesfully deleted, `false` if it failed"
+                nullable: true
+                type: boolean
+              name:
+                type: string
+                example: "Event report 1 soft deleted."
+              message:
+                type: string
+                example: "Event Report 1 could not be soft deleted. Reasons: Errors"
+              url:
+                type: string
+                example: "/eventReport/delete/1"
+              errors:
+                description: "Only present if an error occurred when deleting the event"
+                type: string
+                nullable: true
+                example: "Event Report was not deleted."
+
+    RestoreEventReportResponse:
+      description: "Restore Event Report response"
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              saved:
+                description: "`true` if the event was succesfully deleted, `false` if it failed"
+                type: boolean
+              success:
+                description: "`true` if the event was succesfully deleted, `false` if it failed"
+                nullable: true
+                type: boolean
+              name:
+                type: string
+                example: "Event report 1 restored."
+              message:
+                type: string
+                example: "Event Report 1 could not be restored. Reasons: Errors"
+              url:
+                type: string
+                example: "/eventReport/restore/1"
+              errors:
+                description: "Only present if an error occurred when deleting the event"
+                type: string
+                nullable: true
+                example: "Event Report was not restored."
+
+    EventReportImportFromUrlResponse:
+      description: "Import from URL Event Report response"
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              saved:
+                description: "`true` if the event report was succesfully created, `false` if it failed"
+                type: boolean
+              success:
+                description: "`true` if the event report was succesfully created, `false` if it failed"
+                nullable: true
+                type: boolean
+              name:
+                type: string
+                example: "Event report imported"
+              message:
+                type: string
+                example: "Report downloaded and created"
+              url:
+                type: string
+                example: "/eventReport/importReportFromUrl/1"
+              errors:
+                description: "Only present if an error occurred when deleting the event"
+                type: string
+                nullable: true
+                example: "Could not fetch report from URL. Fetcher module not enabled or could not download the page"
+
    AddEventTagResponse:
      description: "Add event tag response"
      content:
@ -7332,6 +7900,28 @@ components:
                type: string
                example: "/events/unpublish/1"

+    EventReportResponse:
+      description: "An Event Report"
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              EventReport:
+                $ref: "#/components/schemas/EventReport"
+
+    EventReportListResponse:
+      description: "Get a list of Event Report"
+      content:
+        application/json:
+          schema:
+            type: array
+            items:
+              type: object
+              properties:
+                EventReport:
+                  $ref: "#/components/schemas/EventReport"
+
    GalaxyListResponse:
      description: "A list of galaxies"
      content:
--- a/requirements.txt
+++ b/requirements.txt
@ -6,7 +6,7 @@ misp-lib-stix2>=3.0.1.1
 mixbox>=1.0.5
 plyara>=2.1.1
 pydeep2>=0.5.1
-pymisp==2.4.187
+pymisp==2.4.188
 python-magic>=0.4.27
 pyzmq>=25.1.1
 redis>=5.0.1
--- a/tests/logs_fail_regexes.txt
+++ b/tests/logs_fail_regexes.txt
@ -2,4 +2,5 @@
 # Whenever the regex matches, the Logs job will fail and report the error.
 class="cake-error"
 Error: [ParseError]
-Error: [PDOException]
+Error: [PDOException]
+Error: [BadRequestException]
--- a/tests/testlive_comprehensive_local.py
+++ b/tests/testlive_comprehensive_local.py
@ -891,7 +891,7 @@ class TestComprehensive(unittest.TestCase):

    def test_etag(self):
        headers = {
-            'Authorization': self.admin_misp_connector.key,
+            'Authorization': key.strip(),
            'Accept': 'application/json',
            'User-Agent': 'PyMISP',
            'If-None-Match': '',