Merge branch '2.4' into develop

app/Lib/Tools/AWSS3Client.php

@@ -1,6 +1,7 @@
 <?php
 
 use Aws\S3\S3Client;
+use Aws\Exception\AwsException;
 
 class AWSS3Client
 {
@@ -10,11 +11,15 @@ class AWSS3Client
     private function __getSetSettings()
     {
         $settings = array(
-                'enabled' => false,
+                'enable' => false,
                 'bucket_name' => 'my-malware-bucket',
                 'region' => 'eu-west-1',
                 'aws_access_key' => '',
-                'aws_secret_key' => ''
+                'aws_secret_key' => '',
+                'aws_endpoint' => '',
+                'aws_compatible' => false,
+                'aws_ca' => '',
+                'aws_validate_ca' => true
         );
 
         // We have 2 situations
@@ -39,14 +44,53 @@ class AWSS3Client
     public function initTool()
     {
         $settings = $this->__getSetSettings();
-        $s3 = new Aws\S3\S3Client([
+        $s3Config = array(
             'version' => 'latest',
-            'region' => $settings['region']
-        ]);
-
-        $this->__client = $s3;
+            'region' => $settings['region'],
+        );
+        if ($settings['aws_compatible']) {
+            $s3Config = array(
+                 'version' => 'latest',
+                 'region' => $settings['region'],
+                 // MinIO compatibility
+                 // Reference: https://docs.min.io/docs/how-to-use-aws-sdk-for-php-with-minio-server.html
+                 'endpoint' => $settings['aws_endpoint'],
+                 'use_path_style_endpoint' => true,
+                 'credentials' => [
+                    'key'    => $settings['aws_access_key'],
+                    'secret' => $settings['aws_secret_key'],
+                 ],
+            );
+        }
+        // This line should points to server certificate
+        // Generically, this verify is set to false so that any certificate is valid
+        // Reference:
+        //   - https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/guide_configuration.html
+        //   - https://docs.guzzlephp.org/en/5.3/clients.html#verify
+        // Example:
+        // -- Verify certificate
+        //    'http'    => ['verify' => '/usr/lib/ssl/certs/minio.pem'],
+        // -- Do not verify certificate, securitywise, this option is not recommended, however due to 
+        //    internal deployment scheme it is acceptable risk to set this to false
+        //    'http'    => ['verify' => false],
+        // -- Verify againts  built in CA certificates
+        //    'http'    => ['verify' => true],
+        if ($settings['aws_validate_ca']) {
+            $s3Config['http']['verify'] = true;
+            if (!empty($settings['aws_ca'])) {
+                $s3Config['http']['verify'] = $settings['aws_ca'];
+            }
+        } else {
+            $s3Config['http']['verify'] = false;
+        }
+        echo 'Settings=====';
+        var_dump($settings);
+        echo 'S3Config=====';
+        var_dump($s3Config);
+        $s3Client = new Aws\S3\S3Client($s3Config);
+        $this->__client = $s3Client;
         $this->__settings = $settings;
-        return $s3;
+        return $s3Client;
     }
 
     public function exist($key)
@@ -68,12 +112,16 @@ class AWSS3Client
 
     public function download($key)
     {
-        $result = $this->__client->getObject([
-            'Bucket' => $this->__settings['bucket_name'],
-            'Key' => $key
-        ]);
+        try {
+            $result = $this->__client->getObject([
+                'Bucket' => $this->__settings['bucket_name'],
+                'Key' => $key
+            ]);
 
-        return $result['Body'];
+            return $result['Body'];
+        } catch (AwsException $e) {
+            throw new NotFoundException('Could not download object ' . $e->getMessage());
+        }
     }
 
     public function delete($key)

app/Model/Server.php

@@ -6561,9 +6561,41 @@ class Server extends AppModel
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
+                'S3_aws_compatible' => array(
+                    'level' => 2,
+                    'description' => __('Use external AWS compatible system such as MinIO'),
+                    'value' => false,
+                    'errorMessage' => '',
+                    'test' => 'testBool',
+                    'type' => 'boolean'
+                ),
+                'S3_aws_ca' => array(
+                    'level' => 2,
+                    'description' => __('AWS TLS CA, set to empty to use CURL internal trusted certificates or path for custom trusted CA'),
+                    'value' => '',
+                    'errorMessage' => '',
+                    'test' => 'testForEmpty',
+                    'type' => 'string'
+                ),
+                'S3_aws_validate_ca' => array(
+                    'level' => 2,
+                    'description' => __('Validate CA'),
+                    'value' => true,
+                    'errorMessage' => '',
+                    'test' => 'testBool',
+                    'type' => 'boolean'
+                ),
+                'S3_aws_endpoint' => array(
+                    'level' => 2,
+                    'description' => __('Uses external AWS compatible endpoint such as MinIO'),
+                    'value' => '',
+                    'errorMessage' => '',
+                    'test' => 'testForEmpty',
+                    'type' => 'string'
+                ),
                 'S3_bucket_name' => array(
                     'level' => 2,
-                    'description' => __('Bucket name to upload to'),
+                    'description' => __('Bucket name to upload to, please make sure that the bucket exists. We will not create the bucket for you'),
                     'value' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'

docs/INSTALL.rhel8.md

@@ -274,6 +274,8 @@ installCoreRHEL8 () {
 
   # install python-stix dependencies
   $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install ordered-set python-dateutil six weakrefmethod
+  debug "Install misp-stix"
+  ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/misp-stix
 
   # install zmq, redis
   $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U zmq redis

docs/INSTALL.ubuntu1804.md

@@ -147,6 +147,8 @@ installCore () {
 
     # install python-stix dependencies
     ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ordered-set python-dateutil six weakrefmethod
+    debug "Install misp-stix"
+    ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/misp-stix
 
     debug "Install PyMISP"
     ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/PyMISP

docs/INSTALL.ubuntu2004.md

@@ -139,6 +139,8 @@ installCore () {
 
     # install python-stix dependencies
     ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ordered-set python-dateutil six weakrefmethod
+    debug "Install misp-stix"
+    ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/misp-stix
 
     debug "Install PyMISP"
     ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/PyMISP

docs/xINSTALL.ubuntu2204.md

@@ -7,7 +7,7 @@
 
 Make sure you are reading the parsed version of this Document. When in doubt [click here](https://misp.github.io/MISP/INSTALL.ubuntu2004/).
 
-### 0/ MISP Ubuntu 20.04-server install - status
+### 0/ MISP Ubuntu 22.04-server install - status
 -------------------------
 !!! notice
     Installer tested working by [@SteveClement](https://twitter.com/SteveClement) on 20211002
@@ -20,7 +20,7 @@ Make sure you are reading the parsed version of this Document. When in doubt [cl
 ### 1/ Minimal Ubuntu install
 -------------------------
 
-#### Install a minimal Ubuntu 20.04-server system with the software:
+#### Install a minimal Ubuntu 22.04-server system with the software:
 - OpenSSH server
 - This guide assumes a user name of 'misp' with sudo working but can be overwritten by setting the environment variable: *${MISP_USER}*
 
@@ -101,10 +101,21 @@ installDepsPhp80 () {
   libapache2-mod-php7.4 \
   php7.4 php7.4-cli \
   php7.4-dev \
-  php-json php7.4-xml php7.4-mysql php7.4-opcache php7.4-readline php7.4-mbstring php7.4-zip \
-  php-redis php-gnupg \
+  php7.4-json php7.4-xml php7.4-mysql php7.4-opcache php7.4-readline php7.4-mbstring php7.4-zip \
   php7.4-intl php7.4-bcmath \
   php7.4-gd
+  # php-redis php-gnupg \
+
+  # Only needed while 7.4 downgrade
+  sudo apt install libgpgme-dev
+  sudo pecl channel-update pecl.php.net
+  sudo pecl install redis
+  sudo pecl install gnupg
+
+  echo "extension=redis.so" | sudo tee ${PHP_ETC_BASE}/mods-available/redis.ini
+  sudo phpenmod redis
+  echo "extension=gnupg.so" | sudo tee ${PHP_ETC_BASE}/mods-available/gnupg.ini
+  sudo phpenmod gnupg
 
   for key in upload_max_filesize post_max_size max_execution_time max_input_time memory_limit
   do
@@ -143,6 +154,8 @@ installCore () {
 
     # install python-stix dependencies
     ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ordered-set python-dateutil six weakrefmethod
+    debug "Install misp-stix"
+    ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/misp-stix
 
     debug "Install PyMISP"
     ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/PyMISP