From abceaf98aea75bc447297eb8b6feb8ade86c9fef Mon Sep 17 00:00:00 2001 From: Fandi Gunawan <10239907+fandigunawan@users.noreply.github.com> Date: Fri, 8 Oct 2021 23:19:54 +0700 Subject: [PATCH 1/3] Supports MinIO as alternative to AWS S3 --- app/Lib/Tools/AWSS3Client.php | 56 ++++++++++++++++++++++++++++------- app/Model/Server.php | 18 ++++++++++- 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/app/Lib/Tools/AWSS3Client.php b/app/Lib/Tools/AWSS3Client.php index 45941aeca..38ef93c95 100644 --- a/app/Lib/Tools/AWSS3Client.php +++ b/app/Lib/Tools/AWSS3Client.php @@ -1,6 +1,7 @@ 'my-malware-bucket', 'region' => 'eu-west-1', 'aws_access_key' => '', - 'aws_secret_key' => '' + 'aws_secret_key' => '', + 'aws_endpoint' => '', + 'aws_compatible' => false ); // We have 2 situations @@ -39,10 +42,39 @@ class AWSS3Client public function initTool() { $settings = $this->__getSetSettings(); - $s3 = new Aws\S3\S3Client([ - 'version' => 'latest', - 'region' => $settings['region'] - ]); + if ($settings['aws_compatible']) { + $s3 = new Aws\S3\S3Client([ + 'version' => 'latest', + 'region' => $settings['region'], + // MinIO compatibility + // Reference: https://docs.min.io/docs/how-to-use-aws-sdk-for-php-with-minio-server.html + 'endpoint' => $settings['aws_endpoint'], + 'use_path_style_endpoint' => true, + // This line should points to server certificate + // Generically, this verify is set to false so that any certificate is valid + // Reference: + // - https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/guide_configuration.html + // - https://docs.guzzlephp.org/en/5.3/clients.html#verify + // Example: + // -- Verify certificate + // 'http' => ['verify' => '/usr/lib/ssl/certs/minio.pem'], + // -- Do not verify certificate, securitywise, this option is not recommended, however due to + // internal deployment scheme it is acceptable risk to set this to false + // 'http' => ['verify' => false], + // -- Verify againts built in CA certificates + // 'http' => ['verify' => true], + 'http' => ['verify' => false], + 'credentials' => [ + 'key' => $settings['aws_access_key'], + 'secret' => $settings['aws_secret_key'], + ], + ]); + } else { + $s3 = new Aws\S3\S3Client([ + 'version' => 'latest', + 'region' => $settings['region'] + ]); + } $this->__client = $s3; $this->__settings = $settings; @@ -68,12 +100,16 @@ class AWSS3Client public function download($key) { - $result = $this->__client->getObject([ - 'Bucket' => $this->__settings['bucket_name'], - 'Key' => $key - ]); + try { + $result = $this->__client->getObject([ + 'Bucket' => $this->__settings['bucket_name'], + 'Key' => $key + ]); - return $result['Body']; + return $result['Body']; + } catch (AwsException $e) { + throw new NotFoundException('Could not download object ' . $e->getMessage()); + } } public function delete($key) diff --git a/app/Model/Server.php b/app/Model/Server.php index 0c08b5dbd..216367bec 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -6691,9 +6691,25 @@ class Server extends AppModel 'test' => 'testBool', 'type' => 'boolean' ), + 'S3_aws_compatible' => array( + 'level' => 2, + 'description' => __('Use external AWS compatible system such as MinIO'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'S3_aws_endpoint' => array( + 'level' => 2, + 'description' => __('External AWS compatible endpoint such as MinIO'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), 'S3_bucket_name' => array( 'level' => 2, - 'description' => __('Bucket name to upload to'), + 'description' => __('Bucket name to upload to, please make sure that the bucket exists. We will not create the bucket for you'), 'value' => '', 'errorMessage' => '', 'test' => 'testForEmpty', From 1166aa6d49dd376a69d926e2da0f24e0d466ac2d Mon Sep 17 00:00:00 2001 From: Fandi Gunawan <10239907+fandigunawan@users.noreply.github.com> Date: Sun, 7 Nov 2021 18:48:14 +0700 Subject: [PATCH 2/3] Adds default TLS validation to true and supports custom CA path --- app/Lib/Tools/AWSS3Client.php | 64 +++++++++++++++++++++-------------- app/Model/Server.php | 18 +++++++++- 2 files changed, 55 insertions(+), 27 deletions(-) diff --git a/app/Lib/Tools/AWSS3Client.php b/app/Lib/Tools/AWSS3Client.php index 38ef93c95..3fc0358a0 100644 --- a/app/Lib/Tools/AWSS3Client.php +++ b/app/Lib/Tools/AWSS3Client.php @@ -11,13 +11,15 @@ class AWSS3Client private function __getSetSettings() { $settings = array( - 'enabled' => false, + 'enable' => false, 'bucket_name' => 'my-malware-bucket', 'region' => 'eu-west-1', 'aws_access_key' => '', 'aws_secret_key' => '', 'aws_endpoint' => '', - 'aws_compatible' => false + 'aws_compatible' => false, + 'aws_ca' => '', + 'aws_validate_ca' => true ); // We have 2 situations @@ -42,43 +44,53 @@ class AWSS3Client public function initTool() { $settings = $this->__getSetSettings(); + $s3Config = array( + 'version' => 'latest', + 'region' => $settings['region'], + ); if ($settings['aws_compatible']) { - $s3 = new Aws\S3\S3Client([ + $s3Config = array( 'version' => 'latest', 'region' => $settings['region'], // MinIO compatibility // Reference: https://docs.min.io/docs/how-to-use-aws-sdk-for-php-with-minio-server.html 'endpoint' => $settings['aws_endpoint'], 'use_path_style_endpoint' => true, - // This line should points to server certificate - // Generically, this verify is set to false so that any certificate is valid - // Reference: - // - https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/guide_configuration.html - // - https://docs.guzzlephp.org/en/5.3/clients.html#verify - // Example: - // -- Verify certificate - // 'http' => ['verify' => '/usr/lib/ssl/certs/minio.pem'], - // -- Do not verify certificate, securitywise, this option is not recommended, however due to - // internal deployment scheme it is acceptable risk to set this to false - // 'http' => ['verify' => false], - // -- Verify againts built in CA certificates - // 'http' => ['verify' => true], - 'http' => ['verify' => false], 'credentials' => [ 'key' => $settings['aws_access_key'], 'secret' => $settings['aws_secret_key'], ], - ]); - } else { - $s3 = new Aws\S3\S3Client([ - 'version' => 'latest', - 'region' => $settings['region'] - ]); + ); } - - $this->__client = $s3; + // This line should points to server certificate + // Generically, this verify is set to false so that any certificate is valid + // Reference: + // - https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/guide_configuration.html + // - https://docs.guzzlephp.org/en/5.3/clients.html#verify + // Example: + // -- Verify certificate + // 'http' => ['verify' => '/usr/lib/ssl/certs/minio.pem'], + // -- Do not verify certificate, securitywise, this option is not recommended, however due to + // internal deployment scheme it is acceptable risk to set this to false + // 'http' => ['verify' => false], + // -- Verify againts built in CA certificates + // 'http' => ['verify' => true], + if ($settings['aws_validate_ca']) { + $s3Config['http']['verify'] = true; + if (!empty($settings['aws_ca'])) { + $s3Config['http']['verify'] = $settings['aws_ca']; + } + } else { + $s3Config['http']['verify'] = false; + } + echo 'Settings====='; + var_dump($settings); + echo 'S3Config====='; + var_dump($s3Config); + $s3Client = new Aws\S3\S3Client($s3Config); + $this->__client = $s3Client; $this->__settings = $settings; - return $s3; + return $s3Client; } public function exist($key) diff --git a/app/Model/Server.php b/app/Model/Server.php index 216367bec..8c6c5303c 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -6699,9 +6699,25 @@ class Server extends AppModel 'test' => 'testBool', 'type' => 'boolean' ), + 'S3_aws_ca' => array( + 'level' => 2, + 'description' => __('AWS TLS CA, set to empty to use CURL internal trusted certificates or path for custom trusted CA'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'S3_aws_validate_ca' => array( + 'level' => 2, + 'description' => __('Validate CA'), + 'value' => true, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), 'S3_aws_endpoint' => array( 'level' => 2, - 'description' => __('External AWS compatible endpoint such as MinIO'), + 'description' => __('Uses external AWS compatible endpoint such as MinIO'), 'value' => '', 'errorMessage' => '', 'test' => 'testForEmpty', From 50009b6c8b75c4c16d9b5a0fda2f7cabf6680515 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Tue, 9 Nov 2021 12:05:16 +0100 Subject: [PATCH 3/3] fix: [doc] Added missing misp-stix to the documentation --- docs/INSTALL.rhel8.md | 2 ++ docs/INSTALL.ubuntu1804.md | 2 ++ docs/INSTALL.ubuntu2004.md | 2 ++ docs/xINSTALL.ubuntu2204.md | 21 +++++++++++++++++---- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/docs/INSTALL.rhel8.md b/docs/INSTALL.rhel8.md index 92e96192d..bc1ccffbc 100644 --- a/docs/INSTALL.rhel8.md +++ b/docs/INSTALL.rhel8.md @@ -274,6 +274,8 @@ installCoreRHEL8 () { # install python-stix dependencies $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install ordered-set python-dateutil six weakrefmethod + debug "Install misp-stix" + ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/misp-stix # install zmq, redis $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U zmq redis diff --git a/docs/INSTALL.ubuntu1804.md b/docs/INSTALL.ubuntu1804.md index ec132a1ac..b6e237190 100644 --- a/docs/INSTALL.ubuntu1804.md +++ b/docs/INSTALL.ubuntu1804.md @@ -147,6 +147,8 @@ installCore () { # install python-stix dependencies ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ordered-set python-dateutil six weakrefmethod + debug "Install misp-stix" + ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/misp-stix debug "Install PyMISP" ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/PyMISP diff --git a/docs/INSTALL.ubuntu2004.md b/docs/INSTALL.ubuntu2004.md index 2ea2cfb5d..c07fe8e3a 100644 --- a/docs/INSTALL.ubuntu2004.md +++ b/docs/INSTALL.ubuntu2004.md @@ -139,6 +139,8 @@ installCore () { # install python-stix dependencies ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ordered-set python-dateutil six weakrefmethod + debug "Install misp-stix" + ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/misp-stix debug "Install PyMISP" ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/PyMISP diff --git a/docs/xINSTALL.ubuntu2204.md b/docs/xINSTALL.ubuntu2204.md index bd0815cb2..97f33086c 100644 --- a/docs/xINSTALL.ubuntu2204.md +++ b/docs/xINSTALL.ubuntu2204.md @@ -7,7 +7,7 @@ Make sure you are reading the parsed version of this Document. When in doubt [click here](https://misp.github.io/MISP/INSTALL.ubuntu2004/). -### 0/ MISP Ubuntu 20.04-server install - status +### 0/ MISP Ubuntu 22.04-server install - status ------------------------- !!! notice Installer tested working by [@SteveClement](https://twitter.com/SteveClement) on 20211002 @@ -20,7 +20,7 @@ Make sure you are reading the parsed version of this Document. When in doubt [cl ### 1/ Minimal Ubuntu install ------------------------- -#### Install a minimal Ubuntu 20.04-server system with the software: +#### Install a minimal Ubuntu 22.04-server system with the software: - OpenSSH server - This guide assumes a user name of 'misp' with sudo working but can be overwritten by setting the environment variable: *${MISP_USER}* @@ -101,10 +101,21 @@ installDepsPhp80 () { libapache2-mod-php7.4 \ php7.4 php7.4-cli \ php7.4-dev \ - php-json php7.4-xml php7.4-mysql php7.4-opcache php7.4-readline php7.4-mbstring php7.4-zip \ - php-redis php-gnupg \ + php7.4-json php7.4-xml php7.4-mysql php7.4-opcache php7.4-readline php7.4-mbstring php7.4-zip \ php7.4-intl php7.4-bcmath \ php7.4-gd + # php-redis php-gnupg \ + + # Only needed while 7.4 downgrade + sudo apt install libgpgme-dev + sudo pecl channel-update pecl.php.net + sudo pecl install redis + sudo pecl install gnupg + + echo "extension=redis.so" | sudo tee ${PHP_ETC_BASE}/mods-available/redis.ini + sudo phpenmod redis + echo "extension=gnupg.so" | sudo tee ${PHP_ETC_BASE}/mods-available/gnupg.ini + sudo phpenmod gnupg for key in upload_max_filesize post_max_size max_execution_time max_input_time memory_limit do @@ -143,6 +154,8 @@ installCore () { # install python-stix dependencies ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ordered-set python-dateutil six weakrefmethod + debug "Install misp-stix" + ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/misp-stix debug "Install PyMISP" ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/PyMISP