Remove plugin configuration and refactor REST client functions

README.md

@@ -27,11 +27,8 @@ Additionally, this fork features the following improvements:
 -   Fix MISP modules loading of faup library
 -   Fix MISP modules loading of gl library
 -   Add support for new background job system (see https://github.com/MISP/MISP/blob/2.4/docs/background-jobs-migration-guide.md)
--   Add support for exposing locally generated resources
 -   Add support for building specific MISP and MISP-modules commits
--   Add automatic configuration of MISP modules (see `configure_misp.sh`)
 -   Add automatic configuration of sync servers (see `configure_misp.sh`)
--   Add automatic configuration of organizations (see `configure_misp.sh`)
 -   Add autoamtic configuration of authentication keys (see `configure_misp.sh`)
 -   Add direct push of docker images to Docker Hub
 -   Consolidate docker compose files
@@ -85,11 +82,13 @@ Updating the images should be as simple as `docker-compose pull` which, unless c
     -   Certificate Key File: `key.pem`
     -   CA File for Cert Authentication (optional) `ca.pem`
 
--   Directory volume mount and create configs: `/var/www/MISP/app/Config/`
-
 -   Additional directory volume mounts:
-    -   `/var/www/MISP/app/files`
+    -   `./configs`: `/var/www/MISP/app/Config/`
-    -   `/var/www/MISP/.gnupg`
+    -   `./logs`: `/var/www/MISP/app/tmp/logs/`
+    -   `./files`: `/var/www/MISP/app/files/`
+    -   `./gnupg`: `/var/www/MISP/.gnupg/`
+
+-   If you need to automatically run additional steps each time the container starts, create a new file `files/customize_misp.sh`, and replace the variable `${CUSTOM_PATH}` inside `docker-compose.yml` with its parent path.
 
 ### Building
 

docker-compose.yml

@@ -48,10 +48,17 @@ services:
       - "./files/:/var/www/MISP/app/files/"
       - "./ssl/:/etc/nginx/certs/"
       - "./gnupg/:/var/www/MISP/.gnupg/"
+      # customize by replacing ${CUSTOM_PATH} with a path containing 'files/customize_misp.sh'
+      # - "${CUSTOM_PATH}/:/custom/"
     environment:
       - "HOSTNAME=https://localhost"
       - "REDIS_FQDN=redis"
       - "CRON_USER_ID=1"
+      # standard settings
+      - "ADMIN_EMAIL=${ADMIN_EMAIL}"
+      - "ADMIN_KEY=${ADMIN_KEY}"
+      - "ADMIN_ORG=${ADMIN_ORG}"
+      - "GPG_PASSPHRASE=${GPG_PASSPHRASE}"
       # sync server settings
       - "SYNCSERVERS=${SYNCSERVERS}"
       - "SYNCSERVERS_1_NAME=${SYNCSERVERS_1_NAME}"
@@ -64,15 +71,6 @@ services:
           "pull_rules": "{\"tags\":{\"OR\":[],\"NOT\":[]},\"orgs\":{\"OR\":[],\"NOT\":[]},\"url_params\":\"{\\\"searchanalysis\\\": \\\"2\\\"}\"}",
           "pull": true
         }
-      # standard settings
-      - "ADMIN_EMAIL=${ADMIN_EMAIL}"
-      - "ADMIN_KEY=${ADMIN_KEY}"
-      - "ADMIN_ORG=${ADMIN_ORG}"
-      - "GPG_PASSPHRASE=${GPG_PASSPHRASE}"
-      - "NSX_ANALYSIS_API_TOKEN=${NSX_ANALYSIS_API_TOKEN}"
-      - "NSX_ANALYSIS_KEY=${NSX_ANALYSIS_KEY}"
-      - "ORGANIZATIONS=${ORGANIZATIONS}"
-      - "VIRUSTOTAL_KEY=${VIRUSTOTAL_KEY}"
   
   misp-modules:
     image: ostefano/misp-docker:modules-latest

server/Dockerfile

@@ -186,6 +186,7 @@ ARG PHP_VER
     COPY files/etc/supervisor/workers.conf /etc/supervisor/conf.d/50-workers.conf
     COPY files/var/www/html/index.php /var/www/html/index.php
     COPY files/configure_misp.sh /
+    COPY files/rest_client.sh /
     COPY files/entrypoint_fpm.sh /
     COPY files/entrypoint_nginx.sh /
     COPY files/entrypoint_cron.sh /

server/files/configure_misp.sh

@@ -1,11 +1,13 @@
 #!/bin/bash
 
+source /rest_client.sh
+
 [ -z "$ADMIN_EMAIL" ] && ADMIN_EMAIL="admin@admin.test"
 [ -z "$GPG_PASSPHRASE" ] && GPG_PASSPHRASE="passphrase"
 [ -z "$REDIS_FQDN" ] && REDIS_FQDN=redis
 [ -z "$MISP_MODULES_FQDN" ] && MISP_MODULES_FQDN="http://misp-modules"
 
-init_misp_configuration(){
+init_configuration(){
     # Note that we are doing this after enforcing permissions, so we need to use the www-data user for this
     echo "... configuring default settings"
     sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "MISP.redis_host" "$REDIS_FQDN"
@@ -22,7 +24,7 @@ init_misp_configuration(){
     sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "Plugin.Cortex_services_enable" false
 }
 
-init_misp_workers(){
+init_workers(){
     # Note that we are doing this after enforcing permissions, so we need to use the www-data user for this
     echo "... configuring background workers"
     sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "SimpleBackgroundJobs.enabled" true
@@ -36,7 +38,7 @@ init_misp_workers(){
     supervisorctl start misp-workers:*
 }
 
-init_gnupg() {
+configure_gnupg() {
     GPG_DIR=/var/www/MISP/.gnupg
     GPG_ASC=/var/www/MISP/app/webroot/gpg.asc
     GPG_TMP=/tmp/gpg.tmp
@@ -142,35 +144,7 @@ apply_optional_fixes() {
     sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "Plugin.Enrichment_hover_timeout" 5
 }
 
-configure_optional_plugins() {
+update_components() {
-    if [ ! -z "$VIRUSTOTAL_KEY" ]; then
-        echo "... enabling 'virustotal' module ..."
-        sudo -u www-data php /var/www/MISP/tests/modify_config.php modify "{
-            \"Plugin\": {
-                \"Enrichment_virustotal_enabled\": true,
-                \"Enrichment_virustotal_apikey\": \"${VIRUSTOTAL_KEY}\"
-            }
-        }" > /dev/null
-    fi
-
-    if [ ! -z "$VIRUSTOTAL_KEY" ] && [ ! -z "$NSX_ANALYSIS_KEY" ] && [ ! -z "$NSX_ANALYSIS_API_TOKEN" ] && [ ! -z "$ADMIN_KEY" ]; then
-        echo "... enabling 'vmware_nsx' module ..."
-        sudo -u www-data php /var/www/MISP/tests/modify_config.php modify "{
-            \"Plugin\": {
-                \"Enrichment_vmware_nsx_enabled\": true,
-                \"Enrichment_vmware_nsx_analysis_verify_ssl\": \"True\",
-                \"Enrichment_vmware_nsx_analysis_key\": \"${NSX_ANALYSIS_KEY}\",
-                \"Enrichment_vmware_nsx_analysis_api_token\": \"${NSX_ANALYSIS_API_TOKEN}\",
-                \"Enrichment_vmware_nsx_vt_key\": \"${VIRUSTOTAL_KEY}\",
-                \"Enrichment_vmware_nsx_misp_url\": \"${HOSTNAME}\",
-                \"Enrichment_vmware_nsx_misp_verify_ssl\": \"False\",
-                \"Enrichment_vmware_nsx_misp_key\": \"${ADMIN_KEY}\"
-            }
-        }" > /dev/null
-    fi
-}
-
-updateComponents() {
     sudo -u www-data /var/www/MISP/app/Console/cake Admin updateGalaxies
     sudo -u www-data /var/www/MISP/app/Console/cake Admin updateTaxonomies
     sudo -u www-data /var/www/MISP/app/Console/cake Admin updateWarningLists
@@ -178,54 +152,6 @@ updateComponents() {
     sudo -u www-data /var/www/MISP/app/Console/cake Admin updateObjectTemplates "$CRON_USER_ID"
 }
 
-add_organization() {
-    # empty uuid fallbacks to auto-generate
-    curl -s --show-error -k \
-     -H "Authorization: ${ADMIN_KEY}" \
-     -H "Accept: application/json" \
-     -H "Content-type: application/json" \
-     -d "{ \
-        \"uuid\": \"${3}\", \
-        \"name\": \"${1}\", \
-        \"local\": ${2} \
-     }" ${HOSTNAME}/admin/organisations/add
-}
-
-get_organization() {
-    curl -s --show-error -k \
-     -H "Authorization: ${ADMIN_KEY}" \
-     -H "Accept: application/json" \
-     -H "Content-type: application/json" ${HOSTNAME}/organisations/view/${1} | jq -e -r ".Organisation.id // empty"
-}
-
-add_server() {
-    curl -s --show-error -k \
-     -H "Authorization: ${ADMIN_KEY}" \
-     -H "Accept: application/json" \
-     -H "Content-type: application/json" \
-     -d "${1}" ${HOSTNAME}/servers/add
-}
-
-get_server() {
-    curl -s --show-error -k \
-     -H "Authorization: ${ADMIN_KEY}" \
-     -H "Accept: application/json" \
-     -H "Content-type: application/json" ${HOSTNAME}/servers | jq -e -r ".[] | select(.Server[\"name\"] == \"${1}\") | .Server.id"
-}
-
-create_organizations() {
-    SPLITTED_ORGS=$(echo $ORGANIZATIONS | tr ',' '\n')
-    for ORG in $SPLITTED_ORGS; do
-        ORG_ID=$(get_organization ${ORG})
-        if [[ -z $ORG_ID ]]; then
-            echo "... adding organization: $ORG"
-            add_organization $ORG true
-        else
-            echo "... organization $ORG already exists"
-        fi
-    done
-}
-
 create_sync_servers() {
     SPLITTED_SYNCSERVERS=$(echo $SYNCSERVERS | tr ',' '\n')
     for ID in $SPLITTED_SYNCSERVERS; do
@@ -233,24 +159,27 @@ create_sync_servers() {
         UUID="SYNCSERVERS_${ID}_UUID"
         DATA="SYNCSERVERS_${ID}_DATA"
         KEY="SYNCSERVERS_${ID}_KEY"
-        if ! get_server ${!NAME}; then
+        echo "... searching sync server ${!NAME}..."
-            echo "... configuring sync server ${!NAME}..."
+        if ! get_server ${HOSTNAME} ${ADMIN_KEY} ${!NAME}; then
-            add_organization ${!NAME} false ${!UUID}
+            echo "... adding new sync server ${!NAME}..."
-            ORG_ID=$(get_organization ${!UUID})
+            add_organization ${HOSTNAME} ${ADMIN_KEY} ${!NAME} false ${!UUID}
+            ORG_ID=$(get_organization ${HOSTNAME} ${ADMIN_KEY} ${!UUID})
             DATA=$(echo "${!DATA}" | jq --arg org_id ${ORG_ID} --arg name ${!NAME} --arg key ${!KEY} '. + {remote_org_id: $org_id, name: $name, authkey: $key}')
-            add_server "$DATA"
+            add_server ${HOSTNAME} ${ADMIN_KEY} "$DATA"
+        else
+            echo "... found existing sync server ${!NAME}..."
         fi
     done   
 }
 
 
-echo "MISP | Initialize configuration ..." && init_misp_configuration
+echo "MISP | Initialize configuration ..." && init_configuration
 
-echo "MISP | Initialize workers ..." && init_misp_workers
+echo "MISP | Initialize workers ..." && init_workers
 
-echo "MISP | Configure GPG key ..." && init_gnupg
+echo "MISP | Configure GPG key ..." && configure_gnupg
 
-echo "MISP | Running updates ..." && apply_updates
+echo "MISP | Apply updates ..." && apply_updates
 
 echo "MISP | Init default user and organization ..." && init_user
 
@@ -258,13 +187,9 @@ echo "MISP | Resolve critical issues ..." && apply_critical_fixes
 
 echo "MISP | Resolve non-critical issues ..." && apply_optional_fixes
 
-echo "MISP | Creating organizations ..." && create_organizations
+echo "MISP | Create sync servers ..." && create_sync_servers
 
-echo "MISP | Creating sync servers ..." && create_sync_servers
+echo "MISP | Update components ..." && update_components
 
-echo "MISP | Updating components ..." && updateComponents
+echo "MISP | Mark instance live"
-
-echo "MISP | Configure plugins with newly generate admin key ..." && configure_optional_plugins
-
-echo "MISP | Marking instance live"
 sudo -u www-data /var/www/MISP/app/Console/cake Admin live 1

server/files/rest_client.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+if ! command -v jq &> /dev/null
+then
+    echo "aborting. jq could not be found"
+    exit
+fi
+
+if ! command -v curl &> /dev/null
+then
+    echo "aborting. curl could not be found"
+    exit
+fi
+
+add_organization() {
+    # empty uuid fallbacks to auto-generate
+    curl -s --show-error -k \
+     -H "Authorization: ${2}" \
+     -H "Accept: application/json" \
+     -H "Content-type: application/json" \
+     -d "{ \
+        \"uuid\": \"${5}\", \
+        \"name\": \"${3}\", \
+        \"local\": ${4} \
+     }" ${1}/admin/organisations/add
+}
+
+get_organization() {
+    curl -s --show-error -k \
+     -H "Authorization: ${2}" \
+     -H "Accept: application/json" \
+     -H "Content-type: application/json" ${1}/organisations/view/${3} | jq -e -r ".Organisation.id // empty"
+}
+
+add_server() {
+    curl -s --show-error -k \
+     -H "Authorization: ${2}" \
+     -H "Accept: application/json" \
+     -H "Content-type: application/json" \
+     -d "${3}" ${1}/servers/add
+}
+
+get_server() {
+    curl -s --show-error -k \
+     -H "Authorization: ${2}" \
+     -H "Accept: application/json" \
+     -H "Content-type: application/json" ${1}/servers | jq -e -r ".[] | select(.Server[\"name\"] == \"${3}\") | .Server.id"
+}

template.env

@@ -15,11 +15,6 @@ ADMIN_KEY=
 # default to 'passphrase'
 GPG_PASSPHRASE=
 
-# optional and used by some misp-modules
-NSX_ANALYSIS_API_TOKEN=
-NSX_ANALYSIS_KEY=
-VIRUSTOTAL_KEY=
-
 # optional and used by the mail sub-system
 SMARTHOST_ADDRESS=
 SMARTHOST_PORT=
@@ -35,6 +30,3 @@ SYNCSERVERS_1_URL=
 SYNCSERVERS_1_NAME=
 SYNCSERVERS_1_UUID=
 SYNCSERVERS_1_KEY=
-
-# comma separated list of organizations to create (e.g. ORGANIZATIONS="ORG1, ORG2, ORG3")
-ORGANIZATIONS=