diff --git a/README.md b/README.md index d2678f7..e9f754b 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,8 @@ Additionally, this fork features the following improvements: - Fix MISP modules loading of faup library - Fix MISP modules loading of gl library - Add support for new background job system (see https://github.com/MISP/MISP/blob/2.4/docs/background-jobs-migration-guide.md) -- Add support for exposing locally generated resources - Add support for building specific MISP and MISP-modules commits -- Add automatic configuration of MISP modules (see `configure_misp.sh`) - Add automatic configuration of sync servers (see `configure_misp.sh`) -- Add automatic configuration of organizations (see `configure_misp.sh`) - Add autoamtic configuration of authentication keys (see `configure_misp.sh`) - Add direct push of docker images to Docker Hub - Consolidate docker compose files @@ -85,11 +82,13 @@ Updating the images should be as simple as `docker-compose pull` which, unless c - Certificate Key File: `key.pem` - CA File for Cert Authentication (optional) `ca.pem` -- Directory volume mount and create configs: `/var/www/MISP/app/Config/` - - Additional directory volume mounts: - - `/var/www/MISP/app/files` - - `/var/www/MISP/.gnupg` + - `./configs`: `/var/www/MISP/app/Config/` + - `./logs`: `/var/www/MISP/app/tmp/logs/` + - `./files`: `/var/www/MISP/app/files/` + - `./gnupg`: `/var/www/MISP/.gnupg/` + +- If you need to automatically run additional steps each time the container starts, create a new file `files/customize_misp.sh`, and replace the variable `${CUSTOM_PATH}` inside `docker-compose.yml` with its parent path. ### Building diff --git a/docker-compose.yml b/docker-compose.yml index 423c921..5684d3a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,10 +48,17 @@ services: - "./files/:/var/www/MISP/app/files/" - "./ssl/:/etc/nginx/certs/" - "./gnupg/:/var/www/MISP/.gnupg/" + # customize by replacing ${CUSTOM_PATH} with a path containing 'files/customize_misp.sh' + # - "${CUSTOM_PATH}/:/custom/" environment: - "HOSTNAME=https://localhost" - "REDIS_FQDN=redis" - "CRON_USER_ID=1" + # standard settings + - "ADMIN_EMAIL=${ADMIN_EMAIL}" + - "ADMIN_KEY=${ADMIN_KEY}" + - "ADMIN_ORG=${ADMIN_ORG}" + - "GPG_PASSPHRASE=${GPG_PASSPHRASE}" # sync server settings - "SYNCSERVERS=${SYNCSERVERS}" - "SYNCSERVERS_1_NAME=${SYNCSERVERS_1_NAME}" @@ -64,15 +71,6 @@ services: "pull_rules": "{\"tags\":{\"OR\":[],\"NOT\":[]},\"orgs\":{\"OR\":[],\"NOT\":[]},\"url_params\":\"{\\\"searchanalysis\\\": \\\"2\\\"}\"}", "pull": true } - # standard settings - - "ADMIN_EMAIL=${ADMIN_EMAIL}" - - "ADMIN_KEY=${ADMIN_KEY}" - - "ADMIN_ORG=${ADMIN_ORG}" - - "GPG_PASSPHRASE=${GPG_PASSPHRASE}" - - "NSX_ANALYSIS_API_TOKEN=${NSX_ANALYSIS_API_TOKEN}" - - "NSX_ANALYSIS_KEY=${NSX_ANALYSIS_KEY}" - - "ORGANIZATIONS=${ORGANIZATIONS}" - - "VIRUSTOTAL_KEY=${VIRUSTOTAL_KEY}" misp-modules: image: ostefano/misp-docker:modules-latest diff --git a/server/Dockerfile b/server/Dockerfile index fa5d0d4..d7db484 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -186,6 +186,7 @@ ARG PHP_VER COPY files/etc/supervisor/workers.conf /etc/supervisor/conf.d/50-workers.conf COPY files/var/www/html/index.php /var/www/html/index.php COPY files/configure_misp.sh / + COPY files/rest_client.sh / COPY files/entrypoint_fpm.sh / COPY files/entrypoint_nginx.sh / COPY files/entrypoint_cron.sh / diff --git a/server/files/configure_misp.sh b/server/files/configure_misp.sh index a1a0de7..4241308 100755 --- a/server/files/configure_misp.sh +++ b/server/files/configure_misp.sh @@ -1,11 +1,13 @@ #!/bin/bash +source /rest_client.sh + [ -z "$ADMIN_EMAIL" ] && ADMIN_EMAIL="admin@admin.test" [ -z "$GPG_PASSPHRASE" ] && GPG_PASSPHRASE="passphrase" [ -z "$REDIS_FQDN" ] && REDIS_FQDN=redis [ -z "$MISP_MODULES_FQDN" ] && MISP_MODULES_FQDN="http://misp-modules" -init_misp_configuration(){ +init_configuration(){ # Note that we are doing this after enforcing permissions, so we need to use the www-data user for this echo "... configuring default settings" sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "MISP.redis_host" "$REDIS_FQDN" @@ -22,7 +24,7 @@ init_misp_configuration(){ sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "Plugin.Cortex_services_enable" false } -init_misp_workers(){ +init_workers(){ # Note that we are doing this after enforcing permissions, so we need to use the www-data user for this echo "... configuring background workers" sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "SimpleBackgroundJobs.enabled" true @@ -36,7 +38,7 @@ init_misp_workers(){ supervisorctl start misp-workers:* } -init_gnupg() { +configure_gnupg() { GPG_DIR=/var/www/MISP/.gnupg GPG_ASC=/var/www/MISP/app/webroot/gpg.asc GPG_TMP=/tmp/gpg.tmp @@ -142,35 +144,7 @@ apply_optional_fixes() { sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "Plugin.Enrichment_hover_timeout" 5 } -configure_optional_plugins() { - if [ ! -z "$VIRUSTOTAL_KEY" ]; then - echo "... enabling 'virustotal' module ..." - sudo -u www-data php /var/www/MISP/tests/modify_config.php modify "{ - \"Plugin\": { - \"Enrichment_virustotal_enabled\": true, - \"Enrichment_virustotal_apikey\": \"${VIRUSTOTAL_KEY}\" - } - }" > /dev/null - fi - - if [ ! -z "$VIRUSTOTAL_KEY" ] && [ ! -z "$NSX_ANALYSIS_KEY" ] && [ ! -z "$NSX_ANALYSIS_API_TOKEN" ] && [ ! -z "$ADMIN_KEY" ]; then - echo "... enabling 'vmware_nsx' module ..." - sudo -u www-data php /var/www/MISP/tests/modify_config.php modify "{ - \"Plugin\": { - \"Enrichment_vmware_nsx_enabled\": true, - \"Enrichment_vmware_nsx_analysis_verify_ssl\": \"True\", - \"Enrichment_vmware_nsx_analysis_key\": \"${NSX_ANALYSIS_KEY}\", - \"Enrichment_vmware_nsx_analysis_api_token\": \"${NSX_ANALYSIS_API_TOKEN}\", - \"Enrichment_vmware_nsx_vt_key\": \"${VIRUSTOTAL_KEY}\", - \"Enrichment_vmware_nsx_misp_url\": \"${HOSTNAME}\", - \"Enrichment_vmware_nsx_misp_verify_ssl\": \"False\", - \"Enrichment_vmware_nsx_misp_key\": \"${ADMIN_KEY}\" - } - }" > /dev/null - fi -} - -updateComponents() { +update_components() { sudo -u www-data /var/www/MISP/app/Console/cake Admin updateGalaxies sudo -u www-data /var/www/MISP/app/Console/cake Admin updateTaxonomies sudo -u www-data /var/www/MISP/app/Console/cake Admin updateWarningLists @@ -178,54 +152,6 @@ updateComponents() { sudo -u www-data /var/www/MISP/app/Console/cake Admin updateObjectTemplates "$CRON_USER_ID" } -add_organization() { - # empty uuid fallbacks to auto-generate - curl -s --show-error -k \ - -H "Authorization: ${ADMIN_KEY}" \ - -H "Accept: application/json" \ - -H "Content-type: application/json" \ - -d "{ \ - \"uuid\": \"${3}\", \ - \"name\": \"${1}\", \ - \"local\": ${2} \ - }" ${HOSTNAME}/admin/organisations/add -} - -get_organization() { - curl -s --show-error -k \ - -H "Authorization: ${ADMIN_KEY}" \ - -H "Accept: application/json" \ - -H "Content-type: application/json" ${HOSTNAME}/organisations/view/${1} | jq -e -r ".Organisation.id // empty" -} - -add_server() { - curl -s --show-error -k \ - -H "Authorization: ${ADMIN_KEY}" \ - -H "Accept: application/json" \ - -H "Content-type: application/json" \ - -d "${1}" ${HOSTNAME}/servers/add -} - -get_server() { - curl -s --show-error -k \ - -H "Authorization: ${ADMIN_KEY}" \ - -H "Accept: application/json" \ - -H "Content-type: application/json" ${HOSTNAME}/servers | jq -e -r ".[] | select(.Server[\"name\"] == \"${1}\") | .Server.id" -} - -create_organizations() { - SPLITTED_ORGS=$(echo $ORGANIZATIONS | tr ',' '\n') - for ORG in $SPLITTED_ORGS; do - ORG_ID=$(get_organization ${ORG}) - if [[ -z $ORG_ID ]]; then - echo "... adding organization: $ORG" - add_organization $ORG true - else - echo "... organization $ORG already exists" - fi - done -} - create_sync_servers() { SPLITTED_SYNCSERVERS=$(echo $SYNCSERVERS | tr ',' '\n') for ID in $SPLITTED_SYNCSERVERS; do @@ -233,24 +159,27 @@ create_sync_servers() { UUID="SYNCSERVERS_${ID}_UUID" DATA="SYNCSERVERS_${ID}_DATA" KEY="SYNCSERVERS_${ID}_KEY" - if ! get_server ${!NAME}; then - echo "... configuring sync server ${!NAME}..." - add_organization ${!NAME} false ${!UUID} - ORG_ID=$(get_organization ${!UUID}) + echo "... searching sync server ${!NAME}..." + if ! get_server ${HOSTNAME} ${ADMIN_KEY} ${!NAME}; then + echo "... adding new sync server ${!NAME}..." + add_organization ${HOSTNAME} ${ADMIN_KEY} ${!NAME} false ${!UUID} + ORG_ID=$(get_organization ${HOSTNAME} ${ADMIN_KEY} ${!UUID}) DATA=$(echo "${!DATA}" | jq --arg org_id ${ORG_ID} --arg name ${!NAME} --arg key ${!KEY} '. + {remote_org_id: $org_id, name: $name, authkey: $key}') - add_server "$DATA" + add_server ${HOSTNAME} ${ADMIN_KEY} "$DATA" + else + echo "... found existing sync server ${!NAME}..." fi done } -echo "MISP | Initialize configuration ..." && init_misp_configuration +echo "MISP | Initialize configuration ..." && init_configuration -echo "MISP | Initialize workers ..." && init_misp_workers +echo "MISP | Initialize workers ..." && init_workers -echo "MISP | Configure GPG key ..." && init_gnupg +echo "MISP | Configure GPG key ..." && configure_gnupg -echo "MISP | Running updates ..." && apply_updates +echo "MISP | Apply updates ..." && apply_updates echo "MISP | Init default user and organization ..." && init_user @@ -258,13 +187,9 @@ echo "MISP | Resolve critical issues ..." && apply_critical_fixes echo "MISP | Resolve non-critical issues ..." && apply_optional_fixes -echo "MISP | Creating organizations ..." && create_organizations +echo "MISP | Create sync servers ..." && create_sync_servers -echo "MISP | Creating sync servers ..." && create_sync_servers +echo "MISP | Update components ..." && update_components -echo "MISP | Updating components ..." && updateComponents - -echo "MISP | Configure plugins with newly generate admin key ..." && configure_optional_plugins - -echo "MISP | Marking instance live" +echo "MISP | Mark instance live" sudo -u www-data /var/www/MISP/app/Console/cake Admin live 1 diff --git a/server/files/rest_client.sh b/server/files/rest_client.sh new file mode 100644 index 0000000..9b3c107 --- /dev/null +++ b/server/files/rest_client.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +if ! command -v jq &> /dev/null +then + echo "aborting. jq could not be found" + exit +fi + +if ! command -v curl &> /dev/null +then + echo "aborting. curl could not be found" + exit +fi + +add_organization() { + # empty uuid fallbacks to auto-generate + curl -s --show-error -k \ + -H "Authorization: ${2}" \ + -H "Accept: application/json" \ + -H "Content-type: application/json" \ + -d "{ \ + \"uuid\": \"${5}\", \ + \"name\": \"${3}\", \ + \"local\": ${4} \ + }" ${1}/admin/organisations/add +} + +get_organization() { + curl -s --show-error -k \ + -H "Authorization: ${2}" \ + -H "Accept: application/json" \ + -H "Content-type: application/json" ${1}/organisations/view/${3} | jq -e -r ".Organisation.id // empty" +} + +add_server() { + curl -s --show-error -k \ + -H "Authorization: ${2}" \ + -H "Accept: application/json" \ + -H "Content-type: application/json" \ + -d "${3}" ${1}/servers/add +} + +get_server() { + curl -s --show-error -k \ + -H "Authorization: ${2}" \ + -H "Accept: application/json" \ + -H "Content-type: application/json" ${1}/servers | jq -e -r ".[] | select(.Server[\"name\"] == \"${3}\") | .Server.id" +} diff --git a/template.env b/template.env index d2c383c..2e33619 100644 --- a/template.env +++ b/template.env @@ -15,11 +15,6 @@ ADMIN_KEY= # default to 'passphrase' GPG_PASSPHRASE= -# optional and used by some misp-modules -NSX_ANALYSIS_API_TOKEN= -NSX_ANALYSIS_KEY= -VIRUSTOTAL_KEY= - # optional and used by the mail sub-system SMARTHOST_ADDRESS= SMARTHOST_PORT= @@ -35,6 +30,3 @@ SYNCSERVERS_1_URL= SYNCSERVERS_1_NAME= SYNCSERVERS_1_UUID= SYNCSERVERS_1_KEY= - -# comma separated list of organizations to create (e.g. ORGANIZATIONS="ORG1, ORG2, ORG3") -ORGANIZATIONS=