Fix bugs and improve reliability

docker-compose.yml

@@ -48,7 +48,18 @@ services:
       - "REDIS_FQDN=redis"
       - "INIT=true"             # Initialze MISP, things includes, attempting to import SQL and the Files DIR
       - "CRON_USER_ID=1"        # The MISP user ID to run cron jobs as
-#      - "SYNCSERVERS=1 2 3 4"  # The MISP Feed servers to sync in the cron job
+      # Synchronization Servers settings
+      - "SYNCSERVERS=1"
+      - "SYNCSERVERS_1_NAME=${SYNCSERVERS_1_NAME}"
+      - "SYNCSERVERS_1_UUID=${SYNCSERVERS_1_UUID}"
+      - "SYNCSERVERS_1_KEY=${SYNCSERVERS_1_KEY}"
+      - |
+        SYNCSERVERS_1_DATA=
+        {
+          "url": "https://intel.thedfirreport.com/",
+          "pull_rules": "{\"tags\":{\"OR\":[],\"NOT\":[]},\"orgs\":{\"OR\":[],\"NOT\":[]},\"url_params\":\"{\\\"searchanalysis\\\": \\\"2\\\"}\"}",
+          "pull": true
+        }
       # Database Configuration (And their defaults)
 #      - "MYSQL_HOST=db"
 #      - "MYSQL_USER=misp"

server/Dockerfile

@@ -125,6 +125,8 @@ ARG PHP_VER
         libbrotli1 \
         # Unsure we need these
         zip unzip \
+        # Require for advanced an unattended configuration
+        curl jq \
         && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
 # MISP code

server/files/entrypoint_cron.sh

@@ -5,6 +5,9 @@ cat << EOF > /etc/cron.d/misp
 20 2 * * * www-data /var/www/MISP/app/Console/cake Server cacheFeed "$CRON_USER_ID" all >/tmp/cronlog 2>/tmp/cronlog
 30 2 * * * www-data /var/www/MISP/app/Console/cake Server fetchFeed "$CRON_USER_ID" all >/tmp/cronlog 2>/tmp/cronlog
 
+0 0 * * * www-data /var/www/MISP/app/Console/cake Server pullAll "$CRON_USER_ID" >/tmp/cronlog 2>/tmp/cronlog
+0 1 * * * www-data /var/www/MISP/app/Console/cake Server pushAll "$CRON_USER_ID" >/tmp/cronlog 2>/tmp/cronlog
+
 00 3 * * * www-data /var/www/MISP/app/Console/cake Admin updateGalaxies >/tmp/cronlog 2>/tmp/cronlog
 10 3 * * * www-data /var/www/MISP/app/Console/cake Admin updateTaxonomies >/tmp/cronlog 2>/tmp/cronlog
 20 3 * * * www-data /var/www/MISP/app/Console/cake Admin updateWarningLists >/tmp/cronlog 2>/tmp/cronlog
@@ -13,20 +16,6 @@ cat << EOF > /etc/cron.d/misp
 
 EOF
 
-if [ ! -z "$SYNCSERVERS" ];
-then
-    TIME=0
-    for SYNCSERVER in $SYNCSERVERS
-    do
-cat << EOF >> /etc/cron.d/misp
-$TIME 0 * * * www-data /var/www/MISP/app/Console/cake Server pull "$CRON_USER_ID" "$SYNCSERVER">/tmp/cronlog 2>/tmp/cronlog
-$TIME 1 * * * www-data /var/www/MISP/app/Console/cake Server push "$CRON_USER_ID" "$SYNCSERVER">/tmp/cronlog 2>/tmp/cronlog
-EOF
-
-    ((TIME+=5))
-    done
-fi
-
 # Build a fifo buffer for the cron logs, 777 so anyone can write to it
 if [[ ! -p /tmp/cronlog ]]; then
     mkfifo /tmp/cronlog

server/files/entrypoint_internal.sh

@@ -8,7 +8,7 @@ init_gnupg() {
     GPG_ASC=/var/www/MISP/app/webroot/gpg.asc
     GPG_TMP=/tmp/gpg.tmp
 
-    if [ ! -d ${GPG_DIR} ]; then
+    if [ ! -f "${GPG_DIR}/trustdb.gpg" ]; then
         echo "Generating GPG key ... (please be patient, we need some entropy)"
         cat >${GPG_TMP} <<GPGEOF
 %echo Generating a basic OpenPGP key
@@ -23,12 +23,16 @@ Passphrase: $GPG_PASSPHRASE
 GPGEOF
         mkdir ${GPG_DIR}
         gpg --homedir ${GPG_DIR} --gen-key --batch ${GPG_TMP}
-        chown -R www-data:www-data ${GPG_DIR}
         rm -f ${GPG_TMP}
     else
         echo "Using pre-generated GPG key in ${GPG_DIR}"
     fi
 
+    # Fix permissions
+    chown -R www-data:www-data ${GPG_DIR}
+    find ${GPG_DIR} -type f -exec chmod 600 {} \;
+    find ${GPG_DIR} -type d -exec chmod 700 {} \;
+
     if [ ! -f ${GPG_ASC} ]; then
         echo "Exporting GPG key ..."
         sudo -u www-data gpg --homedir ${GPG_DIR} --export --armor ${ADMIN_EMAIL} > ${GPG_ASC}
@@ -175,6 +179,41 @@ class EmailConfig {
 EOT
 }
 
+add_organization() {
+    # empty uuid fallbacks to auto-generate
+    curl -s --show-error -k \
+     -H "Authorization: ${ADMIN_KEY}" \
+     -H "Accept: application/json" \
+     -H "Content-type: application/json" \
+     -d "{ \
+        \"uuid\": \"${3}\", \
+        \"name\": \"${1}\", \
+        \"local\": ${2} \
+     }" ${HOSTNAME}/admin/organisations/add
+}
+
+get_organization() {
+    curl -s --show-error -k \
+     -H "Authorization: ${ADMIN_KEY}" \
+     -H "Accept: application/json" \
+     -H "Content-type: application/json" ${HOSTNAME}/organisations/view/${1} | jq -e -r ".Organisation.id"
+}
+
+add_server() {
+    curl -s --show-error -k \
+     -H "Authorization: ${ADMIN_KEY}" \
+     -H "Accept: application/json" \
+     -H "Content-type: application/json" \
+     -d "${1}" ${HOSTNAME}/servers/add
+}
+
+get_server() {
+    curl -s --show-error -k \
+     -H "Authorization: ${ADMIN_KEY}" \
+     -H "Accept: application/json" \
+     -H "Content-type: application/json" ${HOSTNAME}/servers | jq -e -r ".[] | select(.Server[\"name\"] == \"${1}\") | .Server.id"
+}
+
 
 echo "Customize MISP | Configure email ..." && configure_email
 
@@ -191,5 +230,28 @@ echo "Customize MISP | Customize installation ..." && apply_custom_settings
 # This item last so we had a chance to create the ADMIN_KEY if not specified
 echo "Customize MISP | Configure plugins ..." && configure_plugins
 
+# Create organizations (and silently fail if present already)
+echo "Customize MISP | Creating organizations ..."
+add_organization nuTAU true
+add_organization CBTAU true
+add_organization T-Rex true
+add_organization NDR true
+add_organization MDR true
+
+# Create sync servers
+for ID in $SYNCSERVERS; do
+    NAME="SYNCSERVERS_${ID}_NAME"
+    UUID="SYNCSERVERS_${ID}_UUID"
+    DATA="SYNCSERVERS_${ID}_DATA"
+    KEY="SYNCSERVERS_${ID}_KEY"
+    if ! get_server ${!NAME}; then
+        echo "Customize MISP | Configuring sync server ${!NAME}..."
+        add_organization ${!NAME} false ${!UUID}
+        ORG_ID=$(get_organization ${!UUID})
+        DATA=$(echo "${!DATA}" | jq --arg org_id ${ORG_ID} --arg name ${!NAME} --arg key ${!KEY} '. + {remote_org_id: $org_id, name: $name, authkey: $key}')
+        add_server "$DATA"
+    fi
+done
+
 # Make the instance live
 sudo -u www-data /var/www/MISP/app/Console/cake Admin live 1

server/files/entrypoint_nginx.sh

@@ -199,18 +199,6 @@ if [[ -x /custom-entrypoint.sh ]]; then
     /custom-entrypoint.sh
 fi
 
-if [[ -x /entrypoint_internal.sh ]]; then
-    export ADMIN_EMAIL=${ADMIN_EMAIL}
-    export ADMIN_ORG=${ADMIN_ORG}
-    export GPG_PASSPHRASE=${GPG_PASSPHRASE}
-    export HOSTNAME=${HOSTNAME}
-    export MYSQLCMD=${MYSQLCMD}
-    export NSX_ANALYSIS_API_TOKEN=${NSX_ANALYSIS_API_TOKEN}
-    export NSX_ANALYSIS_KEY=${NSX_ANALYSIS_KEY}
-    export VIRUSTOTAL_KEY=${VIRUSTOTAL_KEY}
-    /entrypoint_internal.sh
-fi
-
 # delete pid file
 [ -f $ENTRYPOINT_PID_FILE ] && rm $ENTRYPOINT_PID_FILE
 
@@ -222,5 +210,32 @@ if [[ "$WARNING53" == true ]]; then
     echo "WARNING - WARNING - WARNING"
 fi
 
+if [[ -x /entrypoint_internal.sh ]]; then
+    ## Re-exporting might not be necessary after all?
+    # export ADMIN_EMAIL=${ADMIN_EMAIL}
+    # export ADMIN_ORG=${ADMIN_ORG}
+    # export ADMIN_KEY=${ADMIN_KEY}
+    # export GPG_PASSPHRASE=${GPG_PASSPHRASE}
+    # export HOSTNAME=${HOSTNAME}
+    # export NSX_ANALYSIS_API_TOKEN=${NSX_ANALYSIS_API_TOKEN}
+    # export NSX_ANALYSIS_KEY=${NSX_ANALYSIS_KEY}
+    # export VIRUSTOTAL_KEY=${VIRUSTOTAL_KEY}
+    # export SYNCSERVERS=${SYNCSERVERS}
+    # for ID in $SYNCSERVERS; do
+    #     NAME="SYNCSERVERS_${ID}_NAME"
+    #     UUID="SYNCSERVERS_${ID}_UUID"
+    #     DATA="SYNCSERVERS_${ID}_DATA"
+    #     KEY="SYNCSERVERS_${ID}_KEY"
+    #     export ${NAME}="${!NAME}"
+    #     export ${UUID}="${!UUID}"
+    #     export ${DATA}="${!DATA}"
+    #     export ${KEY}="${!KEY}"
+    # done
+    export MYSQLCMD=${MYSQLCMD}
+    nginx -g 'daemon on;'
+    /entrypoint_internal.sh
+    killall nginx
+fi
+
 # Start NGINX
 nginx -g 'daemon off;'