From b2f0349ad82fdacf60e3db141200ef09b59aa1d3 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 16 Jul 2020 20:40:35 +0900 Subject: [PATCH 1/6] fix: [typo] verify.txt pointed to the wrong checksum dir --- verify.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/verify.txt b/verify.txt index 368c080..36e3195 100644 --- a/verify.txt +++ b/verify.txt @@ -29,8 +29,8 @@ gpg: Good signature from "CIRCL MISP (CIRCL MISP VM Release Key) Date: Wed, 25 Nov 2020 08:33:43 +0000 Subject: [PATCH 2/6] chg: [upd] updated base ubuntu image --- misp.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misp.json b/misp.json index 10f49c5..244c718 100644 --- a/misp.json +++ b/misp.json @@ -172,11 +172,11 @@ "hostname": "misp", "http_proxy": "{{env `http_proxy`}}", "https_proxy": "{{env `https_proxy`}}", - "iso_checksum": "b957311833d79c14bd6743feac2e641938f99ee96679895a04c703391d1fb6e8c92fe609d9401228c28d85fee7a816967051f5206119e7fe674f8ca798e9267d", + "iso_checksum": "43738d7dfd3e2661e4d55d2e0f9d8150f0687f4335af9b4dac047bf45fafcb4a4831685281fd5a318c5747681c351375d1129094d3f1bf38d88ab4bb49b6c457", "iso_checksum_type": "sha512", - "iso_name": "ubuntu-18.04.4-server-amd64.iso", + "iso_name": "ubuntu-18.04.5-server-amd64.iso", "iso_path": "iso", - "iso_url": "http://cdimage.ubuntu.com/ubuntu/releases/bionic/release/ubuntu-18.04.4-server-amd64.iso", + "iso_url": "http://cdimage.ubuntu.com/ubuntu/releases/bionic/release/ubuntu-18.04.5-server-amd64.iso", "memory": "3072", "no_proxy": "{{env `no_proxy`}}", "ovftool_path": "ovftool", From c9bdb8e78d388d562b6da0b6d3736688d4f1eb6a Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 20 Jan 2021 03:29:25 +0000 Subject: [PATCH 3/6] chg: [sh] Various deploy.sh updates --- .gitignore | 3 +++ config.sh | 32 ++++++++++++++++++++++++++++++++ deploy.sh | 45 +++++++++------------------------------------ scripts/clean.sh | 2 +- 4 files changed, 45 insertions(+), 37 deletions(-) create mode 100644 config.sh diff --git a/.gitignore b/.gitignore index f46ec98..2b45eed 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ packer_cache/ output-virtualbox-iso/ +output-vmware-iso/ scripts/INSTALL.sh* *.checksum misp-deploy.json +packerlog-vbox.txt +packerlog-vmware.txt diff --git a/config.sh b/config.sh new file mode 100644 index 0000000..a36729b --- /dev/null +++ b/config.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Name of the packer +PACKER_NAME="misp" +PACKER_VM="MISP" +NAME="${PACKER_NAME}-packer" + +# Configure your user and remote server +REMOTE=1 +REL_USER="${PACKER_NAME}-release" +REL_SERVER="cpab" + +# GPG Sign +GPG_ENABLED=1 +GPG_KEY="0x34F20B13" + +# Enable debug for packer, omit -debug to disable +##PACKER_DEBUG="-debug" + +# Enable logging and debug for packer +export PACKER_LOG=1 + +REPO="MISP/MISP" +BRANCH="2.4" + +# SHAsums to be computed, note the -- notatiation is for ease of use with rhash +SHA_SUMS="--sha1 --sha256 --sha384 --sha512" + +NAME_OF_INSTALLER="INSTALL.sh" +PATH_TO_INSTALLER="scripts/${NAME_OF_INSTALLER}" +URL_TO_INSTALLER="https://raw.githubusercontent.com/${REPO}/${BRANCH}/INSTALL/${NAME_OF_INSTALLER}" +URL_TO_LICENSE="https://raw.githubusercontent.com/${REPO}/${BRANCH}/LICENSE" diff --git a/deploy.sh b/deploy.sh index 6b4fa42..b352e19 100755 --- a/deploy.sh +++ b/deploy.sh @@ -9,38 +9,10 @@ # Timing creation TIME_START=$(date +%s) -# Name of the packer -PACKER_NAME="misp" -PACKER_VM="MISP" -NAME="${PACKER_NAME}-packer" +# Please adjust config.sh accordingly +source config.sh -# Configure your user and remote server -REMOTE=1 -REL_USER="${PACKER_NAME}-release" -REL_SERVER="cpab" - -# GPG Sign -GPG_ENABLED=1 -GPG_KEY="0x34F20B13" - -# Enable debug for packer, omit -debug to disable -##PACKER_DEBUG="-debug" - -# Enable logging and debug for packer -export PACKER_LOG=0 - -REPO="MISP/MISP" -BRANCH="2.4" - -# SHAsums to be computed, note the -- notatiation is for ease of use with rhash -SHA_SUMS="--sha1 --sha256 --sha384 --sha512" - -NAME_OF_INSTALLER="INSTALL.sh" -PATH_TO_INSTALLER="scripts/${NAME_OF_INSTALLER}" -URL_TO_INSTALLER="https://raw.githubusercontent.com/${REPO}/${BRANCH}/INSTALL/${NAME_OF_INSTALLER}" -URL_TO_LICENSE="https://raw.githubusercontent.com/${REPO}/${BRANCH}/LICENSE" - -### ---- NOT TOUCHY BEOYND THIS POINT, PLEASE --- ### +### ---- NO TOUCHY BEYOND THIS POINT, PLEASE --- ### # TODO: Move into seprate file GOT_PACKER=$(which packer > /dev/null 2>&1; echo $?) @@ -166,7 +138,8 @@ checkInstaller () { if [[ ${chsum} == ${INSTsum} ]] && [[ ${rhash_chk} == 0 ]]; then echo "sha${sum} matches" else - echo "sha${sum}: ${chsum} does not match the installer sum of: ${INSTsum}" + echo "Either: sha${sum}: ${chsum} does not match the installer sum of: ${INSTsum}" + echo "Or: rhash failed on non Zero: ${rhash_chk}" echo "Deleting installer, please run again." rm ${PATH_TO_INSTALLER} exit 1 @@ -214,17 +187,17 @@ if [[ "${LATEST_COMMIT}" != "$(cat /tmp/${PACKER_NAME}-latest.sha)" ]]; then cat ${PACKER_NAME}.json| sed "s|\"vm_name\": \"${PACKER_VM}_demo\",|\"vm_name\": \"${PACKER_VM}_${VER}@${LATEST_COMMIT_SHORT}\",|" > ${PACKER_NAME}-deploy.json # Build virtualbox VM set - PACKER_LOG_PATH="${PWD}/packerlog-vbox.txt" - ($PACKER_RUN build --on-error=cleanup -only=virtualbox-iso ${PACKER_NAME}-deploy.json > /dev/null 2>&1 ; echo $? > /tmp/${PACKER_NAME}-vbox.done) & + export PACKER_LOG_PATH="${PWD}/packerlog-vbox.txt" + ($PACKER_RUN build --on-error=cleanup -only=virtualbox-iso ${PACKER_NAME}-deploy.json ; echo $? > /tmp/${PACKER_NAME}-vbox.done) & # Build vmware VM set - PACKER_LOG_PATH="${PWD}/packerlog-vmware.txt" + export PACKER_LOG_PATH="${PWD}/packerlog-vmware.txt" ($PACKER_RUN build --on-error=cleanup -only=vmware-iso ${PACKER_NAME}-deploy.json ; echo $? > /tmp/${PACKER_NAME}-vmware.done) & # The below waits for the above 2 parallel packer builds to finish while [[ ! -f /tmp/${PACKER_NAME}-vmware.done ]]; do :; done while [[ ! -f /tmp/${PACKER_NAME}-vbox.done ]]; do :; done - +#exit -1 # Prevent uploading only half a build if [[ "$(cat /tmp/${PACKER_NAME}-vbox.done)" == "0" ]] && [[ "$(cat /tmp/${PACKER_NAME}-vmware.done)" == "0" ]]; then # ZIPup all the vmware stuff diff --git a/scripts/clean.sh b/scripts/clean.sh index 58219df..c381107 100644 --- a/scripts/clean.sh +++ b/scripts/clean.sh @@ -22,7 +22,7 @@ cd /var/www/MISP/PyMISP if [ "$?" != "0" ]; then echo "Damage, terrible terrible damage!!!!" >> /tmp/tests-output.txt # TODO: Move the smtp server and e-mail address to a config file - set smtp=smtp://149.13.33.5 ; cat /tmp/tests-output.txt |mail -s "tests/testlive_comprehensive.py failed on autogen-VM" steve.clement@circl.lu + ##set smtp=smtp://149.13.33.5 ; cat /tmp/tests-output.txt |mail -s "tests/testlive_comprehensive.py failed on autogen-VM" steve.clement@circl.lu fi rm -rf tests/viper-test-files From 906f0ee80eeff687ec1a0d728f50893bbc354107 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 20 Jan 2021 03:35:27 +0000 Subject: [PATCH 4/6] chg: [sh] moved depenency check to seperate file --- checkDeps.sh | 19 +++++++++++++++++++ deploy.sh | 19 +------------------ 2 files changed, 20 insertions(+), 18 deletions(-) create mode 100644 checkDeps.sh diff --git a/checkDeps.sh b/checkDeps.sh new file mode 100644 index 0000000..4c9447a --- /dev/null +++ b/checkDeps.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +GOT_PACKER=$(which packer > /dev/null 2>&1; echo $?) +if [[ "${GOT_PACKER}" == 0 ]]; then + echo "Packer detected, version: $(packer -v)" + PACKER_RUN=$(which packer) +else + echo "No packer binary detected, please make sure you installed it from: https://www.packer.io/downloads.html" + exit 1 +fi + +GOT_RHASH=$(which rhash > /dev/null 2>&1; echo $?) +if [[ "${GOT_RHASH}" == 0 ]]; then + echo "rhash detected, version: $(rhash --version)" + RHASH_RUN=$(which rhash) +else + echo "No rhash binary detected, please make sure you installed it." + exit 1 +fi diff --git a/deploy.sh b/deploy.sh index b352e19..8ea4680 100755 --- a/deploy.sh +++ b/deploy.sh @@ -14,24 +14,7 @@ source config.sh ### ---- NO TOUCHY BEYOND THIS POINT, PLEASE --- ### -# TODO: Move into seprate file -GOT_PACKER=$(which packer > /dev/null 2>&1; echo $?) -if [[ "${GOT_PACKER}" == 0 ]]; then - echo "Packer detected, version: $(packer -v)" - PACKER_RUN=$(which packer) -else - echo "No packer binary detected, please make sure you installed it from: https://www.packer.io/downloads.html" - exit 1 -fi - -GOT_RHASH=$(which rhash > /dev/null 2>&1; echo $?) -if [[ "${GOT_RHASH}" == 0 ]]; then - echo "rhash detected, version: $(rhash --version)" - RHASH_RUN=$(which rhash) -else - echo "No rhash binary detected, please make sure you installed it." - exit 1 -fi +source checkDeps.sh # Latest version of misp VER=$(curl -s https://api.github.com/repos/${REPO}/tags |jq -r '.[0] | .name') From 7fdf4318092cbf1b1a859e8f3cc8d61054cb8620 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Fri, 5 Feb 2021 04:31:13 +0000 Subject: [PATCH 5/6] new: [dbg] Added debug mode --- config.sh | 20 ++++++++++++++++++++ deploy.sh | 32 ++++++++++++++++++++++++++------ 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/config.sh b/config.sh index a36729b..7512f56 100644 --- a/config.sh +++ b/config.sh @@ -1,5 +1,13 @@ #!/bin/bash +# Leave empty for NO debug messages, if run with set -x or bash -x it will enable DEBUG by default +DEBUG= + +case "$-" in + *x*) NO_PROGRESS=1; DEBUG=1 ;; + *) NO_PROGRESS=0 ;; +esac + # Name of the packer PACKER_NAME="misp" PACKER_VM="MISP" @@ -30,3 +38,15 @@ NAME_OF_INSTALLER="INSTALL.sh" PATH_TO_INSTALLER="scripts/${NAME_OF_INSTALLER}" URL_TO_INSTALLER="https://raw.githubusercontent.com/${REPO}/${BRANCH}/INSTALL/${NAME_OF_INSTALLER}" URL_TO_LICENSE="https://raw.githubusercontent.com/${REPO}/${BRANCH}/LICENSE" + +if [[ ! -z $DEBUG ]]; then + echo "Debug mode enabled." + echo "-------------------" + echo "" + echo "Some config info:" + echo "Using: $NAME" + [[ ! -z $GPG_ENABLED ]] && echo "GnuPG enabled with key $GPG_KEY" + [[ ! -z $PACKER_LOG ]] && echo "Packer Log enabled." + [[ ! -z $REMOTE ]] && echo "Remote deploy enabled with connection string: $REL_USER@$REL_SERVER" +fi + diff --git a/deploy.sh b/deploy.sh index 8ea4680..d62e9f4 100755 --- a/deploy.sh +++ b/deploy.sh @@ -169,18 +169,38 @@ if [[ "${LATEST_COMMIT}" != "$(cat /tmp/${PACKER_NAME}-latest.sha)" ]]; then # Search and replace for vm_name and make sure we can easily identify the generated VMs cat ${PACKER_NAME}.json| sed "s|\"vm_name\": \"${PACKER_VM}_demo\",|\"vm_name\": \"${PACKER_VM}_${VER}@${LATEST_COMMIT_SHORT}\",|" > ${PACKER_NAME}-deploy.json - # Build virtualbox VM set - export PACKER_LOG_PATH="${PWD}/packerlog-vbox.txt" - ($PACKER_RUN build --on-error=cleanup -only=virtualbox-iso ${PACKER_NAME}-deploy.json ; echo $? > /tmp/${PACKER_NAME}-vbox.done) & + if [[ -z $DEBUG ]]; then + # Build virtualbox VM set + export PACKER_LOG_PATH="${PWD}/packerlog-vbox.txt" + ($PACKER_RUN build --on-error=cleanup -only=virtualbox-iso ${PACKER_NAME}-deploy.json ; echo $? > /tmp/${PACKER_NAME}-vbox.done) & # Build vmware VM set - export PACKER_LOG_PATH="${PWD}/packerlog-vmware.txt" - ($PACKER_RUN build --on-error=cleanup -only=vmware-iso ${PACKER_NAME}-deploy.json ; echo $? > /tmp/${PACKER_NAME}-vmware.done) & + export PACKER_LOG_PATH="${PWD}/packerlog-vmware.txt" + ($PACKER_RUN build --on-error=cleanup -only=vmware-iso ${PACKER_NAME}-deploy.json ; echo $? > /tmp/${PACKER_NAME}-vmware.done) & + + else + echo "Disabling // builds" + # Build virtualbox VM set + export PACKER_LOG_PATH="${PWD}/packerlog-vbox.txt" + ($PACKER_RUN build --on-error=cleanup -only=virtualbox-iso ${PACKER_NAME}-deploy.json ; echo $? > /tmp/${PACKER_NAME}-vbox.done) + + # Build vmware VM set + export PACKER_LOG_PATH="${PWD}/packerlog-vmware.txt" + ($PACKER_RUN build --on-error=cleanup -only=vmware-iso ${PACKER_NAME}-deploy.json ; echo $? > /tmp/${PACKER_NAME}-vmware.done) + TIME_END=$(date +%s) + TIME_DELTA=$(expr ${TIME_END} - ${TIME_START}) + TIME=$(convertSecs ${TIME_DELTA}) + echo "So far the generation took ${TIME}" + echo "" + echo "Waiting for return key..." + read + #exit -1 + fi # The below waits for the above 2 parallel packer builds to finish while [[ ! -f /tmp/${PACKER_NAME}-vmware.done ]]; do :; done while [[ ! -f /tmp/${PACKER_NAME}-vbox.done ]]; do :; done -#exit -1 + # Prevent uploading only half a build if [[ "$(cat /tmp/${PACKER_NAME}-vbox.done)" == "0" ]] && [[ "$(cat /tmp/${PACKER_NAME}-vmware.done)" == "0" ]]; then # ZIPup all the vmware stuff From 022773f658017c2a801610f7bd1fcaf9ccc8a978 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Fri, 5 Feb 2021 04:36:55 +0000 Subject: [PATCH 6/6] chg: [json] Added more waits, needed on this HW --- misp.json | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/misp.json b/misp.json index 244c718..30d3df4 100644 --- a/misp.json +++ b/misp.json @@ -3,21 +3,21 @@ "builders": [ { "boot_command": [ - "{{ user `boot_command_prefix` }}", - "/install/vmlinuz noapic ", - "initrd=/install/initrd.gz ", - "preseed/url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/preseed.cfg ", - "debian-installer=en_US auto locale=en_US kbd-chooser/method=us ", - "hostname={{ user `hostname` }} ", - "grub-installer/bootdev=/dev/sda ", - "fb=false debconf/frontend=noninteractive ", - "keyboard-configuration/modelcode=SKIP keyboard-configuration/layout=USA ", - "keyboard-configuration/variant=USA console-setup/ask_detect=false ", - "passwd/user-fullname={{ user `ssh_fullname` }} ", - "passwd/user-password={{ user `ssh_password` }} ", - "passwd/user-password-again={{ user `ssh_password` }} ", - "passwd/username={{ user `ssh_username` }} ", - "debconf/frontend=noninteractive ", + "{{ user `boot_command_prefix` }}", + "/install/vmlinuz noapic ", + "initrd=/install/initrd.gz ", + "preseed/url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/preseed.cfg ", + "debian-installer=en_US auto locale=en_US kbd-chooser/method=us ", + "hostname={{ user `hostname` }} ", + "grub-installer/bootdev=/dev/sda ", + "fb=false debconf/frontend=noninteractive ", + "keyboard-configuration/modelcode=SKIP keyboard-configuration/layout=USA ", + "keyboard-configuration/variant=USA console-setup/ask_detect=false ", + "passwd/user-fullname={{ user `ssh_fullname` }} ", + "passwd/user-password={{ user `ssh_password` }} ", + "passwd/user-password-again={{ user `ssh_password` }} ", + "passwd/username={{ user `ssh_username` }} ", + "debconf/frontend=noninteractive ", "-- " ], "disk_size": "{{user `disk_size`}}", @@ -69,13 +69,13 @@ { "boot_command": [ "", - "/install/vmlinuz noapic ", - "preseed/url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/preseed.cfg ", - "debian-installer=en_US auto locale=en_US kbd-chooser/method=us ", - "hostname={{user `hostname`}} ", - "fb=false debconf/frontend=noninteractive ", - "keyboard-configuration/modelcode=SKIP keyboard-configuration/layout=USA ", - "keyboard-configuration/variant=USA console-setup/ask_detect=false ", + "/install/vmlinuz noapic ", + "preseed/url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/preseed.cfg ", + "debian-installer=en_US auto locale=en_US kbd-chooser/method=us ", + "hostname={{user `hostname`}} ", + "fb=false debconf/frontend=noninteractive ", + "keyboard-configuration/modelcode=SKIP keyboard-configuration/layout=USA ", + "keyboard-configuration/variant=USA console-setup/ask_detect=false ", "initrd=/install/initrd.gz -- " ], "disk_size": "{{user `disk_size`}}",