From 58497fc71ce9c4b84cd17f076ccb1e042367cfca Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 21 Feb 2019 12:32:05 +0530 Subject: [PATCH] chg: [doc] Move Debian9 stable install instructions to xPerimental status as only Python 3.5 is supported by default but PyMISP needs 3.6 (There are no easy clean ways to install >3.5) chg: [installer] various changes and cleanups in the installer --- INSTALL/INSTALL.debian.sh | 68 +++++++++++++++---- INSTALL/INSTALL.debian.tpl.sh | 17 +++-- INSTALL/INSTALL.debian9.txt | 1 - ...sql.md => xINSTALL.debian9-postgresql.txt} | 0 INSTALL/xINSTALL.debian9.txt | 1 + docs/generic/sudo_etckeeper.md | 8 ++- docs/generic/supportFunctions.md | 35 +++++++++- ...INSTALL.debian9.md => xINSTALL.debian9.md} | 62 +++++++++++------ mkdocs.yml | 4 +- 9 files changed, 149 insertions(+), 47 deletions(-) delete mode 120000 INSTALL/INSTALL.debian9.txt rename INSTALL/{xINSTALL.debian9-postgresql.md => xINSTALL.debian9-postgresql.txt} (100%) create mode 120000 INSTALL/xINSTALL.debian9.txt rename docs/{INSTALL.debian9.md => xINSTALL.debian9.md} (92%) diff --git a/INSTALL/INSTALL.debian.sh b/INSTALL/INSTALL.debian.sh index b08107154..d7e1dd48d 100755 --- a/INSTALL/INSTALL.debian.sh +++ b/INSTALL/INSTALL.debian.sh @@ -344,12 +344,53 @@ checkID () { # pre-install check to make sure what we will be installing on, is ready and not a half installed system preInstall () { +# preInstall needs to be able to be called before ANY action. Install/Upgrade/AddTool +# Pre install wants to be the place too where the following is checked and set via ENV_VAR: +# Check if composer is installed and functioning +# Check if misp db is installed (API call would confirm that the DB indeed works) +# Check apache config (Maybe try to talk to the server via api, this would confirm quite a lot) +# Check if workers are running/installed, maybe kick them if they are not +# /var/www/MISP/app/Config/[bootstrap,databases,core,config].php exists +# /var/www/MISP perms are correct (for $SUDO_WWW useage) +# + + # Check if $PATH_TO_MISP exists and is writable by $WWW_USER + [[ -d "$PATH_TO_MISP" ]] && PATH_TO_MISP_EXISTS=1 && echo "$PATH_TO_MISP exists" + + # .git exists and git is working for $WWW_USER + [[ -d "$PATH_TO_MISP/.git" ]] && PATH_TO_GIT_EXISTS=1 && echo "$PATH_TO_MISP/.git exists" && cd $PATH_TO_MISP && $SUDO_WWW git status + + # .gnupg exists and working correctly + [[ -d "$PATH_TO_MISP/.gnupg" ]] && PATH_TO_GNUPG_EXISTS=1 && echo "$PATH_TO_MISP/.gnupg exists" + + + # Extract username, password and dbname + ##cat database.php |grep -v // |grep -e database -e login -e password |tr -d \' |tr -d \ |tr -d , |tr -d \> + DBPASSWORD_MISP=$(cat database.php |grep -v // |grep -e password |tr -d \' |tr -d \ |tr -d , |tr -d \> |cut -f 2 -d=) + DBUSER_MISP=$(cat database.php |grep -v // |grep -e login |tr -d \' |tr -d \ |tr -d , |tr -d \> |cut -f 2 -d=) + DBNAME=$(cat database.php |grep -v // |grep -e database |tr -d \' |tr -d \ |tr -d , |tr -d \> |cut -f 2 -d=) + AUTH_KEY=$(mysql --disable-column-names -B -u $DBUSER_MISP -p"$DBPASSWORD_MISP" $DBNAME -e 'SELECT authkey FROM users WHERE role_id=1 LIMIT 1') + + # Check if db exists + [[ -d "/var/lib/mysql/$DBNAME" ]] && MISP_DB_DIR_EXISTS=1 && echo "/var/lib/mysql/$DBNAME exists" + echo -e "${RED}Place-holder, not implemented yet.${NC}" exit } # Upgrade function upgrade () { + headerJSON="application/json" + Acc="Accept:" + Autho="Authorization:" + CT="Content-Type:" + MISP_BASEURL="https://127.0.0.1" + cd $PATH_TO_MISP/app ; $SUDO_WWW php composer.phar update $SUDO_WWW php composer.phar self-update + + for URN in $(echo "galaxies warninglists noticelists objectTemplates taxonomies"); do + curl --header "$Autho $AUTH_KEY" --header "$Acc $headerJSON" --header "$CT $headerJSON" -k -X POST $MISP_BASEURL/$URN/update + done + echo -e "${RED}Place-holder, not implemented yet.${NC}" exit } @@ -750,7 +791,7 @@ theEnd () { space echo -e "Enjoy using ${LBLUE}MISP${NC}. For any issues see here: https://github.com/MISP/MISP/issues" space - if [ $UNATTENDED == "1" ]; then + if [[ "$UNATTENDED" == "1" ]]; then echo -e "${RED}Unattended install!${NC}" echo -e "This means we guessed the Base URL, it might be wrong, please double check." space @@ -773,10 +814,14 @@ aptUpgrade () { checkSudoKeeper () { echo "Checking for sudo and installing etckeeper" if [[ ! -f $(which sudo) ]]; then + echo "Please enter your root password below to install etckeeper" su -c "apt install etckeeper -y" + echo "Please enter your root password below to install sudo" su -c "apt install sudo -y" - # TODO: Fix this, user misp might not exist - su -c "adduser misp sudo" + echo "Please enter your root password below to add $MISP_USER to sudo group" + su -c "adduser $MISP_USER sudo" + echo "We added $MISP_USER to group sudo and now we need to log out and in again." + exit else sudo apt install etckeeper -y fi @@ -1510,7 +1555,7 @@ generateInstaller () { cp ../INSTALL.debian.tpl.sh . # Pull code snippets out of Main Install Documents - for f in `echo INSTALL.ubuntu1804.md INSTALL.debian9.md INSTALL.kali.md xINSTALL.debian_testing.md xINSTALL.tsurugi.md xINSTALL.debian9-postgresql.md xINSTALL.ubuntu1804.with.webmin.md`; do + for f in `echo INSTALL.ubuntu1804.md xINSTALL.debian9.md INSTALL.kali.md xINSTALL.debian_testing.md xINSTALL.tsurugi.md xINSTALL.debian9-postgresql.md xINSTALL.ubuntu1804.with.webmin.md`; do xsnippet . ../../docs/${f} done @@ -1579,22 +1624,21 @@ installMISPubuntuSupported () { echo "Proceeding with the installation of MISP core" space - # Set locale if not set - functionLocation('generic/supportFunctions.md') - debug "Checking Locale" - checkLocale - # Set Base URL - functionLocation('generic/supportFunctions.md') [[ -n $CORE ]] || [[ -n $ALL ]] && setBaseURL progress 4 - # Upgrade system to make sure we install the latest packages - functionLocation('INSTALL.ubuntu1804.md') - [[ -n $CORE ]] || [[ -n $ALL ]] && aptUpgrade 2> /dev/null > /dev/null - progress 4 - # Check if sudo is installed and etckeeper - functionLocation('generic/sudo_etckeeper.md') [[ -n $CORE ]] || [[ -n $ALL ]] && checkSudoKeeper 2> /dev/null > /dev/null progress 4 + # Set locale if not set - functionLocation('generic/supportFunctions.md') + checkLocale + + # Upgrade system to make sure we install the latest packages - functionLocation('INSTALL.ubuntu1804.md') + [[ -n $CORE ]] || [[ -n $ALL ]] && aptUpgrade 2> /dev/null > /dev/null + progress 4 + # TODO: Double check how the user is added and subsequently used during the install. # TODO: Work on possibility to install as user X and install MISP for user Y # TODO: Check if logout needed. (run SUDO_USER in installer) diff --git a/INSTALL/INSTALL.debian.tpl.sh b/INSTALL/INSTALL.debian.tpl.sh index 2a1f65b51..7093987bc 100755 --- a/INSTALL/INSTALL.debian.tpl.sh +++ b/INSTALL/INSTALL.debian.tpl.sh @@ -122,7 +122,7 @@ generateInstaller () { cp ../INSTALL.debian.tpl.sh . # Pull code snippets out of Main Install Documents - for f in `echo INSTALL.ubuntu1804.md INSTALL.debian9.md INSTALL.kali.md xINSTALL.debian_testing.md xINSTALL.tsurugi.md xINSTALL.debian9-postgresql.md xINSTALL.ubuntu1804.with.webmin.md`; do + for f in `echo INSTALL.ubuntu1804.md xINSTALL.debian9.md INSTALL.kali.md xINSTALL.debian_testing.md xINSTALL.tsurugi.md xINSTALL.debian9-postgresql.md xINSTALL.ubuntu1804.with.webmin.md`; do xsnippet . ../../docs/${f} done @@ -191,22 +191,21 @@ installMISPubuntuSupported () { echo "Proceeding with the installation of MISP core" space - # Set locale if not set - functionLocation('generic/supportFunctions.md') - debug "Checking Locale" - checkLocale - # Set Base URL - functionLocation('generic/supportFunctions.md') [[ -n $CORE ]] || [[ -n $ALL ]] && setBaseURL progress 4 - # Upgrade system to make sure we install the latest packages - functionLocation('INSTALL.ubuntu1804.md') - [[ -n $CORE ]] || [[ -n $ALL ]] && aptUpgrade 2> /dev/null > /dev/null - progress 4 - # Check if sudo is installed and etckeeper - functionLocation('generic/sudo_etckeeper.md') [[ -n $CORE ]] || [[ -n $ALL ]] && checkSudoKeeper 2> /dev/null > /dev/null progress 4 + # Set locale if not set - functionLocation('generic/supportFunctions.md') + checkLocale + + # Upgrade system to make sure we install the latest packages - functionLocation('INSTALL.ubuntu1804.md') + [[ -n $CORE ]] || [[ -n $ALL ]] && aptUpgrade 2> /dev/null > /dev/null + progress 4 + # TODO: Double check how the user is added and subsequently used during the install. # TODO: Work on possibility to install as user X and install MISP for user Y # TODO: Check if logout needed. (run SUDO_USER in installer) diff --git a/INSTALL/INSTALL.debian9.txt b/INSTALL/INSTALL.debian9.txt deleted file mode 120000 index 79fe716ca..000000000 --- a/INSTALL/INSTALL.debian9.txt +++ /dev/null @@ -1 +0,0 @@ -../docs/INSTALL.debian9.md \ No newline at end of file diff --git a/INSTALL/xINSTALL.debian9-postgresql.md b/INSTALL/xINSTALL.debian9-postgresql.txt similarity index 100% rename from INSTALL/xINSTALL.debian9-postgresql.md rename to INSTALL/xINSTALL.debian9-postgresql.txt diff --git a/INSTALL/xINSTALL.debian9.txt b/INSTALL/xINSTALL.debian9.txt new file mode 120000 index 000000000..23316cb33 --- /dev/null +++ b/INSTALL/xINSTALL.debian9.txt @@ -0,0 +1 @@ +../docs/xINSTALL.debian9.md \ No newline at end of file diff --git a/docs/generic/sudo_etckeeper.md b/docs/generic/sudo_etckeeper.md index 4caeea176..c3381af25 100644 --- a/docs/generic/sudo_etckeeper.md +++ b/docs/generic/sudo_etckeeper.md @@ -7,10 +7,14 @@ checkSudoKeeper () { echo "Checking for sudo and installing etckeeper" if [[ ! -f $(which sudo) ]]; then + echo "Please enter your root password below to install etckeeper" su -c "apt install etckeeper -y" + echo "Please enter your root password below to install sudo" su -c "apt install sudo -y" - # TODO: Fix this, user misp might not exist - su -c "adduser misp sudo" + echo "Please enter your root password below to add $MISP_USER to sudo group" + su -c "adduser $MISP_USER sudo" + echo "We added $MISP_USER to group sudo and now we need to log out and in again." + exit else sudo apt install etckeeper -y fi diff --git a/docs/generic/supportFunctions.md b/docs/generic/supportFunctions.md index 163b58eee..ed849b976 100644 --- a/docs/generic/supportFunctions.md +++ b/docs/generic/supportFunctions.md @@ -221,18 +221,47 @@ preInstall () { # Check if misp db is installed (API call would confirm that the DB indeed works) # Check apache config (Maybe try to talk to the server via api, this would confirm quite a lot) # Check if workers are running/installed, maybe kick them if they are not -# /var/www/MISP exists -# /var/www/MISP/.git exists # /var/www/MISP/app/Config/[bootstrap,databases,core,config].php exists -# /var/www/MISP/.gnupg exists # /var/www/MISP perms are correct (for $SUDO_WWW useage) # + + # Check if $PATH_TO_MISP exists and is writable by $WWW_USER + [[ -d "$PATH_TO_MISP" ]] && PATH_TO_MISP_EXISTS=1 && echo "$PATH_TO_MISP exists" + + # .git exists and git is working for $WWW_USER + [[ -d "$PATH_TO_MISP/.git" ]] && PATH_TO_GIT_EXISTS=1 && echo "$PATH_TO_MISP/.git exists" && cd $PATH_TO_MISP && $SUDO_WWW git status + + # .gnupg exists and working correctly + [[ -d "$PATH_TO_MISP/.gnupg" ]] && PATH_TO_GNUPG_EXISTS=1 && echo "$PATH_TO_MISP/.gnupg exists" + + + # Extract username, password and dbname + ##cat database.php |grep -v // |grep -e database -e login -e password |tr -d \' |tr -d \ |tr -d , |tr -d \> + DBPASSWORD_MISP=$(cat database.php |grep -v // |grep -e password |tr -d \' |tr -d \ |tr -d , |tr -d \> |cut -f 2 -d=) + DBUSER_MISP=$(cat database.php |grep -v // |grep -e login |tr -d \' |tr -d \ |tr -d , |tr -d \> |cut -f 2 -d=) + DBNAME=$(cat database.php |grep -v // |grep -e database |tr -d \' |tr -d \ |tr -d , |tr -d \> |cut -f 2 -d=) + AUTH_KEY=$(mysql --disable-column-names -B -u $DBUSER_MISP -p"$DBPASSWORD_MISP" $DBNAME -e 'SELECT authkey FROM users WHERE role_id=1 LIMIT 1') + + # Check if db exists + [[ -d "/var/lib/mysql/$DBNAME" ]] && MISP_DB_DIR_EXISTS=1 && echo "/var/lib/mysql/$DBNAME exists" + echo -e "${RED}Place-holder, not implemented yet.${NC}" exit } # Upgrade function upgrade () { + headerJSON="application/json" + Acc="Accept:" + Autho="Authorization:" + CT="Content-Type:" + MISP_BASEURL="https://127.0.0.1" + cd $PATH_TO_MISP/app ; $SUDO_WWW php composer.phar update $SUDO_WWW php composer.phar self-update + + for URN in $(echo "galaxies warninglists noticelists objectTemplates taxonomies"); do + curl --header "$Autho $AUTH_KEY" --header "$Acc $headerJSON" --header "$CT $headerJSON" -k -X POST $MISP_BASEURL/$URN/update + done + echo -e "${RED}Place-holder, not implemented yet.${NC}" exit } diff --git a/docs/INSTALL.debian9.md b/docs/xINSTALL.debian9.md similarity index 92% rename from docs/INSTALL.debian9.md rename to docs/xINSTALL.debian9.md index 7cc2df540..a3e21f312 100644 --- a/docs/INSTALL.debian9.md +++ b/docs/xINSTALL.debian9.md @@ -5,14 +5,10 @@ -------------------------------------- !!! notice - Maintained and tested by @SteveClement on 20181023 + Maintained and tested by @SteveClement on 20190221 -{!generic/globalVariables.md!} - -```bash -PHP_ETC_BASE=/etc/php/7.0 -PHP_INI=${PHP_ETC_BASE}/apache2/php.ini -``` +!!! warning + This install document is NOT working. There are Python issues ### 1/ Minimal Debian install ------------------------- @@ -23,6 +19,13 @@ PHP_INI=${PHP_ETC_BASE}/apache2/php.ini {!generic/sudo_etckeeper.md!} +{!generic/globalVariables.md!} + +```bash +PHP_ETC_BASE=/etc/php/7.0 +PHP_INI=${PHP_ETC_BASE}/apache2/php.ini +``` + {!generic/ethX.md!} #### Make sure your system is up2date @@ -48,18 +51,40 @@ sudo postfix reload #### Install all the dependencies (some might already be installed) +You need to update python3.5 to python3.7 for [PyMISP](https://github.com/MISP/PyMISP) to work properly. + +FIXME: The below breaks redis-server and mariadb-server + +```bash +#echo "deb http://ftp.de.debian.org/debian testing main" | sudo tee -a /etc/apt/sources.list +#echo 'APT::Default-Release "stable";' | sudo tee -a /etc/apt/apt.conf.d/00local +#sudo apt update +#sudo apt-get -t testing install python3 +#sudo apt-get -t testing install python3-setuptools python3-dev python3-pip python3-redis python3-zmq virtualenv +``` + ```bash sudo apt install -y \ curl gcc git gnupg-agent make openssl redis-server vim zip libyara-dev \ -python3-setuptools python3-dev python3-pip python3-redis python3-zmq virtualenv \ -mariadb-client \ -mariadb-server \ apache2 apache2-doc apache2-utils \ -libapache2-mod-php7.0 php7.0 php7.0-cli php7.0-mbstring php7.0-dev php7.0-json php7.0-xml php7.0-mysql php7.0-opcache php7.0-readline php-redis php-gnupg \ libpq5 libjpeg-dev libfuzzy-dev ruby asciidoctor \ -jq ntp ntpdate jupyter-notebook imagemagick tesseract-ocr \ +jq ntp ntpdate imagemagick tesseract-ocr \ libxml2-dev libxslt1-dev zlib1g-dev +#sudo systemctl disable redis-server + +#sudo apt -y -f install + +#sudo /etc/init.d/redis-server restart + +sudo apt install -y libapache2-mod-php7.0 php7.0 php7.0-cli php7.0-mbstring php7.0-dev php7.0-json php7.0-xml php7.0-mysql php7.0-opcache php7.0-readline php-redis php-gnupg + +sudo apt install -y \ +mariadb-client \ +mariadb-server + +sudo apt install -y jupyter-notebook + # Start haveged to get more entropy (optional) sudo apt install haveged -y sudo service havegd start @@ -117,6 +142,10 @@ sudo mkdir $PATH_TO_MISP sudo chown www-data:www-data $PATH_TO_MISP cd $PATH_TO_MISP sudo -u www-data git clone https://github.com/MISP/MISP.git $PATH_TO_MISP +sudo -u www-data git submodule update --init --recursive + +# Make git ignore filesystem permission differences for submodules +sudo -u www-data git submodule foreach --recursive git config core.filemode false # Make git ignore filesystem permission differences sudo -u www-data git config core.filemode false @@ -146,18 +175,13 @@ sudo -u www-data ${PATH_TO_MISP}/venv/bin/pip install . cd $PATH_TO_MISP/cti-python-stix2 sudo -u www-data ${PATH_TO_MISP}/venv/bin/pip install . -cd $PATH_TO_MISP -sudo -u www-data git submodule update --init --recursive -# Make git ignore filesystem permission differences for submodules -sudo -u www-data git submodule foreach --recursive git config core.filemode false - # install PyMISP cd $PATH_TO_MISP/PyMISP sudo -u www-data ${PATH_TO_MISP}/venv/bin/pip install . # Install Crypt_GPG and Console_CommandLine -sudo -H -u www-data pear install ${PATH_TO_MISP}/INSTALL/dependencies/Console_CommandLine/package.xml -sudo -H -u www-data pear install ${PATH_TO_MISP}/INSTALL/dependencies/Crypt_GPG/package.xml +sudo pear install ${PATH_TO_MISP}/INSTALL/dependencies/Console_CommandLine/package.xml +sudo pear install ${PATH_TO_MISP}/INSTALL/dependencies/Crypt_GPG/package.xml ``` ### 4/ CakePHP diff --git a/mkdocs.yml b/mkdocs.yml index b48a6fd16..0580b52b4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -11,6 +11,8 @@ repo_name: MISP/MISP repo_url: https://github.com/MISP/MISP edit_uri: "" +#dev_addr: "192.168.10.34:8000" + # Copyright copyright: "Copyright © 2019 MISP Project" @@ -67,13 +69,13 @@ nav: - Home: 'index.md' - Install Guides: - 'Ubuntu 18.04': 'INSTALL.ubuntu1804.md' - - 'Debian stable': 'INSTALL.debian9.md' - 'Kali Linux': 'INSTALL.kali.md' - 'Redhat Enterprise Linux 7': 'INSTALL.rhel7.md' - xInstall Guides: - 'Warning': 'xINSTALL.md' - 'Centos 6': 'xINSTALL.centos6.md' - 'Centos 7': 'xINSTALL.centos7.md' + - 'Debian stable': 'xINSTALL.debian9.md' - 'Debian testing': 'xINSTALL.debian_testing.md' - 'Debian 9 \w postgresql': 'xINSTALL.debian9-postgresql.md' - 'Ubuntu 18.04 \w webmin': 'xINSTALL.ubuntu1804.with.webmin.md'