diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 103a133f6..913b03b75 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -67,84 +67,84 @@ jobs: # Runs a set of commands using the runners shell - name: Install deps run: | - sudo chown $USER:www-data $HOME/.composer - pushd app - sudo -H -u $USER composer config --no-plugins allow-plugins.composer/installers true - sudo -H -u $USER composer install --no-progress - popd - cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php - # Set perms - sudo chown -R $USER:www-data `pwd` - sudo chmod -R 775 `pwd` - sudo chmod -R g+ws `pwd`/app/tmp - sudo chmod -R g+ws `pwd`/app/tmp/cache - sudo chmod -R g+ws `pwd`/app/tmp/cache/persistent - sudo chmod -R g+ws `pwd`/app/tmp/cache/models - sudo chmod -R g+ws `pwd`/app/tmp/logs - sudo chmod -R g+ws `pwd`/app/files - sudo chmod -R g+ws `pwd`/app/files/scripts/tmp - sudo chown -R $USER:www-data `pwd` - # Resque perms - sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp - sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp - # install MySQL - sudo chmod -R 777 `pwd`/INSTALL - mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';" - mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant usage on *.* to misp@'%' identified by 'blah';" - mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant all privileges on misp.* to misp@'%';" - mysql -h 127.0.0.1 --port 3306 -u misp -pblah misp < INSTALL/MYSQL.sql - # configure apache virtual hosts - sudo chmod -R 777 `pwd`/build - sudo mkdir -p /etc/apache2/sites-available - sudo cp -f build/github-action-ci-apache /etc/apache2/sites-available/misp.conf - sudo sed -e "s?%GITHUB_WORKSPACE%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.conf - sudo sed -e "s?%HOST%?${HOST}?g" --in-place /etc/apache2/sites-available/misp.conf - sudo a2dissite 000-default - sudo a2ensite misp.conf - cat /etc/apache2/sites-enabled/misp.conf - sudo a2enmod rewrite - sudo systemctl restart apache2 - # MISP configuration - sudo chmod -R 777 `pwd`/travis - sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php - sudo cp travis/database.php app/Config/database.php - sudo cp app/Config/core.default.php app/Config/core.php - sudo cp app/Config/config.default.php app/Config/config.php - sudo cp travis/email.php app/Config/email.php - # Ensure the perms - sudo chown -R $USER:www-data `pwd`/app/Config - sudo chmod -R 777 `pwd`/app/Config - # GPG setup - sudo mkdir `pwd`/.gnupg - # /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS - sudo cp -a /dev/urandom /dev/random - sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/travis/gpg - sudo gpg --list-secret-keys --homedir `pwd`/.gnupg - # change perms - sudo chown -R $USER:www-data `pwd` - sudo chown -R www-data:www-data `pwd`/.gnupg - sudo chmod -R 700 `pwd`/.gnupg - sudo usermod -a -G www-data $USER - sudo chmod -R 777 `pwd`/app/Plugin/CakeResque/tmp/ - # Ensure the perms of config files - sudo chown -R $USER:www-data `pwd`/app/Config - sudo chmod -R 777 `pwd`/app/Config - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.server_settings_skip_backup_rotate" 1' - sudo chown -R $USER:www-data `pwd`/app/Config - sudo chmod -R 777 `pwd`/app/Config + sudo chown $USER:www-data $HOME/.composer + pushd app + sudo -H -u $USER composer config --no-plugins allow-plugins.composer/installers true + sudo -H -u $USER composer install --no-progress + popd + cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php + # Set perms + sudo chown -R $USER:www-data `pwd` + sudo chmod -R 775 `pwd` + sudo chmod -R g+ws `pwd`/app/tmp + sudo chmod -R g+ws `pwd`/app/tmp/cache + sudo chmod -R g+ws `pwd`/app/tmp/cache/persistent + sudo chmod -R g+ws `pwd`/app/tmp/cache/models + sudo chmod -R g+ws `pwd`/app/tmp/logs + sudo chmod -R g+ws `pwd`/app/files + sudo chmod -R g+ws `pwd`/app/files/scripts/tmp + sudo chown -R $USER:www-data `pwd` + # Resque perms + sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp + sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp + # install MySQL + sudo chmod -R 777 `pwd`/INSTALL + mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';" + mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant usage on *.* to misp@'%' identified by 'blah';" + mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant all privileges on misp.* to misp@'%';" + mysql -h 127.0.0.1 --port 3306 -u misp -pblah misp < INSTALL/MYSQL.sql + # configure apache virtual hosts + sudo chmod -R 777 `pwd`/build + sudo mkdir -p /etc/apache2/sites-available + sudo cp -f build/github-action-ci-apache /etc/apache2/sites-available/misp.conf + sudo sed -e "s?%GITHUB_WORKSPACE%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.conf + sudo sed -e "s?%HOST%?${HOST}?g" --in-place /etc/apache2/sites-available/misp.conf + sudo a2dissite 000-default + sudo a2ensite misp.conf + cat /etc/apache2/sites-enabled/misp.conf + sudo a2enmod rewrite + sudo systemctl restart apache2 + # MISP configuration + sudo chmod -R 777 `pwd`/travis + sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php + sudo cp travis/database.php app/Config/database.php + sudo cp app/Config/core.default.php app/Config/core.php + sudo cp app/Config/config.default.php app/Config/config.php + sudo cp travis/email.php app/Config/email.php + # Ensure the perms + sudo chown -R $USER:www-data `pwd`/app/Config + sudo chmod -R 777 `pwd`/app/Config + # GPG setup + sudo mkdir `pwd`/.gnupg + # /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS + sudo cp -a /dev/urandom /dev/random + sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/travis/gpg + sudo gpg --list-secret-keys --homedir `pwd`/.gnupg + # change perms + sudo chown -R $USER:www-data `pwd` + sudo chown -R www-data:www-data `pwd`/.gnupg + sudo chmod -R 700 `pwd`/.gnupg + sudo usermod -a -G www-data $USER + sudo chmod -R 777 `pwd`/app/Plugin/CakeResque/tmp/ + # Ensure the perms of config files + sudo chown -R $USER:www-data `pwd`/app/Config + sudo chmod -R 777 `pwd`/app/Config + app/Console/cake Admin setSetting "MISP.server_settings_skip_backup_rotate" 1 + sudo chown -R $USER:www-data `pwd`/app/Config + sudo chmod -R 777 `pwd`/app/Config - # fix perms (?) - namei -m /home/runner/work - sudo chmod +x /home/runner/work - sudo chmod +x /home/runner - sudo chmod +x /home - sudo chmod +x / + # fix perms (?) + namei -m /home/runner/work + sudo chmod +x /home/runner/work + sudo chmod +x /home/runner + sudo chmod +x /home + sudo chmod +x / - name: Python setup run: | # Dirty install python stuff python3 -m virtualenv -p python3 ./venv - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python"' + app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python" . ./venv/bin/activate export PYTHONPATH=$PYTHONPATH:./app/files/scripts pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara pytest @@ -152,84 +152,87 @@ jobs: - name: DB Update run: | - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.osuser" $USER' - sudo -E su $USER -c 'app/Console/cake Admin runUpdates' - sudo -E su $USER -c 'app/Console/cake Admin schemaDiagnostics' + app/Console/cake Admin setSetting "MISP.osuser" $USER + app/Console/cake Admin runUpdates + app/Console/cake Admin schemaDiagnostics - name: Configure MISP run: | - sudo -u $USER app/Console/cake User init | sudo tee ./key.txt - echo "AUTH=`cat key.txt`" >> $GITHUB_ENV - sudo -u $USER app/Console/cake Admin setSetting "Session.autoRegenerate" 0 - sudo -u $USER app/Console/cake Admin setSetting "Session.timeout" 600 - sudo -u $USER app/Console/cake Admin setSetting "Session.cookieTimeout" 3600 - sudo -u $USER app/Console/cake Admin setSetting "MISP.host_org_id" 1 - sudo -u $USER app/Console/cake Admin setSetting "MISP.email" "info@admin.test" - sudo -u $USER app/Console/cake Admin setSetting "MISP.disable_emailing" false - sudo -u $USER app/Console/cake Admin setSetting --force "debug" true - sudo -u $USER app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false - sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1" - sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_port" 6379 - sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_database" 13 - sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_password" "" - sudo -u $USER app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test" - sudo -u $USER app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg" - sudo -u $USER app/Console/cake Admin setSetting "GnuPG.password" "travistest" - sudo -u $USER app/Console/cake Admin setSetting "MISP.download_gpg_from_homedir" 1 - sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_host" "127.0.0.1" - sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_port" 6379 - sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_database" 1 - sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_password" "" - sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" 1 - sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_audit_notifications_enable" 1 + app/Console/cake User init | sudo tee ./key.txt + echo "AUTH=`cat key.txt`" >> $GITHUB_ENV + app/Console/cake Admin setSetting "Session.autoRegenerate" 0 + app/Console/cake Admin setSetting "Session.timeout" 600 + app/Console/cake Admin setSetting "Session.cookieTimeout" 3600 + app/Console/cake Admin setSetting "MISP.host_org_id" 1 + app/Console/cake Admin setSetting "MISP.email" "info@admin.test" + app/Console/cake Admin setSetting "MISP.disable_emailing" false + app/Console/cake Admin setSetting --force "debug" true + app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false + app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1" + app/Console/cake Admin setSetting "MISP.redis_port" 6379 + app/Console/cake Admin setSetting "MISP.redis_database" 13 + app/Console/cake Admin setSetting "MISP.redis_password" "" + app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test" + app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg" + app/Console/cake Admin setSetting "GnuPG.password" "travistest" + app/Console/cake Admin setSetting "MISP.download_gpg_from_homedir" 1 + app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_host" "127.0.0.1" + app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_port" 6379 + app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_database" 1 + app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_password" "" + app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" 1 + app/Console/cake Admin setSetting "Plugin.ZeroMQ_audit_notifications_enable" 1 - name: Update Galaxies - run: sudo -E su $USER -c 'app/Console/cake Admin updateGalaxies' + run: app/Console/cake Admin updateGalaxies - name: Update Taxonomies - run: sudo -E su $USER -c 'app/Console/cake Admin updateTaxonomies' + run: app/Console/cake Admin updateTaxonomies - name: Update Warninglists - run: sudo -E su $USER -c 'app/Console/cake Admin updateWarningLists --verbose' + run: app/Console/cake Admin updateWarningLists --verbose - name: Update Noticelists - run: sudo -E su $USER -c 'app/Console/cake Admin updateNoticeLists' + run: app/Console/cake Admin updateNoticeLists - name: Update Object Templates - run: sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates 1' + run: app/Console/cake Admin updateObjectTemplates 1 - name: Turn MISP live - run: sudo -E su $USER -c 'app/Console/cake Admin live 1' + run: app/Console/cake Admin live 1 - name: Check if Redis is ready - run: sudo -E su $USER -c 'app/Console/cake Admin redisReady' + run: app/Console/cake Admin redisReady - name: Start workers run: | - sudo chmod +x app/Console/worker/start.sh - sudo -u www-data 'app/Console/worker/start.sh' + sudo chmod +x app/Console/worker/start.sh + sudo -u www-data 'app/Console/worker/start.sh' - name: Test if apache is working run: | - sudo systemctl status apache2 --no-pager -l - sudo apache2ctl -S - curl http://${HOST} - sudo chmod -R 777 PyMISP - pushd PyMISP - echo 'url = "http://'${HOST}'"' >> tests/keys.py - echo 'key = "'${AUTH}'"' >> tests/keys.py - cat tests/keys.py - popd - . ./venv/bin/activate - pushd tests - bash ./build-test.sh - popd - deactivate + sudo systemctl status apache2 --no-pager -l + sudo apache2ctl -S + curl http://${HOST} + + - name: Check if dependencies working as expected + run: | + sudo chmod -R 777 PyMISP + pushd PyMISP + echo 'url = "http://'${HOST}'"' >> tests/keys.py + echo 'key = "'${AUTH}'"' >> tests/keys.py + cat tests/keys.py + popd + . ./venv/bin/activate + pushd tests + bash ./build-test.sh + popd + deactivate - name: Run PHP tests run: | - ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ -e php,ctp app/ - sudo -u www-data ./app/Vendor/bin/phpunit app/Test/ + ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ -e php,ctp app/ + sudo -u www-data ./app/Vendor/bin/phpunit app/Test/ - name: Clone test files uses: actions/checkout@v4 @@ -237,31 +240,30 @@ jobs: repository: viper-framework/viper-test-files path: PyMISP/tests/viper-test-files - - name: Run tests run: | - pushd tests - ./curl_tests_GH.sh $AUTH $HOST - popd - - sudo chmod -R g+ws `pwd`/app/tmp/logs - - . ./venv/bin/activate - pushd PyMISP - cp tests/keys.py . - python -m pytest -v --durations=0 tests/test_mispevent.py - python -m pytest -v --durations=0 tests/testlive_comprehensive.py - popd - python tests/testlive_security.py -v - python tests/testlive_sync.py - python tests/testlive_comprehensive_local.py -v - cp PyMISP/tests/keys.py PyMISP/examples/events/ - pushd PyMISP/examples/events/ - python ./create_massive_dummy_events.py -l 5 -a 30 - popd - pip install jsonschema - python tools/misp-feed/validate.py - deactivate + pushd tests + ./curl_tests_GH.sh $AUTH $HOST + popd + + sudo chmod -R g+ws `pwd`/app/tmp/logs + + . ./venv/bin/activate + pushd PyMISP + cp tests/keys.py . + python -m pytest -v --durations=0 tests/test_mispevent.py + python -m pytest -v --durations=0 tests/testlive_comprehensive.py + popd + python tests/testlive_security.py -v + python tests/testlive_sync.py + python tests/testlive_comprehensive_local.py -v + cp PyMISP/tests/keys.py PyMISP/examples/events/ + pushd PyMISP/examples/events/ + python ./create_massive_dummy_events.py -l 5 -a 30 + popd + pip install jsonschema + python tools/misp-feed/validate.py + deactivate - name: Check requirements.txt run: python tests/check_requirements.py @@ -270,13 +272,13 @@ jobs: if: ${{ always() }} # update logs_test.sh when adding more logsources here run: | - tail -n +1 `pwd`/app/tmp/logs/* - tail -n +1 /var/log/apache2/*.log + tail -n +1 `pwd`/app/tmp/logs/* + tail -n +1 /var/log/apache2/*.log - sudo -u $USER app/Console/cake Log export /tmp/logs.json.gz --without-changes - zcat /tmp/logs.json.gz + app/Console/cake Log export /tmp/logs.json.gz --without-changes + zcat /tmp/logs.json.gz - name: Errors in Logs if: ${{ always() }} run: | - ./tests/logs_tests.sh \ No newline at end of file + ./tests/logs_tests.sh \ No newline at end of file