new: [test] Build test

.github/workflows/main.yml

@@ -70,9 +70,8 @@ jobs:
               # Repo is missing for unknown reason
               LC_ALL=C.UTF-8 sudo apt-add-repository ppa:ondrej/php -y
               sudo apt-get -y install curl python3 python3-zmq python3-requests python3-pip python3-nose python3-redis python3-lxml apache2 libapache2-mod-php$php_version libfuzzy-dev
-              sudo pip3 install virtualenv # virtualenv must be instaled from pip and not from ubuntu pacckages
-              sudo pip3 install --upgrade -r requirements.txt
-              curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python -
+              sudo pip3 install virtualenv # virtualenv must be instaled from pip and not from ubuntu packages
+              curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python -
               sudo chown $USER:www-data $HOME/.composer
               pushd app
               sudo -H -u $USER php composer.phar install --no-progress
@@ -167,12 +166,16 @@ jobs:
 
       - name: Update Galaxies
         run: sudo -E su $USER -c 'app/Console/cake Admin updateGalaxies'
+
       - name: Update Taxonomies
         run: sudo -E su $USER -c 'app/Console/cake Admin updateTaxonomies'
+
       - name: Update Warninglists
         run: sudo -E su $USER -c 'app/Console/cake Admin updateWarningLists'
+
       - name: Update Noticelists
         run: sudo -E su $USER -c 'app/Console/cake Admin updateNoticeLists'
+
       - name: Update Object Templates
         run: sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates 1'
 
@@ -215,6 +218,13 @@ jobs:
               cat tests/keys.py
               popd
 
+      - name: Build test
+        run: |
+              . ./venv/bin/activate
+              pip install git+https://github.com/kbandla/pydeep.git
+              pushd tests
+              ./build-test.sh
+
       - name: Run PHP tests
         run: |
               ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
@@ -222,7 +232,7 @@ jobs:
 
       - name: Run tests
         run: |
-              source $HOME/.poetry/env # enable poetry binary
+              export PATH=$HOME/.local/env:$PATH # enable poetry binary
 
               # Ensure the perms of config files
               sudo chown -R $USER:www-data `pwd`/app/Config

tests/build-test.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -e
+set -o xtrace
+
+# Check if dependencies for zmq are properly installed
+python3 ./../app/files/scripts/mispzmq/mispzmqtest.py
+
+# Check if all attachments handlers dependencies are correctly installed
+python3 ./../app/files/scripts/generate_file_objects.py -c | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if len([i for i in data.values() if i == True]) == 0 else 1)'
+
+# Try to extract data from file
+python3 ./../app/files/scripts/generate_file_objects.py -p /bin/ls | python3 -c 'import sys, json; data = json.load(sys.stdin); sys.exit(0 if "objects" in data else 1)'
+
+# Test converting stix1 to MISP format
+curl https://stixproject.github.io/documentation/idioms/c2-indicator/indicator-for-c2-ip-address.xml > ./../app/files/scripts/tmp/test-stix1.xml
+python3 ./../app/files/scripts/stix2misp.py test-stix1.xml 1 1 ./../app/files/scripts/synonymsToTagNames.json | python3 -c 'import sys; data = sys.stdin.read().strip(); print(data); sys.exit(0 if data == "1" else 1)'
+rm -f ./../app/files/scripts/tmp/{test-stix1.xml,test-stix1.xml.json}
+
+# Test converting stix2 to MISP format
+curl https://raw.githubusercontent.com/oasis-open/cti-stix2-json-schemas/master/examples/indicator-for-c2-ip-address.json > ./../app/files/scripts/tmp/test-stix2.json
+python3 ./../app/files/scripts/stix2/stix2misp.py ./../tmp/test-stix2.json 1 1 ./../app/files/scripts/synonymsToTagNames.json | python3 -c 'import sys; data = sys.stdin.read().strip(); print(data); sys.exit(0 if data == "1" else 1)'
+rm -f ./../app/files/scripts/tmp/{test-stix2.json,test-stix2.json.stix2}
+
+# Test converting MISP to STIX2
+python3 ./../app/files/scripts/stix2/misp2stix2.py -i event.json | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if data["success"] == 1 else 1)'
+rm -f event.json.out