From 734cfecd0e972ac38aae000d13ea2a8b895e4c22 Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Tue, 12 Feb 2019 17:11:36 +0000 Subject: [PATCH] chg: Move to latest openTaxii --- .travis.yml | 18 ++--- README.md | 74 +++++++++---------- REQUIREMENTS.txt | 7 ++ config/collections.yaml | 11 --- ...{services.yaml => data-configuration.yaml} | 20 +++++ setup.py | 11 +-- 6 files changed, 70 insertions(+), 71 deletions(-) create mode 100644 REQUIREMENTS.txt delete mode 100644 config/collections.yaml rename config/{services.yaml => data-configuration.yaml} (80%) diff --git a/.travis.yml b/.travis.yml index c8bb1b3..775c206 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,25 +1,21 @@ -dist: trusty sudo: required language: python python: - "3.5" - - "nightly" + - "3.6" services: - mysql - docker env: - - OPENTAXII_CONFIG=$TRAVIS_BUILD_DIR/config/config.travis.yaml URL="http://35.163.95.230" APIKEY=pF4Rq3JOHbYAJLMiFDqRPpLxAh3s0PakiSPKWSN5 + - OPENTAXII_CONFIG=$TRAVIS_BUILD_DIR/config/config.travis.yaml URL="http://localhost" APIKEY=testmispapikeytestmispapikeytestmispapik before_install: - sudo apt-get install mysql-server-5.6 mysql-client-core-5.6 mysql-client-5.6 libxml2-dev libxslt1-dev zlib1g-dev - - wget "https://github.com/TAXIIProject/libtaxii/archive/1.1.111.tar.gz" - - tar xf 1.1.111.tar.gz - - cd libtaxii-1.1.111 - - pip install -r requirements.txt + - pip install -r REQUIREMENTS.txt - python setup.py install - cd .. - docker pull floatingghost/misp-testable:v2.4.102 @@ -45,14 +41,10 @@ install: - mysql -u root -e "grant all on taxiiauth.* to 'taxii'@'%' identified by 'some_password';" - mysql -u root -e "grant all on taxiipersist.* to 'taxii'@'%' identified by 'some_password';" # Populate databases - - opentaxii-create-services -c config/services.yaml - - opentaxii-create-collections -c config/collections.yaml - - opentaxii-create-account -u travis -p travis + - opentaxii-sync-data config/data-configuration.yaml script: - opentaxii-run-dev & - pid=$! - sleep 15 - - cd tests - - nosetests . - - cd .. + - for file in tests/*.xml; do; taxii-push --path http://127.0.0.1:9000/services/inbox -f $file --dest my_collection --username admin --password admin; done; - kill -s INT $pid diff --git a/README.md b/README.md index e1d22c3..4e079b3 100644 --- a/README.md +++ b/README.md @@ -8,63 +8,36 @@ along with a callback for when data is sent to the TAXII Server's inbox. ## Installation -### Docker install - -For a really simple sqlite-based installation (plug and play, no persistence) - -```bash -docker pull floatingghost/misp-taxii-server -docker run -it \ - -e PERSIST_CONNECTION_STRING="sqlite:///persist.db" \ - -e AUTH_CONNECTION_STRING="sqlite:///auth.db" \ - -e MISP_URL="https://mymisp" \ - -e MISP_KEY="myapikey" \ - -e TAXII_USER=root \ - -e TAXII_PASS=root \ - -p 9000:9000 \ - floatingghost/misp-taxii-server -``` - -That'll get you set up with a basic server, but is not recommended for production. -Switch the connection strings to use an external database for that. - -This docker image currently just runs the base server with no supplimentary scripts. ### Manual install -Download the repository with ```bash -git clone --recursive https://github.com/MISP/MISP-Taxii-Server -``` - -This will also download the OpenTAXII Server, which you should install with -```bash -# There's some weird bug wherein pip can't parse >=1.1.111 -sudo pip3 install libtaxii==1.1.111 -cd OpenTAXII -sudo python3 setup.py install +git clone https://github.com/MISP/MISP-Taxii-Server +cd MISP-Taxii-Server +pip3 install -r REQUIREMENTS.txt ``` You'll then need to set up your TAXII database. As you're using MISP, you'll likely already have a MySQL environment running. -Run the following commands to create your databases ```bash mysql -u [database user] -p # Enter Database password - mysql> create database taxiiauth; - mysql> create database taxiipersist; - mysql> grant all on taxiiauth.* to 'taxii'@'%' identified by 'some_password'; - mysql> grant all on taxiipersist.* to 'taxii'@'%' identified by 'some_password'; - mysql> exit; ``` +Now configure your TAXII server + +```bash +cp config/config.default.yaml config/config.yaml +``` + Now, with that data, copy `config/config.default.yaml` over to `config/config.yaml` and open it. Edit the `db_connection` parameters to match your environment. Change `auth_api -> parameters -> secret` whilst you're here as well. + Do not forget to set your MISP server's URL and API key at the bottom. If you wish, you can edit the taxii service definitions in `services.yaml`, @@ -82,8 +55,8 @@ pip3 install mysqlclient export OPENTAXII_CONFIG=/path/to/config.yaml export PYTHONPATH=. -opentaxii-create-services -c config/services.yaml -opentaxii-create-collections -c config/collections.yaml +opentaxii-sync-data config/services.yaml +opentaxii-sync-data config/collections.yaml # Create a user account # Set the username and password to whatever you want @@ -121,6 +94,29 @@ Now you have a TAXII server hooked up to MISP, you're able to send STIX files to There is also an experimental feature to push MISP events to the TAXII server when they're published - that's in `scripts/push_published_to_taxii.py`. It seems to work, but may occasionally re-upload duplicate events to MISP. + +### Docker install + +For a really simple sqlite-based installation (plug and play, no persistence) + +```bash +docker pull floatingghost/misp-taxii-server +docker run -it \ + -e PERSIST_CONNECTION_STRING="sqlite:///persist.db" \ + -e AUTH_CONNECTION_STRING="sqlite:///auth.db" \ + -e MISP_URL="https://mymisp" \ + -e MISP_KEY="myapikey" \ + -e TAXII_USER=root \ + -e TAXII_PASS=root \ + -p 9000:9000 \ + floatingghost/misp-taxii-server +``` + +That'll get you set up with a basic server, but is not recommended for production. +Switch the connection strings to use an external database for that. + +This docker image currently just runs the base server with no supplimentary scripts. + ## Automated TAXII -> MISP Sync If you want, there is the ability to synchronise between a remote TAXII server and the local MISP server. diff --git a/REQUIREMENTS.txt b/REQUIREMENTS.txt new file mode 100644 index 0000000..869ec8b --- /dev/null +++ b/REQUIREMENTS.txt @@ -0,0 +1,7 @@ +zmq +pyaml +cabby +mysqlclient +opentaxii +-e git+https://github.com/MISP/PyMISP#egg=pymisp +-e git+https://github.com/MISP/MISP-STIX-Converter.git#egg=misp-stix-converter diff --git a/config/collections.yaml b/config/collections.yaml deleted file mode 100644 index 8652901..0000000 --- a/config/collections.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -collections: - - name: collection - available: true - accept_all_content: true - type: DATA_SET - - service_ids: - - inbox - - collection_management - - poll diff --git a/config/services.yaml b/config/data-configuration.yaml similarity index 80% rename from config/services.yaml rename to config/data-configuration.yaml index 18c0ea0..ae337ca 100644 --- a/config/services.yaml +++ b/config/data-configuration.yaml @@ -46,3 +46,23 @@ services: protocol_bindings: - urn:taxii.mitre.org:protocol:http:1.0 - urn:taxii.mitre.org:protocol:https:1.0 + +collections: + - name: my_collection + available: true + accept_all_content: true + type: DATA_SET + + service_ids: + - inbox + - collection_management + - poll + +accounts: + - username: guest + password: guest + permissions: + my_collection: read + - username: admin + password: admin + is_admin: yes diff --git a/setup.py b/setup.py index 4d0625b..6b86074 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,4 @@ -#!/usr/bin/env python3 - -# Setup script for ThreatIntel Conversion - from setuptools import setup -import os setup( name="misp_taxii_hooks", @@ -12,8 +7,8 @@ setup( author="Hannah Ward", author_email="hannah.ward2@baesystems.com", packages=['misp_taxii_hooks'], - install_requires=["zmq", "misp-stix-converter", "pymisp>=2.4.53", "pyaml>=3.11", "cabby>=0.1", "mysqlclient>=1.3.9", "nose>=1.3.7"], - scripts=["scripts/start-misp-taxii.sh", "scripts/push_published_to_taxii.py", "scripts/install-remote-server.sh", + scripts=["scripts/start-misp-taxii.sh", + "scripts/push_published_to_taxii.py", + "scripts/install-remote-server.sh", "scripts/run-taxii-poll.py"] ) -