From 67bccde9eb5b1d7285c06b3e05f0eb549ed1701f Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 28 Nov 2018 15:51:40 +0900 Subject: [PATCH 1/5] chg: [doc] Added more visual examples for pymisp --- pymisp/README.md | 8 ++++++++ pymisp/keys.py | 6 ++++++ pymisp/last.py | 43 +++++++++++++++++++++++++++++++++++++++++++ quick-start/README.md | 5 +++-- 4 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 pymisp/keys.py create mode 100755 pymisp/last.py diff --git a/pymisp/README.md b/pymisp/README.md index 861f8aa..aad3083 100644 --- a/pymisp/README.md +++ b/pymisp/README.md @@ -66,6 +66,10 @@ vim keys.py Once you are done with it, you are ready to start. +This is how **keys.py** looks: + +{% codesnippet "/pymisp/keys.py", language="python" %}{% endcodesnippet %} + ### Using PyMISP To have a better understanding of how to use PyMISP, we will have a look at one of the existing examples: add\_named\_attribute.py @@ -335,3 +339,7 @@ Allow to import OpenIOC files into MISP easily. It is also possible to set speci * attribute_treemap.py generate a tree-map showing the distribution of the attributes on the MISP instance. * tags_* : these functions help having statistics and graphs about the tag repartition. +#### Simple example on fetching the last events + +{% codesnippet "/pymisp/last.py", language="python" %}{% endcodesnippet %} + diff --git a/pymisp/keys.py b/pymisp/keys.py new file mode 100644 index 0000000..168b765 --- /dev/null +++ b/pymisp/keys.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +misp_url = 'https:///' +misp_key = 'Your MISP auth key' # The MISP auth key can be found on the MISP web interface under the automation section +misp_verifycert = True diff --git a/pymisp/last.py b/pymisp/last.py new file mode 100755 index 0000000..ed07be5 --- /dev/null +++ b/pymisp/last.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key, misp_verifycert +import argparse +import os +import json + + +# Usage for pipe masters: ./last.py -l 5h | jq . + + +def init(url, key): + return PyMISP(url, key, misp_verifycert, 'json') + + +def download_last(m, last, out=None): + result = m.download_last(last) + if out is None: + if 'response' in result: + print(json.dumps(result['response'])) + else: + print('No results for that time period') + exit(0) + else: + with open(out, 'w') as f: + f.write(json.dumps(result['response'])) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Download latest events from a MISP instance.') + parser.add_argument("-l", "--last", required=True, help="can be defined in days, hours, minutes (for example 5d or 12h or 30m).") + parser.add_argument("-o", "--output", help="Output file") + + args = parser.parse_args() + + if args.output is not None and os.path.exists(args.output): + print('Output file already exists, abord.') + exit(0) + + misp = init(misp_url, misp_key) + + download_last(misp, args.last, args.output) diff --git a/quick-start/README.md b/quick-start/README.md index b7cd189..791ae14 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -25,7 +25,10 @@ MISP default credentials: 1. [Change site admin password](https://misp.gitbooks.io/misp-book/content/quick-start/#password-policy) 2. [Activate Feeds](https://www.circl.lu/doc/misp/managing-feeds/) 3. [Setup your User](https://misp.gitbooks.io/misp-book/content/user-management/#first-run-of-the-system) +3.1 Designate a Site Admin and an Org Admin +3.2 Add some contributing users and assign the corresponding Roles 4. [MISP Administration](https://www.circl.lu/doc/misp/administration/) +4.1 Edit your first organisations' name ## Password Policy - [12]: Ensure that the password is at least 12 characters long @@ -33,8 +36,6 @@ MISP default credentials: - [0-9| ]: includes a digit or a special character - [a-z]: at least one lower-case character. -**Last Updated: April, 2018** - If you need a password generator use: - Ubuntu / Debian: [pwgen](https://linux.die.net/man/1/pwgen) - Website: [LastPass PW Generator](https://lastpass.com/generatepassword.php) From 4c3c7bee0a2fc3b96a1b8e007a4433067606bdca Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 28 Nov 2018 16:34:58 +0900 Subject: [PATCH 2/5] chg: [doc] Misp Backup documented. Misp threat_id documented --- administration/README.md | 89 +++++++++++++++++++++++++++++++++++----- automation/README.md | 8 ++++ 2 files changed, 87 insertions(+), 10 deletions(-) diff --git a/administration/README.md b/administration/README.md index bcde2c5..17dfc74 100644 --- a/administration/README.md +++ b/administration/README.md @@ -77,7 +77,7 @@ To list all current users of the system, just click on List Users under the admi * **Display the user:** Display all user's information.
![display user.](figures/display_user.png) -#### Contacting a user: +#### Contacting a user Site admins can use the "Contact users" feature to send all or individual user an e-mail. Users that have a GnuPG key set will receive their e-mails encrypted. When clicking this button on the left, you'll be presented with a form that allows you to specify the type of the e-mail, who it should reach and what the content is using the following options: @@ -98,7 +98,7 @@ Keep in mind that all e-mails sent through this system, in addition to your own Each users belongs to an organisation. As admin, you can manage these organisations. -#### Adding a new organisation: +#### Adding a new organisation To add a new organisation, click on the "Add Organisation" button in the administration menu to the left and fill out the following fields in the view that is loaded: @@ -113,7 +113,7 @@ To add a new organisation, click on the "Add Organisation" button in the adminis * **Type of organisation:** Define the type of the organisation. * **Contacts:** You can add some contact details for the organisation. -#### Listing all organisation: +#### Listing all organisation To list all current organisation of the system, just click on List Organisations under the administration menu to the left. There are 3 tabs in this view to filter local organisations, remote organisations or both. The default view displays local organisations. For all views the following columns of information are available: @@ -138,7 +138,7 @@ To list all current organisation of the system, just click on List Organisations * **View Organisation:** Use this option to display information about the selected organisation. In this view, you can display the user belongs to this organisation and events published by organisation. ![View organisation.](figures/view_org.png) -#### Merge organisations: +#### Merge organisations Merge Organisation menu is available only in the organisation view, under the left menu. Merge one organisation to another will transfer all users and data from one to another. On the left the organisation to merge, on the right the target one. ![Merge organisations.](figures/merge_org.png) @@ -168,13 +168,13 @@ The extra permissions are defined below: * **Perm Regexp Access:** Allows users who have this permission enabled to edit the regular expression table. Be careful when giving out this permission, incorrect regular expressions can be very harmful (infinite loops, loss of data, etc.). * **Perm Template:** Grant access to create or modify templates. -#### Adding a new role: +#### Adding a new role When creating a new role, you will have to enter a name for the role to be created and set up permissions (as described above) using the drop-down menu and related check-boxes. ![Add a new role.](figures/add_role.png) -#### Listing roles: +#### Listing roles By clicking on the List Roles button, you can view a list of all currently registered roles and a list of the permission flags enabled for each. In addition, you can find buttons that allow you to edit and delete said roles. Keep in mind that you will need to first remove every member from a role before you can delete it. @@ -239,6 +239,46 @@ The workers tab shows a list of the workers that MISP can use. You can restart w * **Worker Id**: The ID is made up of the machine name, the PID of the worker and the queue it monitors. * **Status**: Displays OK if the worker is running. If the _schdlr_ worker is the only one not running, make sure that you copy the config file into the cakeresque directory as described in the INSTALL.txt documentation. +#### Worker types + +**cache** + + Role: + Interdependence: + + +**default** + + Role: + Interdependence: + + +**email** + + Role: + Interdependence: + + +**prio** + + Role: + Interdependence: + + +**scheduler** + Role: + Interdependence: + +#### Workers dead + +Even if the workers are dead, any actions related to them are on-hold. Nothing is lost. +Simply restarting the worker will resume any operations. + +You can either relaunch them via the UI or manually by running **sudo -u www-data bash /var/www/MISP/app/Console/worker/start.sh +** on the CLI. +For reference, below is the script in question. + +{% codesnippet "/automation/start_workers.sh", language="bash" %}{% endcodesnippet %} ### Import Blacklist @@ -267,11 +307,11 @@ Administrators can add, edit or delete regular expression rules, these "expressi The signature whitelist view, accessible through the administration menu on the left, allows administrators to create and maintain a list of addresses that are whitelisted from ever being added to the NIDS signatures. Addresses listed here will be commented out when exporting the NIDS list. -#### Whitelisting an address: +#### Whitelisting an address While in the whitelist view, click on New Whitelist on the left to bring up the "add whitelist" view to add a new address. -#### Managing the list: +#### Managing the list When viewing the list of whitelisted addresses, the following data is shown: The ID of the whitelist entry (assigned automatically when a new address is added), the address itself that is being whitelisted and a set of controls allowing you to delete the entry or edit the address. @@ -292,7 +332,7 @@ Generally, the following actions are logged: * **Regexp:** Creation, deletion, modification -#### Browsing logs: +#### Browsing logs Listing all the log entries will display the following columns generated by the users of your organisation (or all organisations in the case of site admins): @@ -604,7 +644,7 @@ The Jobs tab gives you an overview on any currently running jobs or jobs that we Typically this is one of the places you would turn to even some background process might not complete as expected to get an indication on any issues related to user initiated Jobs. For ease of use, you can filter the Jobs by 'All', 'Default', 'Email', 'Cache' -##### Todo: Explain differences Default, Email, Cache +##### TODO: Explain differences Default, Email, Cache You can also purge the entries, either only by completed status or purge all. This is not automated and needs to be done manually. @@ -624,3 +664,32 @@ If you have a dedicated and concious MISP Site Admin she can keep an eye on the For better performance please use a real scheduler like your systems' crontab. As a rule of thumb: If you can click on it, MISP can automate it. + +### MISP Backup + +Currently there exists this backup script simply called [misp-backup.sh](https://github.com/MISP/MISP/tree/2.4/tools/misp-backup) + +All you need is to copy the the sample config and make sure it is correct. Then launch the script. + +```bash +cd /var/www/MISP/tools/misp-backup +sudo -u www-data cp misp-backup.conf.sample misp-backup.conf +sudo ./misp-backup.sh +``` + +Script output: +```bash +/var/www/MISP/tools/misp-backup   2.4 ● $ sudo ./misp-backup.sh +File ./misp-backup.conf exists. +copy of org images and other custom images +MySQL Dump +/var/www/MISP/tools/misp-backup +MISP Backup Completed, OutputDir: /opt/backup +FileName: MISP-Backup-20181128_163215.tar.gz +FullName: /opt/backup/MISP-Backup-20181128_163214.tar.gz +``` +### MISP Restore + +In a similar fashion you can restore your MISP instance with the **misp-restore.sh** script. +Read the script for details. + diff --git a/automation/README.md b/automation/README.md index d510b3e..ec7c9b9 100644 --- a/automation/README.md +++ b/automation/README.md @@ -1641,6 +1641,14 @@ The following optional parameters are expected:
This will populate the comment field of any attribute created using this API.
+The **threat_level_id** is mapped as such: + +``` +0 = high +1 = medium +2 = low +3 = undefined +``` ## Proposals API From b20f8d4a34700c52a41d8e3378fd7e3eec33d71d Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 28 Nov 2018 16:37:46 +0900 Subject: [PATCH 3/5] chg: [doc] Added calibre mention for PDF generation. --- USAGE.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/USAGE.md b/USAGE.md index 27f5d40..92a9359 100644 --- a/USAGE.md +++ b/USAGE.md @@ -48,9 +48,10 @@ Tested on: *Ubuntu 18.04 LTS* *Debian 9.5/sid/testing* git clone git@github.com:MISP/misp-book.git cd misp-book curl -sL https://deb.nodesource.com/setup_10.x | sudo -E bash - -sudo apt-get install -y nodejs -sudo apt-get install -y build-essential +sudo apt install -y nodejs +sudo apt install -y build-essential sudo apt install -y pkg-config libcairo2-dev libgif-dev libjpeg-dev +sudo apt install -y calibre # for generating PDFs npm install gitbook git+https://github.com/SteveClement/plugin-autocover.git gitbook-plugin-github gitbook-plugin-toc gitbook-plugin-anchors gitbook-plugin-alerts gitbook-plugin-search gitbook-plugin-gist gitbook-plugin-advanced-emoji gitbook-plugin-sitemap gitbook-plugin-codesnippet gitbook-plugin-image-class gitbook-plugin-last-modified gitbook-plugin-fontsettings sudo npm install gitbook-cli -g From cdcc5b650ec0932f2e71a898a16b857d6297bde0 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 28 Nov 2018 16:39:31 +0900 Subject: [PATCH 4/5] new: [doc] Added worker script --- automation/start_workers.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 automation/start_workers.sh diff --git a/automation/start_workers.sh b/automation/start_workers.sh new file mode 100755 index 0000000..9b95861 --- /dev/null +++ b/automation/start_workers.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# Check if run as root +if [ "$EUID" -eq 0 ]; then + echo "Please DO NOT run the worker script as root" + exit 1 +fi + +# Extract base directory where this script is and cd into it +cd "${0%/*}" +../cake CakeResque.CakeResque stop --all +../cake CakeResque.CakeResque start --interval 5 --queue default +../cake CakeResque.CakeResque start --interval 5 --queue prio +../cake CakeResque.CakeResque start --interval 5 --queue cache +../cake CakeResque.CakeResque start --interval 5 --queue email +../cake CakeResque.CakeResque startscheduler --interval 5 + +exit 0 From e3aa4894b664ecd5eb55634cd3dab2445d00d120 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Fri, 12 Apr 2019 18:58:42 +0900 Subject: [PATCH 5/5] new: [doc] Added a few more GLOSSARY Items new: [doc] Added External Connector pages --- GLOSSARY.md | 8 ++++++++ SUMMARY.md | 3 ++- connectors/README.md | 7 +++++++ 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 connectors/README.md diff --git a/GLOSSARY.md b/GLOSSARY.md index ecbe45e..1995c15 100644 --- a/GLOSSARY.md +++ b/GLOSSARY.md @@ -39,6 +39,9 @@ Attributes in MISP can be network indicators (e.g. IP address), system indicator ◦ An IDS flag on an attribute allows to determine if an attribute can be automated (such as being exported as an IDS ruleset or used for detection). If the IDS flag is not present, the attribute can be useful for contextualisation only. +## Observable +Some other SIEMs or formats (STIX) use the term observable. This is the same as an attribute in MISP-speak. + ## MISP Event MISP events are encapsulations for contextually linked information @@ -149,6 +152,11 @@ You can add new Roles depending on your use case. The following permissions can ## Scheduled Tasks Certain common tasks can be scheduled for a later execution or for regular recurring executions. These tasks currently include caching all of the export formats, pulling from all eligible instances and pushing to all eligible instances. +## Standard MISP Install +Any MISP instance install that is strongly aligned with our [official install guides](https://misp.github.io/MISP/). +This is mostly to make sure you have a similar folder structure, /var/www/MISP for an Ubuntu Server Install. +It will also be easier to debug any Web Server issues or other system related problems. + ## Sync User A user of a role that grants sync permissions, these users (and their authentication keys) are used to serve as the points of connection between instances. Events pushed to an instance are pushed to a sync user, who then creates the events on the remote instance. Events pulled are added by the sync user that is used to connect the remote instance to your instance. As an administrator, keep in mind that a sync user needs auth key and publish permissions, has to have undergone the mandatory password change and has to have accepted the Terms of Use in order for the sync to work. Please make sure that all of these steps are taken before attempting to push or pull. diff --git a/SUMMARY.md b/SUMMARY.md index 25bfc37..a8dd090 100755 --- a/SUMMARY.md +++ b/SUMMARY.md @@ -20,9 +20,10 @@ * [Sightings](sightings/README.md) - in progress * [Warning lists](warninglists/README.md) - in progress * [Notice lists](noticelists/README.md) - in progress -* [Modules](modules/README.md) - in progress * [Categories and Types](categories-and-types/README.md) * [Synchronisation/Sharing](sharing/README.md) +* [External Connectors](connectors/README.md) +* [Modules](modules/README.md) - in progress * [ZeroMQ - MISP publish-subscribe](misp-zmq/README.md) * [Translations - i18n & l10n](translation/README.md) * [FAQ](faq/README.md) diff --git a/connectors/README.md b/connectors/README.md new file mode 100644 index 0000000..2e58adf --- /dev/null +++ b/connectors/README.md @@ -0,0 +1,7 @@ +# External Connectors + +Below you will find various tweaks and tips when integrating 3rd party connectors. + +## Microsoft Azure Sentinel + +[Azure Sentinel](https://azure.microsoft.com/en-us/services/azure-sentinel/)