From 9d63e427e6ecfdf9ba06c13e47eaaef28271f777 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Thu, 9 Apr 2020 07:59:20 +0200
Subject: [PATCH 01/18] new: [dashboard] COVID active cases backported from
 widget collections

---
 app/Lib/Dashboard/CsseCovidTrendsWidget.php | 40 ++++++++++++++++-----
 app/Lib/Dashboard/CsseCovidWidget.php       | 15 ++++++--
 2 files changed, 43 insertions(+), 12 deletions(-)

diff --git a/app/Lib/Dashboard/CsseCovidTrendsWidget.php b/app/Lib/Dashboard/CsseCovidTrendsWidget.php
index 298276206..f9c9ef279 100644
--- a/app/Lib/Dashboard/CsseCovidTrendsWidget.php
+++ b/app/Lib/Dashboard/CsseCovidTrendsWidget.php
@@ -8,7 +8,7 @@ class CsseCovidTrendsWidget
     public $height = 5;
     public $params = array(
         'event_info' => 'Substring included in the info field of relevant CSSE COVID-19 events.',
-        'type' => 'Type of data used for the widget - confirmed (default), death, recovered, mortality.',
+        'type' => 'Type of data used for the widget - confirmed (default), death, recovered, mortality, active.',
         'insight' => 'Insight type - raw (default), growth, percent.',
         'countries' => 'List of countries to be included (using the names used by the reports, such as Belgium, US, Germany).',
         'timeframe' => 'Timeframe for events taken into account in days (going back from now, using the date field, default 10).'
@@ -18,11 +18,17 @@ class CsseCovidTrendsWidget
 '{
     "event_info": "%CSSE COVID-19 daily report%",
     "type": "confirmed",
-    "insight": "growth",
+    "insight": "raw",
     "countries": ["Luxembourg", "Germany", "Belgium", "France"],
     "timeframe": 20
 }';
-    //public $cacheLifetime = 600;
+
+    private $__countryAliases = array(
+        'Mainland China' => 'China',
+        'Korea, South' => 'South Korea'
+    );
+
+    public $cacheLifetime = 600;
     public $autoRefreshDelay = false;
 
     private $__countries = array();
@@ -88,7 +94,8 @@ class CsseCovidTrendsWidget
                 'confirmed' => 'confirmed cases',
                 'death' => 'mortalities',
                 'recovered' => 'recoveries',
-                'mortality' => 'mortality rate'
+                'mortality' => 'mortality rate',
+                'active' => 'active cases'
             )
         );
         $data['formula'] = sprintf(
@@ -136,21 +143,23 @@ class CsseCovidTrendsWidget
         }
         if (!empty($options['insight']) && $options['insight'] !== 'raw') {
             if ($options['insight'] == 'growth') {
-                foreach ($data as $k => &$countryData) {
+                foreach ($data as $k => $countryData) {
                     foreach ($countryData as $type => &$value) {
-                        if (empty($previous[$k][$type])) {
-                            $previous[$k][$type] = 0;
+                        if (!isset($previous[$k][$type])) {
+                            $previous[$k][$type] = $data[$k][$type];
                         }
                         $data[$k]['growth'] = $data[$k][$type] - $previous[$k][$type];
                     }
                 }
             } else if ($options['insight'] == 'percent') {
-                foreach ($data as $k => &$countryData) {
+                foreach ($data as $k => $countryData) {
                     foreach ($countryData as $type => &$value) {
                         if (empty($previous[$k][$type])) {
                             $previous[$k][$type] = $data[$k][$type];
                         }
-                        $data[$k]['percent'] = ($data[$k][$type] - $previous[$k][$type]) / $previous[$k][$type];
+                        if (!empty($previous[$k][$type])) {
+                            $data[$k]['percent'] = 100 * ($data[$k][$type] - $previous[$k][$type]) / $previous[$k][$type];
+                        }
                     }
                 }
             }
@@ -175,6 +184,15 @@ class CsseCovidTrendsWidget
                     $data[$country][$type] = (empty($data[$country][$type]) ? $temp[$type] : ($data[$country][$type] + $temp[$type]));
                 }
             }
+        } else if ($options['type'] === 'active') {
+            if (empty($data[$country]['active'])) {
+                $data[$country]['active'] = 0;
+            }
+            $data[$country]['active'] =
+                $data[$country]['active'] +
+                (empty($temp['confirmed']) ? 0 : $temp['confirmed']) -
+                (empty($temp['death']) ? 0 : $temp['death']) -
+                (empty($temp['recovered']) ? 0 : $temp['recovered']);
         } else {
             $type = $options['type'];
             if (!empty($temp[$type])) {
@@ -192,6 +210,10 @@ class CsseCovidTrendsWidget
             if (in_array($attribute['object_relation'], $validFields)) {
                 if ($attribute['object_relation'] !== 'country-region') {
                     $attribute['value'] = intval($attribute['value']);
+                } else {
+                    if (isset($this->__countryAliases[$attribute['value']])) {
+                        $attribute['value'] = $this->__countryAliases[$attribute['value']];
+                    }
                 }
                 $temp[$attribute['object_relation']] = $attribute['value'];
             }
diff --git a/app/Lib/Dashboard/CsseCovidWidget.php b/app/Lib/Dashboard/CsseCovidWidget.php
index c0a9657d8..76d890f85 100644
--- a/app/Lib/Dashboard/CsseCovidWidget.php
+++ b/app/Lib/Dashboard/CsseCovidWidget.php
@@ -8,7 +8,7 @@ class CsseCovidWidget
     public $height = 4;
     public $params = array(
         'event_info' => 'Substring included in the info field of relevant CSSE COVID-19 events.',
-        'type' => 'Type of data used for the widget (confirmed, death, recovered, mortality).',
+        'type' => 'Type of data used for the widget (confirmed, death, recovered, mortality, active).',
         'logarithmic' => 'Use a log10 scale for the graph (set via 0/1).',
         'relative' => 'Take the country\'s population size into account (count / 10M)'
     );
@@ -27,10 +27,10 @@ class CsseCovidWidget
         'Holy See' => 'Vatican',
         'Congo (Kinshasa)' => 'Democratic Republic of Congo',
         'Taiwan*' => 'Taiwan',
-        'Korea, South' => 'South Korea'
+        'Korea, South' => 'South Korea',
+        'Mainland China' => 'China'
     );
 
-
     private $__populationData = array();
 
     public function handler($user, $options = array())
@@ -156,6 +156,15 @@ class CsseCovidWidget
                     $data[$country][$type] = (empty($data[$country][$type]) ? $temp[$type] : ($data[$country][$type] + $temp[$type]));
                 }
             }
+        } else if ($options['type'] === 'active') {
+            if (empty($data[$country]['active'])) {
+                $data[$country]['active'] = 0;
+            }
+            $data[$country]['active'] =
+                $data[$country]['active'] +
+                (empty($temp['confirmed']) ? 0 : $temp['confirmed']) -
+                (empty($temp['death']) ? 0 : $temp['death']) -
+                (empty($temp['recovered']) ? 0 : $temp['recovered']);
         } else {
             $type = $options['type'];
             if (!empty($temp[$type])) {

From 7b16a4b5c90ef2726764c49e2b94b20fd9880578 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 13 Apr 2020 10:09:39 +0200
Subject: [PATCH 02/18] chg: [taxonomies] updated

---
 app/files/taxonomies | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/taxonomies b/app/files/taxonomies
index d577ad875..28e7cb79f 160000
--- a/app/files/taxonomies
+++ b/app/files/taxonomies
@@ -1 +1 @@
-Subproject commit d577ad8758713e4d7c0523bbe2bead64c941ebdb
+Subproject commit 28e7cb79f0ec603c232857a3bf7dca519d02cfa1

From 36f32ae95c715fe17aa6d2df86f1ad6cd062ec05 Mon Sep 17 00:00:00 2001
From: Christophe Vandeplas <christophe@vandeplas.com>
Date: Mon, 13 Apr 2020 22:16:09 +0200
Subject: [PATCH 03/18] new: [tool] MISP to Slack messaging using ZMQ

---
 tools/misp-zmq/slackbot.py                 | 211 +++++++++++++++++++++
 tools/misp-zmq/slackbot_settings.py.sample |  24 +++
 tools/misp-zmq/sub.py                      |  15 +-
 3 files changed, 242 insertions(+), 8 deletions(-)
 create mode 100644 tools/misp-zmq/slackbot.py
 create mode 100644 tools/misp-zmq/slackbot_settings.py.sample

diff --git a/tools/misp-zmq/slackbot.py b/tools/misp-zmq/slackbot.py
new file mode 100644
index 000000000..854b1ff7f
--- /dev/null
+++ b/tools/misp-zmq/slackbot.py
@@ -0,0 +1,211 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+'''
+### MISP to Slack ####
+ZMQ client to post events, attributes or sighting updates from a MISP instance to a slack channel.
+
+This tool is part of the MISP core project and released under the GNU Affero
+General Public License v3.0
+
+Copyright (C) 2020 Christophe Vandeplas
+
+For instructions on creating your BOT, please read: https://api.slack.com/bot-users
+Your bot will need the permissions:
+- channels:join
+- chat:write
+- users:write
+
+WARNING WARNING - THIS SCRIPT DOES NOT MAGICALLY RESPECT ACLs
+MAKE SURE YOU SET THE RIGHT FILTERS IN THE SETTINGS
+'''
+
+
+import argparse
+import sys
+import time
+import zmq
+import json
+try:
+    import slack
+except ImportError:
+    exit("Missing slackclient dependency. Please 'pip3 install slackclient'")
+try:
+    from slackbot_settings import channel_name, slack_token, misp_url, misp_is_public, allowed_distributions, allowed_sharing_groups, max_value_len, include_attr, include_obj
+except ImportError:
+    exit("Missing slackbot_settings.py. Please create from 'slackbot_settings.py.sample'")
+
+
+def sanitize_value(s):
+    # very dirty cleanup
+    s = s.replace('http', 'hxxp')
+    s = s.replace('.', '[.]')
+    s = s.replace('@', '[AT]')
+    # truncate long strings
+    return (s[:max_value_len] + '..') if len(s) > max_value_len else s
+
+
+def gen_attrs_text(attrs):
+    attrs_text_lst = []
+    type_value_mapping = {}
+    for a in attrs:
+        try:
+            type_value_mapping[a['type']].add(sanitize_value(a['value']))
+        except Exception:
+            type_value_mapping[a['type']] = set()
+            type_value_mapping[a['type']].add(sanitize_value(a['value']))
+    for k, v in type_value_mapping.items():
+        attrs_text_lst.append(f"- *{k}*: {','.join(v)}")
+    attrs_text = '\n'.join(attrs_text_lst)
+    return attrs_text
+
+
+def publish_event(e):
+    cnt_attr = len(e.get('Attribute') or '')
+    cnt_obj = len(e.get('Object') or '')
+    cnt_tags = len(e.get('Tag') or '')
+    url = misp_url + '/events/view/' + e['id']
+    zmq_message_short = f"New MISP event '{e['info']}' with {cnt_attr} attributes, {cnt_obj} objects and {cnt_tags} tags."
+
+    image_url = 'https://raw.githubusercontent.com/MISP/MISP/2.4/docs/img/misp.png'
+    if misp_is_public:
+        image_url = f"{misp_url}/img/orgs/{e['Orgc']['name']}.png"
+
+    zmq_message_blocks = [
+        {
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": f"*A new MISP <{url}|event> has been published:*\n"
+                        f"Title: {e['info']}\n"
+                        f"Date: {e['date']}\n"
+                        f"Threat Level: {e['threat_level_id']}\n"
+                        f"Contains {cnt_attr} attributes, {cnt_obj} objects and {cnt_tags} tags\n"
+                        f"Full event: <{url}|{url}>"
+            },
+            "accessory": {
+                "type": "image",
+                "image_url": image_url,
+                "alt_text": "MISP or org logo"
+            }
+        }
+    ]
+
+    if 'Tag' in e:
+        tag_block = {
+            "type": "actions",
+            "elements": [
+            ]
+        }
+        tags = set([t['name'] for t in e['Tag']])
+        for a in e['Attribute']:
+            if 'Tag' in a:
+                for t in a['Tag']:
+                    tags.add(t['name'])
+        for o in e['Object']:
+            for a in o['Attribute']:
+                if 'Tag' in a:
+                    for t in a['Tag']:
+                        tags.add(t['name'])
+
+        tags = sorted(tags)
+        for t in tags:
+            t = t.replace('misp-galaxy:', '').replace('mitre-', '')
+            tag_block['elements'].append({
+                "type": "button",
+                "text": {
+                    "type": "plain_text",
+                    "text": t
+                },
+                "value": "#"
+            })
+        zmq_message_blocks.append(tag_block)
+
+    # List attributes
+    if include_attr:
+        zmq_message_blocks.append({"type": "divider"})
+        attrs_text = gen_attrs_text(e['Attribute'])
+        if attrs_text:
+            zmq_message_blocks.append(
+                {
+                    "type": "section",
+                    "text": {
+                        "type": "mrkdwn",
+                        "text": f"*Attributes:*\n{attrs_text}"
+                    }
+                }
+            )
+    # List Objects
+    if include_obj:
+        zmq_message_blocks.append({"type": "divider"})
+        for o in e['Object']:
+            attrs_text = gen_attrs_text(o['Attribute'])
+            if attrs_text:
+                # print(json.dumps(o, indent=2))
+                zmq_message_blocks.append(
+                    {
+                        "type": "section",
+                        "text": {
+                            "type": "mrkdwn",
+                            "text": f"*{o['name'].capitalize()} object:*\n{attrs_text}"
+                        }
+                    }
+                )
+
+    # Send the message
+    client = slack.WebClient(token=slack_token)
+    client.users_setPresence(presence='auto')
+    channel = client.channels_join(name=channel_name)
+    client.chat_postMessage(
+        channel=channel['channel']['id'],
+        text=zmq_message_short,
+        blocks=zmq_message_blocks
+    )
+
+
+parser = argparse.ArgumentParser(description='MISP to Slack bot - ZMQ client to gather events, attributes and sighting updates from a MISP instance')
+parser.add_argument("-p", "--port", default="50000", help='set TCP port of the MISP ZMQ (default: 50000)')
+parser.add_argument("-r", "--host", default="127.0.0.1", help='set host of the MISP ZMQ (default: 127.0.0.1)')
+parser.add_argument("-t", "--sleep", default=0.1, help='sleep time (default: 0.1)', type=int)
+args = parser.parse_args()
+
+port = args.port
+host = args.host
+context = zmq.Context()
+socket = context.socket(zmq.SUB)
+socket.connect("tcp://%s:%s" % (host, port))
+socket.setsockopt(zmq.SUBSCRIBE, b'')
+
+poller = zmq.Poller()
+poller.register(socket, zmq.POLLIN)
+
+while True:
+    socks = dict(poller.poll(timeout=None))
+    if socket in socks and socks[socket] == zmq.POLLIN:
+            message = socket.recv()
+            topic, s, m = message.decode('utf-8').partition(" ")
+
+            try:
+                m_json = json.loads(m)
+            except Exception:
+                sys.stderr.write(f'Ignoring non-json message: {m}')
+                time.sleep(args.sleep)
+                continue
+
+            if 'status' in m_json:
+                pass
+            elif 'Event' in m_json:
+                # print(m_json)
+                e = m_json['Event']
+                if '*' in allowed_distributions or \
+                   (e['distribution'] in allowed_distributions and (
+                        e['distribution'] != '5' or (
+                            '*' in allowed_sharing_groups or e['sharing_group_id'] in allowed_sharing_groups)
+                   )):
+                    print(f"Publishing event {e['id']} on slack")
+                    publish_event(e)
+                else:
+                    print(f"Ignoring event {e['id']} as it has a filtered distribution.")
+            else:
+                print(f'Non supported message: {m}')
+            time.sleep(args.sleep)
diff --git a/tools/misp-zmq/slackbot_settings.py.sample b/tools/misp-zmq/slackbot_settings.py.sample
new file mode 100644
index 000000000..a254e2faa
--- /dev/null
+++ b/tools/misp-zmq/slackbot_settings.py.sample
@@ -0,0 +1,24 @@
+'''
+For instructions on creating your BOT, please read: https://api.slack.com/bot-users
+Your bot will need the permissions:
+- channels:join
+- chat:write
+- users:write
+
+WARNING WARNING - THIS SCRIPT DOES NOT MAGICALLY RESPECT ACLs
+MAKE SURE YOU SET THE RIGHT FILTERS IN THE SETTINGS
+'''
+
+channel_name = '#name'
+slack_token = ''
+
+misp_url = 'https://192.168.1.1'
+misp_is_public = True   # set to False if your MISP instance is on a non-internet reachable location. Shows the org icon of the event owner. Otherwise shows the MISP logo.
+
+# filter for confidentiality
+allowed_distributions = ['0', '1', '2', '3', '4']  # * = all, 0/ my org only, 1/ this community, 2/ connected communities, 3/ all communities, 4/ sharing group
+allowed_sharing_groups = ['*']  # put here the sharing_group_ids that you allow
+
+max_value_len = 25  # truncate values longer than X chars
+include_attr = True  # include attributes in the message
+include_obj = True   # include objects in the message
diff --git a/tools/misp-zmq/sub.py b/tools/misp-zmq/sub.py
index 73b8d21f3..17330a89a 100644
--- a/tools/misp-zmq/sub.py
+++ b/tools/misp-zmq/sub.py
@@ -17,11 +17,11 @@ import pprint
 pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr)
 
 parser = argparse.ArgumentParser(description='Generic ZMQ client to gather events, attributes and sighting updates from a MISP instance')
-parser.add_argument("-s","--stats", default=False, action='store_true', help='print regular statistics on stderr')
-parser.add_argument("-p","--port", default="50000", help='set TCP port of the MISP ZMQ (default: 50000)')
-parser.add_argument("-r","--host", default="127.0.0.1", help='set host of the MISP ZMQ (default: 127.0.0.1)')
-parser.add_argument("-o","--only", action="append", default=None, help="set filter (misp_json, misp_json_event, misp_json_attribute or misp_json_sighting) to limit the output a specific type (default: no filter)")
-parser.add_argument("-t","--sleep", default=0.1, help='sleep time (default: 0.1)', type=int)
+parser.add_argument("-s", "--stats", default=False, action='store_true', help='print regular statistics on stderr')
+parser.add_argument("-p", "--port", default="50000", help='set TCP port of the MISP ZMQ (default: 50000)')
+parser.add_argument("-r", "--host", default="127.0.0.1", help='set host of the MISP ZMQ (default: 127.0.0.1)')
+parser.add_argument("-o", "--only", action="append", default=None, help="set filter (misp_json, misp_json_event, misp_json_attribute or misp_json_sighting) to limit the output a specific type (default: no filter)")
+parser.add_argument("-t", "--sleep", default=0.1, help='sleep time (default: 0.1)', type=int)
 args = parser.parse_args()
 
 if args.only is not None:
@@ -35,7 +35,7 @@ port = args.port
 host = args.host
 context = zmq.Context()
 socket = context.socket(zmq.SUB)
-socket.connect ("tcp://%s:%s" % (host, port))
+socket.connect("tcp://%s:%s" % (host, port))
 socket.setsockopt(zmq.SUBSCRIBE, b'')
 
 poller = zmq.Poller()
@@ -52,9 +52,8 @@ while True:
             if args.only:
                 if topic not in filters:
                         continue
-            print (m)
+            print(m)
             if args.stats:
                 stats[topic] = stats.get(topic, 0) + 1
                 pp.pprint(stats)
             time.sleep(args.sleep)
-

From 8a4d9226abf9ed4c7da2ddd6c2bd3760f7bd01b2 Mon Sep 17 00:00:00 2001
From: Christophe Vandeplas <christophe@vandeplas.com>
Date: Mon, 13 Apr 2020 22:16:09 +0200
Subject: [PATCH 04/18] new: [tool] MISP to Slack messaging using ZMQ

---
 tools/misp-zmq/slackbot.py                 | 211 +++++++++++++++++++++
 tools/misp-zmq/slackbot_settings.py.sample |  24 +++
 tools/misp-zmq/sub.py                      |  15 +-
 3 files changed, 242 insertions(+), 8 deletions(-)
 create mode 100644 tools/misp-zmq/slackbot.py
 create mode 100644 tools/misp-zmq/slackbot_settings.py.sample

diff --git a/tools/misp-zmq/slackbot.py b/tools/misp-zmq/slackbot.py
new file mode 100644
index 000000000..854b1ff7f
--- /dev/null
+++ b/tools/misp-zmq/slackbot.py
@@ -0,0 +1,211 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+'''
+### MISP to Slack ####
+ZMQ client to post events, attributes or sighting updates from a MISP instance to a slack channel.
+
+This tool is part of the MISP core project and released under the GNU Affero
+General Public License v3.0
+
+Copyright (C) 2020 Christophe Vandeplas
+
+For instructions on creating your BOT, please read: https://api.slack.com/bot-users
+Your bot will need the permissions:
+- channels:join
+- chat:write
+- users:write
+
+WARNING WARNING - THIS SCRIPT DOES NOT MAGICALLY RESPECT ACLs
+MAKE SURE YOU SET THE RIGHT FILTERS IN THE SETTINGS
+'''
+
+
+import argparse
+import sys
+import time
+import zmq
+import json
+try:
+    import slack
+except ImportError:
+    exit("Missing slackclient dependency. Please 'pip3 install slackclient'")
+try:
+    from slackbot_settings import channel_name, slack_token, misp_url, misp_is_public, allowed_distributions, allowed_sharing_groups, max_value_len, include_attr, include_obj
+except ImportError:
+    exit("Missing slackbot_settings.py. Please create from 'slackbot_settings.py.sample'")
+
+
+def sanitize_value(s):
+    # very dirty cleanup
+    s = s.replace('http', 'hxxp')
+    s = s.replace('.', '[.]')
+    s = s.replace('@', '[AT]')
+    # truncate long strings
+    return (s[:max_value_len] + '..') if len(s) > max_value_len else s
+
+
+def gen_attrs_text(attrs):
+    attrs_text_lst = []
+    type_value_mapping = {}
+    for a in attrs:
+        try:
+            type_value_mapping[a['type']].add(sanitize_value(a['value']))
+        except Exception:
+            type_value_mapping[a['type']] = set()
+            type_value_mapping[a['type']].add(sanitize_value(a['value']))
+    for k, v in type_value_mapping.items():
+        attrs_text_lst.append(f"- *{k}*: {','.join(v)}")
+    attrs_text = '\n'.join(attrs_text_lst)
+    return attrs_text
+
+
+def publish_event(e):
+    cnt_attr = len(e.get('Attribute') or '')
+    cnt_obj = len(e.get('Object') or '')
+    cnt_tags = len(e.get('Tag') or '')
+    url = misp_url + '/events/view/' + e['id']
+    zmq_message_short = f"New MISP event '{e['info']}' with {cnt_attr} attributes, {cnt_obj} objects and {cnt_tags} tags."
+
+    image_url = 'https://raw.githubusercontent.com/MISP/MISP/2.4/docs/img/misp.png'
+    if misp_is_public:
+        image_url = f"{misp_url}/img/orgs/{e['Orgc']['name']}.png"
+
+    zmq_message_blocks = [
+        {
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": f"*A new MISP <{url}|event> has been published:*\n"
+                        f"Title: {e['info']}\n"
+                        f"Date: {e['date']}\n"
+                        f"Threat Level: {e['threat_level_id']}\n"
+                        f"Contains {cnt_attr} attributes, {cnt_obj} objects and {cnt_tags} tags\n"
+                        f"Full event: <{url}|{url}>"
+            },
+            "accessory": {
+                "type": "image",
+                "image_url": image_url,
+                "alt_text": "MISP or org logo"
+            }
+        }
+    ]
+
+    if 'Tag' in e:
+        tag_block = {
+            "type": "actions",
+            "elements": [
+            ]
+        }
+        tags = set([t['name'] for t in e['Tag']])
+        for a in e['Attribute']:
+            if 'Tag' in a:
+                for t in a['Tag']:
+                    tags.add(t['name'])
+        for o in e['Object']:
+            for a in o['Attribute']:
+                if 'Tag' in a:
+                    for t in a['Tag']:
+                        tags.add(t['name'])
+
+        tags = sorted(tags)
+        for t in tags:
+            t = t.replace('misp-galaxy:', '').replace('mitre-', '')
+            tag_block['elements'].append({
+                "type": "button",
+                "text": {
+                    "type": "plain_text",
+                    "text": t
+                },
+                "value": "#"
+            })
+        zmq_message_blocks.append(tag_block)
+
+    # List attributes
+    if include_attr:
+        zmq_message_blocks.append({"type": "divider"})
+        attrs_text = gen_attrs_text(e['Attribute'])
+        if attrs_text:
+            zmq_message_blocks.append(
+                {
+                    "type": "section",
+                    "text": {
+                        "type": "mrkdwn",
+                        "text": f"*Attributes:*\n{attrs_text}"
+                    }
+                }
+            )
+    # List Objects
+    if include_obj:
+        zmq_message_blocks.append({"type": "divider"})
+        for o in e['Object']:
+            attrs_text = gen_attrs_text(o['Attribute'])
+            if attrs_text:
+                # print(json.dumps(o, indent=2))
+                zmq_message_blocks.append(
+                    {
+                        "type": "section",
+                        "text": {
+                            "type": "mrkdwn",
+                            "text": f"*{o['name'].capitalize()} object:*\n{attrs_text}"
+                        }
+                    }
+                )
+
+    # Send the message
+    client = slack.WebClient(token=slack_token)
+    client.users_setPresence(presence='auto')
+    channel = client.channels_join(name=channel_name)
+    client.chat_postMessage(
+        channel=channel['channel']['id'],
+        text=zmq_message_short,
+        blocks=zmq_message_blocks
+    )
+
+
+parser = argparse.ArgumentParser(description='MISP to Slack bot - ZMQ client to gather events, attributes and sighting updates from a MISP instance')
+parser.add_argument("-p", "--port", default="50000", help='set TCP port of the MISP ZMQ (default: 50000)')
+parser.add_argument("-r", "--host", default="127.0.0.1", help='set host of the MISP ZMQ (default: 127.0.0.1)')
+parser.add_argument("-t", "--sleep", default=0.1, help='sleep time (default: 0.1)', type=int)
+args = parser.parse_args()
+
+port = args.port
+host = args.host
+context = zmq.Context()
+socket = context.socket(zmq.SUB)
+socket.connect("tcp://%s:%s" % (host, port))
+socket.setsockopt(zmq.SUBSCRIBE, b'')
+
+poller = zmq.Poller()
+poller.register(socket, zmq.POLLIN)
+
+while True:
+    socks = dict(poller.poll(timeout=None))
+    if socket in socks and socks[socket] == zmq.POLLIN:
+            message = socket.recv()
+            topic, s, m = message.decode('utf-8').partition(" ")
+
+            try:
+                m_json = json.loads(m)
+            except Exception:
+                sys.stderr.write(f'Ignoring non-json message: {m}')
+                time.sleep(args.sleep)
+                continue
+
+            if 'status' in m_json:
+                pass
+            elif 'Event' in m_json:
+                # print(m_json)
+                e = m_json['Event']
+                if '*' in allowed_distributions or \
+                   (e['distribution'] in allowed_distributions and (
+                        e['distribution'] != '5' or (
+                            '*' in allowed_sharing_groups or e['sharing_group_id'] in allowed_sharing_groups)
+                   )):
+                    print(f"Publishing event {e['id']} on slack")
+                    publish_event(e)
+                else:
+                    print(f"Ignoring event {e['id']} as it has a filtered distribution.")
+            else:
+                print(f'Non supported message: {m}')
+            time.sleep(args.sleep)
diff --git a/tools/misp-zmq/slackbot_settings.py.sample b/tools/misp-zmq/slackbot_settings.py.sample
new file mode 100644
index 000000000..a254e2faa
--- /dev/null
+++ b/tools/misp-zmq/slackbot_settings.py.sample
@@ -0,0 +1,24 @@
+'''
+For instructions on creating your BOT, please read: https://api.slack.com/bot-users
+Your bot will need the permissions:
+- channels:join
+- chat:write
+- users:write
+
+WARNING WARNING - THIS SCRIPT DOES NOT MAGICALLY RESPECT ACLs
+MAKE SURE YOU SET THE RIGHT FILTERS IN THE SETTINGS
+'''
+
+channel_name = '#name'
+slack_token = ''
+
+misp_url = 'https://192.168.1.1'
+misp_is_public = True   # set to False if your MISP instance is on a non-internet reachable location. Shows the org icon of the event owner. Otherwise shows the MISP logo.
+
+# filter for confidentiality
+allowed_distributions = ['0', '1', '2', '3', '4']  # * = all, 0/ my org only, 1/ this community, 2/ connected communities, 3/ all communities, 4/ sharing group
+allowed_sharing_groups = ['*']  # put here the sharing_group_ids that you allow
+
+max_value_len = 25  # truncate values longer than X chars
+include_attr = True  # include attributes in the message
+include_obj = True   # include objects in the message
diff --git a/tools/misp-zmq/sub.py b/tools/misp-zmq/sub.py
index 73b8d21f3..17330a89a 100644
--- a/tools/misp-zmq/sub.py
+++ b/tools/misp-zmq/sub.py
@@ -17,11 +17,11 @@ import pprint
 pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr)
 
 parser = argparse.ArgumentParser(description='Generic ZMQ client to gather events, attributes and sighting updates from a MISP instance')
-parser.add_argument("-s","--stats", default=False, action='store_true', help='print regular statistics on stderr')
-parser.add_argument("-p","--port", default="50000", help='set TCP port of the MISP ZMQ (default: 50000)')
-parser.add_argument("-r","--host", default="127.0.0.1", help='set host of the MISP ZMQ (default: 127.0.0.1)')
-parser.add_argument("-o","--only", action="append", default=None, help="set filter (misp_json, misp_json_event, misp_json_attribute or misp_json_sighting) to limit the output a specific type (default: no filter)")
-parser.add_argument("-t","--sleep", default=0.1, help='sleep time (default: 0.1)', type=int)
+parser.add_argument("-s", "--stats", default=False, action='store_true', help='print regular statistics on stderr')
+parser.add_argument("-p", "--port", default="50000", help='set TCP port of the MISP ZMQ (default: 50000)')
+parser.add_argument("-r", "--host", default="127.0.0.1", help='set host of the MISP ZMQ (default: 127.0.0.1)')
+parser.add_argument("-o", "--only", action="append", default=None, help="set filter (misp_json, misp_json_event, misp_json_attribute or misp_json_sighting) to limit the output a specific type (default: no filter)")
+parser.add_argument("-t", "--sleep", default=0.1, help='sleep time (default: 0.1)', type=int)
 args = parser.parse_args()
 
 if args.only is not None:
@@ -35,7 +35,7 @@ port = args.port
 host = args.host
 context = zmq.Context()
 socket = context.socket(zmq.SUB)
-socket.connect ("tcp://%s:%s" % (host, port))
+socket.connect("tcp://%s:%s" % (host, port))
 socket.setsockopt(zmq.SUBSCRIBE, b'')
 
 poller = zmq.Poller()
@@ -52,9 +52,8 @@ while True:
             if args.only:
                 if topic not in filters:
                         continue
-            print (m)
+            print(m)
             if args.stats:
                 stats[topic] = stats.get(topic, 0) + 1
                 pp.pprint(stats)
             time.sleep(args.sleep)
-

From fb26771e6c68005310bb51250b463a49ffee96a4 Mon Sep 17 00:00:00 2001
From: Christophe Vandeplas <christophe@vandeplas.com>
Date: Mon, 13 Apr 2020 22:25:44 +0200
Subject: [PATCH 05/18] fix: [tool] slackbot cosmetic change

---
 tools/misp-zmq/slackbot.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/misp-zmq/slackbot.py b/tools/misp-zmq/slackbot.py
index 854b1ff7f..4679e9248 100644
--- a/tools/misp-zmq/slackbot.py
+++ b/tools/misp-zmq/slackbot.py
@@ -41,6 +41,7 @@ def sanitize_value(s):
     s = s.replace('http', 'hxxp')
     s = s.replace('.', '[.]')
     s = s.replace('@', '[AT]')
+    s = s.replace('\n', ' ')
     # truncate long strings
     return (s[:max_value_len] + '..') if len(s) > max_value_len else s
 

From c20712fc0ef3747529fb9c5878bcfcb745fe79a0 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Tue, 14 Apr 2020 07:14:39 +0200
Subject: [PATCH 06/18] fix: [restresponse] invalid keyword for controllers
 blocked SQL data to be appended on demand

---
 app/Controller/Component/RestResponseComponent.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Controller/Component/RestResponseComponent.php b/app/Controller/Component/RestResponseComponent.php
index ae665b9b6..295c166b5 100644
--- a/app/Controller/Component/RestResponseComponent.php
+++ b/app/Controller/Component/RestResponseComponent.php
@@ -449,7 +449,7 @@ class RestResponseComponent extends Component
                 }
                 if (Configure::read('debug') > 1 && !empty($this->Controller->sql_dump)) {
                     $this->Log = ClassRegistry::init('Log');
-                    if ($this->Content->sql_dump === 2) {
+                    if ($this->Controller->sql_dump === 2) {
                         $response = array('sql_dump' => $this->Log->getDataSource()->getLog(false, false));
                     } else {
                         $response['sql_dump'] = $this->Log->getDataSource()->getLog(false, false);

From 63d6669deae47e30b57eb21d0591b52d1180b948 Mon Sep 17 00:00:00 2001
From: mokaddem <sami.mokaddem@circl.lu>
Date: Tue, 14 Apr 2020 10:26:49 +0200
Subject: [PATCH 07/18] chg: [event:restSearch] Added
 `includeEventCorrelations` parameter

---
 app/Model/Event.php | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/app/Model/Event.php b/app/Model/Event.php
index 51a5478a4..6805b8eea 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -1873,6 +1873,9 @@ class Event extends AppModel
         if (!empty($options['includeDecayScore'])) {
             $this->DecayingModel = ClassRegistry::init('DecayingModel');
         }
+        if (!isset($options['includeEventCorrelations'])) {
+            $options['includeEventCorrelations'] = true;
+        }
         foreach ($possibleOptions as &$opt) {
             if (!isset($options[$opt])) {
                 $options[$opt] = false;
@@ -2161,7 +2164,9 @@ class Event extends AppModel
             }
             $event = $this->massageTags($event, 'Event', $options['excludeGalaxy']);
             // Let's find all the related events and attach it to the event itself
-            $results[$eventKey]['RelatedEvent'] = $this->getRelatedEvents($user, $event['Event']['id'], $sgids);
+            if (!empty($options['includeEventCorrelations'])) {
+                $results[$eventKey]['RelatedEvent'] = $this->getRelatedEvents($user, $event['Event']['id'], $sgids);
+            }
             // Let's also find all the relations for the attributes - this won't be in the xml export though
             if (!empty($options['includeGranularCorrelations'])) {
                 $results[$eventKey]['RelatedAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], $sgids);

From f038fca80373283736abeb5f2d35ceb98ad37062 Mon Sep 17 00:00:00 2001
From: mokaddem <sami.mokaddem@circl.lu>
Date: Tue, 14 Apr 2020 10:06:55 +0200
Subject: [PATCH 08/18] chg: [logs:search] Added support of JSON return format

---
 app/Controller/LogsController.php | 32 +++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/app/Controller/LogsController.php b/app/Controller/LogsController.php
index 5b58a3c93..a72aa3f74 100644
--- a/app/Controller/LogsController.php
+++ b/app/Controller/LogsController.php
@@ -349,21 +349,25 @@ class LogsController extends AppController
                 }
                 $this->set('list', $list);
 
-                // and store into session
-                $this->Session->write('paginate_conditions_log', $this->paginate);
-                $this->Session->write('paginate_conditions_log_email', $filters['email']);
-                $this->Session->write('paginate_conditions_log_org', $filters['org']);
-                $this->Session->write('paginate_conditions_log_action', $filters['action']);
-                $this->Session->write('paginate_conditions_log_model', $filters['model']);
-                $this->Session->write('paginate_conditions_log_model_id', $filters['model_id']);
-                $this->Session->write('paginate_conditions_log_title', $filters['title']);
-                $this->Session->write('paginate_conditions_log_change', $filters['change']);
-                if (Configure::read('MISP.log_client_ip')) {
-                    $this->Session->write('paginate_conditions_log_ip', $filters['ip']);
-                }
+                if ($this->_isRest()) {
+                    return $this->RestResponse->viewData($list, $this->response->type());
+                } else {
+                    // and store into session
+                    $this->Session->write('paginate_conditions_log', $this->paginate);
+                    $this->Session->write('paginate_conditions_log_email', $filters['email']);
+                    $this->Session->write('paginate_conditions_log_org', $filters['org']);
+                    $this->Session->write('paginate_conditions_log_action', $filters['action']);
+                    $this->Session->write('paginate_conditions_log_model', $filters['model']);
+                    $this->Session->write('paginate_conditions_log_model_id', $filters['model_id']);
+                    $this->Session->write('paginate_conditions_log_title', $filters['title']);
+                    $this->Session->write('paginate_conditions_log_change', $filters['change']);
+                    if (Configure::read('MISP.log_client_ip')) {
+                        $this->Session->write('paginate_conditions_log_ip', $filters['ip']);
+                    }
 
-                // set the same view as the index page
-                $this->render('admin_index');
+                    // set the same view as the index page
+                    $this->render('admin_index');
+                }
             } else {
                 // get from Session
                 $filters['email'] = $this->Session->read('paginate_conditions_log_email');

From 7615497cfa867f71cbf900c5eb2d924e18e50ec8 Mon Sep 17 00:00:00 2001
From: mokaddem <sami.mokaddem@circl.lu>
Date: Tue, 14 Apr 2020 12:15:47 +0200
Subject: [PATCH 09/18] chg: [widgets:multiline] Allow to ctrl+click on labels
 to hide the others

---
 .../dashboard/Widgets/MultiLineChart.ctp      | 24 ++++++++++++++-----
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp b/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp
index 23285b24e..58810271b 100644
--- a/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp
+++ b/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp
@@ -500,13 +500,25 @@ function init<?= $seed ?>() { // variables and functions have their own scope (n
                     return 'translate(' + xpos + ',' + ypos + ')';
                 })
                 .on('click', function(d, i) { 
-                    d.disabled = !d.disabled;
                     var label_text = d.text;
-                    var label_disabled = d.disabled;
-                    data_nodes.filter(function(d) { return d.name === label_text; }).forEach(function(data) {
-                        data.disabled = label_disabled
-                    })
-                    _draw()
+                    if (d3.event.ctrlKey) { // hide all others
+                        data_nodes.filter(function(fd) { return fd.name === label_text; }).forEach(function(data) {
+                            data.disabled = false;
+                        })
+                        data_nodes.filter(function(fd) { return fd.name !== label_text; }).forEach(function(data) {
+                            data.disabled = true;
+                        })
+                        d.disabled = false;
+                        legend_labels.filter(function(fd) { return fd.text !== label_text}).forEach(function(label_data) {
+                            label_data.disabled = true;
+                        })
+                    } else { // hide it
+                        d.disabled = !d.disabled;
+                        data_nodes.filter(function(fd) { return fd.name === label_text; }).forEach(function(data) {
+                            data.disabled = d.disabled;
+                        })
+                    }
+                    _draw();
                 });
         }
     }

From 4ded5a73c4270cbfadf39a4f202cbabce69facd9 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Tue, 14 Apr 2020 15:04:33 +0200
Subject: [PATCH 10/18] new: [database] New MySQL data source added for
 debugging

- MySQLObserver datasource added - prepends all queries with the requested controller/action and user ID for better debugging
---
 app/Model/AppModel.php                        |  8 +++----
 .../Datasource/Database/MysqlObserver.php     | 23 +++++++++++++++++++
 app/Model/Log.php                             |  2 +-
 app/Model/Organisation.php                    |  6 ++---
 app/Model/Server.php                          |  6 ++---
 5 files changed, 34 insertions(+), 11 deletions(-)
 create mode 100644 app/Model/Datasource/Database/MysqlObserver.php

diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php
index b7c9ba417..59402ddf2 100644
--- a/app/Model/AppModel.php
+++ b/app/Model/AppModel.php
@@ -123,7 +123,7 @@ class AppModel extends Model
     public function isAcceptedDatabaseError($errorMessage, $dataSource)
     {
         $isAccepted = false;
-        if ($dataSource == 'Database/Mysql') {
+        if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
             $errorDuplicateColumn = 'SQLSTATE[42S21]: Column already exists: 1060 Duplicate column name';
             $errorDuplicateIndex = 'SQLSTATE[42000]: Syntax error or access violation: 1061 Duplicate key name';
             $errorDropIndex = "/SQLSTATE\[42000\]: Syntax error or access violation: 1091 Can't DROP '[\w]+'; check that column\/key exists/";
@@ -722,7 +722,7 @@ class AppModel extends Model
                 $sqlArray[] = "ALTER TABLE taxonomy_predicates ADD colour varchar(7) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '';";
                 break;
             case '2.4.60':
-                if ($dataSource == 'Database/Mysql') {
+                if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
                     $sqlArray[] = 'CREATE TABLE IF NOT EXISTS `attribute_tags` (
                                 `id` int(11) NOT NULL AUTO_INCREMENT,
                                 `attribute_id` int(11) NOT NULL,
@@ -1595,7 +1595,7 @@ class AppModel extends Model
         $dataSource = $dataSourceConfig['datasource'];
         $this->Log = ClassRegistry::init('Log');
         $indexCheckResult = array();
-        if ($dataSource == 'Database/Mysql') {
+        if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
             $indexCheck = "SELECT INDEX_NAME FROM INFORMATION_SCHEMA.STATISTICS WHERE table_schema=DATABASE() AND table_name='" . $table . "' AND index_name LIKE '" . $field . "%';";
             $indexCheckResult = $this->query($indexCheck);
         } elseif ($dataSource == 'Database/Postgres') {
@@ -1603,7 +1603,7 @@ class AppModel extends Model
             $indexCheckResult[] = array('STATISTICS' => array('INDEX_NAME' => $pgIndexName));
         }
         foreach ($indexCheckResult as $icr) {
-            if ($dataSource == 'Database/Mysql') {
+            if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
                 $dropIndex = 'ALTER TABLE ' . $table . ' DROP INDEX ' . $icr['STATISTICS']['INDEX_NAME'] . ';';
             } elseif ($dataSource == 'Database/Postgres') {
                 $dropIndex = 'DROP INDEX IF EXISTS ' . $icr['STATISTICS']['INDEX_NAME'] . ';';
diff --git a/app/Model/Datasource/Database/MysqlObserver.php b/app/Model/Datasource/Database/MysqlObserver.php
new file mode 100644
index 000000000..2cdfd502d
--- /dev/null
+++ b/app/Model/Datasource/Database/MysqlObserver.php
@@ -0,0 +1,23 @@
+<?php
+    App::uses('Mysql', 'Model/Datasource/Database');
+
+    /*
+     * Overrides the default MySQL database implementation to prepend all queries with debug comments
+     * - Lightweight and doesn't affect default operations, like a protoss observer it remains cloaked
+     *   whilst trying to help detect potential bugs burrowed in our queries
+     */
+    class MysqlObserver extends Mysql {
+        public function execute($sql, $options = array(), $params = array()) {
+            $comment = sprintf(
+                '%s%s%s',
+                empty(Configure::read('CurrentUserId')) ? '' : sprintf(
+                    '[User: %s] ',
+                    intval(Configure::read('CurrentUserId'))
+                ),
+                empty(Configure::read('CurrentController')) ? '' : preg_replace('/[^a-zA-Z0-9_]/', '', Configure::read('CurrentController')) . ' :: ',
+                empty(Configure::read('CurrentAction')) ? '' : preg_replace('/[^a-zA-Z0-9_]/', '', Configure::read('CurrentAction')),
+            );
+            $sql = '/* ' . $comment . ' */ ' . $sql;
+            return parent::execute($sql, $options, $params);
+        }
+    }
diff --git a/app/Model/Log.php b/app/Model/Log.php
index a1b34f2ae..d1ee39d7b 100644
--- a/app/Model/Log.php
+++ b/app/Model/Log.php
@@ -154,7 +154,7 @@ class Log extends AppModel
             $conditions['org'] = $org['Organisation']['name'];
         }
         $conditions['AND']['NOT'] = array('action' => array('login', 'logout', 'changepw'));
-        if ($dataSource == 'Database/Mysql') {
+        if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
             $validDates = $this->find('all', array(
                     'fields' => array('DISTINCT UNIX_TIMESTAMP(DATE(created)) AS Date', 'count(id) AS count'),
                     'conditions' => $conditions,
diff --git a/app/Model/Organisation.php b/app/Model/Organisation.php
index e309faf08..ef4fdfa87 100644
--- a/app/Model/Organisation.php
+++ b/app/Model/Organisation.php
@@ -292,7 +292,7 @@ class Organisation extends AppModel
         $success = true;
         foreach ($this->organisationAssociations as $model => $data) {
             foreach ($data['fields'] as $field) {
-                if ($dataSource == 'Database/Mysql') {
+                if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
                     $sql = 'SELECT `id` FROM `' . $data['table'] . '` WHERE `' . $field . '` = "' . $currentOrg['Organisation']['id'] . '"';
                 } elseif ($dataSource == 'Database/Postgres') {
                     $sql = 'SELECT "id" FROM "' . $data['table'] . '" WHERE "' . $field . '" = "' . $currentOrg['Organisation']['id'] . '"';
@@ -303,13 +303,13 @@ class Organisation extends AppModel
                     if (!empty($dataMoved['values_changed'][$model][$field])) {
                         $this->Log->create();
                         try {
-                            if ($dataSource == 'Database/Mysql') {
+                            if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
                                 $sql = 'UPDATE `' . $data['table'] . '` SET `' . $field . '` = ' . $targetOrg['Organisation']['id'] . ' WHERE `' . $field . '` = ' . $currentOrg['Organisation']['id'] . ';';
                             } elseif ($dataSource == 'Database/Postgres') {
                                 $sql = 'UPDATE "' . $data['table'] . '" SET "' . $field . '" = ' . $targetOrg['Organisation']['id'] . ' WHERE "' . $field . '" = ' . $currentOrg['Organisation']['id'] . ';';
                             }
                             $result = $this->query($sql);
-                            if ($dataSource == 'Database/Mysql') {
+                            if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
                                 $sql = 'UPDATE `' . $data['table'] . '` SET `' . $field . '` = ' . $currentOrg['Organisation']['id'] . ' WHERE `id` IN (' . implode(',', $dataMoved['values_changed'][$model][$field]) . ');';
                             } elseif ($dataSource == 'Database/Postgres') {
                                 $sql = 'UPDATE "' . $data['table'] . '" SET "' . $field . '" = ' . $currentOrg['Organisation']['id'] . ' WHERE "id" IN (' . implode(',', $dataMoved['values_changed'][$model][$field]) . ');';
diff --git a/app/Model/Server.php b/app/Model/Server.php
index 2a698b828..8d84ad665 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -4411,7 +4411,7 @@ class Server extends AppModel
     public function dbSpaceUsage()
     {
         $dataSource = $this->getDataSource()->config['datasource'];
-        if ($dataSource == 'Database/Mysql') {
+        if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
             $sql = sprintf(
                 'select TABLE_NAME, sum((DATA_LENGTH+INDEX_LENGTH)/1024/1024) AS used, sum(DATA_FREE)/1024/1024 AS reclaimable from information_schema.tables where table_schema = %s group by TABLE_NAME;',
                 "'" . $this->getDataSource()->config['database'] . "'"
@@ -4487,7 +4487,7 @@ class Server extends AppModel
             'update_fail_number_reached' => $this->UpdateFailNumberReached(),
             'indexes' => array()
         );
-        if ($dataSource == 'Database/Mysql') {
+        if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
             $dbActualSchema = $this->getActualDBSchema();
             $dbExpectedSchema = $this->getExpectedDBSchema();
             if ($dbExpectedSchema !== false) {
@@ -4645,7 +4645,7 @@ class Server extends AppModel
         $dbActualSchema = array();
         $dbActualIndexes = array();
         $dataSource = $this->getDataSource()->config['datasource'];
-        if ($dataSource == 'Database/Mysql') {
+        if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
             $sqlGetTable = sprintf('SELECT TABLE_NAME FROM information_schema.tables WHERE table_schema = %s;', "'" . $this->getDataSource()->config['database'] . "'");
             $sqlResult = $this->query($sqlGetTable);
             $tables = HASH::extract($sqlResult, '{n}.tables.TABLE_NAME');

From 3fa5c3f3706dacf060f78cd02bb515be34b52c63 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Tue, 14 Apr 2020 15:17:15 +0200
Subject: [PATCH 11/18] fix: [database] added missing file

---
 app/Controller/AppController.php | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
index 15bcecc51..f9a29fe2b 100755
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@@ -182,6 +182,8 @@ class AppController extends Controller
         if (!empty($this->params['named']['disable_background_processing'])) {
             Configure::write('MISP.background_jobs', 0);
         }
+        Configure::write('CurrentController', $this->params['controller']);
+        Configure::write('CurrentAction', $this->params['action']);
         $versionArray = $this->{$this->modelClass}->checkMISPVersion();
         $this->mispVersion = implode('.', array_values($versionArray));
         $this->Security->blackHoleCallback = 'blackHole';
@@ -298,6 +300,7 @@ class AppController extends Controller
         }
 
         if ($this->Auth->user()) {
+            Configure::write('CurrentUserId', $this->Auth->user('id'));
             $this->User->setMonitoring($this->Auth->user());
             if (Configure::read('MISP.log_user_ips')) {
                 $redis = $this->{$this->modelClass}->setupRedis();
@@ -606,7 +609,7 @@ class AppController extends Controller
             ConnectionManager::create('default', $db->config);
         }
         $dataSource = $dataSourceConfig['datasource'];
-        if ($dataSource != 'Database/Mysql' && $dataSource != 'Database/Postgres') {
+        if (!in_array($dataSource, array('Database/Mysql', 'Database/Postgres', 'Database/MysqlObserver'))) {
             throw new Exception('datasource not supported: ' . $dataSource);
         }
     }

From e75828a34b5802cd3d72f1a094e7c22a12d5f2f9 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Tue, 14 Apr 2020 15:18:25 +0200
Subject: [PATCH 12/18] fix: [database] bruteforce check relaxed for datasource

---
 app/Model/Bruteforce.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Model/Bruteforce.php b/app/Model/Bruteforce.php
index ec74a856a..8f214dce2 100644
--- a/app/Model/Bruteforce.php
+++ b/app/Model/Bruteforce.php
@@ -39,7 +39,7 @@ class Bruteforce extends AppModel
         $dataSourceConfig = ConnectionManager::getDataSource('default')->config;
         $dataSource = $dataSourceConfig['datasource'];
         $expire = date('Y-m-d H:i:s', time());
-        if ($dataSource == 'Database/Mysql') {
+        if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
             $sql = 'DELETE FROM bruteforces WHERE `expire` <= "' . $expire . '";';
         } elseif ($dataSource == 'Database/Postgres') {
             $sql = 'DELETE FROM bruteforces WHERE expire <= \'' . $expire . '\';';

From 659e38f6c9cf8d94986ca620e6fe94ee753a5400 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Tue, 14 Apr 2020 15:37:55 +0200
Subject: [PATCH 13/18] fix: [database] made MySQLObserver php < 7.2 compliant

---
 app/Model/Datasource/Database/MysqlObserver.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Model/Datasource/Database/MysqlObserver.php b/app/Model/Datasource/Database/MysqlObserver.php
index 2cdfd502d..4de61f3ba 100644
--- a/app/Model/Datasource/Database/MysqlObserver.php
+++ b/app/Model/Datasource/Database/MysqlObserver.php
@@ -15,7 +15,7 @@
                     intval(Configure::read('CurrentUserId'))
                 ),
                 empty(Configure::read('CurrentController')) ? '' : preg_replace('/[^a-zA-Z0-9_]/', '', Configure::read('CurrentController')) . ' :: ',
-                empty(Configure::read('CurrentAction')) ? '' : preg_replace('/[^a-zA-Z0-9_]/', '', Configure::read('CurrentAction')),
+                empty(Configure::read('CurrentAction')) ? '' : preg_replace('/[^a-zA-Z0-9_]/', '', Configure::read('CurrentAction'))
             );
             $sql = '/* ' . $comment . ' */ ' . $sql;
             return parent::execute($sql, $options, $params);

From 94526fecc8875fc5cbbf97eb896f3b80cf889875 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Tue, 14 Apr 2020 15:57:26 +0200
Subject: [PATCH 14/18] fix: [stix1 import] Fixed object name handling causing
 errors in some cases

- With a wrong object name, the correct function
  was not reached, reaching some unexpected errors
---
 app/files/scripts/stix2misp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/scripts/stix2misp.py b/app/files/scripts/stix2misp.py
index 72d47d786..e4f2cbbf1 100644
--- a/app/files/scripts/stix2misp.py
+++ b/app/files/scripts/stix2misp.py
@@ -1065,7 +1065,7 @@ class StixFromMISPParser(StixParser):
     # Parse STIX object that we know will give MISP objects
     def parse_misp_object_indicator(self, indicator):
         item = indicator.item
-        name = item.title.split(' ')[0]
+        name = item.title.split(': ')[0]
         if name not in ('passive-dns'):
             self.fill_misp_object(item, name, to_ids=True)
         else:

From 7991016039d0465db1f87b260cb48365c9acbfab Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Tue, 14 Apr 2020 18:00:43 +0200
Subject: [PATCH 15/18] fix: [API] event index queries refactored

- fixed ID lookups to be more graceful (IN() instead of OR-d statements)
- removed default sorting which is the default anyway but introduces a massive overhead
---
 app/Controller/EventsController.php | 35 +++++++++++++++++++----------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php
index 857300987..220f1989c 100644
--- a/app/Controller/EventsController.php
+++ b/app/Controller/EventsController.php
@@ -315,11 +315,11 @@ class EventsController extends AppController
                         break;
                     case 'attribute':
                         $event_id_arrays = $this->__filterOnAttributeValue($v);
-                        foreach ($event_id_arrays[0] as $event_id) {
-                            $this->paginate['conditions']['AND']['OR'][] = array('Event.id' => $event_id);
+                        if (!empty($event_id_arrays[0])) {
+                            $this->paginate['conditions']['AND'][] = array('Event.id' => $event_id_arrays[0]);
                         }
-                        foreach ($event_id_arrays[1] as $event_id) {
-                            $this->paginate['conditions']['AND'][] = array('Event.id !=' => $event_id);
+                        if (!empty($event_id_arrays[1])) {
+                            $this->paginate['conditions']['AND'][] = array('Event.id !=' => $event_id_arrays[1]);
                         }
                         break;
                     case 'published':
@@ -342,25 +342,38 @@ class EventsController extends AppController
                         if ($v == "") {
                             continue 2;
                         }
-                        $pieces = explode('|', $v);
+                        if (is_array($v)) {
+                            $pieces = $v;
+                        } else {
+                            $pieces = explode('|', $v);
+                        }
                         $temp = array();
+                        $eventidConditions = array();
                         foreach ($pieces as $piece) {
                             $piece = trim($piece);
                             if ($piece[0] == '!') {
                                 if (strlen($piece) == 37) {
-                                    $this->paginate['conditions']['AND'][] = array('Event.uuid !=' => substr($piece, 1));
+                                    $eventidConditions['NOT']['uuid'][] = substr($piece, 1);
                                 } else {
-                                    $this->paginate['conditions']['AND'][] = array('Event.id !=' => substr($piece, 1));
+                                    $eventidConditions['NOT']['id'][] = substr($piece, 1);
                                 }
                             } else {
                                 if (strlen($piece) == 36) {
-                                    $temp['OR'][] = array('Event.uuid' => $piece);
+                                    $eventidConditions['OR']['uuid'][] = $piece;
                                 } else {
-                                    $temp['OR'][] = array('Event.id' => $piece);
+                                    $eventidConditions['OR']['id'][] = $piece;
                                 }
                             }
                         }
-                        $this->paginate['conditions']['AND'][] = $temp;
+                        foreach ($eventidConditions as $operator => $conditionForOperator) {
+                            foreach ($conditionForOperator as $conditionKey => $conditionValue) {
+                                $lookupKey = 'Event.' . $conditionKey;
+                                if ($operator === 'NOT') {
+                                    $lookupKey = $lookupKey . ' !=';
+                                }
+                                $this->paginate['conditions']['AND'][] = array($lookupKey => $conditionValue);
+                            }
+                        }
                         break;
                     case 'datefrom':
                         if ($v == "") {
@@ -727,8 +740,6 @@ class EventsController extends AppController
                 } else {
                     $rules['order'] = array('Event.' . $passedArgs['sort'] => 'ASC');
                 }
-            } else {
-                $rules['order'] = array('Event.id' => 'DESC');
             }
             $rules['contain'] = $this->paginate['contain'];
             if (isset($this->paginate['conditions'])) {

From ec9338966995c358cd94b7f6faeb236b9d73cfb1 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Wed, 15 Apr 2020 06:21:15 +0200
Subject: [PATCH 16/18] fix: [internal] Added a setting to skip positive
 attribute level filters on the event scope

- when running a large MISP community, it is bound to happen that your instance will be used as the back-end for internal tooling
- often these tools are configured to fetch aggressively, often with heavy consequences on the server load
- some filter that serves mostly edge-case lookups can mistakenly lead to heavy server load for no good reason

We have identified attribute level positive filters on the event scope to be such a filter and made them optionally toggle-able
via the MISP.attribute_fitlers_block_only flag. Turning the setting on will remove all event level filters such as "type" from
being viable filter candidates unless used to block the inclusion of attribute types. Some examples:

"type": {"OR": ["ip-dst", "ip-src", "hostname", "domain"]} would normally return ANY event that has at least one of the listed
attribute types. This is the behaviour that can now be disabled.

"type": {"NOT": ["iban", "cc-number"]} would normally remove any attributes with the given types from the list of returned
events. This functionality is NOT affected by the toggle.
---
 app/Model/Event.php  | 7 ++++++-
 app/Model/Server.php | 9 +++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/app/Model/Event.php b/app/Model/Event.php
index 6805b8eea..c1cddda5a 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -2585,7 +2585,7 @@ class Event extends AppModel
         }
         return $conditions;
     }
-    
+
     public function set_filter_uuid(&$params, $conditions, $options)
     {
         if ($options['scope'] === 'Event') {
@@ -2701,6 +2701,11 @@ class Event extends AppModel
     {
         if (!empty($params[$options['filter']])) {
             $params[$options['filter']] = $this->convert_filters($params[$options['filter']]);
+            if (!empty(Configure::read('MISP.attribute_filters_block_only'))) {
+                if ($options['context'] === 'Event' && !empty($params[$options['filter']]['OR'])) {
+                    unset($params[$options['filter']]['OR']);
+                }
+            }
             $conditions = $this->generic_add_filter($conditions, $params[$options['filter']], 'Attribute.' . $options['filter']);
         }
         return $conditions;
diff --git a/app/Model/Server.php b/app/Model/Server.php
index 454063b11..ea5aef715 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -1072,6 +1072,15 @@ class Server extends AppModel
                                'test' => 'testForNumeric',
                                'type' => 'numeric',
                                'null' => true
+                       ),
+                       'attribute_filters_block_only' => array(
+                               'level' => 1,
+                               'description' => __('This is a performance tweak to change the behaviour of restSearch to use attribute filters solely for blocking. This means that a lookup on the event scope with for example the type field set will be ignored unless it\'s used to strip unwanted attributes from the results. If left disabled, passing [ip-src, ip-dst] for example will return any event with at least one ip-src or ip-dst attribute. This is generally not considered to be too useful and is a heavy burden on the database.'),
+                               'value' => false,
+                               'errorMessage' => '',
+                               'test' => 'testBool',
+                               'type' => 'boolean',
+                               'null' => true
                        )
                 ),
                 'GnuPG' => array(

From b5eb75bb80305bbde94d6e82948a124d3dd0bf00 Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Wed, 15 Apr 2020 14:48:14 +0200
Subject: [PATCH 17/18] chg: [misp-objects] updated to the latest version

---
 app/files/misp-objects | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/misp-objects b/app/files/misp-objects
index 7ef9a2ba5..ef01e6e37 160000
--- a/app/files/misp-objects
+++ b/app/files/misp-objects
@@ -1 +1 @@
-Subproject commit 7ef9a2ba56efc6553a720d6df27c9ee547e24242
+Subproject commit ef01e6e37b025a71b40515bc0a9d4e11fef20798

From 9ca3fafca4f2907aba038b5f6cc865bb325f4c9b Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Wed, 15 Apr 2020 18:04:58 +0200
Subject: [PATCH 18/18] chg: [stix2] Bumped latest STIX2 python library version

---
 cti-python-stix2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cti-python-stix2 b/cti-python-stix2
index 65a943d89..e4f08557e 160000
--- a/cti-python-stix2
+++ b/cti-python-stix2
@@ -1 +1 @@
-Subproject commit 65a943d8929c578041f789665b05810ea68986cb
+Subproject commit e4f08557ec93c589a71a6e4060134661f1c4b2c0