From 87b568526f2917e9facf455cb5194c1b6c55f1e7 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 5 Oct 2019 23:17:19 +0200
Subject: [PATCH 01/43] fix: [internal] Remove unused function

---
 app/Model/Attribute.php | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php
index 648cc4a82..6683d7ca8 100644
--- a/app/Model/Attribute.php
+++ b/app/Model/Attribute.php
@@ -3578,11 +3578,6 @@ class Attribute extends AppModel
         return true;
     }
 
-    public function convertToOpenIOC($user, $attributes)
-    {
-        return $this->IOCExport->buildAll($user, $event);
-    }
-
     private function __createTagSubQuery($tag_id, $blocked = false, $scope = 'Event', $limitAttributeHitsTo = 'event')
     {
         $conditionKey = $blocked ? array('NOT' => array('EventTag.tag_id' => $tag_id)) : array('EventTag.tag_id' => $tag_id);

From e4071f205e00205fd7412a3c7b65f58dc1603b95 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Tue, 1 Oct 2019 15:04:25 +0200
Subject: [PATCH 02/43] new: [internal] Log exact error for GPG diag in error
 log

---
 app/Model/Server.php | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/Model/Server.php b/app/Model/Server.php
index 2a6fd79bc..c5b14963f 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -4386,6 +4386,7 @@ class Server extends AppModel
                     'binary' => Configure::read('GnuPG.binary') ?: '/usr/bin/gpg'
                 ));
             } catch (Exception $e) {
+                $this->logException("Error during initializing GPG.", $e, LOG_NOTICE);
                 $gpgStatus = 2;
                 $continue = false;
             }
@@ -4393,6 +4394,7 @@ class Server extends AppModel
                 try {
                     $key = $gpg->addSignKey(Configure::read('GnuPG.email'), Configure::read('GnuPG.password'));
                 } catch (Exception $e) {
+                    $this->logException("Error during adding GPG signing key.", $e, LOG_NOTICE);
                     $gpgStatus = 3;
                     $continue = false;
                 }
@@ -4402,6 +4404,7 @@ class Server extends AppModel
                     $gpgStatus = 0;
                     $signed = $gpg->sign('test', Crypt_GPG::SIGN_MODE_CLEAR);
                 } catch (Exception $e) {
+                    $this->logException("Error during GPG signing.", $e, LOG_NOTICE);
                     $gpgStatus = 4;
                 }
             }

From 9259d18b6bc89ddf036526a3fc1eac8496a6b1d7 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Tue, 15 Oct 2019 17:31:04 +0200
Subject: [PATCH 03/43] fix: [internal] Remove unused
 ShadowAttributesController method

---
 app/Controller/ShadowAttributesController.php | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/app/Controller/ShadowAttributesController.php b/app/Controller/ShadowAttributesController.php
index 619d5dfca..07380f95b 100644
--- a/app/Controller/ShadowAttributesController.php
+++ b/app/Controller/ShadowAttributesController.php
@@ -197,24 +197,6 @@ class ShadowAttributesController extends AppController
         }
     }
 
-    // If we accept a proposed attachment, then the attachment itself needs to be moved from files/eventId/shadow/shadowId to files/eventId/attributeId
-    private function _moveFile($shadowId, $newId, $eventId)
-    {
-        $attachments_dir = Configure::read('MISP.attachments_dir');
-        if (empty($attachments_dir)) {
-            $attachments_dir = $this->ShadowAttribute->getDefaultAttachments_dir();
-        }
-        $pathOld = $attachments_dir . DS . 'shadow' . DS . $shadowId;
-        $pathNew = $attachments_dir . DS . $newId;
-        if (rename($pathOld, $pathNew)) {
-            return true;
-        } else {
-            $this->Flash->error(__('Moving of the file that this attachment references failed.', true), 'default', array());
-            $this->redirect(array('controller' => 'events', 'action' => 'view', $eventId));
-        }
-    }
-
-
     private function __discard($id)
     {
         $sa = $this->ShadowAttribute->find(

From 7539cbff2c5bd1f2d849a59742aa50c424d63bc1 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 20 Oct 2019 19:08:01 +0200
Subject: [PATCH 04/43] chg: [feed] Use precomputed hashes to speedup attaching
 correlation

---
 app/Model/Feed.php | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/app/Model/Feed.php b/app/Model/Feed.php
index fc83bebda..89a0323ed 100644
--- a/app/Model/Feed.php
+++ b/app/Model/Feed.php
@@ -365,19 +365,12 @@ class Feed extends AppModel
                     if ($scope === 'Server' || $source[$scope]['source_format'] == 'misp') {
                         $pipe = $redis->multi(Redis::PIPELINE);
                         $eventUuidHitPosition = array();
-                        $i = 0;
                         foreach ($objects as $k => $object) {
                             if (isset($object[$scope])) {
                                 foreach ($object[$scope] as $currentFeed) {
                                     if ($source[$scope]['id'] == $currentFeed['id']) {
-                                        $eventUuidHitPosition[$i] = $k;
-                                        $i++;
-                                        if (in_array($object['type'], $compositeTypes)) {
-                                            $value = explode('|', $object['value']);
-                                            $redis->smembers($cachePrefix . 'event_uuid_lookup:' . md5($value[0]));
-                                        } else {
-                                            $redis->smembers($cachePrefix . 'event_uuid_lookup:' . md5($object['value']));
-                                        }
+                                        $eventUuidHitPosition[] = $k;
+                                        $redis->smembers($cachePrefix . 'event_uuid_lookup:' . $hashTable[$k]);
                                     }
                                 }
                             }

From 963269085eaedd204e439afaa2215db9c5ba037c Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Wed, 13 Nov 2019 09:04:55 +0100
Subject: [PATCH 05/43] fix: [stix2 import] Importing stix2-pattern object only
 if the pattern parsing failed

- Also adding the uuid of the stix2-pattern object
- It avoids patterns to be exported twice if we
  export the misp event created from the import
  afterwards
---
 app/files/scripts/stix2/stix2misp.py | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/app/files/scripts/stix2/stix2misp.py b/app/files/scripts/stix2/stix2misp.py
index c5103126c..7b1d9b8eb 100644
--- a/app/files/scripts/stix2/stix2misp.py
+++ b/app/files/scripts/stix2/stix2misp.py
@@ -1180,16 +1180,15 @@ class ExternalStixParser(StixParser):
 
     def parse_external_indicator(self, indicator):
         pattern = indicator.pattern
-        # Deeper analyse of patterns coming when we get examples
-        attribute = {'type': 'stix2-pattern', 'object_relation': 'stix2-pattern', 'value': pattern}
-        misp_object = {'name': 'stix2-pattern', 'meta-category': 'stix2-pattern',
-                       'Attribute': [self.version_attribute, attribute]}
-        self.misp_event.add_object(**misp_object)
         indicator_id = indicator.id.split('--')[1]
-        if hasattr(indicator, 'object_marking_refs'):
-            self.parse_external_pattern(pattern, indicator_id, marking=indicator.object_marking_refs)
-        else:
-            self.parse_external_pattern(pattern, indicator_id)
+        try:
+            if hasattr(indicator, 'object_marking_refs'):
+                self.parse_external_pattern(pattern, indicator_id, marking=indicator.object_marking_refs)
+            else:
+                self.parse_external_pattern(pattern, indicator_id)
+            # Deeper analyse of patterns coming when we get examples
+        except Exception:
+            self.add_stix2_pattern_object(pattern, indicator_id)
 
     def parse_external_observable(self, observable):
         objects = observable.objects
@@ -1225,6 +1224,7 @@ class ExternalStixParser(StixParser):
                         self.pattern_mapping[type_]([p.strip()], marking)
                     except KeyError:
                         print('{} not parsed at the moment'.format(type_), file=sys.stderr)
+                        raise Exception
             else:
                 pattern = [p.strip() for p in pattern.split(' AND ')]
                 types = self.parse_external_pattern_types(pattern)
@@ -1232,6 +1232,9 @@ class ExternalStixParser(StixParser):
                     self.pattern_mapping[types](pattern, marking, uuid=uuid)
                 except KeyError:
                     print('{} not parsed at the moment'.format(types), file=sys.stderr)
+                    raise Exception
+        else:
+            self.add_stix2_pattern_object(pattern, uuid)
 
     @staticmethod
     def parse_external_pattern_types(pattern):
@@ -1460,6 +1463,13 @@ class ExternalStixParser(StixParser):
     ##                             UTILITY FUNCTIONS.                             ##
     ################################################################################
 
+    def add_stix2_pattern_object(self, pattern, uuid):
+        attribute = {'type': 'stix2-pattern', 'object_relation': 'stix2-pattern', 'value': pattern}
+        misp_object = {'name': 'stix2-pattern', 'meta-category': 'stix2-pattern',
+                       'Attribute': [self.version_attribute, attribute],
+                       'uuid': uuid}
+        self.misp_event.add_object(**misp_object)
+
     @staticmethod
     def create_misp_object(attributes, name, uuid=None):
         misp_object = MISPObject(name, misp_objects_path_custom=_MISP_objects_path)

From 4bb4d52a5c56eebd272b74a930ae8507e8bc79a4 Mon Sep 17 00:00:00 2001
From: Steve Clement <steve@localhost.lu>
Date: Fri, 15 Nov 2019 09:50:50 +0900
Subject: [PATCH 06/43] fix: [REST] Python has no 'Null' type, it is called
 'None'

---
 app/Controller/ServersController.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php
index 639b7027f..8fb9e1a8c 100644
--- a/app/Controller/ServersController.php
+++ b/app/Controller/ServersController.php
@@ -1950,7 +1950,7 @@ misp.direct_call(relative_path, body)
             $request['header']['Authorization'],
             $verifyCert,
             $relative,
-            (empty($request['body']) ? 'Null' : $request['body'])
+            (empty($request['body']) ? 'None' : $request['body'])
         );
         return $python_script;
     }

From 35c739980e939febc204fb4d2ce28dabbdd0890a Mon Sep 17 00:00:00 2001
From: Steve Clement <steve@localhost.lu>
Date: Fri, 15 Nov 2019 10:13:49 +0900
Subject: [PATCH 07/43] chg: [REST] Updated to ExpandedPyMISP

---
 app/Controller/ServersController.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php
index 8fb9e1a8c..fa7e485a5 100644
--- a/app/Controller/ServersController.php
+++ b/app/Controller/ServersController.php
@@ -1941,9 +1941,9 @@ misp_verifycert = %s
 relative_path = \'%s\'
 body = %s
 
-from pymisp import PyMISP
+from pymisp import ExpandedPyMISP
 
-misp = PyMISP(misp_url, misp_key, misp_verifycert)
+misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
 misp.direct_call(relative_path, body)
 ',
             $baseurl,

From e4c82eb9ff440220be27130bcbcf2de2102e7e35 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Fri, 15 Nov 2019 14:11:24 +0100
Subject: [PATCH 08/43] fix: [API] adding objects now has better validation
 errors

- instead of silently dropping attributes in certain cases
---
 app/Controller/ObjectsController.php | 9 +++++++--
 app/Model/Attribute.php              | 3 +++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/app/Controller/ObjectsController.php b/app/Controller/ObjectsController.php
index 62014fac1..a274075c7 100644
--- a/app/Controller/ObjectsController.php
+++ b/app/Controller/ObjectsController.php
@@ -198,6 +198,7 @@ class ObjectsController extends AppController
             $this->MispObject->Event->insertLock($this->Auth->user(), $eventId);
         }
         $error = false;
+        $template = false;
         if (!empty($templateId) || !$this->_isRest()) {
             $templates = $this->MispObject->ObjectTemplate->find('all', array(
                 'conditions' => array('ObjectTemplate.id' => $templateId),
@@ -247,10 +248,14 @@ class ObjectsController extends AppController
                 foreach ($object['Attribute'] as $k => $attribute) {
                     unset($object['Attribute'][$k]['id']);
                     $object['Attribute'][$k]['event_id'] = $eventId;
-                    $this->MispObject->Event->Attribute->set($attribute);
+                    $this->MispObject->Event->Attribute->set($object['Attribute'][$k]);
                     if (!$this->MispObject->Event->Attribute->validates()) {
                         if ($this->MispObject->Event->Attribute->validationErrors['value'][0] !== 'Composite type found but the value not in the composite (value1|value2) format.') {
-                            $error = 'Could not save object as at least one attribute has failed validation (' . $attribute['object_relation'] . '). ' . json_encode($this->MispObject->Event->Attribute->validationErrors);
+                            $error = sprintf(
+                                'Could not save object as at least one attribute has failed validation (%s). %s',
+                                isset($attribute['object_relation']) ? $attribute['object_relation'] : 'No object_relation',
+                                json_encode($this->MispObject->Event->Attribute->validationErrors)
+                            );
                         }
                     }
                 }
diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php
index 1a401872c..ee97379a4 100644
--- a/app/Model/Attribute.php
+++ b/app/Model/Attribute.php
@@ -813,9 +813,11 @@ class Attribute extends AppModel
     {
         parent::beforeValidate();
         if (!isset($this->data['Attribute']['type'])) {
+            $this->validationErrors['type'] = ['No type set.'];
             return false;
         }
         if (is_array($this->data['Attribute']['value'])) {
+            $this->validationErrors['type'] = ['Value is an array.'];
             return false;
         }
         App::uses('ComplexTypeTool', 'Tools');
@@ -823,6 +825,7 @@ class Attribute extends AppModel
         $this->data['Attribute']['value'] = $this->complexTypeTool->refangValue($this->data['Attribute']['value'], $this->data['Attribute']['type']);
 
         if (!empty($this->data['Attribute']['object_id']) && empty($this->data['Attribute']['object_relation'])) {
+            $this->validationErrors['type'] = ['Object attribute sent, but no object_relation set.'];
             return false;
         }
         // remove leading and trailing blanks

From be8201a2364f767a10138ffc54e90bfe25b3b504 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Fri, 15 Nov 2019 09:33:37 -0500
Subject: [PATCH 09/43] fix: [stix2 import] Avoids importing an object_relation
 value for single attributes

---
 app/files/scripts/stix2/stix2misp.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/files/scripts/stix2/stix2misp.py b/app/files/scripts/stix2/stix2misp.py
index 7b1d9b8eb..8067ea727 100644
--- a/app/files/scripts/stix2/stix2misp.py
+++ b/app/files/scripts/stix2/stix2misp.py
@@ -1149,6 +1149,7 @@ class ExternalStixParser(StixParser):
                                 ('windows-registry-key',): self.parse_regkey_pattern,
                                 ('x509-certificate',): self.parse_x509_pattern}
         self.pattern_forbidden_relations = (' LIKE ', ' FOLLOWEDBY ', ' MATCHES ', ' ISSUBSET ', ' ISSUPERSET ', ' REPEATS ')
+        self.single_attribute_fields = ('type', 'value', 'to_ids')
 
     def handler(self):
         self.version_attribute = {'type': 'text', 'object_relation': 'version', 'value': self.stix_version}
@@ -1535,6 +1536,7 @@ class ExternalStixParser(StixParser):
     def handle_import_case(self, attributes, name, marking=None, uuid=None):
         if len(attributes) == 1:
             attribute = attributes[0]
+            attribute = {field: attribute[field] for field in self.single_attribute_fields if attribute.get(field)}
             attribute['uuid'] = uuid
             if marking:
                 attribute = self.add_tag_in_attribute(attribute, marking)

From 1cc6a6733596e6f37eac285bdd48ce5cd3636d74 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Sat, 16 Nov 2019 14:06:46 +0100
Subject: [PATCH 10/43] fix: [internal] site admins should not have to be host
 org users to see server correlations

---
 app/Model/Event.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Model/Event.php b/app/Model/Event.php
index 6bddcf7db..2792a1149 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -2157,7 +2157,7 @@ class Event extends AppModel
                     }
                     $event['Attribute'] = $this->Feed->attachFeedCorrelations($event['Attribute'], $user, $event['Event'], $overrideLimit);
                 }
-                if (!empty($options['includeServerCorrelations']) && $user['org_id'] == Configure::read('MISP.host_org_id')) {
+                if (!empty($options['includeServerCorrelations']) && ($user['Role']['perm_site_admin'] || $user['org_id'] == Configure::read('MISP.host_org_id'))) {
                     $this->Feed = ClassRegistry::init('Feed');
                     if (!empty($options['overrideLimit'])) {
                         $overrideLimit = true;

From f3c61a9b505a986bf386acd8d4fabca573c46bf9 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Sat, 16 Nov 2019 14:19:39 +0100
Subject: [PATCH 11/43] fix: [indextable] Fixed the link field

---
 .../genericElements/IndexTable/Fields/links.ctp        | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/app/View/Elements/genericElements/IndexTable/Fields/links.ctp b/app/View/Elements/genericElements/IndexTable/Fields/links.ctp
index 66cc44c02..3d0a972dc 100644
--- a/app/View/Elements/genericElements/IndexTable/Fields/links.ctp
+++ b/app/View/Elements/genericElements/IndexTable/Fields/links.ctp
@@ -2,7 +2,13 @@
     $data_elements = Hash::extract($row, $field['data_path']);
     $links = array();
     foreach ($data_elements as $data) {
-        if (strpos($field['url'], '%s') !== false) {
+        if (!empty($data['name'])) {
+            $field['title'] = $data['name'];
+        }
+        if (!empty($data['url'])) {
+            $data = $data['url'];
+        }
+        if (isset($field['url']) && strpos($field['url'], '%s') !== false) {
             $url = sprintf(
                 $field['url'],
                 $data
@@ -14,7 +20,7 @@
             '<a href="%s" title="%s">%s</a>',
             h($url),
             empty($field['title']) ? h($data) : h($field['title']),
-            h($data)
+            empty($field['title']) ? h($data) : h($field['title'])
         );
     }
     echo implode('<br />', $links);

From a8b5da4be236c79715da4a1010ae7991e3085216 Mon Sep 17 00:00:00 2001
From: mokaddem <sami.mokaddem@circl.lu>
Date: Sat, 16 Nov 2019 13:12:37 -0500
Subject: [PATCH 12/43] chg: [statistics] Added Attribute count

---
 app/Controller/UsersController.php | 3 ++-
 app/View/Users/statistics_orgs.ctp | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php
index d97a89ba1..517b192b3 100644
--- a/app/Controller/UsersController.php
+++ b/app/Controller/UsersController.php
@@ -1756,10 +1756,11 @@ class UsersController extends AppController
             'group' => 'Event.orgc_id',
             'conditions' => array('Event.orgc_id' => array_keys($orgs)),
             'recursive' => -1,
-            'fields' => array('Event.orgc_id', 'count(*)')
+            'fields' => array('Event.orgc_id', 'count(*)', 'sum(Event.attribute_count) as attributeCount')
         ));
         foreach ($events as $event) {
             $orgs[$event['Event']['orgc_id']]['eventCount'] = $event[0]['count(*)'];
+            $orgs[$event['Event']['orgc_id']]['attributeCount'] = $event[0]['attributeCount'];
         }
         unset($events);
         $orgs = Set::combine($orgs, '{n}.name', '{n}');
diff --git a/app/View/Users/statistics_orgs.ctp b/app/View/Users/statistics_orgs.ctp
index 4691a5bba..e705ecb9d 100644
--- a/app/View/Users/statistics_orgs.ctp
+++ b/app/View/Users/statistics_orgs.ctp
@@ -31,6 +31,7 @@
             <th><?php echo __('Name');?></th>
             <th><?php echo __('Users');?></th>
             <th><?php echo __('Events');?></th>
+            <th><?php echo __('Attributes');?></th>
             <th><?php echo __('Nationality');?></th>
             <th><?php echo __('Type');?></th>
             <th><?php echo __('Sector');?></th>
@@ -47,6 +48,7 @@
             <td class="short"><?php echo h($data['name']); ?></td>
             <td class="short"><span class="<?php echo isset($data['userCount']) ? 'blue bold' : 'grey'; ?>"><?php echo isset($data['userCount']) ? h($data['userCount']) : '0';?></span></td>
             <td class="short"><span class="<?php echo isset($data['eventCount']) ? 'blue bold' : 'grey'; ?>"><?php echo isset($data['eventCount']) ? h($data['eventCount']) : '0';?></span></td>
+            <td class="short"><span class="<?php echo isset($data['attributeCount']) ? 'blue bold' : 'grey'; ?>"><?php echo isset($data['attributeCount']) ? h($data['attributeCount']) : '0';?></span></td>
             <td class="shortish"><?php echo isset($data['nationality']) && $data['nationality'] !== 'Not specified' ? h($data['nationality']) : '&nbsp;'; ?></td>
             <td class="shortish"><?php echo isset($data['type']) ? h($data['type']) : '&nbsp;'; ?></td>
             <td class="shortish"><?php echo isset($data['sector']) ? h($data['sector']) : '&nbsp;'; ?></td>

From 806f4437646b75069261ccd1a8fd19df516ad63f Mon Sep 17 00:00:00 2001
From: mokaddem <sami.mokaddem@circl.lu>
Date: Sat, 16 Nov 2019 15:40:02 -0500
Subject: [PATCH 13/43] new: [statistics] Added organisation activity over time

---
 app/Controller/UsersController.php |  1 +
 app/Model/User.php                 | 53 ++++++++++++++++++++++++++++++
 app/View/Users/statistics_orgs.ctp |  8 +++++
 3 files changed, 62 insertions(+)

diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php
index 517b192b3..15d2cf5c0 100644
--- a/app/Controller/UsersController.php
+++ b/app/Controller/UsersController.php
@@ -1761,6 +1761,7 @@ class UsersController extends AppController
         foreach ($events as $event) {
             $orgs[$event['Event']['orgc_id']]['eventCount'] = $event[0]['count(*)'];
             $orgs[$event['Event']['orgc_id']]['attributeCount'] = $event[0]['attributeCount'];
+            $orgs[$event['Event']['orgc_id']]['orgActivity'] = $this->User->getOrgActivity($event['Event']['orgc_id'], array('event_timestamp' => '365d'));
         }
         unset($events);
         $orgs = Set::combine($orgs, '{n}.name', '{n}');
diff --git a/app/Model/User.php b/app/Model/User.php
index 4e5e53563..48a633ce3 100644
--- a/app/Model/User.php
+++ b/app/Model/User.php
@@ -1461,4 +1461,57 @@ class User extends AppModel
 
         return new Crypt_GPG($options);
     }
+
+    public function getOrgActivity($orgId, $params=array())
+    {
+        $conditions = array();
+        $options = array();
+        foreach($params as $paramName => $value) {
+            $options['filter'] = $paramName;
+            $filterParam[$paramName] = $value;
+            $conditions = $this->Event->set_filter_timestamp($filterParam, $conditions, $options);
+        }
+        $conditions['Event.orgc_id'] = $orgId;
+        $events = $this->Event->find('all', array(
+            'recursive' => -1,
+            'fields' => array('Event.orgc_id', 'Event.timestamp', 'Event.attribute_count'),
+            'conditions' => $conditions,
+            'order' => 'Event.timestamp'
+        ));
+        $sparklineData = array();
+        foreach ($events as $event) {
+            $date = date("Y-m-d", $event['Event']['timestamp']);
+            if (!isset($sparklineData[$event['Event']['attribute_count']][$date])) {
+                $sparklineData[$date] = $event['Event']['attribute_count'];
+            } else {
+                $sparklineData[$date] += $event['Event']['attribute_count'];
+            }
+        }
+
+        // get first and last timestamp
+        if (isset($params['from'])) {
+            $startDate = $params['from'];
+        } else {
+            $startDate = $this->resolveTimeDelta($params['event_timestamp']);
+        }
+        if (isset($params['to'])) {
+            $endDate = $params['to'];
+        } else {
+            $endDate = time();
+        }
+        $dates = array();
+        for ($d=$startDate; $d < $endDate; $d=$d+3600*24) { 
+            $dates[] = date('Y-m-d', $d);
+        }
+        $csv = 'Date,Close\n';
+        foreach ($dates as $date) {
+            $csv .= sprintf('%s,%s\n', $date, isset($sparklineData[$date]) ? $sparklineData[$date] : 0);
+        }
+        $data = array(
+            'csv' => $csv,
+            'data' => $sparklineData,
+            'orgId' => $orgId
+        );
+        return $data;
+    }
 }
diff --git a/app/View/Users/statistics_orgs.ctp b/app/View/Users/statistics_orgs.ctp
index e705ecb9d..3bbf4c836 100644
--- a/app/View/Users/statistics_orgs.ctp
+++ b/app/View/Users/statistics_orgs.ctp
@@ -35,6 +35,7 @@
             <th><?php echo __('Nationality');?></th>
             <th><?php echo __('Type');?></th>
             <th><?php echo __('Sector');?></th>
+            <th><?php echo __('Activity (1 year)');?></th>
     </tr>
     <?php
         foreach ($orgs as $data):
@@ -52,6 +53,13 @@
             <td class="shortish"><?php echo isset($data['nationality']) && $data['nationality'] !== 'Not specified' ? h($data['nationality']) : '&nbsp;'; ?></td>
             <td class="shortish"><?php echo isset($data['type']) ? h($data['type']) : '&nbsp;'; ?></td>
             <td class="shortish"><?php echo isset($data['sector']) ? h($data['sector']) : '&nbsp;'; ?></td>
+            <td class="shortish">
+                <?php
+                    if (isset($data['orgActivity'])) {
+                        echo $this->element('sparkline', array('scope' => 'organisation', 'id' => $data['id'], 'csv' => $data['orgActivity']['csv']));
+                    }
+                ?>
+            </td>
         </tr>
     <?php
         endforeach;

From 478015a32a898e1041ba1e1f22518f2f19b101ee Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Sat, 16 Nov 2019 17:43:25 -0500
Subject: [PATCH 14/43] fix: [stix2 export] Exporting stix2-pattern objects as
 pattern

... Instead of failing and being exported as custom object
---
 app/files/scripts/stix2/misp2stix2.py         | 2 +-
 app/files/scripts/stix2/misp2stix2_mapping.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/files/scripts/stix2/misp2stix2.py b/app/files/scripts/stix2/misp2stix2.py
index 4305f8c72..ebf8ab8ab 100644
--- a/app/files/scripts/stix2/misp2stix2.py
+++ b/app/files/scripts/stix2/misp2stix2.py
@@ -1267,7 +1267,7 @@ class StixBuilder():
         return pattern
 
     @staticmethod
-    def resolve_stix2_pattern(attributes):
+    def resolve_stix2_pattern(attributes, _):
         for attribute in attributes:
             if attribute['object_relation'] == 'stix2-pattern':
                 return attribute['value']
diff --git a/app/files/scripts/stix2/misp2stix2_mapping.py b/app/files/scripts/stix2/misp2stix2_mapping.py
index a4e71866c..fa6170703 100644
--- a/app/files/scripts/stix2/misp2stix2_mapping.py
+++ b/app/files/scripts/stix2/misp2stix2_mapping.py
@@ -305,6 +305,7 @@ objectsMapping = {'asn': {'to_call': 'handle_usual_object_name',
                   'registry-key': {'to_call': 'handle_usual_object_name',
                                    'observable': {'0': {'type': 'windows-registry-key'}},
                                    'pattern': "windows-registry-key:{0} = '{1}'"},
+                  'stix2-pattern': {'to_call': 'handle_usual_object_name'},
                   'url': {'to_call': 'handle_usual_object_name',
                           'observable': {'0': {'type': 'url'}},
                           'pattern': "url:{0} = '{1}'"},

From 288df9d8e7273af8253e6ad382a034825cbf0653 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 18 Nov 2019 11:35:10 +0100
Subject: [PATCH 15/43] chg: [internal] Renamed log action name for db worker
 issues to be <= 20 characters in length

- it was a restriction based on the db schema of the log table from before
---
 app/Model/AppModel.php |  8 +++-
 app/Model/Log.php      | 95 ++++++++++++++++++++++--------------------
 2 files changed, 55 insertions(+), 48 deletions(-)

diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php
index 04e3557d0..9f2ee8513 100644
--- a/app/Model/AppModel.php
+++ b/app/Model/AppModel.php
@@ -1297,6 +1297,10 @@ class AppModel extends Model
                 break;
             case 43:
                 $sqlArray[] = "ALTER TABLE sightingdbs ADD namespace varchar(255) DEFAULT '';";
+                break;
+            case 44:
+                $sqlArray[] = "ALTER TABLE object_template_elements CHANGE `disable_correlation` `disable_correlation` tinyint(1);";
+
                 break;
             case 'fixNonEmptySharingGroupID':
                 $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@@ -1658,7 +1662,7 @@ class AppModel extends Model
                             'model' => 'Server',
                             'model_id' => 0,
                             'email' => 'SYSTEM',
-                            'action' => 'update_database_worker',
+                            'action' => 'update_db_worker',
                             'user_id' => 0,
                             'title' => __('Issues executing run_updates'),
                             'change' => __('Database updates are locked. Worker not spawned')
@@ -1716,7 +1720,7 @@ class AppModel extends Model
                             'model' => 'Server',
                             'model_id' => 0,
                             'email' => 'SYSTEM',
-                            'action' => 'update_database_worker',
+                            'action' => 'update_db_worker',
                             'user_id' => 0,
                             'title' => __('Issues executing run_updates'),
                             'change' => __('Updates are locked. Stopping worker gracefully')
diff --git a/app/Model/Log.php b/app/Model/Log.php
index 99ddaa5c7..7060f5454 100644
--- a/app/Model/Log.php
+++ b/app/Model/Log.php
@@ -16,52 +16,55 @@ class Log extends AppModel
     );
     public $validate = array(
             'action' => array(
-            'rule' => array('inList', array(
-                            'accept',
-                            'accept_delegation',
-                            'add',
-                            'admin_email',
-                            'auth',
-                            'auth_fail',
-                            'blacklisted',
-                            'change_pw',
-                            'delete',
-                            'disable',
-                            'discard',
-                            'edit',
-                            'email',
-                            'enable',
-                            'error',
-                            'export',
-                            'file_upload',
-                            'galaxy',
-                            'include_formula',
-                            'login',
-                            'login_fail',
-                            'logout',
-                            'merge',
-                            'pruneUpdateLogs',
-                            'publish',
-                            'publish alert',
-                            'pull',
-                            'purge_events',
-                            'push',
-                            'remove_dead_workers',
-                            'request',
-                            'request_delegation',
-                            'reset_auth_key',
-                            'security',
-                            'serverSettingsEdit',
-                            'tag',
-                            'undelete',
-                            'update',
-                            'update_database',
-                            'update_database_worker',
-                            'upgrade_24',
-                            'upload_sample',
-                            'version_warning',
-                            'warning'
-                        )),
+            'rule' => array(
+                'inList',
+                array( // ensure that the length of the rules is < 20 in length
+                    'accept',
+                    'accept_delegation',
+                    'add',
+                    'admin_email',
+                    'auth',
+                    'auth_fail',
+                    'blacklisted',
+                    'change_pw',
+                    'delete',
+                    'disable',
+                    'discard',
+                    'edit',
+                    'email',
+                    'enable',
+                    'error',
+                    'export',
+                    'file_upload',
+                    'galaxy',
+                    'include_formula',
+                    'login',
+                    'login_fail',
+                    'logout',
+                    'merge',
+                    'pruneUpdateLogs',
+                    'publish',
+                    'publish alert',
+                    'pull',
+                    'purge_events',
+                    'push',
+                    'remove_dead_workers',
+                    'request',
+                    'request_delegation',
+                    'reset_auth_key',
+                    'security',
+                    'serverSettingsEdit',
+                    'tag',
+                    'undelete',
+                    'update',
+                    'update_database',
+                    'update_db_worker',
+                    'upgrade_24',
+                    'upload_sample',
+                    'version_warning',
+                    'warning'
+                )
+            ),
             'message' => 'Options : ...'
         )
     );

From 8008bbbd26a26911d7375182c0baaee7cb60735c Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 18 Nov 2019 11:36:31 +0100
Subject: [PATCH 16/43] new: [sql diagnostics] Started work on a system to
 automatically generate scripts to fix issues

- currently somewhat limited
- requires additional input to generate correct queries, needs an update for the default schemas
- generated, but not exposed for now
---
 app/Model/Server.php | 65 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 65 insertions(+)

diff --git a/app/Model/Server.php b/app/Model/Server.php
index 9fcca9fae..a49040d50 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -4293,9 +4293,69 @@ class Server extends AppModel
         } else {
             $schemaDiagnostic['error'] = sprintf('Diagnostic not available for DataSource `%s`', $dataSource);
         }
+        if (!empty($schemaDiagnostic['diagnostic'])) {
+            foreach ($schemaDiagnostic['diagnostic'] as $table => &$fields) {
+                foreach ($fields as &$field) {
+                    $field = $this->__attachRecoveryQuery($field, $table);
+                }
+            }
+        }
         return $schemaDiagnostic;
     }
 
+    /*
+     * Work in progress, still needs DEFAULT in the schema for it to work correctly
+     * Currently only works for missing_column and column_different
+     * Only currently supported field types are: int, tinyint, varchar, text
+     */
+    private function __attachRecoveryQuery($field, $table)
+    {
+        if ($field['is_critical']) {
+            $length = false;
+            if (in_array($field['error_type'], array('missing_column', 'column_different'))) {
+                if ($field['expected']['data_type'] === 'int') {
+                    $length = 11;
+                } elseif ($field['expected']['data_type'] === 'tinyint') {
+                    $length = 1;
+                } elseif ($field['expected']['data_type'] === 'varchar') {
+                    $length = $field['expected']['character_maximum_length'];
+                } elseif ($field['expected']['data_type'] === 'text') {
+                    $length = null;
+                }
+            }
+            if ($length !== false) {
+                switch($field['error_type']) {
+                    case 'missing_column':
+                        $field['sql'] = sprintf(
+                            'ALTER TABLE `%s` ADD COLUMN `%s` %s%s %s %s;',
+                            $table,
+                            $field['column_name'],
+                            $field['expected']['data_type'],
+                            $length !== null ? sprintf('(%d)', $length) : '',
+                            isset($field['expected']['default']) ? 'DEFAULT "' . $field['expected']['default'] . '"' : '',
+                            $field['expected']['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL',
+                            empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name']
+                        );
+                        break;
+                    case 'column_different':
+                        $field['sql'] = sprintf(
+                            'ALTER TABLE `%s` CHANGE `%s` `%s` %s%s %s %s;',
+                            $table,
+                            $field['column_name'],
+                            $field['column_name'],
+                            $field['expected']['data_type'],
+                            $length !== null ? sprintf('(%d)', $length) : '',
+                            isset($field['expected']['default']) ? 'DEFAULT "' . $field['expected']['default'] . '"' : '',
+                            $field['expected']['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL',
+                            empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name']
+                        );
+                        break;
+                }
+            }
+        }
+        return $field;
+    }
+
     public function getExpectedDBSchema()
     {
         App::uses('Folder', 'Utility');
@@ -4370,6 +4430,7 @@ class Server extends AppModel
             if (!array_key_exists($tableName, $dbActualSchema)) {
                 $dbDiff[$tableName][] = array(
                     'description' => sprintf(__('Table `%s` does not exist'), $tableName),
+                    'error_type' => 'missing_table',
                     'column_name' => $tableName,
                     'is_critical' => true
                 );
@@ -4395,6 +4456,7 @@ class Server extends AppModel
                     }
                     $dbDiff[$tableName][] = array(
                         'description' => sprintf(__('Column `%s` exists but should not'), $additionalKeys),
+                        'error_type' => 'additional_column',
                         'column_name' => $additionalKeys,
                         'is_critical' => false
                     );
@@ -4417,6 +4479,7 @@ class Server extends AppModel
                             $dbDiff[$tableName][] = array(
                                 'description' => sprintf(__('Column `%s` is different'), $columnName),
                                 'column_name' => $column['column_name'],
+                                'error_type' => 'column_different',
                                 'actual' => $keyedActualColumn[$columnName],
                                 'expected' => $column,
                                 'is_critical' => $isCritical
@@ -4426,6 +4489,7 @@ class Server extends AppModel
                         $dbDiff[$tableName][] = array(
                             'description' => sprintf(__('Column `%s` does not exist but should'), $columnName),
                             'column_name' => $columnName,
+                            'error_type' => 'missing_column',
                             'actual' => array(),
                             'expected' => $column,
                             'is_critical' => true
@@ -4438,6 +4502,7 @@ class Server extends AppModel
             $dbDiff[$additionalTable][] = array(
                 'description' => sprintf(__('Table `%s` is an additional table'), $additionalTable),
                 'column_name' => $additionalTable,
+                'error_type' => 'additional_table',
                 'is_critical' => false
             );
         }

From 846b1989c85382cc8e001b563333a484802ba83c Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 18 Nov 2019 15:58:06 +0100
Subject: [PATCH 17/43] fix: [API] fixed notice errors for compact() in PHP
 7.3+

---
 app/Controller/AttributesController.php | 4 ++--
 app/Controller/EventsController.php     | 5 ++---
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php
index c86aada28..b3ae81fca 100644
--- a/app/Controller/AttributesController.php
+++ b/app/Controller/AttributesController.php
@@ -1662,7 +1662,7 @@ class AttributesController extends AppController
                 'request' => $this->request,
                 'named_params' => $this->params['named'],
                 'paramArray' => $paramArray,
-                'ordered_url_params' => compact($paramArray),
+                'ordered_url_params' => @compact($paramArray),
                 'additional_delimiters' => PHP_EOL
             );
             $exception = false;
@@ -1877,7 +1877,7 @@ class AttributesController extends AppController
             'request' => $this->request,
             'named_params' => $this->params['named'],
             'paramArray' => $paramArray,
-            'ordered_url_params' => compact($paramArray)
+            'ordered_url_params' => @compact($paramArray)
         );
         $validFormats = $this->Attribute->validFormats;
         $exception = false;
diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php
index fb66b45d2..b2b75ae51 100644
--- a/app/Controller/EventsController.php
+++ b/app/Controller/EventsController.php
@@ -3205,7 +3205,7 @@ class EventsController extends AppController
             'request' => $this->request,
             'named_params' => $this->params['named'],
             'paramArray' => $paramArray,
-            'ordered_url_params' => compact($paramArray)
+            'ordered_url_params' => @compact($paramArray)
         );
         $exception = false;
         $filters = $this->_harvestParameters($filterData, $exception);
@@ -3497,7 +3497,7 @@ class EventsController extends AppController
             'request' => $this->request,
             'named_params' => $this->params['named'],
             'paramArray' => $paramArray,
-            'ordered_url_params' => compact($paramArray)
+            'ordered_url_params' => @compact($paramArray)
         );
         $exception = false;
         $filters = $this->_harvestParameters($filterData, $exception);
@@ -3543,7 +3543,6 @@ class EventsController extends AppController
             $filename .= '.' . $responseType;
             return $this->RestResponse->viewData($final, $responseType, false, true, $filename, array('X-Result-Count' => $elementCounter, 'X-Export-Module-Used' => $returnFormat, 'X-Response-Format' => $responseType));
         }
-
     }
 
     public function downloadOpenIOCEvent($key, $eventid, $enforceWarninglist = false)

From 4c58602e3a1980fad90ec8199d22108cb326620b Mon Sep 17 00:00:00 2001
From: mokaddem <sami.mokaddem@circl.lu>
Date: Mon, 18 Nov 2019 19:07:43 -0500
Subject: [PATCH 18/43] fix: [attribute:massEdit] Allow removal of non
 exportable tags. Fix #5408

---
 app/Model/AttributeTag.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/Model/AttributeTag.php b/app/Model/AttributeTag.php
index d2d5b11c2..28eed6969 100644
--- a/app/Model/AttributeTag.php
+++ b/app/Model/AttributeTag.php
@@ -172,6 +172,7 @@ class AttributeTag extends AppModel
         $attributes = $this->Attribute->fetchAttributes($user, array(
             'conditions' => $conditions,
             'flatten' => 1,
+            'includeAllTags' => 1
         ));
 
         if (empty($attributes)) {

From a1dcfb193172601ba37992b014f161696dcc1cdb Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Wed, 20 Nov 2019 15:30:06 +0100
Subject: [PATCH 19/43] new: [deprecation] Added a new library to handle
 deprecations

- send X-Deprecation-Warning via the API
- set new Warning flash messages via the UI
- counting the use of these functionalities / API endpoint and / user
  - added a diagnsitic tool to view the outcome of the collection
  - sharing of these collections with the MISP-Project will be optionally available in the future

- two modes of operation:
  - hard deprecation (functions certainly to be removed, reported to the users via API/UI)
  - soft deprecation (gauging interest for the continued use of these functions)
---
 app/Controller/AppController.php              | 15 ++++++-
 .../Component/RestResponseComponent.php       |  3 ++
 app/Controller/ServersController.php          | 11 +++++
 app/View/Elements/Flash/warning.ctp           |  4 ++
 .../Elements/healthElements/diagnostics.ctp   | 17 ++++++--
 .../Servers/view_deprecated_function_use.ctp  | 43 +++++++++++++++++++
 app/webroot/js/misp.js                        | 13 ++++++
 7 files changed, 102 insertions(+), 4 deletions(-)
 create mode 100644 app/View/Elements/Flash/warning.ctp
 create mode 100644 app/View/Servers/view_deprecated_function_use.ctp

diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
index a4446d9da..c8fd3df49 100755
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@@ -91,7 +91,8 @@ class AppController extends Controller
             'Flash',
             'Toolbox',
             'RateLimit',
-            'IndexFilter'
+            'IndexFilter',
+            'Deprecation'
             //,'DebugKit.Toolbar'
     );
 
@@ -473,6 +474,18 @@ class AppController extends Controller
         if ($this->_isRest()) {
             $this->__rateLimitCheck();
         }
+        if ($this->modelClass !== 'CakeError') {
+            $deprecationWarnings = $this->Deprecation->checkDeprecation($this->request->params['controller'], $this->action, $this->{$this->modelClass}, $this->Auth->user('id'));
+            if ($deprecationWarnings) {
+                $deprecationWarnings = __('WARNING: This functionality is deprecated and will be removed in the near future. ') . $deprecationWarnings;
+                if ($this->_isRest()) {
+                    $this->response->header('X-Deprecation-Warning', $deprecationWarnings);
+                    $this->components['RestResponse']['deprecationWarnings'] = $deprecationWarnings;
+                } else {
+                    $this->Flash->warning($deprecationWarnings);
+                }
+            }
+        }
         $this->components['RestResponse']['sql_dump'] = $this->sql_dump;
     }
 
diff --git a/app/Controller/Component/RestResponseComponent.php b/app/Controller/Component/RestResponseComponent.php
index 912e91912..38d4f1bd7 100644
--- a/app/Controller/Component/RestResponseComponent.php
+++ b/app/Controller/Component/RestResponseComponent.php
@@ -486,6 +486,9 @@ class RestResponseComponent extends Component
                 $cakeResponse->header($key, $value);
             }
         }
+        if (!empty($deprecationWarnings)) {
+            $cakeResponse->header('X-Deprecation-Warning', $deprecationWarnings);
+        }
         if ($download) {
             $cakeResponse->download($download);
         }
diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php
index 639b7027f..b9b1dba0d 100644
--- a/app/Controller/ServersController.php
+++ b/app/Controller/ServersController.php
@@ -2207,4 +2207,15 @@ misp.direct_call(relative_path, body)
         }
         return $this->RestResponse->viewData($this->Server->dbSchemaDiagnostic(), $this->response->type());
     }
+
+    public function viewDeprecatedFunctionUse()
+    {
+        $data = $this->Deprecation->getDeprecatedAccessList($this->Server);
+        if ($this->_isRest()) {
+            return $this->RestResponse->viewData($data, $this->response->type());
+        } else {
+            $this->layout = false;
+            $this->set('data', $data);
+        }
+    }
 }
diff --git a/app/View/Elements/Flash/warning.ctp b/app/View/Elements/Flash/warning.ctp
new file mode 100644
index 000000000..d9a31819f
--- /dev/null
+++ b/app/View/Elements/Flash/warning.ctp
@@ -0,0 +1,4 @@
+<div class="alert alert-warning">
+  <button type="button" class="close" data-dismiss="alert">&times;</button>
+  <?php echo h($message); ?>
+</div>
diff --git a/app/View/Elements/healthElements/diagnostics.ctp b/app/View/Elements/healthElements/diagnostics.ctp
index a73dff041..d14963535 100644
--- a/app/View/Elements/healthElements/diagnostics.ctp
+++ b/app/View/Elements/healthElements/diagnostics.ctp
@@ -417,9 +417,20 @@
     <h3><?php echo __('Clean model cache');?></h3>
     <p><?php echo __('If you ever run into issues with missing database fields / tables, please run the following script to clean the model cache.');?></p>
     <?php echo $this->Form->postLink('<span class="btn btn-inverse" style="padding-top:1px;padding-bottom:1px;">' . __('Clean cache') . '</span>', $baseurl . '/events/cleanModelCaches', array('escape' => false));?>
-    <h3><?php echo __('Overwritten objects');?></h3>
-    <p><?php echo __('Prior to 2.4.89, due to a bug a situation could occur where objects got overwritten on a sync pull. This tool allows you to inspect whether you are affected and if yes, remedy the issue.');?></p>
-    <a href="<?php echo $baseurl; ?>/objects/orphanedObjectDiagnostics"><span class="btn btn-inverse"><?php echo __('Reconstruct overwritten objects');?></span></a>
+    <?php
+        echo sprintf(
+            '<h3>%s</h3><p>%s</p><div id="deprecationResults"></div>%s',
+            __('Check for deprecated function usage'),
+            __('In an effort to identify the usage of deprecated functionalities, MISP has started aggregating the count of access requests to these endpoints. Check the frequency of their use below along with the users to potentially warn about better ways of achieving their goals.'),
+            sprintf(
+                '<span class="btn btn-inverse" role="button" tabindex="0" aria-label="%s" title="%s" onClick="%s">%s</span>',
+                __('View deprecated endpoint usage'),
+                __('View deprecated endpoint usage'),
+                'queryDeprecatedEndpointUsage();',
+                __('View deprecated endpoint usage')
+            )
+        );
+    ?>
     <h3><?php echo __('Orphaned attributes');?></h3>
     <p><?php echo __('In some rare cases attributes can remain in the database after an event is deleted becoming orphaned attributes. This means that they do not belong to any event, which can cause issues with the correlation engine (known cases include event deletion directly in the database without cleaning up the attributes and situations involving a race condition with an event deletion happening before all attributes are synchronised over).');?></p>
     <div style="background-color:#f7f7f9;width:400px;">
diff --git a/app/View/Servers/view_deprecated_function_use.ctp b/app/View/Servers/view_deprecated_function_use.ctp
new file mode 100644
index 000000000..cc5e19072
--- /dev/null
+++ b/app/View/Servers/view_deprecated_function_use.ctp
@@ -0,0 +1,43 @@
+<?php
+    foreach ($data as $controller => $controllerData) {
+        echo sprintf(
+            '<div class="bold blue">%s</div>',
+            h($controller)
+        );
+        foreach ($controllerData as $action => $userData) {
+            echo sprintf(
+                '<div class="bold" style="margin-left:8px">%s</div>%s',
+                h($action),
+                sprintf(
+                    '<div style="margin-left:16px;"><span class="bold">Total</span>: %s %s</div>',
+                    h($userData['total']),
+                    sprintf(
+                        '<i class="fas fa-plus-circle" role="button" aria-label="%s" data-toggle="collapse" data-target="#deprecationDetails%s%s"></i>',
+                        __('View details on the usage of %s on the %s controller', h($action), h($controller)),
+                        h($controller),
+                        h($action)
+                    )
+                )
+            );
+            $userDataDiv = '';
+            foreach ($userData as $userId => $count) {
+                if ($userId !== 'total') {
+                    $userDataDiv .= sprintf(
+                        '<div style="margin-left:24px;"><a href="%s" aria-label="%s">%s</a>: %s</div>',
+                        $baseurl . '/admin/users/view/' . h($userId),
+                        __('View user ID ', h($userId)),
+                        __('User #%s', h($userId)),
+                        h($count)
+                    );
+                }
+            }
+            echo sprintf(
+                '<div id="deprecationDetails%s%s" data-toggle="collapse" class="collapse">%s</div>',
+                h($controller),
+                h($action),
+                $userDataDiv
+            );
+        }
+    }
+
+?>
diff --git a/app/webroot/js/misp.js b/app/webroot/js/misp.js
index edcd75ae8..6258a400b 100644
--- a/app/webroot/js/misp.js
+++ b/app/webroot/js/misp.js
@@ -4656,6 +4656,19 @@ function checkRoleEnforceRateLimit() {
     }
 }
 
+function queryDeprecatedEndpointUsage() {
+    $.ajax({
+        url: baseurl + '/servers/viewDeprecatedFunctionUse',
+        type: 'GET',
+        success: function(data) {
+            $('#deprecationResults').html(data);
+        },
+        error: function(data) {
+            handleGenericAjaxResponse({'saved':false, 'errors':['Could not query the deprecation statistics.']});
+        }
+    });
+}
+
 (function(){
     "use strict";
     $(".datepicker").datepicker({

From 7aed94c3917dcf878e53025083d0fa25c62a20fd Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Wed, 20 Nov 2019 15:34:37 +0100
Subject: [PATCH 20/43] fix: [deprecation] Added missing component

---
 .../Component/DeprecationComponent.php        | 96 +++++++++++++++++++
 1 file changed, 96 insertions(+)
 create mode 100644 app/Controller/Component/DeprecationComponent.php

diff --git a/app/Controller/Component/DeprecationComponent.php b/app/Controller/Component/DeprecationComponent.php
new file mode 100644
index 000000000..f2ffff25a
--- /dev/null
+++ b/app/Controller/Component/DeprecationComponent.php
@@ -0,0 +1,96 @@
+<?php
+
+
+class DeprecationComponent extends Component
+{
+    public $redis = false;
+
+    /*
+     *  Deprecated endpoints
+     *  - simple controller->action structure
+     *  - each endpoint can be set to to a deprecation warning message or false
+     */
+    public $deprecatedEndpoints = false;
+
+    public function initialize(Controller $controller) {
+        $this->deprecatedEndpoints = array(
+            'attributes' => array(
+                'bro' => __('Use /attributes/restSearch to export in zeek format.'),
+                'rpz' => __('Use /attributes/restSearch to export RPZ rules.'),
+                'text' => __('Use /attributes/restSearch to export flat indicator lists.')
+            ),
+            'events' => array(
+                'addIOC' => __('Use MISP modules to import in OpenIOC format.'),
+                'csv' => __('Use /events/restSearch to export in CSV format.'),
+                'export' => __('Use the REST client to refine your search conditions and export in any of the given formats with much more control.'),
+                'hids' => __('Use /events/restSearch to export hashes.'),
+                'nids' => __('Use /events/restSearch to export in the various NIDS formats.'),
+                'stix' => __('Use /events/restSearch to export in STIX format.'),
+                'stix2' => __('Use /events/restSearch to export in STIX2 format.'),
+                'xml' => __('Use /events/restSearch to export in XML format. It is highly recommended to use JSON whenever possible.')
+            ),
+            'posts' => array(
+                'add' => false,
+                'index' => false
+            ),
+            'templates' => array(
+                'add' => false,
+                'populateEventFromTemplate' => false
+            ),
+            'whitelists' => array(
+                'admin_add' => false
+            )
+        );
+    }
+
+    public function checkDeprecation($controller, $action, $model, $user_id)
+    {
+        if (isset($this->deprecatedEndpoints[$controller][$action])) {
+            $this->__logDeprecatedAccess($controller, $action, $model, $user_id);
+            if ($this->deprecatedEndpoints[$controller][$action]) {
+                return $this->deprecatedEndpoints[$controller][$action];
+            }
+        }
+        return false;
+    }
+
+    private function __logDeprecatedAccess($controller, $action, $model, $user_id)
+    {
+        $this->redis = $model->setupRedis();
+        if ($this->redis) {
+            @$this->redis->hincrby(
+                'misp:deprecation',
+                sprintf(
+                    '%s:%s:%s',
+                    $controller,
+                    $action,
+                    $user_id
+                ),
+                1
+            );
+            $result = $this->redis->hGetAll('misp:deprecation');
+        }
+        return false;
+    }
+
+    public function getDeprecatedAccessList($model)
+    {
+        $this->redis = $model->setupRedis();
+        if ($this->redis) {
+            $rearranged = array();
+            @$result = $this->redis->hGetAll('misp:deprecation');
+            if (!empty($result)) {
+                foreach ($result as $key => $value) {
+                    $key_components = explode(':', $key);
+                    $rearranged[$key_components[0]][$key_components[1]][$key_components[2]] = intval($value);
+                    if (empty($rearranged[$key_components[0]][$key_components[1]]['total'])) {
+                        $rearranged[$key_components[0]][$key_components[1]]['total'] = intval($value);
+                    } else {
+                        $rearranged[$key_components[0]][$key_components[1]]['total'] += intval($value);
+                    }
+                }
+            }
+        }
+        return $rearranged;
+    }
+}

From af87a64111ac3ae62018097abcf7e7e6c46a125a Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Wed, 20 Nov 2019 15:36:48 +0100
Subject: [PATCH 21/43] fix: [API] bro deprecation message was premature

- needs to be added to restsearch first
---
 app/Controller/Component/DeprecationComponent.php | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/Controller/Component/DeprecationComponent.php b/app/Controller/Component/DeprecationComponent.php
index f2ffff25a..c2b8bc50f 100644
--- a/app/Controller/Component/DeprecationComponent.php
+++ b/app/Controller/Component/DeprecationComponent.php
@@ -15,7 +15,6 @@ class DeprecationComponent extends Component
     public function initialize(Controller $controller) {
         $this->deprecatedEndpoints = array(
             'attributes' => array(
-                'bro' => __('Use /attributes/restSearch to export in zeek format.'),
                 'rpz' => __('Use /attributes/restSearch to export RPZ rules.'),
                 'text' => __('Use /attributes/restSearch to export flat indicator lists.')
             ),

From 965b00d16415e4e9b1fb1f9a037f40fd8cf6d390 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Wed, 20 Nov 2019 15:53:38 +0100
Subject: [PATCH 22/43] chg: [internal] switch intval to (int)

---
 app/Controller/Component/DeprecationComponent.php | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/Controller/Component/DeprecationComponent.php b/app/Controller/Component/DeprecationComponent.php
index c2b8bc50f..849cc9b28 100644
--- a/app/Controller/Component/DeprecationComponent.php
+++ b/app/Controller/Component/DeprecationComponent.php
@@ -81,11 +81,11 @@ class DeprecationComponent extends Component
             if (!empty($result)) {
                 foreach ($result as $key => $value) {
                     $key_components = explode(':', $key);
-                    $rearranged[$key_components[0]][$key_components[1]][$key_components[2]] = intval($value);
+                    $rearranged[$key_components[0]][$key_components[1]][$key_components[2]] = (int)$value;
                     if (empty($rearranged[$key_components[0]][$key_components[1]]['total'])) {
-                        $rearranged[$key_components[0]][$key_components[1]]['total'] = intval($value);
+                        $rearranged[$key_components[0]][$key_components[1]]['total'] = (int)$value;
                     } else {
-                        $rearranged[$key_components[0]][$key_components[1]]['total'] += intval($value);
+                        $rearranged[$key_components[0]][$key_components[1]]['total'] += (int)$value;
                     }
                 }
             }

From 8438db45656ce7c09799b44e57e3795729fb68ba Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Wed, 20 Nov 2019 16:17:18 +0100
Subject: [PATCH 23/43] fix: [user view] server issues fixed

---
 app/Controller/UsersController.php | 7 ++++---
 app/View/Users/admin_view.ctp      | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php
index d97a89ba1..a00b76a62 100644
--- a/app/Controller/UsersController.php
+++ b/app/Controller/UsersController.php
@@ -483,7 +483,8 @@ class UsersController extends AppController
             'conditions' => array('User.id' => $id),
             'contain' => array(
                 'UserSetting',
-                'Role'
+                'Role',
+                'Organisation'
             )
         ));
         if (empty($user)) {
@@ -516,9 +517,9 @@ class UsersController extends AppController
             ), $this->response->type());
             return $this->RestResponse->viewData(array('User' => $user['User']), $this->response->type());
         } else {
-            $temp = $this->User->data['User']['invited_by'];
+            $user2 = $this->User->find('first', array('conditions' => array('User.id' => $user['User']['invited_by']), 'recursive' => -1));
             $this->set('id', $id);
-            $this->set('user2', $this->User->read(null, $temp));
+            $this->set('user2', $user2);
         }
     }
 
diff --git a/app/View/Users/admin_view.ctp b/app/View/Users/admin_view.ctp
index 4800b4c77..bab1219b6 100755
--- a/app/View/Users/admin_view.ctp
+++ b/app/View/Users/admin_view.ctp
@@ -41,7 +41,7 @@ $buttonModifyStatus = $mayModify ? 'button_on':'button_off';
         'key' => __('Authkey'),
         'html' => $authkey_data
     );
-    $table_data[] = array('key' => __('Invited By'), 'value' => $user2['User']['email']);
+    $table_data[] = array('key' => __('Invited By'), 'value' => empty($user2['User']['email']) ? 'N/A' : $user2['User']['email']);
     $org_admin_data = array();
     foreach ($user['User']['orgAdmins'] as $orgAdminId => $orgAdminEmail) {
         $org_admin_data[] = sprintf(

From 7ff656374ee17c6abd6861310c5eeb34126f070a Mon Sep 17 00:00:00 2001
From: Steve Clement <steve@localhost.lu>
Date: Thu, 21 Nov 2019 18:08:41 +0900
Subject: [PATCH 24/43] chg: [doc] Added 2 templates with automatic labelling

---
 .../bug_report.md}                                    | 11 ++++++++---
 .github/ISSUE_TEMPLATE/feature_request.md             |  8 ++++++++
 2 files changed, 16 insertions(+), 3 deletions(-)
 rename .github/{ISSUE_TEMPLATE.md => ISSUE_TEMPLATE/bug_report.md} (71%)
 create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE/bug_report.md
similarity index 71%
rename from .github/ISSUE_TEMPLATE.md
rename to .github/ISSUE_TEMPLATE/bug_report.md
index 215c8cf8e..45c83619e 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -1,6 +1,11 @@
-# This template is meant for bug reports, if you have a feature request, please be as descriptive as possible and delete the template
+---
+name: Bug report
+about: Create a report to help us improve
+labels: bug, needs triage
 
+---
 
+# This template is meant for bug reports, if you have a feature request user the other template.
 
 
 
@@ -16,8 +21,8 @@
 | Questions                 | Answers
 |---------------------------|--------------------
 | Type of issue             | Bug, Question, Feature Request, support...
-| OS version (server)       | Debian, ubuntu, CentOS, RedHat, ...
-| OS version (client)       | XP, Seven, 10, Ubuntu, ...
+| OS version (server)       | Debian, Ubuntu, CentOS, RedHat, ...
+| OS version (client)       | MacOS, Win10, Ubuntu, ...
 | PHP version               | 5.4, 5.5, 5.6, 7.0, 7.1...
 | MISP version / git hash   | 2.4.XX, hash of the commit
 | Browser                   | If applicable
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
new file mode 100644
index 000000000..6cc26a621
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,8 @@
+---
+name: Bug report
+about: Create a report to help us improve
+labels: feature request, needs triage
+
+---
+
+# This template is meant for feature requests. Please be as explicit as possible so the reader of your request can understand what kind of feature you want to see.

From 0b7e2b25ba109abb67f0c7d2e8491220b4190120 Mon Sep 17 00:00:00 2001
From: Steve Clement <steve@localhost.lu>
Date: Thu, 21 Nov 2019 18:11:07 +0900
Subject: [PATCH 25/43] chg: [doc] Better wording

---
 .github/ISSUE_TEMPLATE/feature_request.md | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
index 6cc26a621..d722eb063 100644
--- a/.github/ISSUE_TEMPLATE/feature_request.md
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -1,8 +1,19 @@
 ---
-name: Bug report
-about: Create a report to help us improve
+name: Feature request
+about: Suggest an idea for this project
 labels: feature request, needs triage
 
 ---
 
-# This template is meant for feature requests. Please be as explicit as possible so the reader of your request can understand what kind of feature you want to see.
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.
+

From 79ad82f58bb98c7e95583ff8152322b433bab537 Mon Sep 17 00:00:00 2001
From: StefanKelm <StefanKelm@users.noreply.github.com>
Date: Thu, 21 Nov 2019 10:31:06 +0100
Subject: [PATCH 26/43] Update AdminShell.php

Adding "wwwrun" as a user since it is common under SUSE Linux
---
 app/Console/Command/AdminShell.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/Console/Command/AdminShell.php b/app/Console/Command/AdminShell.php
index 9c42c6850..18aa58b7c 100644
--- a/app/Console/Command/AdminShell.php
+++ b/app/Console/Command/AdminShell.php
@@ -315,13 +315,13 @@ class AdminShell extends AppShell
 
     public function runUpdates() {
         $whoami = exec('whoami');
-        if ($whoami === 'httpd' || $whoami === 'www-data' || $whoami === 'apache') {
+        if ($whoami === 'httpd' || $whoami === 'www-data' || $whoami === 'apache' || $whoami === 'wwwrun') {
             echo 'Executing all updates to bring the database up to date with the current version.' . PHP_EOL;
             $processId = $this->args[0];
             $this->Server->runUpdates(true, false, $processId);
             echo 'All updates completed.' . PHP_EOL;
         } else {
-            die('This OS user is not allowed to run this command.'. PHP_EOL. 'Run it under `www-data` or `httpd`.' . PHP_EOL . 'You tried to run this command as: ' . $whoami . PHP_EOL);
+            die('This OS user is not allowed to run this command.'. PHP_EOL. 'Run it under `www-data` or `httpd` or `apache` or `wwwrun`.' . PHP_EOL . 'You tried to run this command as: ' . $whoami . PHP_EOL);
         }
     }
 

From 5d19b3a2e961394534e11ebc500418ed1044cdcd Mon Sep 17 00:00:00 2001
From: Andras Iklody <andras.iklody@gmail.com>
Date: Thu, 21 Nov 2019 10:41:05 +0100
Subject: [PATCH 27/43] fix: [ACL] added missing function

---
 app/Controller/Component/ACLComponent.php | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php
index a31250c30..094937317 100644
--- a/app/Controller/Component/ACLComponent.php
+++ b/app/Controller/Component/ACLComponent.php
@@ -360,6 +360,7 @@ class ACLComponent extends Component
                     'cache' => array(),
                     'changePriority' => array(),
                     'checkout' => array(),
+                    'clearWorkerQueue' => array(),
                     'createSync' => array('perm_sync'),
                     'delete' => array(),
                     'deleteFile' => array(),
@@ -403,7 +404,7 @@ class ACLComponent extends Component
                     'updateProgress' => array(),
                     'updateSubmodule' => array(),
                     'uploadFile' => array(),
-                    'clearWorkerQueue' => array()
+                    'viewDeprecatedFunctionUse' => array()
             ),
             'shadowAttributes' => array(
                     'accept' => array('perm_add'),

From a3ab148f670d6b4bf9593c4d2d150c725b73dcd8 Mon Sep 17 00:00:00 2001
From: Steve Clement <steve@localhost.lu>
Date: Fri, 22 Nov 2019 11:02:15 +0900
Subject: [PATCH 28/43] new: [doc] Support request template

---
 .github/ISSUE_TEMPLATE/bug_report.md      |  4 +--
 .github/ISSUE_TEMPLATE/support_request.md | 34 +++++++++++++++++++++++
 2 files changed, 36 insertions(+), 2 deletions(-)
 create mode 100644 .github/ISSUE_TEMPLATE/support_request.md

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index 45c83619e..b414d2927 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -1,7 +1,7 @@
 ---
 name: Bug report
 about: Create a report to help us improve
-labels: bug, needs triage
+labels: bug, potential-bug, needs triage
 
 ---
 
@@ -20,7 +20,7 @@ labels: bug, needs triage
 
 | Questions                 | Answers
 |---------------------------|--------------------
-| Type of issue             | Bug, Question, Feature Request, support...
+| Type of issue             | Bug
 | OS version (server)       | Debian, Ubuntu, CentOS, RedHat, ...
 | OS version (client)       | MacOS, Win10, Ubuntu, ...
 | PHP version               | 5.4, 5.5, 5.6, 7.0, 7.1...
diff --git a/.github/ISSUE_TEMPLATE/support_request.md b/.github/ISSUE_TEMPLATE/support_request.md
new file mode 100644
index 000000000..6fd8ded58
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/support_request.md
@@ -0,0 +1,34 @@
+---
+name: Support
+about: Support requests for MISP
+labels: support, needs triage
+
+---
+
+## Please consider the following notes
+- Critical security bugs can be reported, preferably PGP encrypted, by mail to: info@circl.lu
+- Bug reports and feature requests can be filed as an issue on github: https://github.com/MISP/MISP/issues
+- For interactive support please join the Gitter chat on: https://gitter.im/MISP/MISP
+
+The official documentation of MISP can be found here: https://www.circl.lu/doc/misp/
+We also periodically do user/admin/developer trainings and have our training material online: https://www.circl.lu/services/misp-training-materials/
+
+Nevertheless you can of course file a Support request as an issue. Please be as precise as possible and fill the template as detailed as possible too.
+Please remove this text until the line below. Thanks a lot.
+--------8<------
+
+### Work environment
+
+| Questions                 | Answers
+|---------------------------|--------------------
+| Type of issue             | Support
+| OS version (server)       | Debian, Ubuntu, CentOS, RedHat, ...
+| OS version (client)       | MacOS, Win10, Ubuntu, ...
+| PHP version               | 5.4, 5.5, 5.6, 7.0, 7.1... (if relevant)
+| MISP version / git hash   | 2.4.XX, hash of the commit
+| Browser                   | If applicable
+
+### Support Questions
+
+
+### Logs, screenshots, configuration dump, ...

From 8415b69c689609eba96ec5394574993943d9a078 Mon Sep 17 00:00:00 2001
From: Steve Clement <steve@localhost.lu>
Date: Fri, 22 Nov 2019 11:17:02 +0900
Subject: [PATCH 29/43] fix: [doc] composer update missing

---
 docs/xINSTALL.debian10.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/xINSTALL.debian10.md b/docs/xINSTALL.debian10.md
index 7dcd32dcd..d511d5747 100644
--- a/docs/xINSTALL.debian10.md
+++ b/docs/xINSTALL.debian10.md
@@ -1,12 +1,12 @@
 # INSTALLATION INSTRUCTIONS
-## for Debian 10.1 "buster"
+## for Debian 10.2 "buster"
 
 ### 0/ MISP debian stable install - Status
 ------------------------------------
 
 !!! notice
     This is mostly the install [@SteveClement](https://twitter.com/SteveClement) uses for testing, qc and random development.
-    Maintained and tested by @SteveClement on 20191016
+    Maintained and tested by @SteveClement on 20191122
 
 !!! warning
     PHP 7.3.4-2 is not working at the moment with the packaged composer.phar<br />
@@ -202,6 +202,7 @@ $SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === 'a5c698ffe4b
 $SUDO_WWW php composer-setup.php
 $SUDO_WWW php -r "unlink('composer-setup.php');"
 $SUDO_WWW php composer.phar install
+$SUDO_WWW php composer.phar update
 
 # Enable CakeResque with php-redis
 sudo phpenmod redis

From 5bd5adfe1762ffaa8acf44fc75ba3441d908d029 Mon Sep 17 00:00:00 2001
From: Steve Clement <steve@localhost.lu>
Date: Fri, 22 Nov 2019 13:02:26 +0900
Subject: [PATCH 30/43] chg: [doc] Tried to fix viper. Is semi-fixed viper-web
 broken

---
 docs/generic/viper-debian.md | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/docs/generic/viper-debian.md b/docs/generic/viper-debian.md
index 5e4e61f8a..5911d3dc2 100644
--- a/docs/generic/viper-debian.md
+++ b/docs/generic/viper-debian.md
@@ -3,8 +3,10 @@
 
 ```bash
 # <snippet-begin 6_viper.sh>
+# viper-web is broken ATM
 # Main Viper install function
 viper () {
+  export PATH=$PATH:/home/misp/.local/bin
   debug "Installing Viper dependencies"
   cd /usr/local/src/
   sudo apt-get install \
@@ -17,29 +19,25 @@ viper () {
   fi
   echo "Cloning Viper"
   $SUDO_USER git clone https://github.com/viper-framework/viper.git
+  $SUDO_USER git clone https://github.com/viper-framework/viper-web.git
   sudo chown -R $MISP_USER:$MISP_USER viper
+  sudo chown -R $MISP_USER:$MISP_USER viper-web
   cd viper
   echo "Creating virtualenv"
   $SUDO_USER virtualenv -p python3 venv
   echo "Submodule update"
   # TODO: Check for current user install permissions
   $SUDO_USER git submodule update --init --recursive
-  ##$SUDO git submodule update --init --recursive
-  echo "Pip install deps"
-  $SUDO_USER ./venv/bin/pip install SQLAlchemy PrettyTable python-magic
-  echo "pip install scrapy"
-  $SUDO_USER ./venv/bin/pip install scrapy
-  echo "install lief"
-  $SUDO_USER ./venv/bin/pip install https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
-  echo "pip install reqs"
-  $SUDO_USER ./venv/bin/pip install -r requirements.txt
-  $SUDO_USER sed -i '1 s/^.*$/\#!\/usr\/local\/src\/viper\/venv\/bin\/python/' viper-cli
+  echo "pip install deps"
+  $SUDO_USER ./venv/bin/pip install pefile olefile jbxapi Crypto pypdns pypssl r2pipe pdftools virustotal-api SQLAlchemy PrettyTable python-magic scrapy https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
+  $SUDO_USER ./venv/bin/pip install .
+  echo 'update-modules' |/usr/local/src/viper/venv/bin/viper
+  cd /usr/local/src/viper-web
   $SUDO_USER sed -i '1 s/^.*$/\#!\/usr\/local\/src\/viper\/venv\/bin\/python/' viper-web
-  echo "Launching viper-cli"
-  $SUDO_USER /usr/local/src/viper/viper-cli -h > /dev/null
+  $SUDO_USER /usr/local/src/viper/venv/bin/pip install -r requirements.txt
   echo "Launching viper-web"
-  $SUDO_USER /usr/local/src/viper/viper-web -p 8888 -H 0.0.0.0 &
-  echo 'PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/usr/local/src/viper:/var/www/MISP/app/Console"' |sudo tee /etc/environment
+  $SUDO_USER /usr/local/src/viper-web/viper-web -p 8888 -H 0.0.0.0 &
+  echo 'PATH="/home/misp/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/usr/local/src/viper:/var/www/MISP/app/Console"' |sudo tee /etc/environment
   echo ". /etc/environment" >> /home/${MISP_USER}/.profile
 
   # TODO: Perms, MISP_USER_HOME, nasty hack cuz Kali on R00t

From dd963c2e21d61f3c4d85ca0691ec3df00e03eecc Mon Sep 17 00:00:00 2001
From: Richard van den Berg <richard.vandenberg@ncsc.nl>
Date: Fri, 22 Nov 2019 21:53:51 +0100
Subject: [PATCH 31/43] Sync sightings on push, pull and push on add

---
 INSTALL/MYSQL.sql                             |   2 +
 INSTALL/POSTGRESQL-data-initial.sql           |   4 +-
 INSTALL/POSTGRESQL-structure.sql              |   2 +
 app/Console/Command/EventShell.php            |  22 ++++
 app/Console/Command/ServerShell.php           |   2 +-
 app/Controller/Component/ACLComponent.php     |   1 +
 .../Component/RestResponseComponent.php       |  10 +-
 app/Controller/EventsController.php           |  55 ++++++++-
 app/Controller/ServersController.php          |   6 +-
 app/Controller/SightingsController.php        |  26 ++++-
 app/Model/AppModel.php                        |   7 +-
 app/Model/Event.php                           | 104 ++++++++++++++----
 app/Model/Server.php                          |  80 ++++++++++++--
 app/Model/Sighting.php                        |  62 ++++++++++-
 app/View/Servers/add.ctp                      |   1 +
 app/View/Servers/edit.ctp                     |   1 +
 app/View/Servers/index.ctp                    |   2 +
 app/View/Servers/pull.ctp                     |  11 ++
 18 files changed, 356 insertions(+), 42 deletions(-)

diff --git a/INSTALL/MYSQL.sql b/INSTALL/MYSQL.sql
index 86f42f950..44c940f98 100644
--- a/INSTALL/MYSQL.sql
+++ b/INSTALL/MYSQL.sql
@@ -199,6 +199,7 @@ CREATE TABLE IF NOT EXISTS `events` (
   `locked` tinyint(1) NOT NULL DEFAULT 0,
   `threat_level_id` int(11) NOT NULL,
   `publish_timestamp` int(11) NOT NULL DEFAULT 0,
+  `sighting_timestamp` int(11) NOT NULL DEFAULT 0,
   `disable_correlation` tinyint(1) NOT NULL DEFAULT 0,
   `extends_uuid` varchar(40) COLLATE utf8_bin DEFAULT '',
   PRIMARY KEY (`id`),
@@ -821,6 +822,7 @@ CREATE TABLE IF NOT EXISTS `servers` (
   `org_id` int(11) NOT NULL,
   `push` tinyint(1) NOT NULL,
   `pull` tinyint(1) NOT NULL,
+  `push_sightings` tinyint(1) NOT NULL DEFAULT 0,
   `lastpulledid` int(11) DEFAULT NULL,
   `lastpushedid` int(11) DEFAULT NULL,
   `organization` varchar(10) COLLATE utf8_bin DEFAULT NULL,
diff --git a/INSTALL/POSTGRESQL-data-initial.sql b/INSTALL/POSTGRESQL-data-initial.sql
index ca95d6d33..5f28fdfbe 100644
--- a/INSTALL/POSTGRESQL-data-initial.sql
+++ b/INSTALL/POSTGRESQL-data-initial.sql
@@ -100,7 +100,7 @@ COPY public.event_tags (id, event_id, tag_id) FROM stdin;
 -- Data for Name: events; Type: TABLE DATA; Schema: public; Owner: -
 --
 
-COPY public.events (id, org_id, date, info, user_id, uuid, published, analysis, attribute_count, orgc_id, "timestamp", distribution, sharing_group_id, proposal_email_lock, locked, threat_level_id, publish_timestamp, disable_correlation, extends_uuid) FROM stdin;
+COPY public.events (id, org_id, date, info, user_id, uuid, published, analysis, attribute_count, orgc_id, "timestamp", distribution, sharing_group_id, proposal_email_lock, locked, threat_level_id, publish_timestamp, sighting_timestamp, disable_correlation, extends_uuid) FROM stdin;
 \.
 
 
@@ -323,7 +323,7 @@ COPY public.roles (id, name, created, modified, perm_add, perm_modify, perm_modi
 -- Data for Name: servers; Type: TABLE DATA; Schema: public; Owner: -
 --
 
-COPY public.servers (id, name, url, authkey, org_id, push, pull, lastpulledid, lastpushedid, organization, remote_org_id, publish_without_email, unpublish_event, self_signed, pull_rules, push_rules, cert_file, client_cert_file, internal) FROM stdin;
+COPY public.servers (id, name, url, authkey, org_id, push, pull, push_sightings, lastpulledid, lastpushedid, organization, remote_org_id, publish_without_email, unpublish_event, self_signed, pull_rules, push_rules, cert_file, client_cert_file, internal) FROM stdin;
 \.
 
 
diff --git a/INSTALL/POSTGRESQL-structure.sql b/INSTALL/POSTGRESQL-structure.sql
index f7daa9875..22584c2a6 100644
--- a/INSTALL/POSTGRESQL-structure.sql
+++ b/INSTALL/POSTGRESQL-structure.sql
@@ -335,6 +335,7 @@ CREATE TABLE public.events (
     locked boolean DEFAULT false NOT NULL,
     threat_level_id bigint NOT NULL,
     publish_timestamp bigint DEFAULT '0'::bigint NOT NULL,
+    sighting_timestamp bigint DEFAULT '0'::bigint NOT NULL,
     disable_correlation boolean DEFAULT false NOT NULL,
     extends_uuid character varying(40) DEFAULT ''::character varying
 );
@@ -1171,6 +1172,7 @@ CREATE TABLE public.servers (
     org_id bigint NOT NULL,
     push boolean NOT NULL,
     pull boolean NOT NULL,
+    push_sightings boolean DEFAULT false NOT NULL,
     lastpulledid bigint,
     lastpushedid bigint,
     organization character varying(10),
diff --git a/app/Console/Command/EventShell.php b/app/Console/Command/EventShell.php
index c5607bef8..cd58e360b 100644
--- a/app/Console/Command/EventShell.php
+++ b/app/Console/Command/EventShell.php
@@ -508,6 +508,28 @@ class EventShell extends AppShell
         $log->createLogEntry($user, 'publish', 'Event', $id, 'Event (' . $id . '): published.', 'published () => (1)');
     }
 
+    public function publish_sightings() {
+        $id = $this->args[0];
+        $passAlong = $this->args[1];
+        $jobId = $this->args[2];
+        $userId = $this->args[3];
+        $user = $this->User->getAuthUser($userId);
+        $job = $this->Job->read(null, $jobId);
+        $this->Event->Behaviors->unload('SysLogLogable.SysLogLogable');
+        $result = $this->Event->publish_sightings($id, $passAlong);
+        $job['Job']['progress'] = 100;
+        $job['Job']['date_modified'] = date("Y-m-d H:i:s");
+        if ($result) {
+            $job['Job']['message'] = 'Sightings published.';
+        } else {
+            $job['Job']['message'] = 'Sightings published, but the upload to other instances may have failed.';
+        }
+        $this->Job->save($job);
+        $log = ClassRegistry::init('Log');
+        $log->create();
+        $log->createLogEntry($user, 'publish_sightings', 'Event', $id, 'Sightings for event (' . $id . '): published.', 'publish_sightings updated');
+    }
+
     public function enrichment() {
         if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
             die('Usage: ' . $this->Server->command_line_functions['enrichment'] . PHP_EOL);
diff --git a/app/Console/Command/ServerShell.php b/app/Console/Command/ServerShell.php
index 52da69563..058f5cb5d 100644
--- a/app/Console/Command/ServerShell.php
+++ b/app/Console/Command/ServerShell.php
@@ -72,7 +72,7 @@ class ServerShell extends AppShell
                 'status' => 4
         ));
         if (is_array($result)) {
-            $message = sprintf(__('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled.', count($result[0]), count($result[1]), $result[2]));
+            $message = sprintf(__('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled.', count($result[0]), count($result[1]), $result[2], $result[3]));
         } else {
             $message = sprintf(__('ERROR: %s'), $result);
         }
diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php
index 094937317..8c17290e9 100644
--- a/app/Controller/Component/ACLComponent.php
+++ b/app/Controller/Component/ACLComponent.php
@@ -444,6 +444,7 @@ class ACLComponent extends Component
                     'listSightings' => array('*'),
                     'quickDelete' => array('perm_sighting'),
                     'viewSightings' => array('*'),
+                    'bulkSaveSightings' => array('OR' => array('perm_sync', 'perm_sighting')),
                     'quickAdd' => array('perm_sighting')
             ),
             'sightingdb' => array(
diff --git a/app/Controller/Component/RestResponseComponent.php b/app/Controller/Component/RestResponseComponent.php
index 38d4f1bd7..7c0c4d72f 100644
--- a/app/Controller/Component/RestResponseComponent.php
+++ b/app/Controller/Component/RestResponseComponent.php
@@ -171,11 +171,11 @@ class RestResponseComponent extends Component
             'add' => array(
                 'description' => "POST an Server object in JSON format to this API to add a server.",
                 'mandatory' => array('url', 'name', 'remote_org_id', 'authkey'),
-                'optional' => array('push', 'pull', 'push_rules', 'pull_rules', 'submitted_cert', 'submitted_client_cert', 'json')
+                'optional' => array('push', 'pull', 'push_sightings', 'push_rules', 'pull_rules', 'submitted_cert', 'submitted_client_cert', 'json')
             ),
             'edit' => array(
                 'description' => "POST an Server object in JSON format to this API to edit a server.",
-                'optional' => array('url', 'name', 'authkey', 'json', 'push', 'pull', 'push_rules', 'pull_rules', 'submitted_cert', 'submitted_client_cert', 'remote_org_id')
+                'optional' => array('url', 'name', 'authkey', 'json', 'push', 'pull', 'push_sightings', 'push_rules', 'pull_rules', 'submitted_cert', 'submitted_client_cert', 'remote_org_id')
             ),
             'serverSettings' => array(
                 'description' => "Send a GET request to this endpoint to get a full diagnostic along with all currently set settings of the current instance. This will also include the worker status"
@@ -1291,6 +1291,12 @@ class RestResponseComponent extends Component
                 'values' => array(1 => 'True', 0 => 'False' ),
                 'help' => __('Allow the upload of events and their attribute to the server')
             ),
+            'push_sightings' => array(
+                'input' => 'radio',
+                'type' => 'integer',
+                'values' => array(1 => 'True', 0 => 'False' ),
+                'help' => __('Allow the upload of sightings to the server')
+            ),
             'releasability' => array(
                 'input' => 'text',
                 'type' => 'string',
diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php
index b2b75ae51..7a843d7e2 100644
--- a/app/Controller/EventsController.php
+++ b/app/Controller/EventsController.php
@@ -733,7 +733,7 @@ class EventsController extends AppController
             if (!empty($passedArgs['searchminimal'])) {
                 unset($rules['contain']);
                 $rules['recursive'] = -1;
-                $rules['fields'] = array('id', 'timestamp', 'published', 'uuid');
+                $rules['fields'] = array('id', 'timestamp', 'sighting_timestamp', 'published', 'uuid');
                 $rules['contain'] = array('Orgc.uuid');
             }
             $paginationRules = array('page', 'limit', 'sort', 'direction', 'order');
@@ -2572,6 +2572,59 @@ class EventsController extends AppController
         }
     }
 
+    public function publishSightings($id = null)
+    {
+        $id = $this->Toolbox->findIdByUuid($this->Event, $id);
+        $event = fetchEvent(
+            $this->Auth->user(),
+            array(
+                'eventid' => $id,
+                'metadata' => 1
+            )
+        );
+        if (empty($event)) {
+            throw new NotFoundException(__('Invalid event'));
+        }
+        if ($this->request->is('post') || $this->request->is('put')) {
+            $result = $this->Event->publishRouter($id, null, $this->Auth->user(), 'sightings');
+            if (!Configure::read('MISP.background_jobs')) {
+                if (!is_array($result)) {
+                    // redirect to the view event page
+                    $message = 'Sightings published';
+                } else {
+                    $lastResult = array_pop($result);
+                    $resultString = (count($result) > 0) ? implode(', ', $result) . ' and ' . $lastResult : $lastResult;
+                    $errors['failed_servers'] = $result;
+                    $message = sprintf('Sightings published but not pushed to %s, re-try later. If the issue persists, make sure that the correct sync user credentials are used for the server link and that the sync user on the remote server has authentication privileges.', $resultString);
+                }
+            } else {
+                // update the DB to set the published flag
+                // for background jobs, this should be done already
+                $fieldList = array('id', 'info', 'sighting_timestamp');
+                $event['Event']['sighting_timestamp'] = time();
+                $this->Event->save($event, array('fieldList' => $fieldList));
+                $message = 'Job queued';
+            }
+            if ($this->_isRest()) {
+                $this->set('name', 'Publish Sightings');
+                $this->set('message', $message);
+                if (!empty($errors)) {
+                    $this->set('errors', $errors);
+                }
+                $this->set('url', '/events/publishSightings/' . $id);
+                $this->set('id', $id);
+                $this->set('_serialize', array('name', 'message', 'url', 'id', 'errors'));
+            } else {
+                $this->Flash->success($message);
+                $this->redirect(array('action' => 'view', $id));
+            }
+        } else {
+            $this->set('id', $id);
+            $this->set('type', 'publish_sightings');
+            $this->render('ajax/eventPublishConfirmationForm');
+        }
+    }
+
     // Publishes the event without sending an alert email
     public function publish($id = null)
     {
diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php
index b9b1dba0d..069a945a4 100644
--- a/app/Controller/ServersController.php
+++ b/app/Controller/ServersController.php
@@ -250,6 +250,7 @@ class ServersController extends AppController
                     $defaults = array(
                         'push' => 0,
                         'pull' => 0,
+                        'push_sightings' => 0,
                         'caching_enabled' => 0,
                         'json' => '[]',
                         'push_rules' => '[]',
@@ -444,7 +445,7 @@ class ServersController extends AppController
             }
             if (!$fail) {
                 // say what fields are to be updated
-                $fieldList = array('id', 'url', 'push', 'pull', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
+                $fieldList = array('id', 'url', 'push', 'pull', 'push_sightings', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
                 $this->request->data['Server']['id'] = $id;
                 if (isset($this->request->data['Server']['authkey']) && "" != $this->request->data['Server']['authkey']) {
                     $fieldList[] = 'authkey';
@@ -663,13 +664,14 @@ class ServersController extends AppController
             if (!Configure::read('MISP.background_jobs')) {
                 $result = $this->Server->pull($this->Auth->user(), $id, $technique, $s);
                 if (is_array($result)) {
-                    $success = sprintf(__('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled.', count($result[0]), count($result[1]), $result[2]));
+                    $success = sprintf(__('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled.', count($result[0]), count($result[1]), $result[2], $result[3]));
                 } else {
                     $error = $result;
                 }
                 $this->set('successes', $result[0]);
                 $this->set('fails', $result[1]);
                 $this->set('pulledProposals', $result[2]);
+                $this->set('pulledSightings', $result[3]);
             } else {
                 $this->loadModel('Job');
                 $this->Job->create();
diff --git a/app/Controller/SightingsController.php b/app/Controller/SightingsController.php
index a9670e004..29686827d 100644
--- a/app/Controller/SightingsController.php
+++ b/app/Controller/SightingsController.php
@@ -66,7 +66,7 @@ class SightingsController extends AppController
                 $source = isset($this->request->data['source']) ? trim($this->request->data['source']) : '';
             }
             if (!$error) {
-                $result = $this->Sighting->saveSightings($id, $values, $timestamp, $this->Auth->user(), $type, $source);
+                $result = $this->Sighting->saveSightings($id, $values, $timestamp, $this->Auth->user(), $type, $source, false, true);
             }
             if (!is_numeric($result)) {
                 $error = $result;
@@ -422,4 +422,28 @@ class SightingsController extends AppController
         $this->layout = 'ajax';
         $this->render('ajax/view_sightings');
     }
+
+    // Save sightings synced over, restricted to sync users
+    public function bulkSaveSightings($eventId = false)
+    {
+        if ($this->request->is('post')) {
+            if (empty($this->request->data['Sighting'])) {
+                $sightings = $this->request->data;
+            } else {
+                $sightings = $this->request->data['Sighting'];
+            }
+            $saved = $this->Sighting->bulkSaveSightings($eventId, $sightings, $this->Auth->user());
+            if (is_numeric($saved)) {
+                if ($saved > 0) {
+                   return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => $saved . ' sightings added.')), 'status' => 200, 'type' => 'json'));
+                } else {
+                    return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'success' => 'No sightings added.')), 'status' => 200, 'type' => 'json'));
+                }
+            } else {
+                throw new MethodNotAllowedException($saved);
+            }
+        } else {
+            throw new MethodNotAllowedException('This method is only accessible via POST requests.');
+        }
+    }
 }
diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php
index 9f2ee8513..5cb8aa167 100644
--- a/app/Model/AppModel.php
+++ b/app/Model/AppModel.php
@@ -76,7 +76,8 @@ class AppModel extends Model
         21 => false, 22 => false, 23 => false, 24 => false, 25 => false, 26 => false,
         27 => false, 28 => false, 29 => false, 30 => false, 31 => false, 32 => false,
         33 => false, 34 => false, 35 => false, 36 => false, 37 => false, 38 => false,
-        39 => false, 40 => false, 41 => false, 42 => false, 43 => false
+        39 => false, 40 => false, 41 => false, 42 => false, 43 => false, 44 => false,
+        45 => false
     );
 
     public $advanced_updates_description = array(
@@ -1301,6 +1302,10 @@ class AppModel extends Model
             case 44:
                 $sqlArray[] = "ALTER TABLE object_template_elements CHANGE `disable_correlation` `disable_correlation` tinyint(1);";
 
+                break;
+            case 45:
+                $sqlArray[] = "ALTER TABLE `events` ADD `sighting_timestamp` int(11) NOT NULL DEFAULT 0 AFTER `publish_timestamp`;";
+                $sqlArray[] = "ALTER TABLE `servers` ADD `push_sightings` tinyint(1) NOT NULL DEFAULT 0 AFTER `pull`;";
                 break;
             case 'fixNonEmptySharingGroupID':
                 $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
diff --git a/app/Model/Event.php b/app/Model/Event.php
index 2792a1149..d2d548086 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -1063,9 +1063,9 @@ class Event extends AppModel
         return $error;
     }
 
-    private function __executeRestfulEventToServer($event, $server, $resourceId, &$newLocation, &$newTextBody, $HttpSocket)
+    private function __executeRestfulEventToServer($event, $server, $resourceId, &$newLocation, &$newTextBody, $HttpSocket, $scope)
     {
-        $result = $this->restfulEventToServer($event, $server, $resourceId, $newLocation, $newTextBody, $HttpSocket);
+        $result = $this->restfulEventToServer($event, $server, $resourceId, $newLocation, $newTextBody, $HttpSocket, $scope);
         if (is_numeric($result)) {
             $error = $this->__resolveErrorCode($result, $event, $server);
             if ($error) {
@@ -1075,24 +1075,26 @@ class Event extends AppModel
         return true;
     }
 
-    public function uploadEventToServer($event, $server, $HttpSocket = null)
+    public function uploadEventToServer($event, $server, $HttpSocket = null, $scope = 'events')
     {
         $this->Server = ClassRegistry::init('Server');
         $push = $this->Server->checkVersionCompatibility($server['Server']['id'], false, $HttpSocket);
-        if (empty($push['canPush'])) {
+        if ($scope === 'events' && empty($push['canPush'])) {
             return 'The remote user is not a sync user - the upload of the event has been blocked.';
+        } elseif ($scope === 'sightings' && empty($push['canPush']) && empty($push['canSight'])) {
+            return 'The remote user is not a sightings user - the upload of the sightings has been blocked.';
         }
         if (!empty($server['Server']['unpublish_event'])) {
             $event['Event']['published'] = 0;
         }
         $updated = null;
         $newLocation = $newTextBody = '';
-        $result = $this->__executeRestfulEventToServer($event, $server, null, $newLocation, $newTextBody, $HttpSocket);
+        $result = $this->__executeRestfulEventToServer($event, $server, null, $newLocation, $newTextBody, $HttpSocket, $scope);
         if ($result !== true) {
             return $result;
         }
         if (strlen($newLocation)) { // HTTP/1.1 302 Found and Location: http://<newLocation>
-            $result = $this->__executeRestfulEventToServer($event, $server, $newLocation, $newLocation, $newTextBody, $HttpSocket);
+            $result = $this->__executeRestfulEventToServer($event, $server, $newLocation, $newLocation, $newTextBody, $HttpSocket, $scope);
             if ($result !== true) {
                 return $result;
             }
@@ -1181,7 +1183,7 @@ class Event extends AppModel
     }
 
     // Uploads the event and the associated Attributes to another Server
-    public function restfulEventToServer($event, $server, $urlPath, &$newLocation, &$newTextBody, $HttpSocket = null)
+    public function restfulEventToServer($event, $server, $urlPath, &$newLocation, &$newTextBody, $HttpSocket = null, $scope)
     {
         $event = $this->__prepareForPushToServer($event, $server);
         if (is_numeric($event)) {
@@ -1190,7 +1192,11 @@ class Event extends AppModel
         $url = $server['Server']['url'];
         $HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
         $request = $this->setupSyncRequest($server);
-        $uri = $url . '/events' . $this->__getLastUrlPathComponent($urlPath);
+        if ($scope === 'sightings') {
+            $scope .= '/bulkSaveSightings';
+            $urlPath = $event['Event']['uuid'];
+        }
+        $uri = $url . '/' . $scope . $this->__getLastUrlPathComponent($urlPath);
         $data = json_encode($event);
         if (!empty(Configure::read('Security.sync_audit'))) {
             $pushLogEntry = sprintf(
@@ -2095,6 +2101,7 @@ class Event extends AppModel
         if ($options['metadata']) {
             unset($params['contain']['Attribute']);
             unset($params['contain']['ShadowAttribute']);
+            unset($params['contain']['Object']);
         }
         if ($user['Role']['perm_site_admin']) {
             $params['contain']['User'] = array('fields' => 'email');
@@ -2104,7 +2111,6 @@ class Event extends AppModel
             return array();
         }
         // Do some refactoring with the event
-        $this->Sighting = ClassRegistry::init('Sighting');
         $userEmails = array();
         $fields = array(
             'common' => array('distribution', 'sharing_group_id', 'uuid'),
@@ -2269,7 +2275,10 @@ class Event extends AppModel
                 }
                 $event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit, 'Server');
             }
-            $event['Sighting'] = $this->Sighting->attachToEvent($event, $user);
+            if (empty($options['metadata'])) {
+                $this->Sighting = ClassRegistry::init('Sighting');
+                $event['Sighting'] = $this->Sighting->attachToEvent($event, $user);
+            }
             if ($options['includeSightingdb']) {
                 $this->Sightingdb = ClassRegistry::init('Sightingdb');
                 $event = $this->Sightingdb->attachToEvent($event, $user);
@@ -4047,8 +4056,8 @@ class Event extends AppModel
         return true;
     }
 
-    // Uploads this specific event to all remote servers
-    public function uploadEventToServersRouter($id, $passAlong = null)
+    // Uploads this specific event or sightings to all remote servers
+    public function uploadEventToServersRouter($id, $passAlong = null, $scope = 'events')
     {
         $eventOrgcId = $this->find('first', array(
             'conditions' => array('Event.id' => $id),
@@ -4074,6 +4083,11 @@ class Event extends AppModel
         }
         $event = $event[0];
         $event['Event']['locked'] = 1;
+        // attach sightings if needed
+        if ($scope === 'sightings') {
+            $this->Sighting = ClassRegistry::init('Sighting');
+            $event['Sighting'] = $this->Sighting->attachToEvent($event, $elevatedUser);
+        }
         // get a list of the servers
         $this->Server = ClassRegistry::init('Server');
         $conditions = array('push' => 1);
@@ -4092,7 +4106,8 @@ class Event extends AppModel
         $failedServers = array();
         App::uses('SyncTool', 'Tools');
         foreach ($servers as &$server) {
-            if ((!isset($server['Server']['internal']) || !$server['Server']['internal']) && $event['Event']['distribution'] < 2) {
+            if (((!isset($server['Server']['internal']) || !$server['Server']['internal']) && $event['Event']['distribution'] < 2) ||
+                ((!isset($server['Server']['push_sightings']) || !$server['Server']['push_sightings'])) && $scope === 'sightings') {
                 continue;
             }
             $syncTool = new SyncTool();
@@ -4116,7 +4131,7 @@ class Event extends AppModel
                 $event = $this->fetchEvent($elevatedUser, $params);
                 $event = $event[0];
                 $event['Event']['locked'] = 1;
-                $thisUploaded = $this->uploadEventToServer($event, $server, $HttpSocket);
+                $thisUploaded = $this->uploadEventToServer($event, $server, $HttpSocket, $scope);
                 if (!$thisUploaded) {
                     $uploaded = !$uploaded ? $uploaded : $thisUploaded;
                     $failedServers[] = $server['Server']['url'];
@@ -4149,9 +4164,16 @@ class Event extends AppModel
         return $workerType;
     }
 
-    public function publishRouter($id, $passAlong = null, $user)
+    public function publishRouter($id, $passAlong = null, $user, $scope = 'events')
     {
         if (Configure::read('MISP.background_jobs')) {
+            $job_type = 'publish_' . $scope;
+            $function = 'publish';
+            $message = 'Publishing.';
+            if ($scope === 'sightings') {
+                $message = 'Publishing sightings.';
+                $function = 'publish_sightings';
+            }
             $job = ClassRegistry::init('Job');
             $job->create();
             $data = array(
@@ -4162,24 +4184,56 @@ class Event extends AppModel
                     'retries' => 0,
                     'org_id' => $user['org_id'],
                     'org' => $user['Organisation']['name'],
-                    'message' => 'Publishing.',
+                    'message' => $message
             );
             $job->save($data);
             $jobId = $job->id;
             $process_id = CakeResque::enqueue(
                     'prio',
                     'EventShell',
-                    array('publish', $id, $passAlong, $jobId, $user['id']),
+                    array($function, $id, $passAlong, $jobId, $user['id']),
                     true
             );
             $job->saveField('process_id', $process_id);
             return $process_id;
+        } elseif ($scope === 'sightings') {
+            $result = $this->publish_sightings($id, $passAlong);
+            return $result;
         } else {
             $result = $this->publish($id, $passAlong);
             return $result;
         }
     }
 
+    public function publish_sightings($id, $passAlong = null, $jobId = null)
+    {
+        if (is_numeric($id)) {
+            $condition = array('Event.id' => $id);
+        } else {
+            $condition = array('Event.uuid' => $id);
+        }
+        $event = $this->find('first', array(
+            'recursive' => -1,
+            'conditions' => $condition
+        ));
+        if (empty($event)) {
+            return false;
+        }
+        if ($jobId) {
+            $this->Behaviors->unload('SysLogLogable.SysLogLogable');
+        } else {
+            // update the DB to set the sightings timestamp
+            // for background jobs, this should be done already
+            $fieldList = array('id', 'info', 'sighting_timestamp');
+            $event['Event']['sighting_timestamp'] = time();
+            $event['Event']['skip_zmq'] = 1;
+            $event['Event']['skip_kafka'] = 1;
+            $this->save($event, array('fieldList' => $fieldList));
+        }
+        $uploaded = $this->uploadEventToServersRouter($id, $passAlong, 'sightings');
+        return $uploaded;
+    }
+
     // Performs all the actions required to publish an event
     public function publish($id, $passAlong = null, $jobId = null)
     {
@@ -4450,19 +4504,27 @@ class Event extends AppModel
         return false;
     }
 
-    public function removeOlder(&$eventArray)
+    public function removeOlder(&$eventArray, $scope = 'events')
     {
+        if ($scope === 'sightings' ) {
+            $field = 'sighting_timestamp';
+        } else {
+            $field = 'timestamp';
+        }
         $uuidsToCheck = array();
         foreach ($eventArray as $k => &$event) {
             $uuidsToCheck[$event['uuid']] = $k;
         }
         $localEvents = array();
-        $temp = $this->find('all', array('recursive' => -1, 'fields' => array('Event.uuid', 'Event.timestamp', 'Event.locked')));
+        $temp = $this->find('all', array('recursive' => -1, 'fields' => array('Event.uuid', 'Event.' . $field, 'Event.locked')));
         foreach ($temp as $e) {
-            $localEvents[$e['Event']['uuid']] = array('timestamp' => $e['Event']['timestamp'], 'locked' => $e['Event']['locked']);
+            $localEvents[$e['Event']['uuid']] = array($field => $e['Event'][$field], 'locked' => $e['Event']['locked']);
         }
         foreach ($uuidsToCheck as $uuid => $eventArrayId) {
-            if (isset($localEvents[$uuid]) && ($localEvents[$uuid]['timestamp'] >= $eventArray[$eventArrayId]['timestamp'] || !$localEvents[$uuid]['locked'])) {
+            if (isset($localEvents[$uuid])
+                  && ($localEvents[$uuid][$field] >= $eventArray[$eventArrayId][$field]
+                  || ($scope === 'events' && !$localEvents[$uuid]['locked'])))
+            {
                 unset($eventArray[$eventArrayId]);
             }
         }
diff --git a/app/Model/Server.php b/app/Model/Server.php
index a49040d50..0a2b0ad18 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -81,6 +81,16 @@ class Server extends AppModel
                 //'on' => 'create', // Limit validation to 'create' or 'update' operations
             ),
         ),
+        'push_sightings' => array(
+            'boolean' => array(
+                'rule' => array('boolean'),
+                //'message' => 'Your custom message here',
+                'allowEmpty' => true,
+                'required' => false,
+                //'last' => false, // Stop validation after this rule
+                //'on' => 'create', // Limit validation to 'create' or 'update' operations
+            ),
+        ),
         'lastpushedid' => array(
             'numeric' => array(
                 'rule' => array('numeric'),
@@ -2495,6 +2505,11 @@ class Server extends AppModel
             $job->saveField('message', 'Pulling proposals.');
         }
         $pulledProposals = $eventModel->ShadowAttribute->pullProposals($user, $server);
+        if ($jobId) {
+            $job->saveField('progress', 75);
+            $job->saveField('message', 'Pulling sightings.');
+        }
+        $pulledSightings = $eventModel->Sighting->pullSightings($user, $server);
         if ($jobId) {
             $job->saveField('progress', 100);
             $job->saveField('message', 'Pull completed.');
@@ -2511,13 +2526,14 @@ class Server extends AppModel
             'user_id' => $user['id'],
             'title' => 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $email,
             'change' => sprintf(
-                '%s events and %s proposals pulled or updated. %s events failed or didn\'t need an update.',
+                '%s events, %s proposals and %s sightings pulled or updated. %s events failed or didn\'t need an update.',
                 count($successes),
                 $pulledProposals,
+                $pulledSightings,
                 count($fails)
             )
         ));
-        return array($successes, $fails, $pulledProposals);
+        return array($successes, $fails, $pulledProposals, $pulledSightings);
     }
 
     public function filterRuleToParameter($filter_rules)
@@ -2549,7 +2565,7 @@ class Server extends AppModel
 
 
     // Get an array of event_ids that are present on the remote server
-    public function getEventIdsFromServer($server, $all = false, $HttpSocket=null, $force_uuid=false, $ignoreFilterRules = false)
+    public function getEventIdsFromServer($server, $all = false, $HttpSocket=null, $force_uuid=false, $ignoreFilterRules = false, $scope = 'events')
     {
         $url = $server['Server']['url'];
         if ($ignoreFilterRules) {
@@ -2576,8 +2592,21 @@ class Server extends AppModel
                 $eventIds = array();
                 if ($all) {
                     if (!empty($eventArray)) {
-                        foreach ($eventArray as $event) {
-                            $eventIds[] = $event['uuid'];
+                        if ($scope === 'sightings') {
+                            foreach ($eventArray as $event) {
+                                $localEvent = $this->Event->find('first', array(
+                                        'recursive' => -1,
+                                        'fields' => array('Event.uuid', 'Event.sighting_timestamp'),
+                                        'conditions' => array('Event.uuid' => $event['uuid'])
+                                    ));
+                                if (!empty($localEvent) && $localEvent['Event']['sighting_timestamp'] > $event['sighting_timestamp']) {
+                                    $eventIds[] = $event['uuid'];
+                                }
+                            }
+                        } else {
+                            foreach ($eventArray as $event) {
+                                $eventIds[] = $event['uuid'];
+                            }
                         }
                     }
                 } else {
@@ -2617,7 +2646,7 @@ class Server extends AppModel
                             }
                         }
                     }
-                    $this->Event->removeOlder($eventArray);
+                    $this->Event->removeOlder($eventArray, $scope);
                     if (!empty($eventArray)) {
                         foreach ($eventArray as $event) {
                             if ($force_uuid) {
@@ -2722,7 +2751,7 @@ class Server extends AppModel
                 ), // array of conditions
                 'recursive' => -1, //int
                 'contain' => array('EventTag' => array('fields' => array('EventTag.tag_id'))),
-                'fields' => array('Event.id', 'Event.timestamp', 'Event.uuid', 'Event.orgc_id'), // array of field names
+                'fields' => array('Event.id', 'Event.timestamp', 'Event.sighting_timestamp', 'Event.uuid', 'Event.orgc_id'), // array of field names
         );
         $eventIds = $this->Event->find('all', $findParams);
         $eventUUIDsFiltered = $this->getEventIdsForPush($id, $HttpSocket, $eventIds, $user);
@@ -2731,7 +2760,7 @@ class Server extends AppModel
         }
         if (!empty($eventUUIDsFiltered)) {
             $eventCount = count($eventUUIDsFiltered);
-            // now process the $eventIds to pull each of the events sequentially
+            // now process the $eventIds to push each of the events sequentially
             if (!empty($eventUUIDsFiltered)) {
                 $successes = array();
                 $fails = array();
@@ -2779,9 +2808,12 @@ class Server extends AppModel
         }
 
         $this->syncProposals($HttpSocket, $this->data, null, null, $this->Event);
+        $sightingSuccesses = $this->syncSightings($HttpSocket, $this->data, $user, $this->Event);
 
         if (!isset($successes)) {
-            $successes = array();
+            $successes = $sightingSuccesses;
+        } else {
+            $successes = array_merge($successes, $sightingSuccesses);
         }
         if (!isset($fails)) {
             $fails = array();
@@ -2834,6 +2866,33 @@ class Server extends AppModel
         return $uuidList;
     }
 
+    public function syncSightings($HttpSocket, $server, $user, $eventModel)
+    {
+        $successes = array();
+        if (!$server['Server']['push_sightings']) {
+            return $successes;
+        }
+        $this->Sighting = ClassRegistry::init('Sighting');
+        $HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
+        $eventIds = $this->getEventIdsFromServer($server, true, $HttpSocket, false, true, 'sightings');
+        // now process the $eventIds to push each of the events sequentially
+        if (!empty($eventIds)) {
+            // check each event and push sightings when needed
+            foreach ($eventIds as $k => $eventId) {
+                $event = $eventModel->fetchEvent($user, $options = array('event_uuid' => $eventId, 'metadata' => true));
+                if (!empty($event)) {
+                    $event = $event[0];
+                    $event['Sighting'] = $this->Sighting->attachToEvent($event, $user);
+                    $result = $eventModel->uploadEventToServer($event, $server, $HttpSocket, 'sightings');
+                    if ($result === 'Success') {
+                        $successes[] = 'Sightings for event ' .  $event['Event']['id'];
+                    }
+                }
+            }
+        }
+        return $successes;
+    }
+
     public function syncProposals($HttpSocket, $server, $sa_id = null, $event_id = null, $eventModel)
     {
         $saModel = ClassRegistry::init('ShadowAttribute');
@@ -4114,6 +4173,7 @@ class Server extends AppModel
         }
         $remoteVersion = json_decode($response->body, true);
         $canPush = isset($remoteVersion['perm_sync']) ? $remoteVersion['perm_sync'] : false;
+        $canSight = isset($remoteVersion['perm_sighting']) ? $remoteVersion['perm_sighting'] : false;
         $remoteVersion = explode('.', $remoteVersion['version']);
         if (!isset($remoteVersion[0])) {
             $this->Log = ClassRegistry::init('Log');
@@ -4175,7 +4235,7 @@ class Server extends AppModel
                     'title' => ucfirst($issueLevel) . ': ' . $response,
             ));
         }
-        return array('success' => $success, 'response' => $response, 'canPush' => $canPush, 'version' => $remoteVersion);
+        return array('success' => $success, 'response' => $response, 'canPush' => $canPush, 'canSight' => $canSight, 'version' => $remoteVersion);
     }
 
     public function isJson($string)
diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php
index cfc64b000..491e14c50 100644
--- a/app/Model/Sighting.php
+++ b/app/Model/Sighting.php
@@ -337,7 +337,7 @@ class Sighting extends AppModel
         return $attributes;
     }
 
-    public function saveSightings($id, $values, $timestamp, $user, $type = false, $source = false, $sighting_uuid = false)
+    public function saveSightings($id, $values, $timestamp, $user, $type = false, $source = false, $sighting_uuid = false, $publish = false)
     {
         $conditions = array();
         if ($id && $id !== 'stix') {
@@ -402,6 +402,9 @@ class Sighting extends AppModel
                 return json_encode($this->validationErrors);
             }
             $sightingsAdded += $result ? 1 : 0;
+            if ($publish) {
+                $this->Event->publishRouter($sighting['event_id'], null, $user, 'sightings');
+            }
         }
         if ($sightingsAdded == 0) {
             return 'There was nothing to add.';
@@ -756,4 +759,61 @@ class Sighting extends AppModel
         fclose($tmpfile);
         return $final;
     }
+
+    // Bulk save sightings
+    public function bulkSaveSightings($eventId, $sightings, $user, $passAlong = null)
+    {
+        if (!is_numeric($eventId)) {
+             $eventId = $this->Event->field('id', array('uuid' => $eventId));
+        }
+        $event = $this->Event->fetchEvent($user, array(
+             'eventid' => $eventId,
+             'metadata' => 1,
+             'flatten' => true
+        ));
+        if (empty($event)) {
+            return 'Event not found or not accesible by this user.';
+        }
+        $saved = 0;
+        foreach ($sightings as $s) {
+            $result = $this->saveSightings($s['attribute_uuid'], false, $s['date_sighting'], $user, $s['type'], $s['source'], $s['uuid']);
+            if (is_numeric($result)) {
+                $saved += $result;
+            }
+        }
+        if ($saved > 0) {
+            $this->Event->publishRouter($eventId, $passAlong, $user, 'sightings');
+        }
+        return $saved;
+    }
+
+    public function pullSightings($user, $server)
+    {
+        $HttpSocket = $this->setupHttpSocket($server);
+        $this->Server = ClassRegistry::init('Server');
+        $eventIds = $this->Server->getEventIdsFromServer($server, false, $HttpSocket, false, false, 'sightings');
+        $saved = 0;
+        // now process the $eventIds to pull each of the events sequentially
+        if (!empty($eventIds)) {
+            // download each event and save sightings
+            foreach ($eventIds as $k => $eventId) {
+                $event = $this->Event->downloadEventFromServer($eventId, $server);
+                $sightings = array();
+                if(!empty($event) && !empty($event['Event']['Attribute'])) {
+                    foreach($event['Event']['Attribute'] as $attribute) {
+                        if(!empty($attribute['Sighting'])) {
+                            $sightings = array_merge($sightings, $attribute['Sighting']);
+                        }
+                    }
+                }
+                if(!empty($event) && !empty($sightings)) {
+                    $result = $this->bulkSaveSightings($event['Event']['uuid'], $sightings, $user, $server['Server']['id']);
+                    if (is_numeric($result)) {
+                        $saved += $result;
+                    }
+                }
+            }
+        }
+        return $saved;
+    }
 }
diff --git a/app/View/Servers/add.ctp b/app/View/Servers/add.ctp
index 6ff65db61..52ad43afa 100644
--- a/app/View/Servers/add.ctp
+++ b/app/View/Servers/add.ctp
@@ -80,6 +80,7 @@
         echo '<h4 class="input clear">' . __('Enabled synchronisation methods') . '</h4>';
         echo $this->Form->input('push', array());
         echo $this->Form->input('pull', array());
+        echo $this->Form->input('push_sightings', array());
         echo $this->Form->input('caching_enabled', array());
         echo '<div class = "input clear" style="width:100%;"><hr /></div>';
         echo $this->Form->input('unpublish_event', array(
diff --git a/app/View/Servers/edit.ctp b/app/View/Servers/edit.ctp
index 448259fce..dc332e43c 100644
--- a/app/View/Servers/edit.ctp
+++ b/app/View/Servers/edit.ctp
@@ -80,6 +80,7 @@
         echo '<h4 class="input clear">' . __('Enabled synchronisation methods') . '</h4>';
         echo $this->Form->input('push', array());
         echo $this->Form->input('pull', array());
+        echo $this->Form->input('push_sightings', array());
         echo $this->Form->input('caching_enabled', array());
         echo '<div class = "input clear" style="width:100%;"><hr /><h4>' . __('Misc settings') . '</h4></div>';
         echo $this->Form->input('unpublish_event', array(
diff --git a/app/View/Servers/index.ctp b/app/View/Servers/index.ctp
index 24cbb185e..3570315fd 100644
--- a/app/View/Servers/index.ctp
+++ b/app/View/Servers/index.ctp
@@ -27,6 +27,7 @@
             <th><?php echo $this->Paginator->sort('internal');?></th>
             <th><?php echo $this->Paginator->sort('push');?></th>
             <th><?php echo $this->Paginator->sort('pull');?></th>
+            <th><?php echo $this->Paginator->sort('push_sightings', 'Push Sightings');?></th>
             <th><?php echo $this->Paginator->sort('caching_enabled', 'Cache');?></th>
             <th><?php echo $this->Paginator->sort('unpublish_event (push event)');?></th>
             <th><?php echo $this->Paginator->sort('publish_without_email (pull event)');?></th>
@@ -106,6 +107,7 @@ foreach ($servers as $row_pos => $server):
 <td><span class="<?php echo ($server['Server']['internal']? 'icon-ok' : 'icon-remove'); ?>" role="img" aria-label="<?php echo ($server['Server']['internal']? __('Yes') : __('No')); ?>" title="<?php echo ($server['Server']['internal']? __('Internal instance that ignores distribution level degradation *WARNING: Only use this setting if you have several internal instances and the sync link is to an internal extension of the current MISP community*') : __('Normal sync link to an external MISP instance. Distribution degradation will follow the normal rules.')); ?>"></span></td>
         <td><span class="<?php echo ($server['Server']['push']? 'icon-ok' : 'icon-remove'); ?>" role="img" aria-label="<?php echo ($server['Server']['push']? __('Yes') : __('No')); ?>"></span><span class="short <?php if (!$server['Server']['push'] || empty($ruleDescription['push'])) echo "hidden"; ?>" data-toggle="popover" title="Distribution List" data-content="<?php echo $ruleDescription['push']; ?>"> (<?php echo __('Rules');?>)</span></td>
         <td><span class="<?php echo ($server['Server']['pull']? 'icon-ok' : 'icon-remove'); ?>" role="img" aria-label="<?php echo ($server['Server']['pull']? __('Yes') : __('No')); ?>"></span><span class="short <?php if (!$server['Server']['pull'] || empty($ruleDescription['pull'])) echo "hidden"; ?>" data-toggle="popover" title="Distribution List" data-content="<?php echo $ruleDescription['pull']; ?>"> (<?php echo __('Rules');?>)</span></td>
+        <td class="short"><span class="<?php echo ($server['Server']['push_sightings'] ? 'icon-ok' : 'icon-remove'); ?>" role="img" aria-label="<?php echo ($server['Server']['push_sightings'] ? __('Yes') : __('No')); ?>"></span></td>
         <td>
             <?php
                 if ($server['Server']['caching_enabled']) {
diff --git a/app/View/Servers/pull.ctp b/app/View/Servers/pull.ctp
index 4d73dfe44..3dc7c5b94 100644
--- a/app/View/Servers/pull.ctp
+++ b/app/View/Servers/pull.ctp
@@ -32,6 +32,17 @@ else:?>
     </ul>
     <?php
 endif;?>
+    <h2><?php echo __('Sightings pulled');?></h2>
+    <?php
+if (0 == count($pulledSightings)):?>
+    <p><?php echo __('No sightings pulled');?></p>
+    <?php
+else:?>
+    <ul>
+    <?php foreach ($pulledSightins as $e => $p) echo '<li>Event ' . $e . ' : ' . $p . ' sighting(s).</li>'; ?>
+    </ul>
+    <?php
+endif;?>
 
 </div>
 <?php

From bcf58018f30f8b9bdb540830a49a2b9369e43a1c Mon Sep 17 00:00:00 2001
From: Steve Clement <steve@localhost.lu>
Date: Sun, 24 Nov 2019 17:27:59 +0900
Subject: [PATCH 32/43] chg: [doc] Minor note on composer update

---
 docs/xINSTALL.debian10.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/xINSTALL.debian10.md b/docs/xINSTALL.debian10.md
index d511d5747..ae46a4407 100644
--- a/docs/xINSTALL.debian10.md
+++ b/docs/xINSTALL.debian10.md
@@ -202,6 +202,7 @@ $SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === 'a5c698ffe4b
 $SUDO_WWW php composer-setup.php
 $SUDO_WWW php -r "unlink('composer-setup.php');"
 $SUDO_WWW php composer.phar install
+# The following is potentially not needed, but just here in case of Keyboard/Chair failures
 $SUDO_WWW php composer.phar update
 
 # Enable CakeResque with php-redis

From cfa478923aa44e69953f7078e05f6cae2e219ee9 Mon Sep 17 00:00:00 2001
From: Steve Clement <steve@localhost.lu>
Date: Sun, 24 Nov 2019 17:31:37 +0900
Subject: [PATCH 33/43] chg: [installer] Installer checksum updates

---
 INSTALL/INSTALL.sh        | 26 ++++++++++++--------------
 INSTALL/INSTALL.sh.sfv    |  6 +++---
 INSTALL/INSTALL.sh.sha1   |  2 +-
 INSTALL/INSTALL.sh.sha256 |  2 +-
 INSTALL/INSTALL.sh.sha384 |  2 +-
 INSTALL/INSTALL.sh.sha512 |  2 +-
 6 files changed, 19 insertions(+), 21 deletions(-)

diff --git a/INSTALL/INSTALL.sh b/INSTALL/INSTALL.sh
index dcca598df..575f71598 100644
--- a/INSTALL/INSTALL.sh
+++ b/INSTALL/INSTALL.sh
@@ -1807,8 +1807,10 @@ ssdeep () {
   sudo service apache2 restart
 }
 
+# viper-web is broken ATM
 # Main Viper install function
 viper () {
+  export PATH=$PATH:/home/misp/.local/bin
   debug "Installing Viper dependencies"
   cd /usr/local/src/
   sudo apt-get install \
@@ -1821,29 +1823,25 @@ viper () {
   fi
   echo "Cloning Viper"
   $SUDO_USER git clone https://github.com/viper-framework/viper.git
+  $SUDO_USER git clone https://github.com/viper-framework/viper-web.git
   sudo chown -R $MISP_USER:$MISP_USER viper
+  sudo chown -R $MISP_USER:$MISP_USER viper-web
   cd viper
   echo "Creating virtualenv"
   $SUDO_USER virtualenv -p python3 venv
   echo "Submodule update"
   # TODO: Check for current user install permissions
   $SUDO_USER git submodule update --init --recursive
-  ##$SUDO git submodule update --init --recursive
-  echo "Pip install deps"
-  $SUDO_USER ./venv/bin/pip install SQLAlchemy PrettyTable python-magic
-  echo "pip install scrapy"
-  $SUDO_USER ./venv/bin/pip install scrapy
-  echo "install lief"
-  $SUDO_USER ./venv/bin/pip install https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
-  echo "pip install reqs"
-  $SUDO_USER ./venv/bin/pip install -r requirements.txt
-  $SUDO_USER sed -i '1 s/^.*$/\#!\/usr\/local\/src\/viper\/venv\/bin\/python/' viper-cli
+  echo "pip install deps"
+  $SUDO_USER ./venv/bin/pip install pefile olefile jbxapi Crypto pypdns pypssl r2pipe pdftools virustotal-api SQLAlchemy PrettyTable python-magic scrapy https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
+  $SUDO_USER ./venv/bin/pip install .
+  echo 'update-modules' |/usr/local/src/viper/venv/bin/viper
+  cd /usr/local/src/viper-web
   $SUDO_USER sed -i '1 s/^.*$/\#!\/usr\/local\/src\/viper\/venv\/bin\/python/' viper-web
-  echo "Launching viper-cli"
-  $SUDO_USER /usr/local/src/viper/viper-cli -h > /dev/null
+  $SUDO_USER /usr/local/src/viper/venv/bin/pip install -r requirements.txt
   echo "Launching viper-web"
-  $SUDO_USER /usr/local/src/viper/viper-web -p 8888 -H 0.0.0.0 &
-  echo 'PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/usr/local/src/viper:/var/www/MISP/app/Console"' |sudo tee /etc/environment
+  $SUDO_USER /usr/local/src/viper-web/viper-web -p 8888 -H 0.0.0.0 &
+  echo 'PATH="/home/misp/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/usr/local/src/viper:/var/www/MISP/app/Console"' |sudo tee /etc/environment
   echo ". /etc/environment" >> /home/${MISP_USER}/.profile
 
   # TODO: Perms, MISP_USER_HOME, nasty hack cuz Kali on R00t
diff --git a/INSTALL/INSTALL.sh.sfv b/INSTALL/INSTALL.sh.sfv
index 87f4e6b15..fc2cc8a34 100644
--- a/INSTALL/INSTALL.sh.sfv
+++ b/INSTALL/INSTALL.sh.sfv
@@ -1,5 +1,5 @@
-; Generated by RHash v1.3.8 on 2019-11-09 at 13:08.33
+; Generated by RHash v1.3.8 on 2019-11-24 at 17:30.46
 ; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
 ;
-;        99021  13:08.33 2019-11-09 INSTALL.sh
-INSTALL.sh A607BC5BA7F4C81E97A5BEA6FAAE35C289899625 D9BE26A0A2DF33755382BA5090F7EF8F7212FE80771B3149FBEB94CE13530856 33C3C7F6C2B327E2E73392A7B35518622D6AEDD265B5CB210DD3E0723F0A8FE4328CFF4883F605198B09E197E917B214 F1CBEEE0471B9813591F71655290A38B2DADB5BA86D365C87AF44263E9D4223B719194CA7BF283BA584C31F4C04209F012C56B3063F36C00DD7E386C0D53622E
+;        99082  17:30.46 2019-11-24 INSTALL.sh
+INSTALL.sh 5BB0CEB0AB45AF769C8A3B044F9A494E8733B1CB 9402BCF66DD2C8A82B8871C5C414A5710D5FAA0B1AD40BB0EDEC57A8883F52F7 616975D3EC3CA34C590570F272AC244535ECECCF535B66AA765B4E36C68E78649E65E5D719977D18B6D69FF59F709CC0 5BAA423F8306B0B2E16FC91380E1E551550F31ED013A38233B049040AAB81579BAD4E3C81203C0EC324BC17010BC1063E1D67BDDF45A922EC7CEC3A551AA49EE
diff --git a/INSTALL/INSTALL.sh.sha1 b/INSTALL/INSTALL.sh.sha1
index c1c873587..fc5abbc45 100644
--- a/INSTALL/INSTALL.sh.sha1
+++ b/INSTALL/INSTALL.sh.sha1
@@ -1 +1 @@
-a607bc5ba7f4c81e97a5bea6faae35c289899625  INSTALL.sh
+5bb0ceb0ab45af769c8a3b044f9a494e8733b1cb  INSTALL.sh
diff --git a/INSTALL/INSTALL.sh.sha256 b/INSTALL/INSTALL.sh.sha256
index 633aa1639..133dc7125 100644
--- a/INSTALL/INSTALL.sh.sha256
+++ b/INSTALL/INSTALL.sh.sha256
@@ -1 +1 @@
-d9be26a0a2df33755382ba5090f7ef8f7212fe80771b3149fbeb94ce13530856  INSTALL.sh
+9402bcf66dd2c8a82b8871c5c414a5710d5faa0b1ad40bb0edec57a8883f52f7  INSTALL.sh
diff --git a/INSTALL/INSTALL.sh.sha384 b/INSTALL/INSTALL.sh.sha384
index 6ad852094..469d59424 100644
--- a/INSTALL/INSTALL.sh.sha384
+++ b/INSTALL/INSTALL.sh.sha384
@@ -1 +1 @@
-33c3c7f6c2b327e2e73392a7b35518622d6aedd265b5cb210dd3e0723f0a8fe4328cff4883f605198b09e197e917b214  INSTALL.sh
+616975d3ec3ca34c590570f272ac244535ececcf535b66aa765b4e36c68e78649e65e5d719977d18b6d69ff59f709cc0  INSTALL.sh
diff --git a/INSTALL/INSTALL.sh.sha512 b/INSTALL/INSTALL.sh.sha512
index 2f69b01d8..c9c05905a 100644
--- a/INSTALL/INSTALL.sh.sha512
+++ b/INSTALL/INSTALL.sh.sha512
@@ -1 +1 @@
-f1cbeee0471b9813591f71655290a38b2dadb5ba86d365c87af44263e9d4223b719194ca7bf283ba584c31f4c04209f012c56b3063f36c00dd7e386c0d53622e  INSTALL.sh
+5baa423f8306b0b2e16fc91380e1e551550f31ed013a38233b049040aab81579bad4e3c81203c0ec324bc17010bc1063e1d67bddf45a922ec7cec3a551aa49ee  INSTALL.sh

From c4566d653b02ff0a6d0a47adee55c6d3e85e6231 Mon Sep 17 00:00:00 2001
From: chrisr3d <chris.studer.68@gmail.com>
Date: Sun, 24 Nov 2019 09:47:20 -0500
Subject: [PATCH 34/43] fix: [stix2 export] Adding attribute type or object
 name in the custom object id

- Should fix #5410
---
 app/files/scripts/stix2/misp2stix2.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/app/files/scripts/stix2/misp2stix2.py b/app/files/scripts/stix2/misp2stix2.py
index ebf8ab8ab..efcc83693 100644
--- a/app/files/scripts/stix2/misp2stix2.py
+++ b/app/files/scripts/stix2/misp2stix2.py
@@ -448,8 +448,9 @@ class StixBuilder():
         self.append_object(course_of_action)
 
     def add_custom(self, attribute):
-        custom_object_id = "x-misp-object--{}".format(attribute['uuid'])
-        custom_object_type = "x-misp-object-{}".format(attribute['type'].replace('|', '-').replace(' ', '-').lower())
+        attribute_type = attribute['type'].replace('|', '-').replace(' ', '-').lower()
+        custom_object_id = "x-misp-object-{}--{}".format(attribute_type, attribute['uuid'])
+        custom_object_type = "x-misp-object-{}".format(attribute_type)
         labels, markings = self.create_labels(attribute)
         custom_object_args = {'id': custom_object_id, 'x_misp_category': attribute['category'], 'labels': labels,
                               'x_misp_timestamp': self.get_datetime_from_timestamp(attribute['timestamp']),
@@ -586,8 +587,8 @@ class StixBuilder():
         self.append_object(vulnerability)
 
     def add_object_custom(self, misp_object, to_ids):
-        custom_object_id = 'x-misp-object--{}'.format(misp_object['uuid'])
         name = misp_object['name']
+        custom_object_id = 'x-misp-object-{}--{}'.format(name, misp_object['uuid'])
         custom_object_type = 'x-misp-object-{}'.format(name)
         category = misp_object.get('meta-category')
         labels = self.create_object_labels(name, category, to_ids)

From 5dc9d6925f00b5e45018396ff8d7eb417cfd3488 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 25 Nov 2019 09:21:52 +0100
Subject: [PATCH 35/43] fix: [UI] duplicate entries in the attribute
 correlation column on the event view, fixes #5421

---
 .../Elements/Events/View/attribute_correlations.ctp    | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/app/View/Elements/Events/View/attribute_correlations.ctp b/app/View/Elements/Events/View/attribute_correlations.ctp
index 94dfbef2e..334ef9349 100644
--- a/app/View/Elements/Events/View/attribute_correlations.ctp
+++ b/app/View/Elements/Events/View/attribute_correlations.ctp
@@ -1,6 +1,16 @@
 <?php
 $i = 0;
 $linkColour = ($scope == 'Attribute') ? 'red' : 'white';
+// remove duplicates
+$relatedEvents = array();
+foreach ($event['Related' . $scope][$object['id']] as $k => $relatedAttribute) {
+    if (isset($relatedEvents[$relatedAttribute['id']])) {
+        unset($event['Related' . $scope][$object['id']][$k]);
+    } else {
+        $relatedEvents[$relatedAttribute['id']] = true;
+    }
+}
+$event['Related' . $scope][$object['id']] = array_values($event['Related' . $scope][$object['id']]);
 $count = count($event['Related' . $scope][$object['id']]);
 foreach ($event['Related' . $scope][$object['id']] as $relatedAttribute) {
   if ($i == 4 && $count > 5) {

From 95f17d6acde409b09d50789350d06417df22b9d7 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 25 Nov 2019 14:32:22 +0100
Subject: [PATCH 36/43] fix: [sync] Some minor changes to the sighting push

- correctly handle anonymisation
- only push sightings, not rest of the event (decide on sender side)
- handle receiving sanitised sightings
---
 app/Controller/EventsController.php           |   2 +-
 .../Database/MysqlExtendedLogging.php         | 921 ++++++++++++++++++
 app/Model/Event.php                           |  51 +-
 app/Model/Sighting.php                        |  19 +-
 4 files changed, 978 insertions(+), 15 deletions(-)
 create mode 100644 app/Model/Datasource/Database/MysqlExtendedLogging.php

diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php
index 7a843d7e2..cd10c07c4 100644
--- a/app/Controller/EventsController.php
+++ b/app/Controller/EventsController.php
@@ -2575,7 +2575,7 @@ class EventsController extends AppController
     public function publishSightings($id = null)
     {
         $id = $this->Toolbox->findIdByUuid($this->Event, $id);
-        $event = fetchEvent(
+        $event = $this->Event->fetchEvent(
             $this->Auth->user(),
             array(
                 'eventid' => $id,
diff --git a/app/Model/Datasource/Database/MysqlExtendedLogging.php b/app/Model/Datasource/Database/MysqlExtendedLogging.php
new file mode 100644
index 000000000..b24db73f6
--- /dev/null
+++ b/app/Model/Datasource/Database/MysqlExtendedLogging.php
@@ -0,0 +1,921 @@
+<?php
+/**
+ * MySQL layer for DBO
+ *
+ * CakePHP(tm) : Rapid Development Framework (https://cakephp.org)
+ * Copyright (c) Cake Software Foundation, Inc. (https://cakefoundation.org)
+ *
+ * Licensed under The MIT License
+ * For full copyright and license information, please see the LICENSE.txt
+ * Redistributions of files must retain the above copyright notice.
+ *
+ * @copyright     Copyright (c) Cake Software Foundation, Inc. (https://cakefoundation.org)
+ * @link          https://cakephp.org CakePHP(tm) Project
+ * @package       Cake.Model.Datasource.Database
+ * @since         CakePHP(tm) v 0.10.5.1790
+ * @license       https://opensource.org/licenses/mit-license.php MIT License
+ */
+
+App::uses('DboSource', 'Model/Datasource');
+
+/**
+ * MySQL DBO driver object
+ *
+ * Provides connection and SQL generation for MySQL RDMS
+ *
+ * @package       Cake.Model.Datasource.Database
+ */
+class MysqlExtendedLogging extends DboSource {
+
+/**
+ * Datasource description
+ *
+ * @var string
+ */
+	public $description = "MySQL DBO Driver";
+
+/**
+ * Base configuration settings for MySQL driver
+ *
+ * @var array
+ */
+	protected $_baseConfig = array(
+		'persistent' => true,
+		'host' => 'localhost',
+		'login' => 'root',
+		'password' => '',
+		'database' => 'cake',
+		'port' => '3306',
+		'flags' => array()
+	);
+
+/**
+ * Reference to the PDO object connection
+ *
+ * @var PDO
+ */
+	protected $_connection = null;
+
+/**
+ * Start quote
+ *
+ * @var string
+ */
+	public $startQuote = "`";
+
+/**
+ * End quote
+ *
+ * @var string
+ */
+	public $endQuote = "`";
+
+/**
+ * use alias for update and delete. Set to true if version >= 4.1
+ *
+ * @var bool
+ */
+	protected $_useAlias = true;
+
+/**
+ * List of engine specific additional field parameters used on table creating
+ *
+ * @var array
+ */
+	public $fieldParameters = array(
+		'charset' => array('value' => 'CHARACTER SET', 'quote' => false, 'join' => ' ', 'column' => false, 'position' => 'beforeDefault'),
+		'collate' => array('value' => 'COLLATE', 'quote' => false, 'join' => ' ', 'column' => 'Collation', 'position' => 'beforeDefault'),
+		'comment' => array('value' => 'COMMENT', 'quote' => true, 'join' => ' ', 'column' => 'Comment', 'position' => 'afterDefault'),
+		'unsigned' => array(
+			'value' => 'UNSIGNED',
+			'quote' => false,
+			'join' => ' ',
+			'column' => false,
+			'position' => 'beforeDefault',
+			'noVal' => true,
+			'options' => array(true),
+			'types' => array('integer', 'smallinteger', 'tinyinteger', 'float', 'decimal', 'biginteger')
+		)
+	);
+
+/**
+ * List of table engine specific parameters used on table creating
+ *
+ * @var array
+ */
+	public $tableParameters = array(
+		'charset' => array('value' => 'DEFAULT CHARSET', 'quote' => false, 'join' => '=', 'column' => 'charset'),
+		'collate' => array('value' => 'COLLATE', 'quote' => false, 'join' => '=', 'column' => 'Collation'),
+		'engine' => array('value' => 'ENGINE', 'quote' => false, 'join' => '=', 'column' => 'Engine'),
+		'comment' => array('value' => 'COMMENT', 'quote' => true, 'join' => '=', 'column' => 'Comment'),
+	);
+
+/**
+ * MySQL column definition
+ *
+ * @var array
+ * @link https://dev.mysql.com/doc/refman/5.7/en/data-types.html MySQL Data Types
+ */
+	public $columns = array(
+		'primary_key' => array('name' => 'NOT NULL AUTO_INCREMENT'),
+		'string' => array('name' => 'varchar', 'limit' => '255'),
+		'text' => array('name' => 'text'),
+		'enum' => array('name' => 'enum'),
+		'biginteger' => array('name' => 'bigint', 'limit' => '20'),
+		'integer' => array('name' => 'int', 'limit' => '11', 'formatter' => 'intval'),
+		'smallinteger' => array('name' => 'smallint', 'limit' => '6', 'formatter' => 'intval'),
+		'tinyinteger' => array('name' => 'tinyint', 'limit' => '4', 'formatter' => 'intval'),
+		'float' => array('name' => 'float', 'formatter' => 'floatval'),
+		'decimal' => array('name' => 'decimal', 'formatter' => 'floatval'),
+		'datetime' => array('name' => 'datetime', 'format' => 'Y-m-d H:i:s', 'formatter' => 'date'),
+		'timestamp' => array('name' => 'timestamp', 'format' => 'Y-m-d H:i:s', 'formatter' => 'date'),
+		'time' => array('name' => 'time', 'format' => 'H:i:s', 'formatter' => 'date'),
+		'date' => array('name' => 'date', 'format' => 'Y-m-d', 'formatter' => 'date'),
+		'binary' => array('name' => 'blob'),
+		'boolean' => array('name' => 'tinyint', 'limit' => '1')
+	);
+
+/**
+ * Mapping of collation names to character set names
+ *
+ * @var array
+ */
+	protected $_charsets = array();
+
+/**
+ * Connects to the database using options in the given configuration array.
+ *
+ * MySQL supports a few additional options that other drivers do not:
+ *
+ * - `unix_socket` Set to the path of the MySQL sock file. Can be used in place
+ *   of host + port.
+ * - `ssl_key` SSL key file for connecting via SSL. Must be combined with `ssl_cert`.
+ * - `ssl_cert` The SSL certificate to use when connecting via SSL. Must be
+ *   combined with `ssl_key`.
+ * - `ssl_ca` The certificate authority for SSL connections.
+ *
+ * @return bool True if the database could be connected, else false
+ * @throws MissingConnectionException
+ */
+	public function connect() {
+		$config = $this->config;
+		$this->connected = false;
+
+		$flags = $config['flags'] + array(
+			PDO::ATTR_PERSISTENT => $config['persistent'],
+			PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => true,
+			PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION
+		);
+
+		if (!empty($config['encoding'])) {
+			$flags[PDO::MYSQL_ATTR_INIT_COMMAND] = 'SET NAMES ' . $config['encoding'];
+		}
+		if (!empty($config['ssl_key']) && !empty($config['ssl_cert'])) {
+			$flags[PDO::MYSQL_ATTR_SSL_KEY] = $config['ssl_key'];
+			$flags[PDO::MYSQL_ATTR_SSL_CERT] = $config['ssl_cert'];
+		}
+		if (!empty($config['ssl_ca'])) {
+			$flags[PDO::MYSQL_ATTR_SSL_CA] = $config['ssl_ca'];
+		}
+		if (empty($config['unix_socket'])) {
+			$dsn = "mysql:host={$config['host']};port={$config['port']};dbname={$config['database']}";
+		} else {
+			$dsn = "mysql:unix_socket={$config['unix_socket']};dbname={$config['database']}";
+		}
+
+		try {
+			$this->_connection = new PDO(
+				$dsn,
+				$config['login'],
+				$config['password'],
+				$flags
+			);
+			$this->connected = true;
+			if (!empty($config['settings'])) {
+				foreach ($config['settings'] as $key => $value) {
+					$this->_execute("SET $key=$value");
+				}
+			}
+		} catch (PDOException $e) {
+			throw new MissingConnectionException(array(
+				'class' => get_class($this),
+				'message' => $e->getMessage()
+			));
+		}
+
+		$this->_charsets = array();
+		$this->_useAlias = (bool)version_compare($this->getVersion(), "4.1", ">=");
+
+		return $this->connected;
+	}
+
+/**
+ * Check whether the MySQL extension is installed/loaded
+ *
+ * @return bool
+ */
+	public function enabled() {
+		return in_array('mysql', PDO::getAvailableDrivers());
+	}
+
+/**
+ * Returns an array of sources (tables) in the database.
+ *
+ * @param mixed $data List of tables.
+ * @return array Array of table names in the database
+ */
+	public function listSources($data = null) {
+		$cache = parent::listSources();
+		if ($cache) {
+			return $cache;
+		}
+		$result = $this->_execute('SHOW TABLES FROM ' . $this->name($this->config['database']));
+
+		if (!$result) {
+			$result->closeCursor();
+			return array();
+		}
+		$tables = array();
+
+		while ($line = $result->fetch(PDO::FETCH_NUM)) {
+			$tables[] = $line[0];
+		}
+
+		$result->closeCursor();
+		parent::listSources($tables);
+		return $tables;
+	}
+
+/**
+ * Builds a map of the columns contained in a result
+ *
+ * @param PDOStatement $results The results to format.
+ * @return void
+ */
+	public function resultSet($results) {
+		$this->map = array();
+		$numFields = $results->columnCount();
+		$index = 0;
+
+		while ($numFields-- > 0) {
+			$column = $results->getColumnMeta($index);
+			if ($column['len'] === 1 && (empty($column['native_type']) || $column['native_type'] === 'TINY')) {
+				$type = 'boolean';
+			} else {
+				$type = empty($column['native_type']) ? 'string' : $column['native_type'];
+			}
+			if (!empty($column['table']) && strpos($column['name'], $this->virtualFieldSeparator) === false) {
+				$this->map[$index++] = array($column['table'], $column['name'], $type);
+			} else {
+				$this->map[$index++] = array(0, $column['name'], $type);
+			}
+		}
+	}
+
+/**
+ * Fetches the next row from the current result set
+ *
+ * @return mixed array with results fetched and mapped to column names or false if there is no results left to fetch
+ */
+	public function fetchResult() {
+		if ($row = $this->_result->fetch(PDO::FETCH_NUM)) {
+			$resultRow = array();
+			foreach ($this->map as $col => $meta) {
+				list($table, $column, $type) = $meta;
+				$resultRow[$table][$column] = $row[$col];
+				if ($type === 'boolean' && $row[$col] !== null) {
+					$resultRow[$table][$column] = $this->boolean($resultRow[$table][$column]);
+				}
+			}
+			return $resultRow;
+		}
+		$this->_result->closeCursor();
+		return false;
+	}
+
+/**
+ * Gets the database encoding
+ *
+ * @return string The database encoding
+ */
+	public function getEncoding() {
+		return $this->_execute('SHOW VARIABLES LIKE ?', array('character_set_client'))->fetchObject()->Value;
+	}
+
+/**
+ * Query charset by collation
+ *
+ * @param string $name Collation name
+ * @return string|false Character set name
+ */
+	public function getCharsetName($name) {
+		if ((bool)version_compare($this->getVersion(), "5", "<")) {
+			return false;
+		}
+		if (isset($this->_charsets[$name])) {
+			return $this->_charsets[$name];
+		}
+		$r = $this->_execute(
+			'SELECT CHARACTER_SET_NAME FROM INFORMATION_SCHEMA.COLLATIONS WHERE COLLATION_NAME = ?',
+			array($name)
+		);
+		$cols = $r->fetch(PDO::FETCH_ASSOC);
+
+		if (isset($cols['CHARACTER_SET_NAME'])) {
+			$this->_charsets[$name] = $cols['CHARACTER_SET_NAME'];
+		} else {
+			$this->_charsets[$name] = false;
+		}
+		return $this->_charsets[$name];
+	}
+
+/**
+ * Returns an array of the fields in given table name.
+ *
+ * @param Model|string $model Name of database table to inspect or model instance
+ * @return array Fields in table. Keys are name and type
+ * @throws CakeException
+ */
+	public function describe($model) {
+		$key = $this->fullTableName($model, false);
+		$cache = parent::describe($key);
+		if ($cache) {
+			return $cache;
+		}
+		$table = $this->fullTableName($model);
+
+		$fields = false;
+		$cols = $this->_execute('SHOW FULL COLUMNS FROM ' . $table);
+		if (!$cols) {
+			throw new CakeException(__d('cake_dev', 'Could not describe table for %s', $table));
+		}
+
+		while ($column = $cols->fetch(PDO::FETCH_OBJ)) {
+			$fields[$column->Field] = array(
+				'type' => $this->column($column->Type),
+				'null' => ($column->Null === 'YES' ? true : false),
+				'default' => $column->Default,
+				'length' => $this->length($column->Type)
+			);
+			if (in_array($fields[$column->Field]['type'], $this->fieldParameters['unsigned']['types'], true)) {
+				$fields[$column->Field]['unsigned'] = $this->_unsigned($column->Type);
+			}
+			if (in_array($fields[$column->Field]['type'], array('timestamp', 'datetime')) &&
+				in_array(strtoupper($column->Default), array('CURRENT_TIMESTAMP', 'CURRENT_TIMESTAMP()'))
+			) {
+				$fields[$column->Field]['default'] = null;
+			}
+			if (!empty($column->Key) && isset($this->index[$column->Key])) {
+				$fields[$column->Field]['key'] = $this->index[$column->Key];
+			}
+			foreach ($this->fieldParameters as $name => $value) {
+				if (!empty($column->{$value['column']})) {
+					$fields[$column->Field][$name] = $column->{$value['column']};
+				}
+			}
+			if (isset($fields[$column->Field]['collate'])) {
+				$charset = $this->getCharsetName($fields[$column->Field]['collate']);
+				if ($charset) {
+					$fields[$column->Field]['charset'] = $charset;
+				}
+			}
+		}
+		$this->_cacheDescription($key, $fields);
+		$cols->closeCursor();
+		return $fields;
+	}
+
+/**
+ * Generates and executes an SQL UPDATE statement for given model, fields, and values.
+ *
+ * @param Model $model The model to update.
+ * @param array $fields The fields to update.
+ * @param array $values The values to set.
+ * @param mixed $conditions The conditions to use.
+ * @return bool
+ */
+	public function update(Model $model, $fields = array(), $values = null, $conditions = null) {
+		if (!$this->_useAlias) {
+			return parent::update($model, $fields, $values, $conditions);
+		}
+
+		if (!$values) {
+			$combined = $fields;
+		} else {
+			$combined = array_combine($fields, $values);
+		}
+
+		$alias = $joins = false;
+		$fields = $this->_prepareUpdateFields($model, $combined, empty($conditions), !empty($conditions));
+		$fields = implode(', ', $fields);
+		$table = $this->fullTableName($model);
+
+		if (!empty($conditions)) {
+			$alias = $this->name($model->alias);
+			if ($model->name === $model->alias) {
+				$joins = implode(' ', $this->_getJoins($model));
+			}
+		}
+		$conditions = $this->conditions($this->defaultConditions($model, $conditions, $alias), true, true, $model);
+
+		if ($conditions === false) {
+			return false;
+		}
+
+		if (!$this->execute($this->renderStatement('update', compact('table', 'alias', 'joins', 'fields', 'conditions')))) {
+			$model->onError();
+			return false;
+		}
+		return true;
+	}
+
+/**
+ * Generates and executes an SQL DELETE statement for given id/conditions on given model.
+ *
+ * @param Model $model The model to delete from.
+ * @param mixed $conditions The conditions to use.
+ * @return bool Success
+ */
+	public function delete(Model $model, $conditions = null) {
+		if (!$this->_useAlias) {
+			return parent::delete($model, $conditions);
+		}
+		$alias = $this->name($model->alias);
+		$table = $this->fullTableName($model);
+		$joins = implode(' ', $this->_getJoins($model));
+
+		if (empty($conditions)) {
+			$alias = $joins = false;
+		}
+		$complexConditions = $this->_deleteNeedsComplexConditions($model, $conditions);
+		if (!$complexConditions) {
+			$joins = false;
+		}
+
+		$conditions = $this->conditions($this->defaultConditions($model, $conditions, $alias), true, true, $model);
+		if ($conditions === false) {
+			return false;
+		}
+		if ($this->execute($this->renderStatement('delete', compact('alias', 'table', 'joins', 'conditions'))) === false) {
+			$model->onError();
+			return false;
+		}
+		return true;
+	}
+
+/**
+ * Checks whether complex conditions are needed for a delete with the given conditions.
+ *
+ * @param Model $model The model to delete from.
+ * @param mixed $conditions The conditions to use.
+ * @return bool Whether or not complex conditions are needed
+ */
+	protected function _deleteNeedsComplexConditions(Model $model, $conditions) {
+		$fields = array_keys($this->describe($model));
+		foreach ((array)$conditions as $key => $value) {
+			if (in_array(strtolower(trim($key)), $this->_sqlBoolOps, true)) {
+				if ($this->_deleteNeedsComplexConditions($model, $value)) {
+					return true;
+				}
+			} elseif (strpos($key, $model->alias) === false && !in_array($key, $fields, true)) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+/**
+ * Sets the database encoding
+ *
+ * @param string $enc Database encoding
+ * @return bool
+ */
+	public function setEncoding($enc) {
+		return $this->_execute('SET NAMES ' . $enc) !== false;
+	}
+
+/**
+ * Returns an array of the indexes in given datasource name.
+ *
+ * @param string $model Name of model to inspect
+ * @return array Fields in table. Keys are column and unique
+ */
+	public function index($model) {
+		$index = array();
+		$table = $this->fullTableName($model);
+		$old = version_compare($this->getVersion(), '4.1', '<=');
+		if ($table) {
+			$indexes = $this->_execute('SHOW INDEX FROM ' . $table);
+			// @codingStandardsIgnoreStart
+			// MySQL columns don't match the cakephp conventions.
+			while ($idx = $indexes->fetch(PDO::FETCH_OBJ)) {
+				if ($old) {
+					$idx = (object)current((array)$idx);
+				}
+				if (!isset($index[$idx->Key_name]['column'])) {
+					$col = array();
+					$index[$idx->Key_name]['column'] = $idx->Column_name;
+
+					if ($idx->Index_type === 'FULLTEXT') {
+						$index[$idx->Key_name]['type'] = strtolower($idx->Index_type);
+					} else {
+						$index[$idx->Key_name]['unique'] = (int)($idx->Non_unique == 0);
+					}
+				} else {
+					if (!empty($index[$idx->Key_name]['column']) && !is_array($index[$idx->Key_name]['column'])) {
+						$col[] = $index[$idx->Key_name]['column'];
+					}
+					$col[] = $idx->Column_name;
+					$index[$idx->Key_name]['column'] = $col;
+				}
+				if (!empty($idx->Sub_part)) {
+					if (!isset($index[$idx->Key_name]['length'])) {
+						$index[$idx->Key_name]['length'] = array();
+					}
+					$index[$idx->Key_name]['length'][$idx->Column_name] = $idx->Sub_part;
+				}
+			}
+			// @codingStandardsIgnoreEnd
+			$indexes->closeCursor();
+		}
+		return $index;
+	}
+
+/**
+ * Generate a MySQL Alter Table syntax for the given Schema comparison
+ *
+ * @param array $compare Result of a CakeSchema::compare()
+ * @param string $table The table name.
+ * @return string|false String of alter statements to make.
+ */
+	public function alterSchema($compare, $table = null) {
+		if (!is_array($compare)) {
+			return false;
+		}
+		$out = '';
+		$colList = array();
+		foreach ($compare as $curTable => $types) {
+			$indexes = $tableParameters = $colList = array();
+			if (!$table || $table === $curTable) {
+				$out .= 'ALTER TABLE ' . $this->fullTableName($curTable) . " \n";
+				foreach ($types as $type => $column) {
+					if (isset($column['indexes'])) {
+						$indexes[$type] = $column['indexes'];
+						unset($column['indexes']);
+					}
+					if (isset($column['tableParameters'])) {
+						$tableParameters[$type] = $column['tableParameters'];
+						unset($column['tableParameters']);
+					}
+					switch ($type) {
+						case 'add':
+							foreach ($column as $field => $col) {
+								$col['name'] = $field;
+								$alter = 'ADD ' . $this->buildColumn($col);
+								if (isset($col['after'])) {
+									$alter .= ' AFTER ' . $this->name($col['after']);
+								}
+								$colList[] = $alter;
+							}
+							break;
+						case 'drop':
+							foreach ($column as $field => $col) {
+								$col['name'] = $field;
+								$colList[] = 'DROP ' . $this->name($field);
+							}
+							break;
+						case 'change':
+							foreach ($column as $field => $col) {
+								if (!isset($col['name'])) {
+									$col['name'] = $field;
+								}
+								$alter = 'CHANGE ' . $this->name($field) . ' ' . $this->buildColumn($col);
+								if (isset($col['after'])) {
+									$alter .= ' AFTER ' . $this->name($col['after']);
+								}
+								$colList[] = $alter;
+							}
+							break;
+					}
+				}
+				$colList = array_merge($colList, $this->_alterIndexes($curTable, $indexes));
+				$colList = array_merge($colList, $this->_alterTableParameters($curTable, $tableParameters));
+				$out .= "\t" . implode(",\n\t", $colList) . ";\n\n";
+			}
+		}
+		return $out;
+	}
+
+/**
+ * Generate a "drop table" statement for the given table
+ *
+ * @param type $table Name of the table to drop
+ * @return string Drop table SQL statement
+ */
+	protected function _dropTable($table) {
+		return 'DROP TABLE IF EXISTS ' . $this->fullTableName($table) . ";";
+	}
+
+/**
+ * Generate MySQL table parameter alteration statements for a table.
+ *
+ * @param string $table Table to alter parameters for.
+ * @param array $parameters Parameters to add & drop.
+ * @return array Array of table property alteration statements.
+ */
+	protected function _alterTableParameters($table, $parameters) {
+		if (isset($parameters['change'])) {
+			return $this->buildTableParameters($parameters['change']);
+		}
+		return array();
+	}
+
+/**
+ * Format indexes for create table
+ *
+ * @param array $indexes An array of indexes to generate SQL from
+ * @param string $table Optional table name, not used
+ * @return array An array of SQL statements for indexes
+ * @see DboSource::buildIndex()
+ */
+	public function buildIndex($indexes, $table = null) {
+		$join = array();
+		foreach ($indexes as $name => $value) {
+			$out = '';
+			if ($name === 'PRIMARY') {
+				$out .= 'PRIMARY ';
+				$name = null;
+			} else {
+				if (!empty($value['unique'])) {
+					$out .= 'UNIQUE ';
+				}
+				$name = $this->startQuote . $name . $this->endQuote;
+			}
+			if (isset($value['type']) && strtolower($value['type']) === 'fulltext') {
+				$out .= 'FULLTEXT ';
+			}
+			$out .= 'KEY ' . $name . ' (';
+
+			if (is_array($value['column'])) {
+				if (isset($value['length'])) {
+					$vals = array();
+					foreach ($value['column'] as $column) {
+						$name = $this->name($column);
+						if (isset($value['length'])) {
+							$name .= $this->_buildIndexSubPart($value['length'], $column);
+						}
+						$vals[] = $name;
+					}
+					$out .= implode(', ', $vals);
+				} else {
+					$out .= implode(', ', array_map(array(&$this, 'name'), $value['column']));
+				}
+			} else {
+				$out .= $this->name($value['column']);
+				if (isset($value['length'])) {
+					$out .= $this->_buildIndexSubPart($value['length'], $value['column']);
+				}
+			}
+			$out .= ')';
+			$join[] = $out;
+		}
+		return $join;
+	}
+
+/**
+ * Generate MySQL index alteration statements for a table.
+ *
+ * @param string $table Table to alter indexes for
+ * @param array $indexes Indexes to add and drop
+ * @return array Index alteration statements
+ */
+	protected function _alterIndexes($table, $indexes) {
+		$alter = array();
+		if (isset($indexes['drop'])) {
+			foreach ($indexes['drop'] as $name => $value) {
+				$out = 'DROP ';
+				if ($name === 'PRIMARY') {
+					$out .= 'PRIMARY KEY';
+				} else {
+					$out .= 'KEY ' . $this->startQuote . $name . $this->endQuote;
+				}
+				$alter[] = $out;
+			}
+		}
+		if (isset($indexes['add'])) {
+			$add = $this->buildIndex($indexes['add']);
+			foreach ($add as $index) {
+				$alter[] = 'ADD ' . $index;
+			}
+		}
+		return $alter;
+	}
+
+/**
+ * Format length for text indexes
+ *
+ * @param array $lengths An array of lengths for a single index
+ * @param string $column The column for which to generate the index length
+ * @return string Formatted length part of an index field
+ */
+	protected function _buildIndexSubPart($lengths, $column) {
+		if ($lengths === null) {
+			return '';
+		}
+		if (!isset($lengths[$column])) {
+			return '';
+		}
+		return '(' . $lengths[$column] . ')';
+	}
+
+/**
+ * Returns a detailed array of sources (tables) in the database.
+ *
+ * @param string $name Table name to get parameters
+ * @return array Array of table names in the database
+ */
+	public function listDetailedSources($name = null) {
+		$condition = '';
+		if (is_string($name)) {
+			$condition = ' WHERE name = ' . $this->value($name);
+		}
+		$result = $this->_connection->query('SHOW TABLE STATUS ' . $condition, PDO::FETCH_ASSOC);
+
+		if (!$result) {
+			$result->closeCursor();
+			return array();
+		}
+		$tables = array();
+		foreach ($result as $row) {
+			$tables[$row['Name']] = (array)$row;
+			unset($tables[$row['Name']]['queryString']);
+			if (!empty($row['Collation'])) {
+				$charset = $this->getCharsetName($row['Collation']);
+				if ($charset) {
+					$tables[$row['Name']]['charset'] = $charset;
+				}
+			}
+		}
+		$result->closeCursor();
+		if (is_string($name) && isset($tables[$name])) {
+			return $tables[$name];
+		}
+		return $tables;
+	}
+
+/**
+ * Converts database-layer column types to basic types
+ *
+ * @param string $real Real database-layer column type (i.e. "varchar(255)")
+ * @return string Abstract column type (i.e. "string")
+ */
+	public function column($real) {
+		if (is_array($real)) {
+			$col = $real['name'];
+			if (isset($real['limit'])) {
+				$col .= '(' . $real['limit'] . ')';
+			}
+			return $col;
+		}
+
+		$col = str_replace(')', '', $real);
+		$limit = $this->length($real);
+		if (strpos($col, '(') !== false) {
+			list($col, $vals) = explode('(', $col);
+		}
+
+		if (in_array($col, array('date', 'time', 'datetime', 'timestamp'))) {
+			return $col;
+		}
+		if (($col === 'tinyint' && $limit === 1) || $col === 'boolean') {
+			return 'boolean';
+		}
+		if (strpos($col, 'bigint') !== false || $col === 'bigint') {
+			return 'biginteger';
+		}
+		if (strpos($col, 'tinyint') !== false) {
+			return 'tinyinteger';
+		}
+		if (strpos($col, 'smallint') !== false) {
+			return 'smallinteger';
+		}
+		if (strpos($col, 'int') !== false) {
+			return 'integer';
+		}
+		if (strpos($col, 'char') !== false || $col === 'tinytext') {
+			return 'string';
+		}
+		if (strpos($col, 'text') !== false) {
+			return 'text';
+		}
+		if (strpos($col, 'blob') !== false || $col === 'binary') {
+			return 'binary';
+		}
+		if (strpos($col, 'float') !== false || strpos($col, 'double') !== false) {
+			return 'float';
+		}
+		if (strpos($col, 'decimal') !== false || strpos($col, 'numeric') !== false) {
+			return 'decimal';
+		}
+		if (strpos($col, 'enum') !== false) {
+			return "enum($vals)";
+		}
+		if (strpos($col, 'set') !== false) {
+			return "set($vals)";
+		}
+		return 'text';
+	}
+
+/**
+ * {@inheritDoc}
+ */
+	public function value($data, $column = null, $null = true) {
+		$value = parent::value($data, $column, $null);
+		if (is_numeric($value) && substr($column, 0, 3) === 'set') {
+			return $this->_connection->quote($value);
+		}
+		return $value;
+	}
+
+/**
+ * Gets the schema name
+ *
+ * @return string The schema name
+ */
+	public function getSchemaName() {
+		return $this->config['database'];
+	}
+
+/**
+ * Check if the server support nested transactions
+ *
+ * @return bool
+ */
+	public function nestedTransactionSupported() {
+		return $this->useNestedTransactions && version_compare($this->getVersion(), '4.1', '>=');
+	}
+
+/**
+ * Check if column type is unsigned
+ *
+ * @param string $real Real database-layer column type (i.e. "varchar(255)")
+ * @return bool True if column is unsigned, false otherwise
+ */
+	protected function _unsigned($real) {
+		return strpos(strtolower($real), 'unsigned') !== false;
+	}
+
+/**
+ * Inserts multiple values into a table. Uses a single query in order to insert
+ * multiple rows.
+ *
+ * @param string $table The table being inserted into.
+ * @param array $fields The array of field/column names being inserted.
+ * @param array $values The array of values to insert. The values should
+ *   be an array of rows. Each row should have values keyed by the column name.
+ *   Each row must have the values in the same order as $fields.
+ * @return bool
+ */
+	public function insertMulti($table, $fields, $values) {
+		$table = $this->fullTableName($table);
+		$holder = implode(', ', array_fill(0, count($fields), '?'));
+		$fields = implode(', ', array_map(array($this, 'name'), $fields));
+		$pdoMap = array(
+			'integer' => PDO::PARAM_INT,
+			'float' => PDO::PARAM_STR,
+			'boolean' => PDO::PARAM_BOOL,
+			'string' => PDO::PARAM_STR,
+			'text' => PDO::PARAM_STR
+		);
+		$columnMap = array();
+		$rowHolder = "({$holder})";
+		$sql = "INSERT INTO {$table} ({$fields}) VALUES ";
+		$countRows = count($values);
+		for ($i = 0; $i < $countRows; $i++) {
+			if ($i !== 0) {
+				$sql .= ',';
+			}
+			$sql .= " $rowHolder";
+		}
+		$statement = $this->_connection->prepare($sql);
+		foreach ($values[key($values)] as $key => $val) {
+			$type = $this->introspectType($val);
+			$columnMap[$key] = $pdoMap[$type];
+		}
+		$valuesList = array();
+		$i = 1;
+		foreach ($values as $value) {
+			foreach ($value as $col => $val) {
+				$valuesList[] = $val;
+				$statement->bindValue($i, $val, $columnMap[$col]);
+				$i++;
+			}
+		}
+		$result = $statement->execute();
+		$statement->closeCursor();
+		if ($this->fullDebug) {
+			$this->logQuery($sql, $valuesList);
+		}
+		return $result;
+	}
+}
diff --git a/app/Model/Event.php b/app/Model/Event.php
index d2d548086..e327da4ae 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -1075,6 +1075,26 @@ class Event extends AppModel
         return true;
     }
 
+    public function uploadSightingsToServer($sightings, $server, $event_uuid, $HttpSocket = null)
+    {
+        $HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
+        $request = $this->setupSyncRequest($server);
+        $uri = $server['Server']['url'] . '/sightings/bulkSaveSightings/' . $event_uuid;
+        $data = json_encode($sightings);
+        if (!empty(Configure::read('Security.sync_audit'))) {
+            $pushLogEntry = sprintf(
+                "==============================================================\n\n[%s] Pushing Sightings for Event #%s to Server #%d:\n\n%s\n\n",
+                date("Y-m-d H:i:s"),
+                $event_uuid,
+                $server['Server']['id'],
+                $data
+            );
+            file_put_contents(APP . 'files/scripts/tmp/debug_server_' . $server['Server']['id'] . '.log', $pushLogEntry, FILE_APPEND);
+        }
+        $response = $HttpSocket->post($uri, $data, $request);
+        return $this->__handleRestfulEventToServerResponse($response, $newLocation, $newTextBody);
+    }
+
     public function uploadEventToServer($event, $server, $HttpSocket = null, $scope = 'events')
     {
         $this->Server = ClassRegistry::init('Server');
@@ -1208,6 +1228,7 @@ class Event extends AppModel
             );
             file_put_contents(APP . 'files/scripts/tmp/debug_server_' . $server['Server']['id'] . '.log', $pushLogEntry, FILE_APPEND);
         }
+        debug($data);
         $response = $HttpSocket->post($uri, $data, $request);
         return $this->__handleRestfulEventToServerResponse($response, $newLocation, $newTextBody);
     }
@@ -4083,11 +4104,6 @@ class Event extends AppModel
         }
         $event = $event[0];
         $event['Event']['locked'] = 1;
-        // attach sightings if needed
-        if ($scope === 'sightings') {
-            $this->Sighting = ClassRegistry::init('Sighting');
-            $event['Sighting'] = $this->Sighting->attachToEvent($event, $elevatedUser);
-        }
         // get a list of the servers
         $this->Server = ClassRegistry::init('Server');
         $conditions = array('push' => 1);
@@ -4131,14 +4147,31 @@ class Event extends AppModel
                 $event = $this->fetchEvent($elevatedUser, $params);
                 $event = $event[0];
                 $event['Event']['locked'] = 1;
-                $thisUploaded = $this->uploadEventToServer($event, $server, $HttpSocket, $scope);
+                // attach sightings if needed
+                if ($scope === 'sightings') {
+                    $this->Sighting = ClassRegistry::init('Sighting');
+                    $fakeSyncUser = array(
+                        'org_id' => $server['Server']['remote_org_id'],
+                        'Role' => array(
+                            'perm_site_admin' => 0
+                        )
+                    );
+                    $sightings = $this->Sighting->attachToEvent($event, $fakeSyncUser);
+                    if (!empty($sightings)) {
+                        $thisUploaded = $this->uploadSightingsToServer($sightings, $server, $event['Event']['uuid'], $HttpSocket);
+                    } else {
+                        $thisUploaded = true;
+                    }
+                } else {
+                    $thisUploaded = $this->uploadEventToServer($event, $server, $HttpSocket, $scope);
+                    if (isset($this->data['ShadowAttribute'])) {
+                        $this->Server->syncProposals($HttpSocket, $server, null, $id, $this);
+                    }
+                }
                 if (!$thisUploaded) {
                     $uploaded = !$uploaded ? $uploaded : $thisUploaded;
                     $failedServers[] = $server['Server']['url'];
                 }
-                if (isset($this->data['ShadowAttribute'])) {
-                    $this->Server->syncProposals($HttpSocket, $server, null, $id, $this);
-                }
             }
         }
         if (!$uploaded) {
diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php
index 491e14c50..39281f8a7 100644
--- a/app/Model/Sighting.php
+++ b/app/Model/Sighting.php
@@ -248,10 +248,9 @@ class Sighting extends AppModel
             return array();
         }
         $anonymise = Configure::read('Plugin.Sightings_anonymise');
-
         foreach ($sightings as $k => $sighting) {
             if (
-                $sighting['Sighting']['org_id'] == 0 && !empty($sighting['Organisation']) ||
+                ($sighting['Sighting']['org_id'] == 0 && !empty($sighting['Organisation'])) ||
                 $anonymise
             ) {
                 if ($sighting['Sighting']['org_id'] != $user['org_id']) {
@@ -337,7 +336,7 @@ class Sighting extends AppModel
         return $attributes;
     }
 
-    public function saveSightings($id, $values, $timestamp, $user, $type = false, $source = false, $sighting_uuid = false, $publish = false)
+    public function saveSightings($id, $values, $timestamp, $user, $type = false, $source = false, $sighting_uuid = false, $publish = false, $saveOnBehalfOf = false)
     {
         $conditions = array();
         if ($id && $id !== 'stix') {
@@ -379,7 +378,7 @@ class Sighting extends AppModel
             $sighting = array(
                     'attribute_id' => $attribute['Attribute']['id'],
                     'event_id' => $attribute['Attribute']['event_id'],
-                    'org_id' => $user['org_id'],
+                    'org_id' => ($saveOnBehalfOf === false) ? $user['org_id'] : $saveOnBehalfOf,
                     'date_sighting' => $timestamp,
                     'type' => $type,
                     'source' => $source
@@ -776,7 +775,17 @@ class Sighting extends AppModel
         }
         $saved = 0;
         foreach ($sightings as $s) {
-            $result = $this->saveSightings($s['attribute_uuid'], false, $s['date_sighting'], $user, $s['type'], $s['source'], $s['uuid']);
+            $saveOnBehalfOf = false;
+            if ($user['Role']['perm_sync']) {
+                if (isset($s['org_id'])) {
+                    if ($s['org_id'] != 0 && !empty($s['Organisation'])) {
+                        $saveOnBehalfOf = $this->Event->Orgc->captureOrg($s['Organisation'], $user);
+                    } else {
+                        $saveOnBehalfOf = 0;
+                    }
+                }
+            }
+            $result = $this->saveSightings($s['attribute_uuid'], false, $s['date_sighting'], $user, $s['type'], $s['source'], $s['uuid'], false, $saveOnBehalfOf);
             if (is_numeric($result)) {
                 $saved += $result;
             }

From 8ee304eff9757dc36ea4321e254b1884dfe17bfa Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 25 Nov 2019 15:45:40 +0100
Subject: [PATCH 37/43] fix: [sync] Set org_id to 0 on proposal push if the
 sighting is anonymised

- correctly prevents the remote side from misattributing the sighting to the sync user's org
---
 app/Model/Event.php | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/app/Model/Event.php b/app/Model/Event.php
index e327da4ae..52ef9f991 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -1080,6 +1080,11 @@ class Event extends AppModel
         $HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
         $request = $this->setupSyncRequest($server);
         $uri = $server['Server']['url'] . '/sightings/bulkSaveSightings/' . $event_uuid;
+        foreach ($sightings as &$sighting) {
+            if (!isset($sighting['org_id'])) {
+                $sighting['org_id'] = '0';
+            }
+        }
         $data = json_encode($sightings);
         if (!empty(Configure::read('Security.sync_audit'))) {
             $pushLogEntry = sprintf(

From bdfe59766a54e36d71672ee90b4ecc2f9a73a381 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 25 Nov 2019 16:02:54 +0100
Subject: [PATCH 38/43] chg: [cleanup] debug() removed

---
 app/Model/Event.php | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/Model/Event.php b/app/Model/Event.php
index 52ef9f991..96c705058 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -1233,7 +1233,6 @@ class Event extends AppModel
             );
             file_put_contents(APP . 'files/scripts/tmp/debug_server_' . $server['Server']['id'] . '.log', $pushLogEntry, FILE_APPEND);
         }
-        debug($data);
         $response = $HttpSocket->post($uri, $data, $request);
         return $this->__handleRestfulEventToServerResponse($response, $newLocation, $newTextBody);
     }

From dbc229c83ca7c05885aee3ccb1f56e5e79be4942 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 25 Nov 2019 16:21:42 +0100
Subject: [PATCH 39/43] new: [sync] Added sighting sync publish button to the
 event view

---
 app/Controller/EventsController.php                    |  2 +-
 .../Elements/genericElements/SideMenu/side_menu.ctp    | 10 ++++++++++
 app/View/Events/ajax/eventPublishConfirmationForm.ctp  |  2 ++
 app/webroot/js/misp.js                                 |  1 +
 4 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php
index cd10c07c4..691dbdd96 100644
--- a/app/Controller/EventsController.php
+++ b/app/Controller/EventsController.php
@@ -2620,7 +2620,7 @@ class EventsController extends AppController
             }
         } else {
             $this->set('id', $id);
-            $this->set('type', 'publish_sightings');
+            $this->set('type', 'publishSightings');
             $this->render('ajax/eventPublishConfirmationForm');
         }
     }
diff --git a/app/View/Elements/genericElements/SideMenu/side_menu.ctp b/app/View/Elements/genericElements/SideMenu/side_menu.ctp
index 9c74b48e9..08befe110 100644
--- a/app/View/Elements/genericElements/SideMenu/side_menu.ctp
+++ b/app/View/Elements/genericElements/SideMenu/side_menu.ctp
@@ -143,6 +143,16 @@
                         'class' => (isset($event['Event']['published']) && (1 == $event['Event']['published'] && $mayModify)) ? '' : 'hidden',
                         'text' => __('Unpublish')
                     ));
+                    if (!empty($event['Event']['published']) && $me['Role']['perm_sighting']) {
+                        echo $this->element('/genericElements/SideMenu/side_menu_link', array(
+                            'onClick' => array(
+                                'function' => 'publishPopup',
+                                'params' => array($event['Event']['id'], 'sighting')
+                            ),
+                            'class' => 'publishButtons',
+                            'text' => __('Publish Sightings')
+                        ));
+                    }
                     if (Configure::read('MISP.delegation')) {
                         if ((Configure::read('MISP.unpublishedprivate') || (isset($event['Event']['distribution']) && $event['Event']['distribution'] == 0)) && (!isset($delegationRequest) || !$delegationRequest) && ($isSiteAdmin || (isset($isAclDelegate) && $isAclDelegate))) {
                             echo $this->element('/genericElements/SideMenu/side_menu_link', array(
diff --git a/app/View/Events/ajax/eventPublishConfirmationForm.ctp b/app/View/Events/ajax/eventPublishConfirmationForm.ctp
index a228c4050..03489aafd 100644
--- a/app/View/Events/ajax/eventPublishConfirmationForm.ctp
+++ b/app/View/Events/ajax/eventPublishConfirmationForm.ctp
@@ -16,6 +16,8 @@
                 echo '<p>' . __('Are you sure this event is complete and everyone should be informed?') . '</p>';
             } else if ($type === 'unpublish') {
                 echo '<p>' . __('Are you sure you wish to unpublish the event?') . '</p>';
+            } else if ($type === 'publishSightings') {
+                echo '<p>' . __('Are you sure you wish publish and synchronise all sightings attached to this event?') . '</p>';
             } else {
                 echo '<p>' . __('Publish but do NOT send alert email? Only for minor changes!') . '</p>';
             }
diff --git a/app/webroot/js/misp.js b/app/webroot/js/misp.js
index 6258a400b..ec2dbbfe3 100644
--- a/app/webroot/js/misp.js
+++ b/app/webroot/js/misp.js
@@ -78,6 +78,7 @@ function publishPopup(id, type) {
     var action = "alert";
     if (type == "publish") action = "publish";
     if (type == "unpublish") action = "unpublish";
+    if (type == "sighting") action = "publishSightings";
     var destination = 'attributes';
     $.get( "/events/" + action + "/" + id, function(data) {
         $("#confirmation_box").html(data);

From 7ab85143dea9897804857e94d95eac66d6f7290c Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 25 Nov 2019 16:49:07 +0100
Subject: [PATCH 40/43] fix: [ACL] added /events/publishSightings

---
 app/Controller/Component/ACLComponent.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php
index 8c17290e9..7750a35b8 100644
--- a/app/Controller/Component/ACLComponent.php
+++ b/app/Controller/Component/ACLComponent.php
@@ -161,6 +161,7 @@ class ACLComponent extends Component
                     'nids' => array('*'),
                     'proposalEventIndex' => array('*'),
                     'publish' => array('perm_publish'),
+                    'publishSightings' => array('perm_sighting'),
                     'pushEventToZMQ' => array('perm_publish_zmq'),
                     'pushEventToKafka' => array('perm_publish_kafka'),
                     'pushProposals' => array('perm_sync'),

From eb7c534cc14f6ffd615057631e1cb652174bc4a3 Mon Sep 17 00:00:00 2001
From: Christophe Vandeplas <christophe@vandeplas.com>
Date: Tue, 26 Nov 2019 10:42:49 +0100
Subject: [PATCH 41/43] fix: [tag] do not show actions column for non-admins

---
 app/View/Tags/index.ctp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/View/Tags/index.ctp b/app/View/Tags/index.ctp
index 4b781adc9..4d84376b8 100644
--- a/app/View/Tags/index.ctp
+++ b/app/View/Tags/index.ctp
@@ -72,7 +72,7 @@
             <th><?php echo __('Tagged attributes');?></th>
             <th><?php echo __('Activity');?></th>
             <th><?php echo __('Favourite');?></th>
-            <?php if ($isAclTagEditor): ?>
+            <?php if ($isSiteAdmin): ?>
             <th class="actions"><?php echo __('Actions');?></th>
             <?php endif; ?>
     </tr><?php

From e05dc512a437284f14624da23cca4a829a76aebf Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Tue, 26 Nov 2019 11:36:49 +0100
Subject: [PATCH 42/43] fix: [security] tightened checks for restricting users
 from tagging data they shouldn't be allowed to tag

As reported by Christophe Vandeplas
---
 app/Controller/TagsController.php | 67 +++++++++++++++++++------------
 1 file changed, 41 insertions(+), 26 deletions(-)

diff --git a/app/Controller/TagsController.php b/app/Controller/TagsController.php
index abdbe6c40..4fdcddfc7 100644
--- a/app/Controller/TagsController.php
+++ b/app/Controller/TagsController.php
@@ -860,41 +860,48 @@ class TagsController extends AppController
         $this->render('/Servers/json/simple');
     }
 
-    private function __findObjectByUuid($object_uuid, &$type)
+    private function __findObjectByUuid($object_uuid, &$type, $scope = 'modify')
     {
         $this->loadModel('Event');
-        $object = $this->Event->find('first', array(
-            'conditions' => array(
-                'Event.uuid' => $object_uuid,
-            ),
-            'fields' => array('Event.orgc_id', 'Event.id'),
-            'recursive' => -1
+        if (!$this->userRole['perm_tagger']) {
+            throw new MethodNotAllowedException(__('This functionality requires tagging permission.'));
+        }
+        $object = $this->Event->fetchEvent($this->Auth->user(), array(
+            'event_uuid' => $object_uuid,
+            'metadata' => 1
         ));
         $type = 'Event';
         if (!empty($object)) {
+            $object = $object[0];
             if (
+                $scope !== 'view' &&
                 !$this->_isSiteAdmin() &&
-                !$this->userRole['perm_tagger'] &&
-                $object['Event']['orgc_id'] != $this->Auth->user('org_id')
+                !$object['Event']['orgc_id'] != $this->Auth->user('org_id')
             ) {
-                throw new MethodNotAllowedException('Invalid Target.');
+                throw new MethodNotAllowedException(__('Invalid Target.'));
             }
         } else {
             $type = 'Attribute';
-            $object = $this->Event->Attribute->find('first', array(
-                'conditions' => array(
-                    'Attribute.uuid' => $object_uuid,
-                ),
-                'fields' => array('Attribute.id'),
-                'recursive' => -1,
-                'contain' => array('Event.orgc_id')
-            ));
+            $object = $this->Event->Attribute->fetchAttributes(
+                $this->Auth->user(),
+                array(
+                    'conditions' => array(
+                        'Attribute.uuid' => $object_uuid
+                    ),
+                    'flatten' => 1
+                )
+            );
             if (!empty($object)) {
-                if (!$this->_isSiteAdmin() && !$this->userRole['perm_tagger'] && $object['Event']['orgc_id'] != $this->Auth->user('org_id')) {
-                    throw new MethodNotAllowedException('Invalid Target.');
+                $object = $object[0];
+                if (
+                    $scope !== 'view' &&
+                    !$this->_isSiteAdmin() &&
+                    !$object['Event']['orgc_id'] != $this->Auth->user('org_id')
+                ) {
+                    throw new MethodNotAllowedException(__('Invalid Target.'));
                 }
             } else {
-                throw new MethodNotAllowedException('Invalid Target.');
+                throw new MethodNotAllowedException(__('Invalid Target.'));
             }
         }
         return $object;
@@ -932,11 +939,11 @@ class TagsController extends AppController
                 $local = $this->request->data['local'];
             }
         }
-        if (!is_bool($local)) {
-            throw new InvalidArgumentException('Invalid local flag');
+        if (!empty($local) && $this->Auth->user('org_id') != Configure::read('MISP.host_org_id')) {
+            throw new MethodNotAllowedException(__('Local tags can only be added by users of the host organisation.'));
         }
         $objectType = '';
-        $object = $this->__findObjectByUuid($uuid, $objectType);
+        $object = $this->__findObjectByUuid($uuid, $objectType, $local ? 'view' : 'modify');
         $existingTag = $this->Tag->find('first', array('conditions' => $conditions, 'recursive' => -1));
         if (empty($existingTag)) {
             if (!is_numeric($tag)) {
@@ -1038,9 +1045,9 @@ class TagsController extends AppController
             throw new MethodNotAllowedException('Invalid Tag.');
         }
         $objectType = '';
-        $object = $this->__findObjectByUuid($uuid, $objectType);
+        $object = $this->__findObjectByUuid($uuid, $objectType, 'view');
         if (empty($object)) {
-            throw new MethodNotAllowedException('Invalid Target.');
+            throw new MethodNotAllowedException(__('Invalid Target.'));
         }
         $connectorObject = $objectType . 'Tag';
         $this->loadModel($objectType);
@@ -1052,6 +1059,14 @@ class TagsController extends AppController
         ));
         if (empty($existingAssociation)) {
             throw new MethodNotAllowedException('Could not remove tag as it is not attached to the target ' . $objectType);
+        } else {
+            if (empty($existingAssociation[$objectType . 'Tag']['local'])) {
+                $object = $this->__findObjectByUuid($uuid, $objectType);
+            } else {
+                if ($object['Event']['orgc_id'] !== $this->Auth->user('org_id') && $this->Auth->user('org_id') != Configure::read('MISP.host_org_id')) {
+                    throw new MethodNotAllowedException(__('Insufficient privileges to remove local tags from events you do not own.'));
+                }
+            }
         }
         $result = $this->$objectType->$connectorObject->delete($existingAssociation[$connectorObject]['id']);
         if ($result) {

From bd17bdfb6b9e634ad2349733ca0bcf23ae99bb52 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Tue, 26 Nov 2019 12:34:22 +0100
Subject: [PATCH 43/43] fix: [UI] includeSightingdb flag not set correctly in
 the event attribute index

---
 app/Controller/EventsController.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php
index 691dbdd96..e8171b4f2 100644
--- a/app/Controller/EventsController.php
+++ b/app/Controller/EventsController.php
@@ -1253,7 +1253,7 @@ class EventsController extends AppController
         if (isset($filters['deleted'])) {
             $deleted = $filters['deleted'] == 2 ? 0 : 1;
         }
-        $this->set('includeSightingdb', (!empty($filters['includeSightingdb'] && Configure::read('Plugin.Sightings_sighting_db_enable'))));
+        $this->set('includeSightingdb', (!empty($filters['includeSightingdb']) && Configure::read('Plugin.Sightings_sighting_db_enable')));
         $this->set('deleted', $deleted);
         $this->set('typeGroups', array_keys($this->Event->Attribute->typeGroupings));
         $this->set('attributeFilter', isset($filters['attributeFilter']) ? $filters['attributeFilter'] : 'all');