Merge branch 'develop' of github.com:MISP/MISP into feature-periodic-notification

app/Controller/AttributesController.php

@@ -2848,7 +2848,7 @@ class AttributesController extends AppController
 
             $event = $this->Attribute->Event->find('first', [
                 'recursive' => -1,
-                'conditons' => ['Event.id' => $attribute['Attribute']['event_id']]
+                'conditions' => ['Event.id' => $attribute['Attribute']['event_id']]
             ]);
             if (!$this->_isRest()) {
                 $this->Attribute->Event->insertLock($this->Auth->user(), $attribute['Attribute']['event_id']);

app/Controller/ServersController.php

@@ -322,7 +322,8 @@ class ServersController extends AppController
                         'json' => '[]',
                         'push_rules' => $defaultPushRules,
                         'pull_rules' => $defaultPullRules,
-                        'self_signed' => 0
+                        'self_signed' => 0,
+                        'remove_missing_tags' => 0
                     );
                     foreach ($defaults as $default => $dvalue) {
                         if (!isset($this->request->data['Server'][$default])) {
@@ -514,7 +515,7 @@ class ServersController extends AppController
             }
             if (!$fail) {
                 // say what fields are to be updated
-                $fieldList = array('id', 'url', 'push', 'pull', 'push_sightings', 'push_galaxy_clusters', 'pull_galaxy_clusters', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
+                $fieldList = array('id', 'url', 'push', 'pull', 'push_sightings', 'push_galaxy_clusters', 'pull_galaxy_clusters', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'remove_missing_tags', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
                 $this->request->data['Server']['id'] = $id;
                 if (isset($this->request->data['Server']['authkey']) && "" != $this->request->data['Server']['authkey']) {
                     $fieldList[] = 'authkey';

app/Lib/Tools/BackgroundJobsTool.php

@@ -629,6 +629,10 @@ class BackgroundJobsTool
             ];
         }
 
+        if (!isset($this->settings['supervisor_host'])) {
+            throw new RuntimeException("Required option `supervisor_host` for BackgroundJobsTool is not set.");
+        }
+
         $host = null;
         if (substr($this->settings['supervisor_host'], 0, 5) === 'unix:') {
             if (!defined('CURLOPT_UNIX_SOCKET_PATH')) {

app/Model/AppModel.php

@@ -83,7 +83,7 @@ class AppModel extends Model
         75 => false, 76 => true, 77 => false, 78 => false, 79 => false, 80 => false,
         81 => false, 82 => false, 83 => false, 84 => false, 85 => false, 86 => false,
         87 => false, 88 => false, 89 => false, 90 => false, 91 => false, 92 => false,
-        93 => false, 94 => false, 95 => true,
+        93 => false, 94 => false, 95 => true, 96 => true,
     );
 
     const ADVANCED_UPDATES_DESCRIPTION = array(
@@ -1860,11 +1860,14 @@ class AppModel extends Model
                 $sqlArray[] = "ALTER TABLE `over_correlating_values` MODIFY `value` varchar(191) NOT NULL;";
                 break;
             case 95:
+                $sqlArray[] = "ALTER TABLE `servers` ADD `remove_missing_tags` tinyint(1) NOT NULL DEFAULT 0 AFTER `skip_proxy`;";
+                break;
+            case 96:
                 $sqlArray[] = "ALTER TABLE `users`
-                        ADD COLUMN `notification_daily`     tinyint(1) NOT NULL DEFAULT 0,
+                    ADD COLUMN `notification_daily`     tinyint(1) NOT NULL DEFAULT 0,
-                        ADD COLUMN `notification_weekly`    tinyint(1) NOT NULL DEFAULT 0,
+                    ADD COLUMN `notification_weekly`    tinyint(1) NOT NULL DEFAULT 0,
-                        ADD COLUMN `notification_monthly`   tinyint(1) NOT NULL DEFAULT 0
+                    ADD COLUMN `notification_monthly`   tinyint(1) NOT NULL DEFAULT 0
-                    ;";
+                ;";
                 break;
             case 'fixNonEmptySharingGroupID':
                 $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@@ -2349,14 +2352,14 @@ class AppModel extends Model
                     $job = ClassRegistry::init('Job');
                     $jobId = $job->createJob(
                             'SYSTEM',
-                            Job::WORKER_PRIO,
+                            Job::WORKER_UPDATE,
                             'run_updates',
                             'command: ' . implode(',', $updates),
                             'Updating.'
                         );
 
                     $this->getBackgroundJobsTool()->enqueue(
-                        BackgroundJobsTool::PRIO_QUEUE,
+                        BackgroundJobsTool::UPDATE_QUEUE,
                         BackgroundJobsTool::CMD_ADMIN,
                         [
                             'runUpdates',

app/Model/Attribute.php

@@ -2943,7 +2943,7 @@ class Attribute extends AppModel
         return $attribute;
     }
 
-    public function editAttribute($attribute, array $event, $user, $objectId, $log = false, $force = false, &$nothingToChange = false)
+    public function editAttribute($attribute, array $event, $user, $objectId, $log = false, $force = false, &$nothingToChange = false, $server = null)
     {
         $eventId = $event['Event']['id'];
         $attribute['event_id'] = $eventId;
@@ -3026,11 +3026,14 @@ class Attribute extends AppModel
         }
         if ($user['Role']['perm_tagger']) {
             /*
-                We should uncomment the line below in the future once we have tag soft-delete
+                We should unwrap the line below and remove the server option in the future once we have tag soft-delete
                 A solution to still keep the behavior for previous instance could be to not soft-delete the Tag if the remote instance
                 has a version below x
             */
-            // $this->AttributeTag->pruneOutdatedAttributeTagsFromSync(isset($attribute['Tag']) ? $attribute['Tag'] : array(), $existingAttribute['AttributeTag']);
+            if (isset($server) && isset($server['Server']['remove_missing_tags']) && $server['Server']['remove_missing_tags']) {
+                $this->AttributeTag->pruneOutdatedAttributeTagsFromSync(isset($attribute['Tag']) ? $attribute['Tag'] : array(), $existingAttribute['AttributeTag']);
+            }
+
             if (isset($attribute['Tag'])) {
                 foreach ($attribute['Tag'] as $tag) {
                     $tag_id = $this->AttributeTag->Tag->captureTag($tag, $user);

app/Model/Event.php

@@ -3758,6 +3758,7 @@ class Event extends AppModel
                     'Server.unpublish_event',
                     'Server.publish_without_email',
                     'Server.internal',
+                    'Server.remove_missing_tags'
                 )
             ));
         } else {
@@ -3864,7 +3865,7 @@ class Event extends AppModel
                 $data['Event']['Attribute'] = array_values($data['Event']['Attribute']);
                 foreach ($data['Event']['Attribute'] as $attribute) {
                     $nothingToChange = false;
-                    $result = $this->Attribute->editAttribute($attribute, $saveResult, $user, 0, false, $force, $nothingToChange);
+                    $result = $this->Attribute->editAttribute($attribute, $saveResult, $user, 0, false, $force, $nothingToChange, $server);
                     if ($result !== true) {
                         $validationErrors['Attribute'][] = $result;
                     }

app/Model/Job.php

@@ -11,7 +11,8 @@ class Job extends AppModel
     const WORKER_EMAIL = 'email',
         WORKER_PRIO = 'prio',
         WORKER_DEFAULT = 'default',
-        WORKER_CACHE = 'cache';
+        WORKER_CACHE = 'cache',
+        WORKER_UPDATE = 'update';
 
     public $belongsTo = array(
         'Org' => array(

app/Plugin/AadAuth/Controller/Component/Auth/AadAuthenticateAuthenticate.php

@@ -303,34 +303,42 @@ class AadAuthenticateAuthenticate extends BaseAuthenticate
 				'Authorization' => 'Bearer ' . $authdata["access_token"]
 			]
 		];
+				
+		$has_next_page = true;
 		$url = self::$auth_provider_user . "/v1.0/me/memberOf";
+		while ($has_next_page) {
+			$response = (new HttpSocket())->get($url, array(), $options);
 
-		$response = (new HttpSocket())->get($url, null, $options);
+			if (!$response->isOk()) {
-
+				$this->_log("warning", "Error received during user group data fetch.");
-		if (!$response->isOk()) {
+				$this->_logHttpError("debug", $url, $response);
-			$this->_log("warning", "Error received during user group data fetch.");
+				return false;
-			$this->_logHttpError("debug", $url, $response);
-			return false;
-		}
-
-		$groupdata = json_decode($response->body, true);  //This should now contain your logged on user memberOf (groups) information
-		if (isset($groupdata["error"])) {
-			$this->_log("warning", "Group data fetch contained an error.");
-			$this->_log("debug", "Response: " . json_encode($groupdata["error"]));
-			return false;
-		}
-
-		// Now check if the user has any of the MISP AAD groups enabled
-		foreach ($groupdata["value"] as $group) {
-			$groupdisplayName = $group["displayName"];
-			if ($groupdisplayName == self::$misp_siteadmin) {
-				return self::$misp_siteadmin;
 			}
-			if ($groupdisplayName == self::$misp_orgadmin) {
+
-				return self::$misp_orgadmin;
+			$groupdata = json_decode($response->body, true);  //This should now contain your logged on user memberOf (groups) information
+			if (isset($groupdata["error"])) {
+				$this->_log("warning", "Group data fetch contained an error.");
+				$this->_log("debug", "Response: " . json_encode($groupdata["error"]));
+				return false;
 			}
-			if ($groupdisplayName == self::$misp_user) {
+
-				return self::$misp_user;
+			// Now check if the user has any of the MISP AAD groups enabled
+			foreach ($groupdata["value"] as $group) {
+				$groupdisplayName = $group["displayName"];
+				if ($groupdisplayName == self::$misp_siteadmin) {
+					return self::$misp_siteadmin;
+				}
+				if ($groupdisplayName == self::$misp_orgadmin) {
+					return self::$misp_orgadmin;
+				}
+				if ($groupdisplayName == self::$misp_user) {
+					return self::$misp_user;
+				}
+			}
+
+			$has_next_page = array_key_exists("@odata.nextLink", $groupdata);
+			if ($has_next_page) {
+				$url = $groupdata["@odata.nextLink"];
 			}
 		}
 

app/View/Feeds/add.ctp

@@ -73,6 +73,7 @@ echo $this->element('genericElements/Form/genericForm', [
                 'field' => 'orgc_id',
                 'label' => __('Creator organisation'),
                 'options' => $dropdownData['orgs'],
+                'value' => $this->request->params['action'] === 'add' ? $me['org_id'] : '',
                 'type' => 'dropdown',
                 'div' => ['id' => 'OrgcDiv', 'style' => 'display:none', 'class' => 'optionalField'],
                 'class' => 'form-control span6'
@@ -80,7 +81,7 @@ echo $this->element('genericElements/Form/genericForm', [
             [
                 'field' => 'fixed_event',
                 'label' => __('Target Event'),
-                'options' => ['New Event Each Pull', 'Fixed Event'],
+                'options' => ['Fixed Event', 'New Event Each Pull'],
                 'type' => 'dropdown',
                 'div' => ['id' => 'TargetDiv', 'style' => 'display:none', 'class' => 'optionalField'],
                 'class' => 'form-control span6'

app/View/Servers/edit.ctp

@@ -108,6 +108,11 @@
         ));
         echo '<div class="input clear"></div>';
         echo $this->Form->input('skip_proxy', array('type' => 'checkbox', 'label' => 'Skip proxy (if applicable)'));
+        echo '<div class="input clear"></div>';
+        echo $this->Form->input('remove_missing_tags', array(
+            'type' => 'checkbox',
+            'label' => __('Remove Missing Attribute Tags (not recommended)'),
+        ));
     ?>
     <div class="clear">
         <p>
@@ -244,7 +249,8 @@ var formInfoValues = {
     'ServerPublishWithoutEmail' : '<?php echo __('Publish new event without email (working with Pull event).');?>',
     'ServerSubmittedCert' : "<?php echo __('You can also upload a certificate file if the instance you are trying to connect to has its own signing authority.');?>",
     'ServerSubmittedClientCert' : "<?php echo __('You can also upload a client certificate file if the instance you are trying to connect requires this.');?>",
-    'ServerSelfSigned' : "<?php echo __('Click this, if you would like to allow a connection despite the other instance using a self-signed certificate (not recommended).');?>"
+    'ServerSelfSigned' : "<?php echo __('Click this, if you would like to allow a connection despite the other instance using a self-signed certificate (not recommended).');?>",
+    'ServerRemoveMissingTags': "<?php echo __('Remove any global tags from attributes on local instance that are not present on an updated event being received from the server. Any missing global tags will be removed, local tags are unaffected as is pushing events (working with Pull event).');?>"
 };
 
 var rules = {
@@ -277,7 +283,7 @@ $(function() {
         serverOrgTypeChange();
     });
 
-    $("#ServerUrl, #ServerOrganization, #ServerName, #ServerAuthkey, #ServerPush, #ServerPull, #ServerUnpublishEvent, #ServerPublishWithoutEmail, #ServerSubmittedCert, #ServerSubmittedClientCert, #ServerSelfSigned")
+    $("#ServerUrl, #ServerOrganization, #ServerName, #ServerAuthkey, #ServerPush, #ServerPull, #ServerUnpublishEvent, #ServerPublishWithoutEmail, #ServerSubmittedCert, #ServerSubmittedClientCert, #ServerSelfSigned, #ServerRemoveMissingTags")
         .on('mouseleave', function() {
         $(this).popover('destroy');
     }).on('mouseover', function(e) {

db_schema.json

@@ -5719,6 +5719,17 @@
                 "column_default": null,
                 "extra": ""
             },
+            {
+                "column_name": "remove_missing_tags",
+                "is_nullable": "NO",
+                "data_type": "tinyint",
+                "character_maximum_length": null,
+                "numeric_precision": "3",
+                "collation_name": null,
+                "column_type": "tinyint(1)",
+                "column_default": "0",
+                "extra": ""
+            },
             {
                 "column_name": "pull_rules",
                 "is_nullable": "NO",