Merge branch '2.4' of github.com:MISP/MISP into zoidberg-timeline

.travis.yml

@@ -102,7 +102,7 @@ install:
     - sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.cookieTimeout" 3600'
     - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.host_org_id" 1'
     - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.email" "info@admin.test"'
-    - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.disable_emailing" true'
+    - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.disable_emailing" false'
     - sudo -E su $USER -c 'app/Console/cake Admin setSetting "debug" true'
     - sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false'
     - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1"'
@@ -160,10 +160,9 @@ script:
     - ./curl_tests.sh $AUTH
     - popd
     - pushd PyMISP
+    - git submodule init
+    - git submodule update
     - pipenv install -d
-    - pushd tests
-    - git clone https://github.com/viper-framework/viper-test-files.git
-    - popd
     - pipenv run python tests/testlive_comprehensive.py
     - pipenv run python tests/test.py
     - pipenv run python tests/test_mispevent.py

INSTALL/INSTALL.sh

@@ -2464,6 +2464,7 @@ x86_64-debian-stretch
 x86_64-debian-buster
 x86_64-ubuntu-bionic
 x86_64-kali-2019.2
+x86_64-kali-2019.3
 armv6l-raspbian-stretch
 armv7l-raspbian-stretch
 armv7l-debian-jessie

INSTALL/INSTALL.sh.sfv

@@ -1,5 +1,5 @@
-; Generated by RHash v1.3.8 on 2019-09-19 at 11:45.46
+; Generated by RHash v1.3.8 on 2019-10-12 at 23:45.03
 ; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
 ;
-;        98996  11:45.46 2019-09-19 INSTALL.sh
+;        99015  23:45.03 2019-10-12 INSTALL.sh
-INSTALL.sh 590EBA6FCA2E8F2B5044843DF22FEED524ED3C7E 2B3E6DC0191D5D3AFD685E6ACF3E02D15261770E8E17A21F882EC2F07E908D55 13EB5B9EF3AEE3C96558DE068A0210F938491322F0AAD10DB958D9DCFA93B5313044013084E56647586AFCF8DE07C7D1 B169AA7096FAAF21E8CB8B7B7C9296C4CDB25E4B540637C15C040722EE52A2A8D2C4E8122CCEC25AFAD99F9CEC065DE14899E7C1DD571DE2B6127F08C6B79229
+INSTALL.sh 0A4A963E90CD83550F59CBC270CEF3DB3E25DBDF D292D42E306A3E19FCD9A6C10B1A28F56584810D49420592A9A6644BB3EFB2E6 7453482ADF587B9760B23A0563FADE99568E497CF810BAE9CD7E44F0025D8D6BFE1B17DD2C790A6CCD72130A72CD59C6 E6BEACA555D8752D210757BE8A372AF7BA8675CA293810B8A5ADEE905FC44D41CDAD60225203D8E93B037D520703996E03E62B501EB766290C452716BB125C4F

INSTALL/INSTALL.sh.sha1

@@ -1 +1 @@
-590eba6fca2e8f2b5044843df22feed524ed3c7e  INSTALL.sh
+0a4a963e90cd83550f59cbc270cef3db3e25dbdf  INSTALL.sh

INSTALL/INSTALL.sh.sha256

@@ -1 +1 @@
-2b3e6dc0191d5d3afd685e6acf3e02d15261770e8e17a21f882ec2f07e908d55  INSTALL.sh
+d292d42e306a3e19fcd9a6c10b1a28f56584810d49420592a9a6644bb3efb2e6  INSTALL.sh

INSTALL/INSTALL.sh.sha384

@@ -1 +1 @@
-13eb5b9ef3aee3c96558de068a0210f938491322f0aad10db958d9dcfa93b5313044013084e56647586afcf8de07c7d1  INSTALL.sh
+7453482adf587b9760b23a0563fade99568e497cf810bae9cd7e44f0025d8d6bfe1b17dd2c790a6ccd72130a72cd59c6  INSTALL.sh

INSTALL/INSTALL.sh.sha512

@@ -1 +1 @@
-b169aa7096faaf21e8cb8b7b7c9296c4cdb25e4b540637c15c040722ee52a2a8d2c4e8122ccec25afad99f9cec065de14899e7c1dd571de2b6127f08c6b79229  INSTALL.sh
+e6beaca555d8752d210757be8a372af7ba8675ca293810b8a5adee905fc44d41cdad60225203d8e93b037d520703996e03e62b501eb766290c452716bb125c4f  INSTALL.sh

INSTALL/INSTALL.tpl.sh

@@ -678,6 +678,7 @@ x86_64-debian-stretch
 x86_64-debian-buster
 x86_64-ubuntu-bionic
 x86_64-kali-2019.2
+x86_64-kali-2019.3
 armv6l-raspbian-stretch
 armv7l-raspbian-stretch
 armv7l-debian-jessie

INSTALL/MYSQL.sql

@@ -1358,7 +1358,7 @@ INSERT INTO `admin_settings` (`id`, `setting`, `value`) VALUES
 
 INSERT INTO `feeds` (`id`, `provider`, `name`, `url`, `distribution`, `default`, `enabled`) VALUES
 (1, 'CIRCL', 'CIRCL OSINT Feed', 'https://www.circl.lu/doc/misp/feed-osint', 3, 1, 0),
-(2, 'Botvrij.eu', 'The Botvrij.eu Data', 'http://www.botvrij.eu/data/feed-osint', 3, 1, 0);
+(2, 'Botvrij.eu', 'The Botvrij.eu Data', 'https://www.botvrij.eu/data/feed-osint', 3, 1, 0);
 
  INSERT INTO `regexp` (`id`, `regexp`, `replacement`, `type`) VALUES
  (1, '/.:.ProgramData./i', '%ALLUSERSPROFILE%\\\\', 'ALL'),

INSTALL/POSTGRESQL-data-initial.sql

@@ -118,7 +118,7 @@ COPY public.favourite_tags (id, tag_id, user_id) FROM stdin;
 
 COPY public.feeds (id, name, provider, url, rules, enabled, distribution, sharing_group_id, tag_id, "default", source_format, fixed_event, delta_merge, event_id, publish, override_ids, settings, input_source, delete_local_file, lookup_visible, headers, caching_enabled) FROM stdin;
 1	CIRCL OSINT Feed	CIRCL	https://www.circl.lu/doc/misp/feed-osint	\N	f	3	0	0	t	misp	f	f	0	f	f	\N	network	f	f	\N	f
-2	The Botvrij.eu Data	Botvrij.eu	http://www.botvrij.eu/data/feed-osint	\N	f	3	0	0	t	misp	f	f	0	f	f	\N	network	f	f	\N	f
+2	The Botvrij.eu Data	Botvrij.eu	https://www.botvrij.eu/data/feed-osint	\N	f	3	0	0	t	misp	f	f	0	f	f	\N	network	f	f	\N	f
 \.
 
 

INSTALL/misplogrotate.te

@@ -1,8 +1,16 @@
-module misplogrotate 1.0;
+module misplogrotate 1.2;
 require {
+	type httpd_t;
 	type logrotate_t;
+	type httpd_log_t;
+	type httpd_sys_script_t;
 	type httpd_sys_content_t;
-	class dir { ioctl read getattr lock search open };
+	type httpd_sys_rw_content_t;
+	class dir { ioctl read getattr lock search open remove_name };
+	class file { unlink write };
 }
 #============= logrotate_t ==============
 allow logrotate_t httpd_sys_content_t:dir { ioctl read getattr lock search open };
+allow logrotate_t httpd_sys_rw_content_t:dir { ioctl read getattr lock search open };
+allow httpd_t httpd_log_t:dir remove_name;
+allow { httpd_t httpd_sys_script_t } httpd_log_t:file { unlink write };

PyMISP

@@ -1 +1 @@
-Subproject commit de6a64ba45b56cef1a233df306da411c49801c03
+Subproject commit 204fd6ba8cc916844156c1819c8375f6bbbca995

VERSION.json

@@ -1 +1 @@
-{"major":2, "minor":4, "hotfix":116}
+{"major":2, "minor":4, "hotfix":117}

app/Config/config.default.php

@@ -17,7 +17,7 @@ $config = array(
 			'org'                            => 'ORGNAME',
 			'showorg'                        => true,
 			'threatlevel_in_email_subject'   => true,
-			'email_subject_TLP_string'       => 'TLP Amber',
+            'email_subject_TLP_string'       => 'tlp:amber',
 			'email_subject_tag'              => 'tlp',
 			'email_subject_include_tag_name' => true,
 			'background_jobs'                => true,
@@ -141,7 +141,7 @@ $config = array(
 	/*
 	'ApacheSecureAuth' => // Configuration for kerberos authentication
 		array(
-			'apacheEnv'          => 'REMOTE_USER',           // If proxy variable = HTTP_REMOTE_USER
+			'apacheEnv'          => 'REMOTE_USER',           // If proxy variable = HTTP_REMOTE_USER, If BasicAuth ldap = PHP_AUTH_USER
 			'ldapServer'         => 'ldap://example.com',   // FQDN or IP
 			'ldapProtocol'       => 3,
 			'ldapNetworkTimeout' => -1,  // use -1 for unlimited network timeout

app/Console/Command/ServerShell.php

@@ -6,6 +6,32 @@ class ServerShell extends AppShell
 {
     public $uses = array('Server', 'Task', 'Job', 'User', 'Feed');
 
+    public function list() {
+        $res = ['servers'=>[]];
+
+        $servers = $this->Server->find('all', [
+            'fields' => ['Server.id', 'Server.name', 'Server.url'],
+            'recursive' => 0
+        ]);
+        foreach ($servers as $server)
+            $res['servers'][] = $server['Server'];
+
+        echo json_encode($res) . PHP_EOL;
+    }
+
+    public function test() {
+        if (empty($this->args[0])) {
+            die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Test'] . PHP_EOL);
+        }
+
+        $serverId = intval($this->args[0]);
+        $res = @$this->Server->runConnectionTest($serverId);
+        if (!empty($res['message']))
+            $res['message'] = json_decode($res['message']);
+
+        echo json_encode($res) . PHP_EOL;
+    }
+
     public function pull() {
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['pull'] . PHP_EOL);

app/Controller/AppController.php

@@ -46,8 +46,8 @@ class AppController extends Controller
 
     public $helpers = array('Utility', 'OrgImg', 'FontAwesome', 'UserName');
 
-    private $__queryVersion = '89';
+    private $__queryVersion = '90';
-    public $pyMispVersion = '2.4.114';
+    public $pyMispVersion = '2.4.117';
     public $phpmin = '7.0';
     public $phprec = '7.2';
     public $isApiAuthed = false;
@@ -471,7 +471,9 @@ class AppController extends Controller
         }
         $this->set('notifications', $notifications);
         $this->ACL->checkAccess($this->Auth->user(), Inflector::variable($this->request->params['controller']), $this->action);
-        $this->__rateLimitCheck();
+        if ($this->_isRest()) {
+            $this->__rateLimitCheck();
+        }
     }
 
     private function __rateLimitCheck()

app/Controller/AttributesController.php

@@ -80,18 +80,10 @@ class AttributesController extends AppController
             return $this->RestResponse->viewData($attributes, $this->response->type());
         }
         $org_ids = array();
-        $tag_ids = array();
         $orgs = $this->Attribute->Event->Orgc->find('list', array(
                 'conditions' => array('Orgc.id' => $org_ids),
                 'fields' => array('Orgc.id', 'Orgc.name')
         ));
-        if (!empty($tag_ids)) {
-            $tags = $this->Attribute->AttributeTag->Tag->find('all', array(
-                'conditions' => array('Tag.id' => $tag_ids),
-                'recursive' => -1,
-                'fields' => array('Tag.id', 'Tag.name', 'Tag.colour')
-            ));
-        }
         if (!$this->_isRest()) {
             $temp = $this->__searchUI($attributes);
             $this->loadModel('Galaxy');
@@ -1645,25 +1637,6 @@ class AttributesController extends AppController
         }
     }
 
-    // Deletes this specific attribute from all remote servers
-    private function __deleteAttributeFromServers($uuid)
-    {
-        // get a list of the servers with push active
-        $this->loadModel('Server');
-        $servers = $this->Server->find('all', array('conditions' => array('push' => 1)));
-
-        // iterate over the servers and upload the attribute
-        if (empty($servers)) {
-            return;
-        }
-        App::uses('SyncTool', 'Tools');
-        foreach ($servers as $server) {
-            $syncTool = new SyncTool();
-            $HttpSocket = $syncTool->setupHttpSocket($server);
-            $this->Attribute->deleteAttributeFromServer($uuid, $server, $HttpSocket);
-        }
-    }
-
     public function search($continue = false)
     {
         $this->set('attrDescriptions', $this->Attribute->fieldDescriptions);
@@ -2986,6 +2959,7 @@ class AttributesController extends AppController
 
     public function addTag($id = false, $tag_id = false)
     {
+        $this->Taxonomy = $log = ClassRegistry::init('Taxonomy');
         $rearrangeRules = array(
             'request' => false,
             'Attribute' => false,
@@ -3126,6 +3100,20 @@ class AttributesController extends AppController
                         $fails++;
                         continue;
                     }
+                    $tagsOnAttribute = $this->Attribute->AttributeTag->find('all', array(
+                        'conditions' => array(
+                            'AttributeTag.attribute_id' => $id,
+                            'AttributeTag.local' => $local
+                        ),
+                        'contain' => 'Tag',
+                        'fields' => array('Tag.name'),
+                        'recursive' => -1
+                    ));
+                    $exclusiveTestPassed = $this->Taxonomy->checkIfNewTagIsAllowedByTaxonomy($tag['Tag']['name'], Hash::extract($tagsOnAttribute, '{n}.Tag.name'));
+                    if (!$exclusiveTestPassed) {
+                        $fails++;
+                        continue;
+                    }
                     $this->Attribute->AttributeTag->create();
                     if ($this->Attribute->AttributeTag->save(array('attribute_id' => $id, 'tag_id' => $tag_id, 'event_id' => $eventId, 'local' => $local))) {
                         if (!$local) {

app/Controller/EventsController.php

@@ -1150,6 +1150,7 @@ class EventsController extends AppController
 
         // remove galaxies tags
         $this->loadModel('GalaxyCluster');
+        $this->loadModel('Taxonomy');
         $cluster_names = $this->GalaxyCluster->find('list', array('fields' => array('GalaxyCluster.tag_name'), 'group' => array('GalaxyCluster.tag_name', 'GalaxyCluster.id')));
         foreach ($event['Object'] as $k => $object) {
             if (isset($object['Attribute'])) {
@@ -1159,6 +1160,14 @@ class EventsController extends AppController
                             unset($event['Object'][$k]['Attribute'][$k2]['AttributeTag'][$k3]);
                         }
                     }
+                    $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']);
+                    foreach ($tagConflicts['global'] as $tagConflict) {
+                        $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy'];
+                    }
+                    foreach ($tagConflicts['local'] as $tagConflict) {
+                        $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy'];
+                    }
+                    $event['Object'][$k]['Attribute'][$k2]['tagConflicts'] = $tagConflicts;
                 }
             }
         }
@@ -1168,6 +1177,14 @@ class EventsController extends AppController
                     unset($event['Attribute'][$k]['AttributeTag'][$k2]);
                 }
             }
+            $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']);
+            foreach ($tagConflicts['global'] as $tagConflict) {
+                $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy'];
+            }
+            foreach ($tagConflicts['local'] as $tagConflict) {
+                $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy'];
+            }
+            $event['Attribute'][$k]['tagConflicts'] = $tagConflicts;
         }
         if (empty($this->passedArgs['sort'])) {
             $filters['sort'] = 'timestamp';
@@ -1258,12 +1275,14 @@ class EventsController extends AppController
 
     private function __viewUI($event, $continue, $fromEvent)
     {
+        $this->loadModel('Taxonomy');
         $filterData = array(
             'request' => $this->request,
             'paramArray' => $this->acceptedFilteringNamedParams,
             'named_params' => $this->params['named']
         );
         $exception = false;
+        $warningTagConflicts = array();
         $filters = $this->_harvestParameters($filterData, $exception);
 
         $this->loadModel('GalaxyCluster');
@@ -1364,6 +1383,16 @@ class EventsController extends AppController
                 unset($event['EventTag'][$k]);
             }
         }
+
+        $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($event['EventTag']);
+        foreach ($tagConflicts['global'] as $tagConflict) {
+            $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy'];
+        }
+        foreach ($tagConflicts['local'] as $tagConflict) {
+            $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy'];
+        }
+        $this->set('tagConflicts', $tagConflicts);
+
         $startDate = null;
         $modificationMap = array();
         foreach ($event['Attribute'] as $k => $attribute) {
@@ -1380,6 +1409,14 @@ class EventsController extends AppController
                     unset($event['Attribute'][$k]['AttributeTag'][$k2]);
                 }
             }
+            $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']);
+            foreach ($tagConflicts['global'] as $tagConflict) {
+                $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy'];
+            }
+            foreach ($tagConflicts['local'] as $tagConflict) {
+                $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy'];
+            }
+            $event['Attribute'][$k]['tagConflicts'] = $tagConflicts;
         }
         $attributeTagsName = $this->Event->Attribute->AttributeTag->extractAttributeTagsNameFromEvent($event, 'both');
         $this->set('attributeTags', array_values($attributeTagsName['tags']));
@@ -1405,9 +1442,18 @@ class EventsController extends AppController
                             unset($event['Object'][$k]['Attribute'][$k2]['AttributeTag'][$k3]);
                         }
                     }
+                    $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']);
+                    foreach ($tagConflicts['global'] as $tagConflict) {
+                        $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy'];
+                    }
+                    foreach ($tagConflicts['local'] as $tagConflict) {
+                        $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy'];
+                    }
+                    $event['Object'][$k]['Attribute'][$k2]['tagConflicts'] = $tagConflicts;
                 }
             }
         }
+        $this->set('warningTagConflicts', $warningTagConflicts);
         $filters['sort'] = 'timestamp';
         $filters['direction'] = 'desc';
         if (isset($filters['distribution'])) {
@@ -3715,6 +3761,7 @@ class EventsController extends AppController
 
     public function addTag($id = false, $tag_id = false)
     {
+        $this->loadModel('Taxonomy');
         $rearrangeRules = array(
                 'request' => false,
                 'Event' => false,
@@ -3829,6 +3876,20 @@ class EventsController extends AppController
                     $error = __('Tag is already attached to this event.');
                     continue;
                 }
+                $tagsOnEvent = $this->Event->EventTag->find('all', array(
+                    'conditions' => array(
+                        'EventTag.event_id' => $id,
+                        'EventTag.local' => $local
+                    ),
+                    'contain' => 'Tag',
+                    'fields' => array('Tag.name'),
+                    'recursive' => -1
+                ));
+                $exclusiveTestPassed = $this->Taxonomy->checkIfNewTagIsAllowedByTaxonomy($tag['Tag']['name'], Hash::extract($tagsOnEvent, '{n}.Tag.name'));
+                if (!$exclusiveTestPassed) {
+                    $fail = __('Tag is not allowed due to taxonomy exclusivity settings');
+                    continue;
+                }
                 $this->Event->EventTag->create();
                 if ($this->Event->EventTag->save(array('event_id' => $id, 'tag_id' => $tag_id, 'local' => $local))) {
                     if (!$local) {
@@ -4456,12 +4517,12 @@ class EventsController extends AppController
                     'checkbox_set' => '/events/restSearch/stix/eventid:' . $id . '/withAttachments:1'
             ),
             'stix_json' => array(
-                    'url' => '/events/restSearch/stix/eventid:' . $id . '.json',
+                    'url' => '/events/restSearch/stix-json/eventid:' . $id,
                     'text' => 'STIX JSON (metadata + all attributes)',
                     'requiresPublished' => false,
                     'checkbox' => true,
                     'checkbox_text' => 'Encode Attachments',
-                    'checkbox_set' => '/events/restSearch/stix/withAttachments:1/eventid:' . $id . '.json'
+                    'checkbox_set' => '/events/restSearch/stix-json/withAttachments:1/eventid:' . $id
             ),
             'stix2_json' => array(
                     'url' => '/events/restSearch/stix2/eventid:' . $id,

app/Controller/TagCollectionsController.php

@@ -237,6 +237,9 @@ class TagCollectionsController extends AppController
         $RearrangeTool = new RequestRearrangeTool();
         $this->request->data = $RearrangeTool->rearrangeArray($this->request->data, $rearrangeRules);
         if ($id === false) {
+            if (!isset($this->request->data['tag_collection'])) {
+                throw new NotFoundException(__('Invalid tag collection'));
+            }
             $id = $this->request->data['tag_collection'];
         }
         if (!$this->request->is('post')) {
@@ -247,6 +250,9 @@ class TagCollectionsController extends AppController
             $this->render('/Events/add_tag');
         } else {
             if ($tag_id === false) {
+                if (!isset($this->request->data['tag'])) {
+                    throw new NotFoundException(__('Invalid tag'));
+                }
                 $tag_id = $this->request->data['tag'];
             }
             $conditions = array();

app/Controller/TagsController.php

@@ -459,6 +459,7 @@ class TagsController extends AppController
     public function showEventTag($id)
     {
         $this->loadModel('EventTag');
+        $this->loadModel('Taxonomy');
         if (!$this->EventTag->Event->checkIfAuthorised($this->Auth->user(), $id)) {
             throw new MethodNotAllowedException('Invalid event.');
         }
@@ -487,6 +488,8 @@ class TagsController extends AppController
                 'conditions' => array('Event.id' => $id)
         ));
         $this->set('required_taxonomies', $this->EventTag->Event->getRequiredTaxonomies());
+        $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($tags);
+        $this->set('tagConflicts', $tagConflicts);
         $this->set('event', $event);
         $this->layout = 'ajax';
         $this->render('/Events/ajax/ajaxTags');
@@ -496,6 +499,7 @@ class TagsController extends AppController
     {
         $this->helpers[] = 'TextColour';
         $this->loadModel('AttributeTag');
+        $this->loadModel('Taxonomy');
 
         $this->Tag->AttributeTag->Attribute->id = $id;
         if (!$this->Tag->AttributeTag->Attribute->exists()) {
@@ -528,6 +532,8 @@ class TagsController extends AppController
         $this->set('event', $event);
         $this->set('attributeTags', $attributeTags);
         $this->set('attributeId', $id);
+        $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attributeTags);
+        $this->set('tagConflicts', $tagConflicts);
         $this->layout = 'ajax';
         $this->render('/Attributes/ajax/ajaxAttributeTags');
     }

app/Controller/UserSettingsController.php

@@ -103,7 +103,15 @@ class UserSettingsController extends AppController
             return $this->RestResponse->viewData($userSettings, $this->response->type());
         } else {
             $this->paginate['conditions'] = $conditions;
-            $this->set('data', $this->paginate());
+            $data = $this->paginate();
+            foreach ($data as $k => $v) {
+                if (!empty($this->UserSetting->validSettings[$v['UserSetting']['setting']])) {
+                    $data[$k]['UserSetting']['restricted'] = empty($this->UserSetting->validSettings[$v['UserSetting']['setting']]['restricted']) ? '' : $this->UserSetting->validSettings[$v['UserSetting']['setting']]['restricted'];
+                } else {
+                    $data[$k]['UserSetting']['restricted'] = array();
+                }
+            }
+            $this->set('data', $data);
             $this->set('context', empty($context) ? 'null' : $context);
         }
     }
@@ -138,6 +146,15 @@ class UserSettingsController extends AppController
 
     public function setSetting($user_id = false, $setting = false)
     {
+        if (!empty($setting)) {
+            if (!$this->UserSetting->checkSettingValidity($setting)) {
+                throw new MethodNotAllowedException(__('Invalid setting.'));
+            }
+            $settingPermCheck = $this->UserSetting->checkSettingAccess($this->Auth->user(), $setting);
+            if ($settingPermCheck !== true) {
+                throw new MethodNotAllowedException(__('This setting is restricted and requires the following permission(s): %s', $settingPermCheck));
+            }
+        }
         // handle POST requests
         if ($this->request->is('post')) {
             // massage the request to allow for unencapsulated POST requests via the API
@@ -168,16 +185,22 @@ class UserSettingsController extends AppController
                     $userSetting['user_id'] = $this->request->data['UserSetting']['user_id'];
                 }
             }
-            if (empty($this->request->data['UserSetting']['setting'])) {
+            if (empty($this->request->data['UserSetting']['setting']) || !isset($this->request->data['UserSetting']['setting'])) {
                 throw new MethodNotAllowedException(__('This endpoint expects both a setting and a value to be set.'));
-            } else {
-                if (!$this->UserSetting->checkSettingValidity($this->request->data['UserSetting']['setting'])) {
-                    throw new MethodNotAllowedException(__('Invalid setting.'));
-                }
-                $userSetting['setting'] = $this->request->data['UserSetting']['setting'];
             }
-            $userSetting['value'] = empty($this->request->data['UserSetting']['value']) ? '' :
+            if (!$this->UserSetting->checkSettingValidity($this->request->data['UserSetting']['setting'])) {
-                json_encode(json_decode($this->request->data['UserSetting']['value'], true));
+                throw new MethodNotAllowedException(__('Invalid setting.'));
+            }
+            $settingPermCheck = $this->UserSetting->checkSettingAccess($this->Auth->user(), $this->request->data['UserSetting']['setting']);
+            if ($settingPermCheck !== true) {
+                throw new MethodNotAllowedException(__('This setting is restricted and requires the following permission(s): %s', $settingPermCheck));
+            }
+            $userSetting['setting'] = $this->request->data['UserSetting']['setting'];
+            if ($this->request->data['UserSetting']['value'] !== '') {
+                $userSetting['value'] = json_encode(json_decode($this->request->data['UserSetting']['value'], true));
+            } else {
+                $userSetting['value'] = '';
+            }
             $existingSetting = $this->UserSetting->find('first', array(
                 'recursive' => -1,
                 'conditions' => array(
@@ -243,6 +266,7 @@ class UserSettingsController extends AppController
             if (!empty($user_id) && $this->request->is('get')) {
                 $this->request->data['UserSetting']['user_id'] = $user_id;
             }
+            $this->set('setting', $setting);
             $this->set('users', $users);
             $this->set('validSettings', $validSettings);
         }
@@ -301,6 +325,10 @@ class UserSettingsController extends AppController
         if (!$checkAccess) {
             throw new NotFoundException(__('Invalid user setting.'));
         }
+        $settingPermCheck = $this->UserSetting->checkSettingAccess($this->Auth->user(), $userSetting['UserSetting']['setting']);
+        if ($settingPermCheck !== true) {
+            throw new MethodNotAllowedException(__('This setting is restricted and requires the following permission(s): %s', $settingPermCheck));
+        }
         if ($this->request->is('post') || $this->request->is('delete')) {
             // Delete the setting that we were after.
             $result = $this->UserSetting->delete($userSetting['UserSetting']['id']);

app/Controller/UsersController.php

@@ -48,12 +48,19 @@ class UsersController extends AppController
             ));
             $id = $userid['User']['id'];
         }
-        $this->User->id = $id;
+        $user = $this->User->read(null, $id);
-        $this->User->recursive = 0;
+        $user = $this->User->find('first', array(
-        if (!$this->User->exists()) {
+            'recursive' => -1,
+            'conditions' => array('User.id' => $id),
+            'contain' => array(
+                'UserSetting',
+                'Role',
+                'Organisation'
+            )
+        ));
+        if (empty($user)) {
             throw new NotFoundException(__('Invalid user'));
         }
-        $user = $this->User->read(null, $id);
         if (!empty($user['User']['gpgkey'])) {
             $pgpDetails = $this->User->verifySingleGPG($user);
             $user['User']['pgp_status'] = isset($pgpDetails[2]) ? $pgpDetails[2] : 'OK';
@@ -62,9 +69,15 @@ class UsersController extends AppController
         if ($this->_isRest()) {
             unset($user['User']['server_id']);
             $user['User']['password'] = '*****';
+            $temp = array();
+            foreach ($user['UserSetting'] as $k => $v) {
+                $temp[$v['setting']] = $v['value'];
+            }
+            $user['UserSetting'] = $temp;
             return $this->RestResponse->viewData(array(
                 'User' => $user['User'],
-                'Role' => $user['Role']
+                'Role' => $user['Role'],
+                'UserSetting' => $user['UserSetting']
             ), $this->response->type());
         } else {
             $this->set('user', $user);
@@ -465,11 +478,17 @@ class UsersController extends AppController
 
     public function admin_view($id = null)
     {
-        $this->User->id = $id;
+        $user = $this->User->find('first', array(
-        if (!$this->User->exists()) {
+            'recursive' => -1,
+            'conditions' => array('User.id' => $id),
+            'contain' => array(
+                'UserSetting',
+                'Role'
+            )
+        ));
+        if (empty($user)) {
             throw new NotFoundException(__('Invalid user'));
         }
-        $user = $this->User->read(null, $id);
         if (!empty($user['User']['gpgkey'])) {
             $pgpDetails = $this->User->verifySingleGPG($user);
             $user['User']['pgp_status'] = isset($pgpDetails[2]) ? $pgpDetails[2] : 'OK';
@@ -485,6 +504,16 @@ class UsersController extends AppController
         }
         if ($this->_isRest()) {
             $user['User']['password'] = '*****';
+            $temp = array();
+            foreach ($user['UserSetting'] as $k => $v) {
+                $temp[$v['setting']] = $v['value'];
+            }
+            $user['UserSetting'] = $temp;
+            return $this->RestResponse->viewData(array(
+                'User' => $user['User'],
+                'Role' => $user['Role'],
+                'UserSetting' => $user['UserSetting']
+            ), $this->response->type());
             return $this->RestResponse->viewData(array('User' => $user['User']), $this->response->type());
         } else {
             $temp = $this->User->data['User']['invited_by'];

app/Lib/Export/Stix1Export.php

@@ -15,13 +15,13 @@ class Stix1Export extends StixExport
         $this->__org = escapeshellarg(Configure::read('MISP.org'));
         $framing_file = $this->__scripts_dir . 'misp_framing.py ';
         $my_server = ClassRegistry::init('Server');
-        return $my_server->getPythonVersion() . ' ' . $framing_file . $this->__return_type . ' ' . $this->__baseurl . ' ' . $this->__org . ' xml' . $this->__end_of_cmd;
+        return $my_server->getPythonVersion() . ' ' . $framing_file . $this->__return_type . ' ' . $this->__baseurl . ' ' . $this->__org . ' ' . $this->__return_format . ' ' . $this->__end_of_cmd;
     }
 
     protected function __parse_misp_events($filename)
     {
         $scriptFile = $this->__scripts_dir . $this->__script_name;
         $my_server = ClassRegistry::init('Server');
-        return shell_exec($my_server->getPythonVersion() . ' ' . $scriptFile . ' ' . $filename . ' xml ' . $this->__baseurl . ' ' . $this->__org . $this->__end_of_cmd);
+        return shell_exec($my_server->getPythonVersion() . ' ' . $scriptFile . ' ' . $filename . ' ' . $this->__return_format . ' ' . $this->__baseurl . ' ' . $this->__org . $this->__end_of_cmd);
     }
 }

app/Lib/Export/StixExport.php

@@ -2,6 +2,11 @@
 
 class StixExport
 {
+    public $additional_params = array(
+        'includeEventTags' => 1,
+        'includeGalaxy' => 1
+    );
+    protected $__return_format = 'json';
     protected $__scripts_dir = APP . 'files/scripts/';
     protected $__end_of_cmd = ' 2>' . APP . 'tmp/logs/exec-errors.log';
     protected $__return_type = null;
@@ -54,6 +59,11 @@ class StixExport
     public function header($options = array())
     {
         $this->__return_type = $options['returnFormat'];
+        if ($this->__return_type == 'stix-json') {
+            $this->__return_type = 'stix';
+        } else if ($this->__return_type == 'stix') {
+            $this->__return_format = 'xml';
+        }
         $framing_cmd = $this->initiate_framing_params();
         $randomFileName = $this->__generateRandomFileName();
         $this->__tmp_dir = $this->__scripts_dir . 'tmp/';

app/Lib/Tools/EventGraphTool.php

@@ -265,7 +265,7 @@
 
             foreach ($object as $obj) {
                 $toPush = array(
-                    'id' => $obj['id'],
+                    'id' => sprintf('o-%s', $obj['id']),
                     'uuid' => $obj['uuid'],
                     'type' => $obj['name'],
                     'label' => '',
@@ -290,8 +290,8 @@
                     $toPush = array(
                         'id' => $rel['id'],
                         'uuid' => $rel['uuid'],
-                        'from' => $obj['id'],
+                        'from' => sprintf('o-%s', $obj['id']),
-                        'to' => $rel['referenced_id'],
+                        'to' => $rel['referenced_type'] == 1 ? sprintf('o-%s', $rel['referenced_id']) : $rel['referenced_id'],
                         'type' => $rel['relationship_type'],
                         'comment' => $rel['comment'],
                         'event_id' => $rel['event_id'],
@@ -356,7 +356,7 @@
 
             foreach ($object as $obj) {
                 $toPush = array(
-                    'id' => $obj['id'],
+                    'id' => sprintf('o-%s', $obj['id']),
                     'uuid' => $obj['uuid'],
                     'type' => $obj['name'],
                     'Attribute' => $obj['Attribute'],
@@ -382,7 +382,7 @@
                         if (!in_array($tag['name'], $added_value)) {
                             $toPush = array(
                                 'id' => "tag_edge_id_" . $i,
-                                'from' => $obj['id'],
+                                'from' => sprintf('o-%s', $obj['id']),
                                 'to' => $tag['name'],
                             );
                             $tagSet[$tag['name']] = $tag;
@@ -466,7 +466,7 @@
 
             foreach ($object as $obj) {
                 $toPush = array(
-                    'id' => $obj['id'],
+                    'id' => sprintf('o-%s', $obj['id']),
                     'uuid' => $obj['uuid'],
                     'type' => $obj['name'],
                     'Attribute' => $obj['Attribute'],
@@ -491,7 +491,7 @@
                     if (!in_array($keyVal, $added_value)) {
                         $toPush = array(
                             'id' => "keyType_edge_id_" . $i,
-                            'from' => $obj['id'],
+                            'from' => sprintf('o-%s', $obj['id']),
                             'to' => "keyType_" . $keyVal,
                         );
                         array_push($added_value, $keyVal);

app/Model/AppModel.php

@@ -142,7 +142,7 @@ class AppModel extends Model
                 break;
             case '2.4.27':
                 $newFeeds = array(
-                    array('provider' => 'Botvrij.eu', 'name' => 'The Botvrij.eu Data','url' => 'http://www.botvrij.eu/data/feed-osint', 'enabled' => 0)
+                    array('provider' => 'Botvrij.eu', 'name' => 'The Botvrij.eu Data','url' => 'https://www.botvrij.eu/data/feed-osint', 'enabled' => 0)
                 );
                 $this->__addNewFeeds($newFeeds);
                 break;

app/Model/Behavior/RegexpBehavior.php

@@ -4,37 +4,40 @@ App::uses('Regexp', 'Model');
 
 /**
  * Behavior to regexp all string fields in a model
- *
  */
 class RegexpBehavior extends ModelBehavior
 {
-    private $__allRegexp = array();
+    private $__allRegexp = null;
 
     public $excluded_types = array('sigma', 'float');
 
-    public function setup(Model $model, $config = null)
-    {
-        $regexp = new Regexp();
-        $this->__allRegexp = $regexp->find('all', array('order' => 'id ASC'));
-    }
-
     /**
      * replace the current value according to the regexp rules, or block blacklisted regular expressions
      *
      * @param Model $Model
-     * @param unknown_type $array
+     * @param string $type
+     * @param string $value
+     * @return string
      */
     public function runRegexp(Model $Model, $type, $value)
     {
         if (in_array($type, $this->excluded_types)) {
             return $value;
         }
+
+        if ($this->__allRegexp === null) {
+            $regexp = new Regexp();
+            $this->__allRegexp = $regexp->find('all', array('order' => 'id ASC'));
+        }
+
         foreach ($this->__allRegexp as $regexp) {
-            if (!empty($regexp['Regexp']['replacement']) && !empty($regexp['Regexp']['regexp']) && ($regexp['Regexp']['type'] === 'ALL' || $regexp['Regexp']['type'] === $type)) {
+            if ($regexp['Regexp']['type'] === 'ALL' || $regexp['Regexp']['type'] === $type) {
-                $value = preg_replace($regexp['Regexp']['regexp'], $regexp['Regexp']['replacement'], $value);
+                if (!empty($regexp['Regexp']['replacement']) && !empty($regexp['Regexp']['regexp'])) {
-            }
+                    $value = preg_replace($regexp['Regexp']['regexp'], $regexp['Regexp']['replacement'], $value);
-            if (empty($regexp['Regexp']['replacement']) && preg_match($regexp['Regexp']['regexp'], $value) && ($regexp['Regexp']['type'] === 'ALL' || $regexp['Regexp']['type'] === $type)) {
+                }
-                return false;
+                if (empty($regexp['Regexp']['replacement']) && preg_match($regexp['Regexp']['regexp'], $value)) {
+                    return false;
+                }
             }
         }
         return $value;

app/Model/DecayingModelsFormulas/Base.php

@@ -124,5 +124,3 @@ abstract class DecayingModelBase
     // Return a True if the attribute should be marked as decayed
     abstract public function isDecayed($model, $attribute, $score);
 }
-
-?>

app/Model/DecayingModelsFormulas/Polynomial.php

@@ -24,4 +24,3 @@ class Polynomial extends DecayingModelBase
         return $threshold > $score;
     }
 }
-?>

app/Model/DecayingModelsFormulas/PolynomialExtended.php

@@ -51,4 +51,3 @@ class PolynomialExtended extends Polynomial
         return parent::isDecayed($model, $attribute, $score);
     }
 }
-?>

app/Model/Event.php

@@ -121,7 +121,15 @@ class Event extends AppModel
                     'scope' => 'Event',
                     'requiresPublished' => 1,
                     'params' => array('returnFormat' => 'stix', 'includeAttachments' => 1),
-                    'description' => 'Click this to download an a STIX document containing the STIX version of all events and attributes that you have access to.'
+                    'description' => 'Click this to download a STIX document containing the STIX version of all events and attributes that you have access to.'
+            ),
+            'stix-json' => array(
+                    'extension' => '.json',
+                    'type' => 'STIX',
+                    'scope' => 'Event',
+                    'requiresPublished' => 1,
+                    'params' => array('returnFormat' => 'stix', 'includeAttachments' => 1),
+                    'description' => 'Click this to download a STIX document containing the STIX version of all events and attributes that you have access to.'
             ),
             'stix2' => array(
                     'extension' => '.json',
@@ -129,7 +137,7 @@ class Event extends AppModel
                     'scope' => 'Event',
                     'requiresPublished' => 1,
                     'params' => array('returnFormat' => 'stix2', 'includeAttachments' => 1),
-                    'description' => 'Click this to download an a STIX2 document containing the STIX2 version of all events and attributes that you have access to.'
+                    'description' => 'Click this to download a STIX2 document containing the STIX2 version of all events and attributes that you have access to.'
             ),
             'rpz' => array(
                     'extension' => '.txt',
@@ -175,6 +183,7 @@ class Event extends AppModel
         'text' => array('text', 'TextExport', 'txt'),
         'csv' => array('csv', 'CsvExport', 'csv'),
         'stix' => array('xml', 'Stix1Export', 'xml'),
+        'stix-json' => array('json', 'Stix1Export', 'json'),
         'stix2' => array('json', 'Stix2Export', 'json'),
         'yara' => array('txt', 'YaraExport', 'yara'),
         'yara-json' => array('json', 'YaraExport', 'json'),
@@ -3171,7 +3180,7 @@ class Event extends AppModel
             $bodyevent = $temp[0];
             $body = $temp[1];
             $result = true;
-            $tplColorString = !empty(Configure::read('MISP.email_subject_TLP_string')) ? Configure::read('MISP.email_subject_TLP_string') : "TLP Amber";
+            $tplColorString = !empty(Configure::read('MISP.email_subject_TLP_string')) ? Configure::read('MISP.email_subject_TLP_string') : "tlp:amber";
             $subject = "[" . Configure::read('MISP.org') . " MISP] Need info about event " . $id . " - ".$tplColorString;
             $result = $this->User->sendEmail($reporter, $bodyevent, $body, $subject, $user) && $result;
         }

app/Model/Post.php

@@ -118,7 +118,7 @@ class Post extends AppModel
         $bodyDetail .= "The following message was added: \n";
         $bodyDetail .= "\n";
         $bodyDetail .= $message . "\n";
-        $tplColorString = !empty(Configure::read('MISP.email_subject_TLP_string')) ? Configure::read('MISP.email_subject_TLP_string') : "TLP Amber";
+        $tplColorString = !empty(Configure::read('MISP.email_subject_TLP_string')) ? Configure::read('MISP.email_subject_TLP_string') : "tlp:amber";
         $subject = "[" . Configure::read('MISP.org') . " MISP] New post in discussion " . $post['Post']['thread_id'] . " - ".$tplColorString;
         foreach ($orgMembers as $recipient) {
             $this->User->sendEmail($recipient, $bodyDetail, $body, $subject);

app/Model/Server.php

@@ -134,7 +134,9 @@ class Server extends AppModel
                     'Push' => 'MISP/app/Console/cake Server push [user_id] [server_id]',
                     'Cache feeds for quick lookups' => 'MISP/app/Console/cake Server cacheFeed [user_id] [feed_id|all|csv|text|misp]',
                     'Fetch feeds as local data' => 'MISP/app/Console/cake Server fetchFeed [user_id] [feed_id|all|csv|text|misp]',
-                    'Run enrichment' => 'MISP/app/Console/cake Event enrichEvent [user_id] [event_id] [json_encoded_module_list]'
+                    'Run enrichment' => 'MISP/app/Console/cake Event enrichEvent [user_id] [event_id] [json_encoded_module_list]',
+                    'Test' => 'MISP/app/Console/cake Server test [server_id]',
+                    'List' => 'MISP/app/Console/cake Server list'
                 ),
                 'description' => __('If you would like to automate tasks such as caching feeds or pulling from server instances, you can do it using the following command line tools. Simply execute the given commands via the command line / create cron jobs easily out of them.'),
                 'header' => __('Automating certain console tasks')
@@ -428,7 +430,7 @@ class Server extends AppModel
                         'email_subject_TLP_string' => array(
                                 'level' => 2,
                                 'description' => __('This is the TLP string for e-mails when email_subject_tag is not found.'),
-                                'value' => 'TLP Amber',
+                                'value' => 'tlp:amber',
                                 'errorMessage' => '',
                                 'test' => 'testForEmpty',
                                 'type' => 'string',
@@ -2671,11 +2673,11 @@ class Server extends AppModel
         } elseif ("incremental" == $technique) {
             $eventid_conditions_key = 'Event.id >';
             $eventid_conditions_value = $this->data['Server']['lastpushedid'];
-        } elseif (true == $technique) {
+        } elseif (intval($technique) !== 0) {
             $eventid_conditions_key = 'Event.id';
             $eventid_conditions_value = intval($technique);
         } else {
-            $this->redirect(array('action' => 'index'));
+            throw new InvalidArgumentException("Technique parameter must be 'full', 'incremental' or event ID.");
         }
         $sgs = $this->Event->SharingGroup->find('all', array(
             'recursive' => -1,
@@ -2683,13 +2685,11 @@ class Server extends AppModel
         ));
         $sgIds = array();
         foreach ($sgs as $k => $sg) {
-            if (!$this->Event->SharingGroup->checkIfServerInSG($sg, $this->data)) {
+            if ($this->Event->SharingGroup->checkIfServerInSG($sg, $this->data)) {
-                unset($sgs[$k]);
+                $sgIds[] = $sg['SharingGroup']['id'];
-                continue;
             }
-            $sgIds[] = $sg['SharingGroup']['id'];
         }
-        if (!isset($sgIds) || empty($sgIds)) {
+        if (empty($sgIds)) {
             $sgIds = array(-1);
         }
         $findParams = array(
@@ -4338,7 +4338,7 @@ class Server extends AppModel
     public function stixDiagnostics(&$diagnostic_errors, &$stixVersion, &$cyboxVersion, &$mixboxVersion, &$maecVersion, &$stix2Version, &$pymispVersion)
     {
         $result = array();
-        $expected = array('stix' => '>1.2.0.6', 'cybox' => '>2.1.0.18.dev0', 'mixbox' => '1.0.3', 'maec' => '>4.1.0.14', 'stix2' => '1.1.3', 'pymisp' => '>2.4.93');
+        $expected = array('stix' => '>1.2.0.6', 'cybox' => '>2.1.0.18.dev0', 'mixbox' => '1.0.3', 'maec' => '>4.1.0.14', 'stix2' => '>1.2.0', 'pymisp' => '>2.4.93');
         // check if the STIX and Cybox libraries are working using the test script stixtest.py
         $scriptResult = shell_exec($this->getPythonVersion() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'stixtest.py');
         $scriptResult = json_decode($scriptResult, true);

app/Model/SharingGroup.php

@@ -189,7 +189,11 @@ class SharingGroup extends AppModel
                         'conditions' => array('id' => $sg['SharingGroup']['org_id'])
                     ));
                 }
-                $sg['Organisation'] = $this->__sgoCache[$sg['SharingGroup']['org_id']]['Organisation'];
+                if(isset($this->__sgoCache[$sg['SharingGroup']['org_id']]['Organisation'])) {
+                        $sg['Organisation'] = $this->__sgoCache[$sg['SharingGroup']['org_id']]['Organisation'];
+                } else {
+                        $sg['Organisation'] = '';
+                }
                 if (!empty($sg['SharingGroupOrg'])) {
                     foreach ($sg['SharingGroupOrg'] as &$sgo) {
                         if (!isset($this->__sgoCache[$sgo['org_id']])) {

app/Model/Taxonomy.php

@@ -101,7 +101,7 @@ class Taxonomy extends AppModel
             }
             $this->deleteAll(array('Taxonomy.namespace' => $current['Taxonomy']['namespace']));
         }
-        $taxonomy['Taxonomy'] = array('namespace' => $vocab['namespace'], 'description' => $vocab['description'], 'version' => $vocab['version'], 'enabled' => $enabled);
+        $taxonomy['Taxonomy'] = array('namespace' => $vocab['namespace'], 'description' => $vocab['description'], 'version' => $vocab['version'], 'exclusive' => $vocab['exclusive'], 'enabled' => $enabled);
         $predicateLookup = array();
         foreach ($vocab['predicates'] as $k => $predicate) {
             $taxonomy['Taxonomy']['TaxonomyPredicate'][$k] = $predicate;
@@ -489,27 +489,29 @@ class Taxonomy extends AppModel
         return $taxonomies;
     }
 
-    public function getTaxonomyForTag($tagName, $metaOnly = false)
+    public function getTaxonomyForTag($tagName, $metaOnly = false, $fullTaxonomy = False)
     {
         if (preg_match('/^[^:="]+:[^:="]+="[^:="]+"$/i', $tagName)) {
             $temp = explode(':', $tagName);
             $pieces = array_merge(array($temp[0]), explode('=', $temp[1]));
             $pieces[2] = trim($pieces[2], '"');
+            $contain = array(
+                'TaxonomyPredicate' => array(
+                    'TaxonomyEntry' => array()
+                )
+            );
+            if (!$fullTaxonomy) {
+                $contain['TaxonomyPredicate']['conditions'] = array(
+                    'LOWER(TaxonomyPredicate.value)' => strtolower($pieces[1])
+                );
+                $contain['TaxonomyPredicate']['TaxonomyEntry']['conditions'] = array(
+                    'LOWER(TaxonomyEntry.value)' => strtolower($pieces[2])
+                );
+            }
             $taxonomy = $this->find('first', array(
                 'recursive' => -1,
                 'conditions' => array('LOWER(Taxonomy.namespace)' => strtolower($pieces[0])),
-                'contain' => array(
+                'contain' => $contain
-                    'TaxonomyPredicate' => array(
-                        'conditions' => array(
-                            'LOWER(TaxonomyPredicate.value)' => strtolower($pieces[1])
-                        ),
-                        'TaxonomyEntry' => array(
-                            'conditions' => array(
-                                'LOWER(TaxonomyEntry.value)' => strtolower($pieces[2])
-                            )
-                        )
-                    )
-                )
             ));
             if ($metaOnly && !empty($taxonomy)) {
                 return array('Taxonomy' => $taxonomy['Taxonomy']);
@@ -517,16 +519,16 @@ class Taxonomy extends AppModel
             return $taxonomy;
         } elseif (preg_match('/^[^:="]+:[^:="]+$/i', $tagName)) {
             $pieces = explode(':', $tagName);
+            $contain = array('TaxonomyPredicate' => array());
+            if (!$fullTaxonomy) {
+                $contain['TaxonomyPredicate']['conditions'] = array(
+                    'LOWER(TaxonomyPredicate.value)' => strtolower($pieces[1])
+                );
+            }
             $taxonomy = $this->find('first', array(
                 'recursive' => -1,
                 'conditions' => array('LOWER(Taxonomy.namespace)' => strtolower($pieces[0])),
-                'contain' => array(
+                'contain' => $contain
-                    'TaxonomyPredicate' => array(
-                        'conditions' => array(
-                            'LOWER(TaxonomyPredicate.value)' => strtolower($pieces[1])
-                        )
-                    )
-                )
             ));
             if ($metaOnly && !empty($taxonomy)) {
                 return array('Taxonomy' => $taxonomy['Taxonomy']);
@@ -536,4 +538,97 @@ class Taxonomy extends AppModel
             return false;
         }
     }
+
+    // Remove the value for triple component tags or the predicate for double components tags
+    public function stripLastTagComponent($tagName)
+    {
+        $shortenedTag = '';
+        if (preg_match('/^[^:="]+:[^:="]+="[^:="]+"$/i', $tagName)) {
+            $shortenedTag = explode('=', $tagName)[0];
+        } elseif (preg_match('/^[^:="]+:[^:="]+$/i', $tagName)) {
+            $shortenedTag = explode(':', $tagName)[0];
+        }
+        return $shortenedTag;
+    }
+
+    public function checkIfNewTagIsAllowedByTaxonomy($newTagName, $tagNameList=array())
+    {
+        $newTagShortened = $this->stripLastTagComponent($newTagName);
+        $prefixIsFree = true;
+        foreach ($tagNameList as $tagName) {
+            $tagShortened = $this->stripLastTagComponent($tagName);
+            if ($newTagShortened == $tagShortened) {
+                $prefixIsFree = false;
+            }
+        }
+        if (!$prefixIsFree) {
+            // at this point, we have a duplicated namespace(-predicate)
+            $taxonomy = $this->getTaxonomyForTag($newTagName);
+            if (!empty($taxonomy['Taxonomy']['exclusive'])) {
+                return false; // only one tag of this taxonomy is allowed
+            } elseif (!empty($taxonomy['TaxonomyPredicate'][0]['exclusive'])) {
+                return false; // only one tag belonging to this predicate is allowed
+            }
+        }
+        return true;
+    }
+
+    public function checkIfTagInconsistencies($tagList)
+    {
+        $eventTags = array();
+        $localEventTags = array();
+        foreach($tagList as $tag) {
+            if ($tag['local'] == 0) {
+                $eventTags[] = $tag['Tag']['name'];
+            } else {
+                $localEventTags[] = $tag['Tag']['name'];
+            }
+        }
+        $tagConflicts = $this->getTagConflicts($eventTags);
+        $localTagConflicts = $this->getTagConflicts($localEventTags);
+        return array(
+            'global' => $tagConflicts,
+            'local' => $localTagConflicts
+        );
+    }
+
+    public function getTagConflicts($tagNameList)
+    {
+        $potentiallyConflictingTaxonomy = array();
+        $conflictingTaxonomy = array();
+        foreach ($tagNameList as $tagName) {
+            $tagShortened = $this->stripLastTagComponent($tagName);
+            if (isset($potentiallyConflictingTaxonomy[$tagShortened])) {
+                $potentiallyConflictingTaxonomy[$tagShortened]['taxonomy'] = $this->getTaxonomyForTag($tagName);
+                $potentiallyConflictingTaxonomy[$tagShortened]['count']++;
+            } else {
+                $potentiallyConflictingTaxonomy[$tagShortened] = array(
+                    'count' => 1
+                );
+            }
+            $potentiallyConflictingTaxonomy[$tagShortened]['tagNames'][] = $tagName;
+        }
+        foreach ($potentiallyConflictingTaxonomy as $potTaxonomy) {
+            if ($potTaxonomy['count'] > 1) {
+                $taxonomy = $potTaxonomy['taxonomy'];
+                if (isset($taxonomy['Taxonomy']['exclusive']) && $taxonomy['Taxonomy']['exclusive']) {
+                    $conflictingTaxonomy[] = array(
+                        'tags' => $potTaxonomy['tagNames'],
+                        'taxonomy' => $taxonomy,
+                        'conflict' => sprintf(__('Taxonomy `%s` is an exclusive Taxonomy'), $taxonomy['Taxonomy']['namespace'])
+                    );
+                } elseif (isset($taxonomy['TaxonomyPredicate'][0]['exclusive']) && $taxonomy['TaxonomyPredicate'][0]['exclusive']) {
+                    $conflictingTaxonomy[] = array(
+                        'tags' => $potTaxonomy['tagNames'],
+                        'taxonomy' => $taxonomy,
+                        'conflict' => sprintf(
+                            __('Predicate `%s` is exclusive'),
+                            $taxonomy['TaxonomyPredicate'][0]['value']
+                        )
+                    );
+                }
+            }
+        }
+        return $conflictingTaxonomy;
+    }
 }

app/Model/User.php

@@ -609,7 +609,19 @@ class User extends AppModel
             throw new NotFoundException('Invalid user ID.');
         }
         $conditions = array('User.id' => $id);
-        $user = $this->find('first', array('conditions' => $conditions, 'recursive' => -1,'contain' => array('Organisation', 'Role', 'Server')));
+        $user = $this->find(
+            'first',
+            array(
+                'conditions' => $conditions,
+                'recursive' => -1,
+                'contain' => array(
+                    'Organisation',
+                    'Role',
+                    'Server',
+                    'UserSetting'
+                )
+            )
+        );
         if (empty($user)) {
             return $user;
         }
@@ -617,6 +629,7 @@ class User extends AppModel
         $user['User']['Role'] = $user['Role'];
         $user['User']['Organisation'] = $user['Organisation'];
         $user['User']['Server'] = $user['Server'];
+        $user['User']['UserSetting'] = $user['UserSetting'];
         unset($user['Organisation'], $user['Role'], $user['Server']);
         return $user['User'];
     }

app/Model/UserSetting.php

@@ -45,6 +45,10 @@ class UserSetting extends AppModel
                     )
                 )
             )
+        ),
+        'dashboard_access' => array(
+            'placeholder' => 1,
+            'restricted' => 'perm_site_admin'
         )
     );
 
@@ -56,7 +60,11 @@ class UserSetting extends AppModel
         if (empty($this->data['UserSetting']['timestamp'])) {
             $this->data['UserSetting']['timestamp'] = time();
         }
-        if (!empty($this->data['UserSetting']['value']) && $this->data['UserSetting']['value'] !== 'null') {
+        if (
+            isset($this->data['UserSetting']['value']) &&
+            $this->data['UserSetting']['value'] !== '' &&
+            $this->data['UserSetting']['value'] !== 'null'
+        ) {
             if (is_array($this->data['UserSetting']['value'])) {
                 $this->data['UserSetting']['value'] = json_encode($this->data['UserSetting']['value']);
             }
@@ -80,6 +88,29 @@ class UserSetting extends AppModel
         return isset($this->validSettings[$setting]);
     }
 
+    public function checkSettingAccess($user, $setting)
+    {
+        if (!empty($this->validSettings[$setting]['restricted'])) {
+            $role_check = $this->validSettings[$setting]['restricted'];
+            if (!is_array($role_check)) {
+                $role_check = array($role_check);
+            }
+            $userHasValidRole = false;
+            foreach ($role_check as $role) {
+                if (!empty($user['Role'][$role])) {
+                    return true;
+                }
+            }
+            if (!$userHasValidRole) {
+                foreach ($role_check as &$role) {
+                    $role = substr($role, 5);
+                }
+                return implode(', ', $role_check);
+            }
+        }
+        return true;
+    }
+
     /*
      * canModify expects an auth user object or a user ID and a loaded setting as input parameters
      * check if the user can modify/remove the given entry

app/View/Attributes/ajax/attributeViewFieldForm.ctp

@@ -1,8 +1,8 @@
 <?php
 if ($value === 'No') {
-    echo '<input type="checkbox" disabled></input>';
+    echo '<input type="checkbox" disabled>';
 } else if ($value === 'Yes') {
-    echo '<input type="checkbox" checked disabled></input>';
+    echo '<input type="checkbox" checked disabled>';
 } else {
     echo nl2br(h($value)) . '&nbsp;';
 }

app/View/Communities/view.ctp

@@ -56,12 +56,3 @@
 <?php
     echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'sync', 'menuItem' => 'view_community'));
 ?>
-<script type="text/javascript">
-    <?php
-        $startingTab = 'description';
-        if (!$local) $startingTab = 'events';
-    ?>
-    $(document).ready(function () {
-        organisationViewContent('<?php echo $startingTab; ?>', '<?php echo h($id);?>');
-    });
-</script>

app/View/DecayingModel/decaying_tool.ctp

@@ -5,21 +5,21 @@
         <div class="span9 form-inline" style="border: 1px solid #ddd; border-radius: 4px; margin-bottom: 15px;">
             <div style="border-bottom: 1px solid #ddd;">
                 <label class="checkbox inline">
-                    <input id="table_toggle_all_type" type="checkbox"></input>
+                    <input id="table_toggle_all_type" type="checkbox">
                     <?php echo __('Show All Types'); ?>
                 </label>
                 <label class="checkbox inline">
-                    <input id="table_toggle_objects" type="checkbox"></input>
+                    <input id="table_toggle_objects" type="checkbox">
                     <?php echo __('Show MISP Objects'); ?>
                 </label>
-                <input id="table_type_search" class="input" type="text" placeholder="<?php echo __('Search Attribute Type'); ?>"></input>
+                <input id="table_type_search" class="input" type="text" placeholder="<?php echo __('Search Attribute Type'); ?>">
                 <button class="btn btn-primary btn-small" onclick="decayingTool.restoreSelection()"><span class="fa fa-history"></span></button>
             </div>
             <div class="AttributeTypeTableContainer">
                 <table id="table_attribute_type" class="table table-striped table-bordered">
                     <thead>
                         <tr>
-                            <th><input id="checkAll" type="checkbox" title="<?php echo __('Check all'); ?>"></input></th>
+                            <th><input id="checkAll" type="checkbox" title="<?php echo __('Check all'); ?>"></th>
                             <th><?php echo __('Attribute Type'); ?></th>
                             <th><?php echo __('Category'); ?></th>
                             <th><?php echo __('Model ID'); ?></th>
@@ -41,7 +41,7 @@
                                 }
                             ?>
                             <tr class="<?php echo $class; ?>">
-                                <td><input type="checkbox"></input></td>
+                                <td><input type="checkbox"></td>
                                 <td class="useCursorPointer isFilteringField isAttributeTypeField">
                                     <?php if(isset($info['isObject']) && $info['isObject'] && !(isset($info['isAttribute']) && $info['isAttribute'])): ?>
                                         <it class="fa fa-cube" title="<?php echo __('Belong to a MISP Object'); ?>"></it>
@@ -85,17 +85,17 @@
                                 <span class="add-on param-name" data-toggle="tooltip" data-placement="left" style="min-width: 100px;" title="<?php echo isset($config['info']) ? h($config['info']) : ''?>">
                                     <?php echo h($config['name']) . (isset($config['greek']) ? ' <strong>' . h($config['greek']).'</strong>' : ''); ?>
                                 </span>
-                                <input id="input_<?php echo h($param); ?>" class="input-mini" type="number" min=0 step=<?php echo h($config['step']); ?> value=<?php echo h($config['value']); ?> max=<?php echo isset($config['max']) ? h($config['max']) : ''; ?> oninput="refreshGraph(this);" ></input>
+                                <input id="input_<?php echo h($param); ?>" class="input-mini" type="number" min=0 step=<?php echo h($config['step']); ?> value=<?php echo h($config['value']); ?> max=<?php echo isset($config['max']) ? h($config['max']) : ''; ?> oninput="refreshGraph(this);">
-                                <span class="add-on"><input id="input_<?php echo h($param); ?>_range" type="range" min=0 <?php echo isset($config['max']) ? 'max=' . h($config['max']) : '' ?> step=<?php echo h($config['step']); ?> value=<?php echo h($config['value']); ?> oninput="$('#input_<?php echo h($param); ?>').val(this.value).trigger('input');"></input></span>
+                                <span class="add-on"><input id="input_<?php echo h($param); ?>_range" type="range" min=0 <?php echo isset($config['max']) ? 'max=' . h($config['max']) : '' ?> step=<?php echo h($config['step']); ?> value=<?php echo h($config['value']); ?> oninput="$('#input_<?php echo h($param); ?>').val(this.value).trigger('input');"></span>
                                 <?php if (isset($config['unit'])): ?>
                                     <span class="add-on"><?php echo h($config['unit']); ?></span>
                                 <?php endif; ?>
 
                             </div>
                         <?php endforeach; ?>
-                        <input id="input_default_base_score" value=0 class="hidden"></input>
+                        <input id="input_default_base_score" value=0 class="hidden">
                         <div class="input-append" style="margin-bottom: 0px;">
-                            <input id="input_base_score_config" class="hidden" value="[]"></input>
+                            <input id="input_base_score_config" class="hidden" value="[]">
                             <button class="btn btn-primary" style="border-radius: 4px 0px 0px 4px;" onclick="decayingTool.toggleBasescoreForm()">
                                 <span class="fa fa-tags"> <?php echo __('Adjust base  score'); ?></span>
                             </button>

app/View/DecayingModel/decaying_tool_basescore.ctp

@@ -1,11 +1,11 @@
 <div id="basescore_configurator" class="row">
     <div class="span8" class="taxonomyTableContainer">
-        <input id="table_taxonomy_search" class="input" style="width: 250px; margin: 0px;" type="text" placeholder="<?php echo __('Search Taxonomy'); ?>"></input>
+        <input id="table_taxonomy_search" class="input" style="width: 250px; margin: 0px;" type="text" placeholder="<?php echo __('Search Taxonomy'); ?>">
         <i class="fa fa-times useCursorPointer" title="<?php echo __('Clear search field'); ?>" onclick="$('#table_taxonomy_search').val('').trigger('input');"></i>
         <span style="float: right; margin-top: 6px;" class="badge badge-info"><b><?php echo h($taxonomies_not_having_numerical_value); ?></b><?php echo __(' not having numerical value'); ?></span>
         <div class="input-prepend" style="margin: 4px;">
             <span class="add-on"><?php echo __('Default basescore') ?></span>
-            <input id="base_score_default_value" type="number" min=0 max=100 class="input-mini" value="0" placeholder="0"></input>
+            <input id="base_score_default_value" type="number" min=0 max=100 class="input-mini" value="0" placeholder="0">
         </div>
         <table id="tableTaxonomy" class="table table-striped table-bordered table-condensed">
             <thead>
@@ -49,8 +49,8 @@
                                     </div>
                                 </td>
                                 <td>
-                                    <input id="slider_<?php echo h($name) ?>" data-taxonomyname="<?php echo sprintf('%s:%s', h($name), h($predicate['value'])); ?>" type="range" min=0 max=100 step=1 value="<?php echo isset($taxonomy['value']) ? h($taxonomy['value']) : 0 ?>" onchange="sliderChanged(this);" oninput="sliderChanged(this);"></input>
+                                    <input id="slider_<?php echo h($name) ?>" data-taxonomyname="<?php echo sprintf('%s:%s', h($name), h($predicate['value'])); ?>" type="range" min=0 max=100 step=1 value="<?php echo isset($taxonomy['value']) ? h($taxonomy['value']) : 0 ?>" onchange="sliderChanged(this);" oninput="sliderChanged(this);">
-                                    <input type="number" min=0 max=100 step=1 value="<?php echo isset($taxonomy['value']) ? h($taxonomy['value']) : 0 ?>" class="taxonomySlider" data-taxonomyname="<?php echo sprintf('%s:%s', h($name), h($predicate['value'])); ?>" onchange="inputChanged(this);" oninput="inputChanged(this);"></input>
+                                    <input type="number" min=0 max=100 step=1 value="<?php echo isset($taxonomy['value']) ? h($taxonomy['value']) : 0 ?>" class="taxonomySlider" data-taxonomyname="<?php echo sprintf('%s:%s', h($name), h($predicate['value'])); ?>" onchange="inputChanged(this);" oninput="inputChanged(this);">
                                 </td>
                             </tr>
                         <?php else: // numerical_value on predicate ?>
@@ -77,8 +77,8 @@
                                     </div>
                                 </td>
                                 <td>
-                                    <input id="slider_<?php echo h($name) ?>" data-taxonomyname="<?php echo h($name); ?>" type="range" min=0 max=100 step=1 value="<?php echo isset($taxonomy['value']) ? h($taxonomy['value']) : 0 ?>" onchange="sliderChanged(this);" oninput="sliderChanged(this);"></input>
+                                    <input id="slider_<?php echo h($name) ?>" data-taxonomyname="<?php echo h($name); ?>" type="range" min=0 max=100 step=1 value="<?php echo isset($taxonomy['value']) ? h($taxonomy['value']) : 0 ?>" onchange="sliderChanged(this);" oninput="sliderChanged(this);">
-                                    <input type="number" min=0 max=100 step=1 value="<?php echo isset($taxonomy['value']) ? h($taxonomy['value']) : 0 ?>" class="taxonomySlider" data-taxonomyname="<?php echo h($name); ?>" onchange="inputChanged(this);" oninput="inputChanged(this);"></input>
+                                    <input type="number" min=0 max=100 step=1 value="<?php echo isset($taxonomy['value']) ? h($taxonomy['value']) : 0 ?>" class="taxonomySlider" data-taxonomyname="<?php echo h($name); ?>" onchange="inputChanged(this);" oninput="inputChanged(this);">
                                 </td>
                             </tr>
                             <?php break; ?>

app/View/Elements/Events/View/eventFilteringQueryBuilder.ctp

@@ -1,7 +1,7 @@
 <div id="eventFilteringQBWrapper" style="padding: 5px; display: none; border: 1px solid #dddddd; border-bottom: 0px;">
     <div id="eventFilteringQB" style="overflow-y: auto; padding-right: 5px; resize: vertical; max-height: 750px; height: 400px;"></div>
     <div style="display: flex; justify-content: flex-end; margin-top: 5px;">
-            <input id="eventFilteringQBLinkInput" class="form-control" style="width: 66%;"></input>
+            <input id="eventFilteringQBLinkInput" class="form-control" style="width: 66%;">
             <button id="eventFilteringQBLinkCopy" type="button" class="btn btn-inverse" style="margin-right: 5px; margin-left: 5px;" onclick="clickMessage(this);"> <i class="fa fa-clipboard"></i> <?php echo h('Copy to clipboard'); ?> </button>
             <button id="eventFilteringQBSubmit" type="button" class="btn btn-success" style="margin-right: 5px;"> <i class="fa fa-filter"></i> <?php echo h('Filter'); ?> </button>
             <button id="eventFilteringQBClear" type="button" class="btn btn-xs btn-danger" style="" title="<?php echo h('Clear filtering rules'); ?>"> <i class="fa fa-times"></i> <?php echo h('Clear'); ?> </button>

app/View/Elements/Events/View/row_attribute.ctp

@@ -159,7 +159,7 @@
     </td>
     <td class="short">
       <div class="attributeTagContainer" id="#Attribute_<?php echo h($object['id']);?>_tr .attributeTagContainer">
-        <?php echo $this->element('ajaxTags', array('attributeId' => $object['id'], 'tags' => $object['AttributeTag'], 'tagAccess' => ($isSiteAdmin || $mayModify || $me['org_id'] == $event['Event']['org_id']), 'context' => $context, 'scope' => 'attribute')); ?>
+        <?php echo $this->element('ajaxTags', array('attributeId' => $object['id'], 'tags' => $object['AttributeTag'], 'tagAccess' => ($isSiteAdmin || $mayModify || $me['org_id'] == $event['Event']['org_id']), 'context' => $context, 'scope' => 'attribute', 'tagConflicts' => isset($object['tagConflicts']) ? $object['tagConflicts'] : array())); ?>
       </div>
     </td>
     <?php
@@ -323,7 +323,7 @@
     <td class="short">
       <div id = "Attribute_<?php echo $object['id']; ?>_to_ids_placeholder" class = "inline-field-placeholder"></div>
       <div id = "Attribute_<?php echo $object['id']; ?>_to_ids_solid" class="inline-field-solid">
-        <input type="checkbox" class="toids-toggle" id="toids_toggle_<?php echo h($object['id']); ?>" data-attribute-id="<?php echo h($object['id']); ?>" aria-label="<?php echo __('Toggle IDS flag');?>" title="<?php echo __('Toggle IDS flag');?>" <?php echo $object['to_ids'] ? 'checked' : ''; ?> ></input>
+        <input type="checkbox" class="toids-toggle" id="toids_toggle_<?php echo h($object['id']); ?>" data-attribute-id="<?php echo h($object['id']); ?>" aria-label="<?php echo __('Toggle IDS flag');?>" title="<?php echo __('Toggle IDS flag');?>" <?php echo $object['to_ids'] ? 'checked' : ''; ?> >
       </div>
     </td>
     <td class="short" onmouseenter="quickEditHover(this, '<?php echo $editScope; ?>', '<?php echo $object['id']; ?>', 'distribution', <?php echo $event['Event']['id'];?>);">

app/View/Elements/Events/View/value_field.ctp

@@ -5,7 +5,7 @@ switch ($object['type']) {
     case 'attachment':
     case 'malware-sample':
         if ($object['type'] === 'attachment' && isset($object['image'])) {
-            if (extension_loaded('gd')) {
+            if ($object['image'] === true) {
                 $img = '<it class="fa fa-spin fa-spinner" style="font-size: large; left: 50%; top: 50%;"></it>';
                 $img .= '<img class="screenshot screenshot-collapsed useCursorPointer img-rounded hidden" src="' . $baseurl . '/attributes/viewPicture/' . h($object['id']) . '/1' . '" title="' . h($object['value']) . '" onload="$(this).show(200); $(this).parent().find(\'.fa-spinner\').remove();"/>';
                 echo $img;
@@ -24,7 +24,8 @@ switch ($object['type']) {
                 $filename = $filenameHash[0];
             }
 
-            $url = array('controller' => 'attributes', 'action' => 'download', $object['id']);
+            $controller = isset($object['objectType']) && $object['objectType'] === 'proposal' ? 'shadow_attributes' : 'attributes';
+            $url = array('controller' => $controller, 'action' => 'download', $object['id']);
             echo $this->Html->link($filename, $url, array('class' => $linkClass));
             if (isset($filenameHash[1])) {
                 echo '<br />' . $filenameHash[1];
@@ -58,7 +59,7 @@ switch ($object['type']) {
         } else {
             $sigDisplay = str_replace("\r", '', h($sigDisplay));
             $sigDisplay = str_replace(" ", '&nbsp;', $sigDisplay);
-            echo h($sigDisplay);
+            echo $sigDisplay;
         }
         break;
 

app/View/Elements/Objects/object_similarities.ctp

@@ -86,7 +86,7 @@ if (!isset($simple_flattened_attribute_noval) || !isset($simple_flattened_attrib
         <div class="blueElement" style="padding: 4px 5px;">
             <div style="text-align: center; position: relative;">
                 <?php if (isset($merge_button_functionname)): ?>
-                    <input type="button" class="btn <?php echo $btn_style; ?>" onclick="<?php echo h($merge_button_functionname); ?>(this)" data-objectid="<?php echo h($object['Object']['id']) ?>" data-updatetemplate="<?php echo $temp_comparison == 'below' ? 'true' : 'false'; ?>" value="<?php echo $temp_text; ?>" <?php echo $temp_comparison == 'above' ? 'disabled' : ''; ?>></input>
+                    <input type="button" class="btn <?php echo $btn_style; ?>" onclick="<?php echo h($merge_button_functionname); ?>(this)" data-objectid="<?php echo h($object['Object']['id']) ?>" data-updatetemplate="<?php echo $temp_comparison == 'below' ? 'true' : 'false'; ?>" value="<?php echo $temp_text; ?>" <?php echo $temp_comparison == 'above' ? 'disabled' : ''; ?>>
                 <?php endif; ?>
                 <?php if (isset($similar_object_similarity_amount[$object['Object']['id']])): ?>
                     <span class="badge badge-inverse" style="position: absolute; right: 0;" title="<?php echo __('Similarity amount') ?>">

app/View/Elements/ajaxTags.ctp

@@ -162,4 +162,36 @@
         '<span class="tag-list-container">%s</span>',
         $tagData
     );
+    $tagConflictData = '';
+    if (!empty($tagConflicts['global'])) {
+        $tagConflictData .= '<div><div class="alert alert-error tag-conflict-notice">';
+        $tagConflictData .= '<i class="fas fa-globe-americas icon"></i>';
+        $tagConflictData .= '<div class="text-container">';
+        foreach ($tagConflicts['global'] as $tagConflict) {
+            $tagConflictData .= sprintf(
+                '<strong>%s</strong></br>',
+                h($tagConflict['conflict'])
+            );
+            foreach ($tagConflict['tags'] as $tag) {
+                $tagConflictData .= sprintf('<span class="apply_css_arrow nowrap">%s</span></br>', h($tag));
+            }
+        }
+        $tagConflictData .= '</div></div></span>';
+    }
+    if (!empty($tagConflicts['local'])) {
+        $tagConflictData .= '<div><div class="alert alert-error tag-conflict-notice">';
+        $tagConflictData .= '<i class="fas fa-user icon"></i>';
+        $tagConflictData .= '<div class="text-container">';
+        foreach ($tagConflicts['local'] as $tagConflict) {
+            $tagConflictData .= sprintf(
+                '<strong>%s</strong></br>',
+                h($tagConflict['conflict'])
+            );
+            foreach ($tagConflict['tags'] as $tag) {
+                $tagConflictData .= sprintf('<span class="apply_css_arrow nowrap">%s</span></br>', h($tag));
+            }
+        }
+        $tagConflictData .= '</div></div></span>';
+    }
+    echo $tagConflictData;
 ?>

app/View/Elements/eventattributetoolbar.ctp

@@ -100,7 +100,7 @@
                         'id' => 'multi-accept-button',
                         'title' => __('Accept selected Proposals'),
                         'class' => 'mass-proposal-select hidden',
-                        'fa-icon' => 'check-circl',
+                        'fa-icon' => 'check-circle',
                         'onClick' => 'multiSelectAction',
                         'onClickParams' => array($event['Event']['id'], 'acceptProposals')
                     ),

app/View/Elements/galaxyQuickView.ctp

@@ -151,9 +151,5 @@
                 $(this).parent().children('.collapse-status-container').children('.collapse-status').html('+');
             }
         });
-        $('.delete-cluster').click(function() {
-            var tagName = $(this).data('tag-name');
-            removeTag($id = false, $tag_id = false, $galaxy = false);
-        });
     });
 </script>

app/View/Elements/galaxyQuickViewMini.ctp

@@ -146,9 +146,5 @@ $(document).ready(function () {
             $(this).children('span').html('+');
         }
     });
-    $('.delete-cluster').click(function() {
-        var tagName = $(this).data('tag-name');
-        removeTag($id = false, $tag_id = false, $galaxy = false);
-    });
 });
 </script>

app/View/Elements/genericElements/IndexTable/Fields/generic_field.ctp

@@ -1,3 +1,11 @@
 <?php
-    echo h(Hash::extract($row, $field['data_path'])[0]);
+    $data = Hash::extract($row, $field['data_path']);
+    if (is_array($data)) {
+        if (count($data) > 1) {
+            $data = implode(', ', $data);
+        } else {
+            $data = $data[0];
+        }
+    }
+    echo h($data);
 ?>

app/View/Elements/genericElements/IndexTable/Fields/json.ctp

@@ -1,5 +1,9 @@
 <?php
     $data = h(Hash::extract($row, $field['data_path']));
+    // I feed dirty for this...
+    if (is_array($data) && count($data) === 1 && isset($data[0])) {
+        $data = $data[0];
+    }
     echo sprintf(
         '<div class="json_container_%s"></div>',
         h($k)

app/View/Elements/genericElements/ListTopBar/element_embedded.ctp

@@ -33,7 +33,7 @@
             '<li><a class="%s %s" id="%s" href="%s" %s %s %s %s>%s%s%s</a></li>',
             empty($data['class']) ? '' : h($data['class']),
             empty($data['active']) ? '' : 'background-blue',   // Change the default class for highlighted/active toggles here
-            empty($data['id']) ? '' : 'id="' . h($data['id']) . '"',
+            empty($data['id']) ? '' :  h($data['id']),
             empty($data['url']) ? '#' : h($data['url']),    // prevent default is passed if the url is not set
             empty($onClick) ? '' : $onClick,    // pass $data['onClick'] for the function name to call and $data['onClickParams'] for the parameter list
             empty($dataFields) ? '' : $dataFields,

app/View/Elements/serverRuleElements/pull.ctp

@@ -15,7 +15,7 @@
                         <span title="<?php echo __('Remove tag from the list of tags to allow');?>" role="button" tabindex="0" aria-label="<?php echo __('Remove tag from the list of tags to allow');?>" class="btn btn-inverse" id="tagspullLeftRight" style="padding:2px;" onClick="serverRuleMoveFilter('pull', 'tags', 'Left', 'Middle');">&gt;&gt;</span>
                     </td>
                     <td style="width:120px;">
-                        <input id="tagspullNewValue" style="width:180px;"></input>
+                        <input id="tagspullNewValue" style="width:180px;">
                     </td>
                     <td style="width:50px;text-align:center;">
                         <span title="<?php echo __('Remove tag from the list of tags to block');?>" role="button" tabindex="0" aria-label="<?php echo __('Remove tag from the list of tags to block');?>" class="btn btn-inverse" id="tagspullRightLeft" style="padding:2px;" onClick="serverRuleMoveFilter('pull', 'tags', 'Right', 'Middle');">&lt;&lt;</span>
@@ -44,7 +44,7 @@
                         <span title="<?php echo __('Remove organisation to the list of tags to allow');?>" role="button" tabindex="0" aria-label="<?php echo __('Remove organisation form the list of organisations to allow');?>" class="btn btn-inverse" id="orgspullLeftRight" style="padding:2px;" onClick="serverRuleMoveFilter('pull', 'orgs', 'Left', 'Middle');">&gt;&gt;</span>
                     </td>
                     <td style="width:120px;">
-                        <input id="orgspullNewValue" style="width:180px;"></input>
+                        <input id="orgspullNewValue" style="width:180px;">
                     </td>
                     <td style="width:50px;text-align:center;">
                         <span title="<?php echo __('Remove organisation from the list of tags to allow');?>" role="button" tabindex="0" aria-label="<?php echo __('Remove organisation from the list of organisations to block');?>" class="btn btn-inverse" id="orgspullRightLeft" style="padding:2px;" onClick="serverRuleMoveFilter('pull', 'orgs', 'Right', 'Middle');">&lt;&lt;</span>

app/View/Elements/view_galaxy_matrix.ctp

@@ -88,7 +88,7 @@ foreach($tabs as $tabName => $column):
         <span id="matrix-heatmap-maxval"><?php echo h($maxScore); ?></span>
     </div>
     <?php endif; ?>
-    <label style="display: inline-block; margin-left: 30px;"><input type="checkbox" id="checkbox_attackMatrix_showAll" checked><span class="fa fa-filter"><?php echo __('Show all');?></span></input></label>
+    <label style="display: inline-block; margin-left: 30px;"><input type="checkbox" id="checkbox_attackMatrix_showAll" checked><span class="fa fa-filter"><?php echo __('Show all');?></span></label>
 </div>
 
 <?php if (isset($eventId)): ?>

app/View/Events/ajax/ajaxTags.ctp

@@ -6,6 +6,7 @@
     echo $this->element('ajaxTags', array(
         'event' => $event,
         'tags' => $tags,
-        'tagAccess' => ($isSiteAdmin || $mayModify)
+        'tagAccess' => ($isSiteAdmin || $mayModify),
+        'tagConflicts' => $tagConflicts
     ));
 ?>

app/View/Events/upload_analysis_file.ctp

@@ -229,7 +229,7 @@ function processString(text)
         }
         $("#individualLines").find('tbody')
             .append($('<tr>')
-                .append($('<td>').html('<input type="checkbox" class="select"></input>'))
+                .append($('<td>').html('<input type="checkbox" class="select">'))
                 .append($('<td>').text(filepath))
                 .append($('<td>').text(size))
                 .append($('<td>').text(activity))

app/View/Events/view.ctp

@@ -133,7 +133,8 @@
                         'event' => $event,
                         'tags' => $event['EventTag'],
                         'tagAccess' => ($isSiteAdmin || $mayModify || $me['org_id'] == $event['Event']['orgc_id']),
-                        'required_taxonomies' => $required_taxonomies
+                        'required_taxonomies' => $required_taxonomies,
+                        'tagConflicts' => $tagConflicts
                     )
                 )
             )
@@ -197,7 +198,7 @@
         );
         $table_data[] = array(
             'key' => __('First recorded change'),
-            'value' => date('Y-m-d H:i:s', $oldest_timestamp)
+            'value' => (!$oldest_timestamp) ? '' : date('Y-m-d H:i:s', $oldest_timestamp)
         );
         $table_data[] = array(
             'key' => __('Last change'),
@@ -330,6 +331,41 @@
             <?php echo $this->element('genericElements/viewMetaTable', array('table_data' => $table_data)); ?>
         </div>
         <div class="related span4">
+
+            <?php if (!empty($warningTagConflicts)): ?>
+                <div class="warning_container" style="width:80%;">
+                    <h4 class="red"><?php echo __('Warning: Taxonomy inconsistencies');?></h4>
+                    <?php echo '<ul>'; ?>
+                    <?php
+                        foreach ($warningTagConflicts as $taxonomy) {
+                            echo sprintf('<li><a href="%s/taxonomies/view/%s" title="">%s</a></li>', $baseurl, h($taxonomy['Taxonomy']['id']), h($taxonomy['Taxonomy']['namespace']), h($taxonomy['Taxonomy']['description']));
+                            echo '<ul>';
+                            if ($taxonomy['Taxonomy']['exclusive']) {
+                                echo sprintf(
+                                    '<li>%s</li>', 
+                                    sprintf(
+                                        ('%s is an exclusive taxonomy. Only one Tag of this taxonomy is allowed on an element.'),
+                                        sprintf('<strong>%s</strong>', h($taxonomy['Taxonomy']['namespace']))
+                                    )
+                                );
+                            } else {
+                                foreach ($taxonomy['TaxonomyPredicate'] as $predicate) {
+                                    echo sprintf(
+                                        '<li>%s</li>', 
+                                        sprintf(
+                                            ('%s is an exclusive taxonomy predicate. Only one Tag of this predicate is allowed on an element'),
+                                            sprintf('<strong>%s</strong>', h($predicate['value']))
+                                        )
+                                    );
+                                }
+                            }
+                            echo '</ul>';
+                        }
+                    ?>
+                    <?php echo '</ul>' ?>
+                </div>
+            <?php endif; ?>
+
             <?php
                 if (!empty($event['RelatedEvent'])):
             ?>

app/View/Feeds/add.ctp

@@ -67,11 +67,11 @@
                         <fieldset>
                             <div class="input">
                                 <label for="BasicAuthUsername"><?php echo __('Username');?></label>
-                                <input class="form-control" type="text" id="BasicAuthUsername"></input><br />
+                                <input class="form-control" type="text" id="BasicAuthUsername"><br />
                             </div>
                             <div class="input">
                                 <label for ="BasicAuthPassword"><?php echo __('Password');?></label>
-                                <input class="form-control" type="text" id="BasicAuthPassword"></input><br />
+                                <input class="form-control" type="text" id="BasicAuthPassword"><br />
                             </div>
                         </fieldset>
                         <span class="btn-inverse btn" onClick="add_basic_auth();" style="line-height:10px; padding: 4px 4px;"><?php echo __('Add basic auth header'); ?></span>

app/View/Feeds/edit.ctp

@@ -48,11 +48,11 @@
                         <fieldset>
                             <div class="input">
                                 <label for="BasicAuthUsername"><?php echo __('Username');?></label>
-                                <input class="form-control" type="text" id="BasicAuthUsername"></input><br />
+                                <input class="form-control" type="text" id="BasicAuthUsername"><br />
                             </div>
                             <div class="input">
                                 <label for ="BasicAuthPassword"><?php echo __('Password');?></label>
-                                <input class="form-control" type="text" id="BasicAuthPassword"></input><br />
+                                <input class="form-control" type="text" id="BasicAuthPassword"><br />
                             </div>
                         </fieldset>
                         <span class="btn-inverse btn" onClick="add_basic_auth();" style="line-height:10px; padding: 4px 4px;"><?php echo __('Add basic auth header'); ?></span>

app/View/GalaxyClusters/ajax/index.ctp

@@ -38,7 +38,7 @@
     <span style="border-right:0px !important;">
         <span id="quickFilterButton" role="button" tabindex="0" aria-label="Filter value galaxie cluster" class="tabMenuFilterFieldButton useCursorPointer"
             onClick="quickFilter('', '<?php echo $baseurl . '/galaxies/view/' . $galaxy_id; ?>');"><?php echo __('Filter');?></span>
-        <input class="tabMenuFilterField" type="text" id="quickFilterField"></input>
+        <input class="tabMenuFilterField" type="text" id="quickFilterField">
     </span>
 </div>
 <table class="table table-striped table-hover table-condensed">

app/View/Helper/FontAwesomeHelper.php

@@ -446,4 +446,3 @@ App::uses('AppHelper', 'View/Helper');
             }
         }
     }
-?>

app/View/Helper/GenericPickerHelper.php

@@ -113,4 +113,3 @@ class GenericPickerHelper extends AppHelper {
         return $template;
     }
 }
-?>

app/View/Helper/HighlightHelper.php

@@ -40,4 +40,3 @@ App::uses('AppHelper', 'View/Helper');
 
         }
     }
-?>

app/View/Helper/OrgImgHelper.php

@@ -58,4 +58,3 @@ App::uses('AppHelper', 'View/Helper');
             }
         }
     }
-?>

app/View/Helper/PivotHelper.php

@@ -80,5 +80,3 @@ App::uses('AppHelper', 'View/Helper');
             return $height + $heightToAdd;
         }
     }
-
-?>

app/View/Helper/TextColourHelper.php

@@ -17,4 +17,3 @@ App::uses('AppHelper', 'View/Helper');
             }
         }
     }
-?>

app/View/Helper/UserNameHelper.php

@@ -21,4 +21,3 @@ App::uses('AppHelper', 'View/Helper');
             return '';
         }
     }
-?>

app/View/Helper/UtilityHelper.php

@@ -9,4 +9,4 @@ App::uses('AppHelper', 'View/Helper');
             return $string;
         }
     }
-?>
+

app/View/Helper/XmlOutputHelper.php

@@ -22,4 +22,3 @@ App::uses('AppHelper', 'View/Helper');
             }
         }
     }
-?>

app/View/Organisations/ajax/sg_org_row_empty.ctp

@@ -7,7 +7,7 @@
         $checked = '';
         if ($extend) $checked = 'checked';
 ?>
-    <td id="orgExtend<?php echo $id;?>"><input title="<?php echo __('Mark organisation as sharing group extender');?>" id="orgExtendInput<?php echo $id;?>" type="checkbox" onClick="sharingGroupExtendOrg(<?php echo $id;?>)" <?php echo h($checked);?>></input></td>
+    <td id="orgExtend<?php echo $id;?>"><input title="<?php echo __('Mark organisation as sharing group extender');?>" id="orgExtendInput<?php echo $id;?>" type="checkbox" onClick="sharingGroupExtendOrg(<?php echo $id;?>)" <?php echo h($checked);?>></td>
 <?php else: ?>
     <td id="orgExtend<?php echo $id;?>" ><span id="orgExtendSpan<?php echo $id;?>" class="icon-ok"></span></td>
 <?php endif; ?>

app/View/SharingGroups/add.ctp

@@ -49,13 +49,13 @@
         ?>
         <div id="page1_content" class="multi-page-form-div tabContent" style="width:544px;">
             <label for="SharingGroupName"><?php echo __('Name');?></label>
-            <input type="text" class="input-xxlarge" placeholder="<?php echo __('Example: Multinational sharing group');?>" id="SharingGroupName"></input>
+            <input type="text" class="input-xxlarge" placeholder="<?php echo __('Example: Multinational sharing group');?>" id="SharingGroupName">
             <label for="SharingGroupReleasability"><?php echo __('Releasable to');?></label>
-            <input type="text" class="input-xxlarge" placeholder="<?php echo __('Example: Community1, Organisation1, Organisation2');?>" id="SharingGroupReleasability"></input>
+            <input type="text" class="input-xxlarge" placeholder="<?php echo __('Example: Community1, Organisation1, Organisation2');?>" id="SharingGroupReleasability">
             <label for="SharingGroupDescription"><?php echo __('Description');?></label>
             <textarea class="input-xxlarge" placeholder="<?php echo __('A description of the sharing group.');?>" cols="30" rows="6" id="SharingGroupDescription"></textarea>
             <div style="display:block;">
-                <input type="checkbox" style="float:left;" title="<?php echo __('Active sharing groups can be selected by users of the local instance when creating events. Generally, sharing groups received through synchronisation will have this disabled until manually enabled.');?>" value="1" id="SharingGroupActive" checked></input>
+                <input type="checkbox" style="float:left;" title="<?php echo __('Active sharing groups can be selected by users of the local instance when creating events. Generally, sharing groups received through synchronisation will have this disabled until manually enabled.');?>" value="1" id="SharingGroupActive" checked>
                 <label for="SharingGroupActive" style="padding-left:20px;"><?php echo __('Make the sharing group selectable (active)');?></label>
             </div>
             <span role="button" tabindex="0" aria-label="<?php echo __('Next page');?>" title="<?php echo __('Next page');?>" class="btn btn-inverse" onClick="simpleTabPage(2);"><?php echo __('Next page');?></span>
@@ -79,7 +79,7 @@
         </div>
         <div id="page3_content" class="multi-page-form-div tabContent" style="display:none;width:544px;">
             <div style="display:block;">
-                <input type="checkbox" style="float:left;" title="<?php echo __('Enable roaming mode for this sharing group. Roaming mode will allow the sharing group to be passed to any instance where the remote recipient is contained in the organisation list. It is preferred to list the recipient instances instead.');?>" value="1" id="SharingGroupRoaming"></input>
+                <input type="checkbox" style="float:left;" title="<?php echo __('Enable roaming mode for this sharing group. Roaming mode will allow the sharing group to be passed to any instance where the remote recipient is contained in the organisation list. It is preferred to list the recipient instances instead.');?>" value="1" id="SharingGroupRoaming">
                 <label for="SharingGroupRoaming" style="padding-left:20px;"><?php echo __('<b>Enable roaming mode</b> for this sharing group (pass the event to any connected instance where the sync connection is tied to an organisation contained in the SG organisation list).');?></label>
             </div>
             <div id="serverList">

app/View/SharingGroups/edit.ctp

@@ -49,13 +49,13 @@
         ?>
         <div id="page1_content" class="multi-page-form-div tabContent" style="width:544px;">
             <label for="SharingGroupName"><?php echo __('Name');?></label>
-            <input type="text" class="input-xxlarge" placeholder="<?php echo __('Example: Multinational sharing group');?>" id="SharingGroupName" value="<?php echo h($sharingGroup['SharingGroup']['name']); ?>"></input>
+            <input type="text" class="input-xxlarge" placeholder="<?php echo __('Example: Multinational sharing group');?>" id="SharingGroupName" value="<?php echo h($sharingGroup['SharingGroup']['name']); ?>">
             <label for="SharingGroupReleasability"><?php echo __('Releasable to');?></label>
-            <input type="text" class="input-xxlarge" placeholder="<?php echo __('Example: Community1, Organisation1, Organisation2');?>" id="SharingGroupReleasability" value="<?php echo h($sharingGroup['SharingGroup']['releasability']); ?>"></input>
+            <input type="text" class="input-xxlarge" placeholder="<?php echo __('Example: Community1, Organisation1, Organisation2');?>" id="SharingGroupReleasability" value="<?php echo h($sharingGroup['SharingGroup']['releasability']); ?>">
             <label for="SharingGroupDescription"><?php echo __('Description');?></label>
             <textarea class="input-xxlarge" placeholder="<?php echo __('A description of the sharing group.');?>" cols="30" rows="6" id="SharingGroupDescription"><?php echo h($sharingGroup['SharingGroup']['description']); ?></textarea>
             <div style="display:block;">
-                <input type="checkbox" style="float:left;" title="<?php echo __('Active sharing groups can be selected by users of the local instance when creating events. Generally, sharing groups received through synchronisation will have this disabled until manually enabled.');?>" <?php if ($sharingGroup['SharingGroup']['active']) echo "checked"; ?> id="SharingGroupActive"></input>
+                <input type="checkbox" style="float:left;" title="<?php echo __('Active sharing groups can be selected by users of the local instance when creating events. Generally, sharing groups received through synchronisation will have this disabled until manually enabled.');?>" <?php if ($sharingGroup['SharingGroup']['active']) echo "checked"; ?> id="SharingGroupActive">
                 <label for="SharingGroupActive" style="padding-left:20px;"><?php echo __('Make the sharing group selectable (active)');?></label>
             </div>
             <span role="button" tabindex="0" aria-label="<?php echo __('Next page');?>" title="<?php echo __('Next page');?>" class="btn btn-inverse" onClick="simpleTabPage(2);"><?php echo __('Next page');?></span>
@@ -87,7 +87,7 @@
             }
         ?>
             <div style="display:block;">
-                <input type="checkbox" style="float:left;" title="<?php echo __('Enable roaming mode for this sharing group. Roaming mode will allow the sharing group to be passed to any instance where the remote recipient is contained in the organisation list. It is preferred to list the recipient instances instead.');?>" <?php echo $checked; ?> id="SharingGroupRoaming"></input>
+                <input type="checkbox" style="float:left;" title="<?php echo __('Enable roaming mode for this sharing group. Roaming mode will allow the sharing group to be passed to any instance where the remote recipient is contained in the organisation list. It is preferred to list the recipient instances instead.');?>" <?php echo $checked; ?> id="SharingGroupRoaming">
                 <label for="SharingGroupRoaming" style="padding-left:20px;"><?php echo __('<b>Enable roaming mode</b> for this sharing group (pass the event to any connected instance where the sync connection is tied to an organisation contained in the SG organisation list).');?></label>
             </div>
             <div id="serverList" <?php echo $serverDivVisibility; ?>>

app/View/Tags/index.ctp

@@ -54,7 +54,7 @@
         <?php endif;?>
         <span style="border-right:0px !important;">
             <span id="quickFilterButton" role="button" tabindex="0" aria-label="<?php echo __('Filter user tags');?>" class="tabMenuFilterFieldButton useCursorPointer" onClick="quickFilter(<?php echo h($passedArgs); ?>, '<?php echo $baseurl . '/tags/index'; ?>');"><?php echo __('Filter');?></span>
-            <input class="tabMenuFilterField" type="text" id="quickFilterField"></input>
+            <input class="tabMenuFilterField" type="text" id="quickFilterField">
         </span>
     </div>
     <table class="table table-striped table-hover table-condensed">

app/View/UserSettings/index.ctp

@@ -69,6 +69,11 @@
                     'sort' => 'type',
                     'element' => 'json',
                     'data_path' => 'UserSetting.value'
+                ),
+                array(
+                    'name' => __('Restricted to'),
+                    'sort' => 'type',
+                    'data_path' => 'UserSetting.restricted'
                 )
             ),
             'title' => __('User settings management'),

app/View/UserSettings/set_setting.ctp

@@ -19,7 +19,9 @@
                 array(
                     'div' => 'clear',
                     'class' => 'input input-xxlarge',
-                    'options' => array_combine(array_keys($validSettings), array_keys($validSettings))
+                    'options' => array_combine(array_keys($validSettings), array_keys($validSettings)),
+                    'default' => $setting,
+                    'disabled' => (boolean)$setting
                 )
             ),
             $this->Form->input(

app/View/Users/statistics_galaxymatrix.ctp

@@ -24,9 +24,9 @@
         </div>
         <div style="display: inline-block;">
             <label>Dates</label>
-            <input id="dateFrom" class="datepicker" placeholder="from" value="<?php echo isset($dateFrom) ? h($dateFrom) : ''; ?>"></input>
+            <input id="dateFrom" class="datepicker" placeholder="from" value="<?php echo isset($dateFrom) ? h($dateFrom) : ''; ?>">
             <i class="fas fa-arrow-right"></i>
-            <input id="dateTo" class="datepicker" placeholder="to" value="<?php echo isset($dateTo) ? h($dateTo) : ''; ?>"></input>
+            <input id="dateTo" class="datepicker" placeholder="to" value="<?php echo isset($dateTo) ? h($dateTo) : ''; ?>">
         </div>
         <button id="btnSubmit" class="btn btn-primary"><?php echo __('Submit') ?></button>
     </div>

app/files/feed-metadata/defaults.json

@@ -29,7 +29,7 @@
       "id": "2",
       "name": "The Botvrij.eu Data",
       "provider": "Botvrij.eu",
-      "url": "http://www.botvrij.eu/data/feed-osint",
+      "url": "https://www.botvrij.eu/data/feed-osint",
       "rules": "{\"tags\":{\"OR\":[],\"NOT\":[]},\"orgs\":{\"OR\":[],\"NOT\":[]}}",
       "enabled": true,
       "distribution": "3",
@@ -1969,5 +1969,41 @@
             "force_to_ids": false,
             "cache_timestamp": "1568901075"
         }
+    },
+    {
+        "Feed": {
+            "id": "115",
+            "name": "Metasploit exploits with CVE assigned",
+            "provider": "eCrimeLabs",
+            "url": "https:\/\/feeds.ecrimelabs.net\/data\/metasploit-cve",
+            "rules": "{\"tags\":{\"OR\":[],\"NOT\":[]},\"orgs\":{\"OR\":[],\"NOT\":[]}}",
+            "enabled": true,
+            "distribution": "0",
+            "sharing_group_id": "0",
+            "tag_id": "0",
+            "default": false,
+            "source_format": "csv",
+            "fixed_event": true,
+            "delta_merge": true,
+            "event_id": "",
+            "publish": true,
+            "override_ids": false,
+            "settings": "{\"csv\":{\"value\":\"\",\"delimiter\":\",\"},\"common\":{\"excluderegex\":\"\"}}",
+            "input_source": "network",
+            "delete_local_file": false,
+            "lookup_visible": true,
+            "headers": "",
+            "caching_enabled": true,
+            "force_to_ids": false,
+            "cache_timestamp": "1571206806"
+        },
+        "Tag": {
+          "id": "615",
+          "name": "osint:source-type=\"block-or-filter-list\"",
+          "colour": "#004577",
+          "exportable": true,
+          "org_id": "0",
+          "hide_tag": false
+        }
     }
 ]

app/files/misp-galaxy

@@ -1 +1 @@
-Subproject commit ac8236d16dca06076150fc9d6e7ec544645a676f
+Subproject commit 64a35698033c471f0aebb375fe4d08addede8227

app/files/misp-objects

@@ -1 +1 @@
-Subproject commit ffc120106c4ba9ed3b2fd5ae18d41f730e61b3ab
+Subproject commit 58d6722f5e276a0ec5889e6e67316b7401960542

app/files/scripts/generate_file_objects.py

@@ -5,7 +5,7 @@ import argparse
 import json
 
 try:
-    from pymisp import MISPEncode
+    from pymisp import pymisp_json_default
     from pymisp.tools import make_binary_objects
 except ImportError:
     pass
@@ -51,7 +51,7 @@ def make_objects(path):
         to_return['objects'].append(fo)
         if fo.ObjectReference:
             to_return['references'] += fo.ObjectReference
-    return json.dumps(to_return, cls=MISPEncode)
+    return json.dumps(to_return, default=pymisp_json_default)
 
 
 if __name__ == '__main__':

app/files/scripts/stix2/stix2misp.py

@@ -48,9 +48,25 @@ class StixParser():
     def load_data(self, filename, version, event, args):
         self.filename = filename
         self.stix_version = version
-        for object_type in special_parsing:
+        self.event = defaultdict(dict)
-            setattr(self, object_type.replace('-', '_'), event.pop(object_type) if object_type in event else {})
+        self.event['relationship'] = defaultdict(list)
-        self.event = event
+        mapping = {'custom_object': self.__load_custom,
+                   'marking-definition': self.__load_marking,
+                   'relationship': self.__load_relationship,
+                   'report': self.__load_report}
+        mapping.update({object_type: self.__load_usual_object for object_type in ('indicator', 'observed-data', 'vulnerability', 'course-of-action')})
+        mapping.update({galaxy_type: self.__load_galaxy for galaxy_type in galaxy_types})
+        for parsed_object in event.objects:
+            try:
+                object_type = parsed_object._type
+            except AttributeError:
+                object_type = parsed_object['type']
+            if object_type.startswith('x-misp-object'):
+                object_type = 'custom_object'
+            try:
+                mapping[object_type](parsed_object)
+            except KeyError:
+                self.__load_usual_object(parsed_object)
         try:
             event_distribution = args[0]
             if not isinstance(event_distribution, int):
@@ -68,23 +84,56 @@ class StixParser():
         self.misp_event.distribution = event_distribution
         self._attribute_distribution = attribute_distribution
 
+    def __load_custom(self, parsed_object):
+        self.event['custom_object'][parsed_object['id'].split('--')[1]] = parsed_object
+
+    def __load_galaxy(self, parsed_object):
+        try:
+            self.galaxy[parsed_object['id'].split('--')[1]] = {'object': parsed_object, 'used': False}
+        except AttributeError:
+            self.galaxy = {parsed_object['id'].split('--')[1]: {'object': parsed_object, 'used': False}}
+
+    def __load_marking(self, parsed_object):
+        try:
+            self.marking_definition[parsed_object['id'].split('--')[1]] = {'object': parsed_object, 'used': False}
+        except AttributeError:
+            self.marking_definition = {parsed_object['id'].split('--')[1]: {'object': parsed_object, 'used': False}}
+
+    def __load_relationship(self, parsed_object):
+        try:
+            self.relationship[parsed_object.source_ref.split('--')[1]].append(parsed_object)
+        except AttributeError:
+            self.relationship = defaultdict(list)
+            self.relationship[parsed_object.source_ref.split('--')[1]].append(parsed_object)
+
+    def __load_report(self, parsed_object):
+        try:
+            self.report[parsed_object['id'].split('--')[1]] = parsed_object
+        except AttributeError:
+            self.report = {parsed_object['id'].split('--')[1]: parsed_object}
+
+    def __load_usual_object(self, parsed_object):
+        self.event[parsed_object._type][parsed_object['id'].split('--')[1]] = parsed_object
+
     def general_handler(self):
         self.outputname = '{}.stix2'.format(self.filename)
-        self.build_from_STIX_with_report() if self.report else self.build_from_STIX_without_report()
+        self.build_from_STIX_with_report() if hasattr(self, 'report') else self.build_from_STIX_without_report()
         self.set_distribution()
-        for galaxy in self.galaxy.values():
+        if hasattr(self, 'galaxy'):
-            if galaxy['used'] == False:
+            for galaxy in self.galaxy.values():
-                self.misp_event['Galaxy'].append(self.parse_galaxies(galaxy['object']))
+                if galaxy['used'] == False:
-        for marking in self.marking_definition.values():
+                    self.misp_event['Galaxy'].append(self.parse_galaxies(galaxy['object']))
-            if marking['used'] == False:
+        if hasattr(self, 'marking_definition'):
-                try:
+            for marking in self.marking_definition.values():
-                    self.misp_event.add_tag(self.parse_marking(marking['object']))
+                if marking['used'] == False:
-                except PyMISPInvalidFormat:
+                    try:
-                    continue
+                        self.misp_event.add_tag(self.parse_marking(marking['object']))
+                    except PyMISPInvalidFormat:
+                        continue
 
     def build_from_STIX_with_report(self):
         report_attributes = defaultdict(set)
-        for report in self.report.values():
+        for ruuid, report in self.report.items():
             try:
                 report_attributes['orgs'].add(report.created_by_ref.split('--')[1])
             except AttributeError:
@@ -99,8 +148,11 @@ class StixParser():
             for ref in report.object_refs:
                 object_type, uuid = ref.split('--')
                 if object_type not in special_parsing and object_type not in galaxy_types:
-                    object2parse = self.event[object_type][uuid]
+                    try:
-                    self.parsing_process(object2parse, object_type)
+                        object2parse = self.event[object_type][uuid]
+                        self.parsing_process(object2parse, object_type)
+                    except KeyError:
+                        continue
         if len(report_attributes['orgs']) == 1:
             identity = self.event['identity'][report_attributes['orgs'].pop()]
             self.misp_event['Org'] = {'name': identity['name']}
@@ -115,7 +167,7 @@ class StixParser():
 
     def build_from_STIX_without_report(self):
         for object_type, objects in self.event.items():
-            for _, _object in objects.items():
+            for _object in objects.values():
                 self.parsing_process(_object, object_type)
         self.misp_event.info = "Imported with MISP import script for {}.".format(self.stix_version)
 
@@ -338,11 +390,13 @@ class StixParser():
         return attributes
 
     @staticmethod
-    def extract_data_from_file(objects):
+    def split_file_observable(objects):
         data = None
         for value in objects.values():
             if isinstance(value, stix2.Artifact):
                 data = value.payload_bin
+            elif isinstance(value, stix2.Directory):
+                data = value
             elif isinstance(value, stix2.File):
                 file = value
         return file, data
@@ -520,7 +574,7 @@ class StixFromMISPParser(StixParser):
                                 'WindowsPEBinaryFile': {'observable': self.observable_pe, 'pattern': self.pattern_pe},
                                 'x509': {'observable': self.attributes_from_x509_observable, 'pattern': self.pattern_x509}}
         self.object_from_refs = {'course-of-action': self.parse_MISP_course_of_action, 'vulnerability': self.parse_vulnerability,
-                                 'x-misp-object': self.parse_custom}
+                                 'custom_object': self.parse_custom}
         self.object_from_refs.update(dict.fromkeys(['indicator', 'observed-data'], self.parse_usual_object))
         self.attributes_fetcher_mapping = {'indicator': self.fetch_attributes_from_indicator,
                                            'observed-data': self.fetch_attributes_from_observable,
@@ -773,7 +827,7 @@ class StixFromMISPParser(StixParser):
 
     def observable_file(self, observable):
         if len(observable) > 1:
-            file, data = self.extract_data_from_file(observable)
+            file, data = self.split_file_observable(observable)
             if data is not None:
                 return self.attributes_from_file_observable(file, data)
         return self.attributes_from_file_observable(observable['0'])
@@ -1063,6 +1117,7 @@ class ExternalStixParser(StixParser):
                                    ('domain-name', 'ipv4-addr', 'ipv6-addr', 'network-traffic'): self.parse_ip_port_or_network_socket_observable,
                                    ('domain-name', 'network-traffic'): self.parse_network_socket_observable,
                                    ('domain-name', 'network-traffic', 'url'): self.parse_url_object_observable,
+                                   ('email-addr',): self.parse_email_address_observable,
                                    ('email-addr', 'email-message'): self.parse_email_observable,
                                    ('email-addr', 'email-message', 'file'): self.parse_email_observable,
                                    ('email-message',): self.parse_email_observable,
@@ -1078,9 +1133,12 @@ class ExternalStixParser(StixParser):
                                    ('url',): self.parse_url_observable,
                                    ('user-account',): self.parse_user_account_observable,
                                    ('windows-registry-key',): self.parse_regkey_observable}
-        self.pattern_mapping = {('domain-name',): self.parse_domain_ip_port_pattern,
+        self.pattern_mapping = {('directory',): self.parse_file_pattern,
+                                ('directory', 'file'): self.parse_file_pattern,
+                                ('domain-name',): self.parse_domain_ip_port_pattern,
                                 ('domain-name', 'ipv4-addr', 'url'): self.parse_domain_ip_port_pattern,
                                 ('domain-name', 'ipv6-addr', 'url'): self.parse_domain_ip_port_pattern,
+                                ('email-addr',): self.parse_email_address_pattern,
                                 ('file',): self.parse_file_pattern,
                                 ('ipv4-addr',): self.parse_ip_address_pattern,
                                 ('ipv6-addr',): self.parse_ip_address_pattern,
@@ -1283,9 +1341,23 @@ class ExternalStixParser(StixParser):
                 attributes.append(self.append_email_attribute(m_key, m_value, to_ids))
         self.handle_import_case(attributes, 'email', marking, uuid)
 
+    def parse_email_address_observable(self, objects, marking, uuid):
+        mapping = to_attribute_mapping
+        attributes = [{'type': 'email-dst', 'object_relation': 'to', 'to_ids': True, 'value': _object.value} for _object in objects.values()]
+        self.handle_import_case(attributes, 'email', marking, uuid)
+
+    def parse_email_address_pattern(self, pattern, marking=None, uuid=None):
+        pattern_types, pattern_values = self.get_types_and_values_from_pattern(pattern)
+        attributes = self.fill_pattern_attributes(pattern_types, pattern_values, email_mapping)
+        self.handle_import_case(attributes, 'email', marking, uuid)
+
     def parse_file_observable(self, objects, marking, uuid):
-        _object = objects['0']
+        file, directory = self.split_file_observable(objects)
-        attributes = self.attributes_from_file_observable(_object)
+        attributes = self.attributes_from_file_observable(file)
+        if directory is not None and directory.path:
+            mapping = file_mapping['path']
+            attributes.append({'type': mapping['type'], 'object_relation': mapping['relation'],
+                               'value': directory.path, 'to_ids': False})
         if hasattr(_object, 'extensions') and 'windows-pebinary-ext' in _object.extensions:
             self.handle_pe_case(_object.extensions['windows-pebinary-ext'], attributes, uuid)
         else:
@@ -1294,10 +1366,13 @@ class ExternalStixParser(StixParser):
     def parse_file_pattern(self, pattern, marking=None, uuid=None):
         pattern_types, pattern_values = self.get_types_and_values_from_pattern(pattern)
         attributes = self.attributes_from_file_pattern(pattern_types, pattern_values)
-        self.handle_import_case(attributes, 'file', marking, uuid)
+        if any((attribute['object_relation'] == 'path' for attribute in attributes)):
+            self.object_case_import(attributes, 'file', uuid)
+        else:
+            self.handle_import_case(attributes, 'file', marking, uuid)
 
     def parse_file_object_observable(self, objects, marking, uuid):
-        file, data = self.extract_data_from_file(objects)
+        file, data = self.split_file_observable(objects)
         attributes = self.attributes_from_file_observable(file, data)
         if hasattr(file, 'extensions') and 'windows-pebinary-ext' in file.extensions:
             self.handle_pe_case(file.extensions['windows-pebinary-ext'], attributes, uuid)
@@ -1439,7 +1514,10 @@ class ExternalStixParser(StixParser):
             try:
                 type_, value = p.split('=')
             except ValueError:
-                type_, value = p.split(' = ')
+                try:
+                    type_, value = p.split(' = ')
+                except ValueError:
+                    type_, value = p.split(' = \'')
             types.append(type_.strip())
             values.append(value.strip().strip('\''))
         return types, values
@@ -1470,42 +1548,19 @@ class ExternalStixParser(StixParser):
             return "{}:{}".format(marking_type, getattr(marking.definition, marking_type))
 
 
-def from_misp(reports):
+def from_misp(stix_objects):
-    for _, o in reports.items():
+    for o in stix_objects:
-        if 'misp:tool="misp2stix2"' in o.get('labels'):
+        if o._type == "report" and 'misp:tool="misp2stix2"' in o.get('labels'):
             return True
     return False
 
 def main(args):
-    stix_event = defaultdict(dict)
-    stix_event['relationship'] = defaultdict(list)
     filename = os.path.join(os.path.dirname(args[0]), args[1])
     with open(filename, 'rt', encoding='utf-8') as f:
         event = stix2.parse(f.read(), allow_custom=True, interoperability=True)
-    for parsed_object in event.objects:
-        try:
-            object_type = parsed_object._type
-        except AttributeError:
-            object_type = parsed_object['type']
-        if object_type.startswith('x-misp-object'):
-            object_type = 'x-misp-object'
-        if object_type == 'relationship':
-            stix_event[object_type][parsed_object.source_ref.split('--')[1]].append(parsed_object)
-        else:
-            uuid = parsed_object['id'].split('--')[1]
-            if object_type in galaxy_types:
-                parsed_object = {'object': parsed_object, 'used': False}
-                object_type = 'galaxy'
-            elif object_type == 'marking-definition':
-                parsed_object = {'object': parsed_object, 'used': False}
-                # object_type = object_type
-            stix_event[object_type][uuid] = parsed_object
-    if not stix_event:
-        print(json.dumps({'success': 0, 'message': 'There is no valid STIX object to import'}))
-        sys.exit(1)
     stix_version = 'STIX {}'.format(event.get('spec_version'))
-    stix_parser = StixFromMISPParser() if from_misp(stix_event['report']) else ExternalStixParser()
+    stix_parser = StixFromMISPParser() if from_misp(event.objects) else ExternalStixParser()
-    stix_parser.load_data(filename, stix_version, stix_event, args[2:])
+    stix_parser.load_data(filename, stix_version, event, args[2:])
     stix_parser.handler()
     stix_parser.saveFile()
     print(1)

app/files/scripts/stix2/stix2misp_mapping.py

@@ -143,6 +143,7 @@ mime_type_attribute_mapping = {'type': 'mime-type', 'relation': 'mimetype'}
 modified_attribute_mapping = {'type': 'datetime', 'relation': 'last-modified'}
 number_sections_mapping = {'type': 'counter', 'relation': 'number-sections'}
 password_mapping = {'type': 'text', 'relation': 'password'}
+path_attribute_mapping = {'type': 'text', 'relation': 'path'}
 pe_type_mapping = {'type': 'text', 'relation': 'type'}
 pid_attribute_mapping = {'type': 'text', 'relation': 'pid'}
 process_command_line_mapping = {'type': 'text', 'relation': 'command-line'}
@@ -204,12 +205,15 @@ email_mapping = {'date': email_date_attribute_mapping,
                  'from_ref': from_attribute_mapping,
                  'email-message:from_ref': from_attribute_mapping,
                  'body_multipart': body_multipart_mapping,
-                 'email-message:body_multipart[*].body_raw_ref.name': body_multipart_mapping}
+                 'email-message:body_multipart[*].body_raw_ref.name': body_multipart_mapping,
+                 'email-addr:value': to_attribute_mapping}
 
 file_mapping = {'mime_type': mime_type_attribute_mapping,
                 'file:mime_type': mime_type_attribute_mapping,
                 'name': filename_attribute_mapping,
                 'file:name': filename_attribute_mapping,
+                'path': path_attribute_mapping,
+                'directory:path': path_attribute_mapping,
                 'size': size_attribute_mapping,
                 'file:size': size_attribute_mapping}
 
@@ -257,13 +261,13 @@ process_mapping = {'name': process_name_mapping,
                    'process:child_refs': {'type': 'text', 'relation': 'child-pid'}}
 
 regkey_mapping = {'data': data_attribute_mapping,
-                  'windows-registry-key:data': data_attribute_mapping,
+                  'windows-registry-key:values.data': data_attribute_mapping,
                   'data_type': data_type_attribute_mapping,
-                  'windows-registry-key:data_type': data_type_attribute_mapping,
+                  'windows-registry-key:values.data_type': data_type_attribute_mapping,
                   'modified': modified_attribute_mapping,
                   'windows-registry-key:modified': modified_attribute_mapping,
                   'name': regkey_name_attribute_mapping,
-                  'windows-registry-key:name': regkey_name_attribute_mapping,
+                  'windows-registry-key:values.name': regkey_name_attribute_mapping,
                   'key': key_attribute_mapping,
                   'windows-registry-key:key': key_attribute_mapping,
                   'windows-registry-key:value': {'type': 'text', 'relation': 'hive'}

app/files/taxonomies

@@ -1 +1 @@
-Subproject commit 33e55b6db3dc73d9f877c8880c13f9f7685625ec
+Subproject commit 9b31f7d02d21f7557435e28c2296e62c52f97e2f

app/webroot/css/main.css

@@ -2467,3 +2467,33 @@ table tr:hover .down-expand-button {
 #notice_message {
    margin: 10px;
 }
+
+.tag-conflict-notice {
+    margin: 5px 0px;
+    padding-right: 14px;
+    position: relative;
+}
+
+.tag-conflict-notice div.text-container {
+    overflow: auto;
+}
+
+.attributeTagContainer .tag-conflict-notice {
+    max-width: 350px;
+    overflow: auto;
+}
+
+.tag-conflict-notice .apply_css_arrow {
+    display: inline-block;
+    padding: 0px 6px;
+}
+
+.tag-conflict-notice i.icon {
+    position: absolute;
+    padding: 3px;
+    right: 0px;
+    top: 0px;
+    background-color: #3b3b3b;
+    color: white;
+    border-radius: 3px;
+}

app/webroot/js/action_table.js

@@ -247,7 +247,7 @@ class ActionTable {
 		var td = document.createElement('td');
 		var btn = document.createElement('button');
 		btn.classList.add("btn", "btn-danger");
-		btn.innerHTML = '<span class="fa fa-trash-o"></span>';
+		btn.innerHTML = '<span class="fa fa-trash"></span>';
 		btn.type = "button";
 		btn.setAttribute('rowID', tr.id);
 		if (that.row_action_button_style.tooltip !== undefined) {

app/webroot/js/decayingTool.js

@@ -1017,7 +1017,7 @@ ModelTable.prototype = {
         }
         var is_row_selected = $('#saveForm #save-model-button').data('modelid') == model.DecayingModel.id;
         return cells_html = [
-            this._gen_td('<input type="checkbox" onchange="decayingTool.refreshSaveButton()" style="margin:0" ' + (is_row_selected ? 'checked' : 'disabled') + '></input>', 'DMCheckbox'),
+            this._gen_td('<input type="checkbox" onchange="decayingTool.refreshSaveButton()" style="margin:0" ' + (is_row_selected ? 'checked' : 'disabled') + '>', 'DMCheckbox'),
             this._gen_td_link('/decayingModel/view/'+model.DecayingModel.id, this._h(model.DecayingModel.id), 'DMId'),
             this._gen_td(
                 this._h(model.DecayingModel.name) + (model.DecayingModel.default ? '<img src="/img/orgs/MISP.png" width="24" height="24" style="padding-bottom:3px;" title="Default Model from MISP Project" />' : '') ,

app/webroot/js/event-graph.js

@@ -1401,7 +1401,7 @@ class DataHandler {
         eventGraph.menu_filter.items["table_attr_value"].add_options("table_control_select_attr_value", available_object_references);
     }
 
-    fetch_data_and_update(stabilize, callback) {
+    fetch_data_and_update(stabilize, updateOnly, callback) {
         eventGraph.network_loading(true, loadingText_fetching);
         $.when(this.fetch_objects_template()).done(function() {
             var filtering_rules = eventGraph.get_filtering_rules();
@@ -1419,7 +1419,9 @@ class DataHandler {
                 data: JSON.stringify( payload ),
                 processData: false,
                 success: function( data, textStatus, jQxhr ){
-                    eventGraph.reset_graphs(true);
+                    if (updateOnly === undefined || updateOnly === false) {
+                        eventGraph.reset_graphs(true);
+                    }
                     eventGraph.is_filtered = (filtering_rules.presence.length > 0 || filtering_rules.value.length > 0);
                     eventGraph.first_draw = true;
                     // update object state
@@ -1574,7 +1576,8 @@ class MispInteraction {
         if (!that.can_create_reference(edgeData.from) || !that.can_be_referenced(edgeData.to)) {
             return;
         }
-        genericPopup('/objectReferences/add/'+edgeData.from, '#popover_form', function() {
+        var edgeFromId = edgeData.from.startsWith('o-') ? edgeData.from.substr(2) : edgeData.from;
+        genericPopup('/objectReferences/add/'+edgeFromId, '#popover_form', function() {
             $('#ObjectReferenceReferencedUuid').val(uuid);
             objectReferenceInput();
         });
@@ -1639,6 +1642,7 @@ class MispInteraction {
         var selected_nodes = nodeData.nodes;
         for (var nodeID of selected_nodes) {
             var node = this.nodes.get(nodeID)
+            nodeID = nodeID.startsWith('o-') ? nodeID.substr(2) : nodeID;
             if (node.group.slice(0, 9) == "attribute") {
                 deleteObject('attributes', 'delete', nodeID, scope_id);
             } else if (node.group == "object") {
@@ -1651,6 +1655,7 @@ class MispInteraction {
         var that = mispInteraction;
         var id = nodeData.id
         var group = nodes.get(id).group;
+        id = id.startsWith('o-') ? id.substr(2) : id;
         if (group.slice(0, 9) == 'attribute') {
             simplePopup('/attributes/edit/'+id);
         } else if (group == 'object') {
@@ -1863,7 +1868,7 @@ function genericPopupCallback(result) {
     // sucess and eventgraph is enabled
     if (result == "success" && dataHandler !== undefined) {
         mispInteraction.apply_callback();
-        dataHandler.fetch_data_and_update(false);
+        dataHandler.fetch_data_and_update(false, true);
     }
 }
 
@@ -1960,7 +1965,7 @@ function import_graph_from_json(data) {
         $('#checkbox_physics_enable').prop('checked', data.physics.enabled);
 
         // update data
-        dataHandler.fetch_data_and_update(false, function() {
+        dataHandler.fetch_data_and_update(false, false, function() {
             eventGraph.nodes.update(data.nodes);
             eventGraph.expand_previous_expansion(data.nodes);
             eventGraph.hiddenNode.clear();
@@ -2182,7 +2187,7 @@ $(document).on("keyup", function(evt) {
 });
 
 eventGraph.update_scope();
-dataHandler.fetch_data_and_update(true, function() {
+dataHandler.fetch_data_and_update(true, false, function() {
     var $select = $('#network-typeahead');
     dataHandler.get_typeaheadData_search().forEach(function(element) {
         var $option = $('<option></option>');

app/webroot/js/misp.js

@@ -2872,7 +2872,7 @@ function sharingGroupPopulateOrganisations() {
         if (org.removable == 1) {
             html += '<input id="orgExtend' + id + '" type="checkbox" onClick="sharingGroupExtendOrg(' + id + ')" ';
             if (org.extend) html+= 'checked';
-            html += '></input>';
+            html += '>';
         } else {
             html += '<span class="icon-ok"></span>'
         }
@@ -2897,7 +2897,7 @@ function sharingGroupPopulateServers() {
         html += '<td>';
         html += '<input id="serverAddOrgs' + id + '" type="checkbox" onClick="sharingGroupServerAddOrgs(' + id + ')" ';
         if (server.all_orgs) html += 'checked';
-        html += '></input>';
+        html += '>';
         html +='</td>';
         html += '<td class="actions short">';
         if (server.removable == 1) html += '<span class="icon-trash" onClick="sharingGroupRemoveServer(' + id + ')"></span>';

app/webroot/js/network-distribution-graph.js

@@ -258,7 +258,7 @@
                 var $div = '<div id="sharingNetworkWrapper" class="fixedRightPanel hidden">'
                     + '<div class="eventgraph_header" style="border-radius: 5px; display: flex;">'
                     + '<it class="fa fa-circle-o" style="margin: auto 10px; font-size: x-large"></it>'
-                    + '<input type="text" id="sharingNetworkTargetId" class="center-in-network-header network-typeahead" style="width: 200px;" disabled></input>';
+                    + '<input type="text" id="sharingNetworkTargetId" class="center-in-network-header network-typeahead" style="width: 200px;" disabled>';
                 if (allow_interactive_picking) {
                     $div += '<div class="form-group" style="margin: auto 10px;"><div class="checkbox">'
                         + '<label style="user-select: none;"><input id="interactive_picking_mode" type="checkbox" title="Click on a element to see how it is distributed" style="margin-top: 4px;">Enable interactive picking mode</label>'

cti-python-stix2

@@ -1 +1 @@
-Subproject commit a739c1154e6c2be0ebb619c581b62481025d7ecc
+Subproject commit e2a4129ad343eb8d98bd8bda36411486fe3a8d63

docs/INSTALL.rhel7.md

@@ -29,7 +29,7 @@ Make sure you are reading the parsed version of this Document. When in doubt [cl
 
 !!! notice
     Maintenance for CentOS 7 will end on: June 30th, 2024 [Source[0]](https://wiki.centos.org/About/Product) [Source[1]](https://linuxlifecycle.com/)
-    CentOS 7.6-1810 [NetInstallURL](http://mirror.centos.org/centos/7.6.1810/os/x86_64/)
+    CentOS 7-1908 [NetInstallURL](http://mirror.centos.org/centos/7/os/x86_64/)
 
 This document details the steps to install MISP on Red Hat Enterprise Linux 7.x (RHEL 7.x) and CentOS 7.x.
 At time of this writing it was tested on versions 7.6 for both.
@@ -158,6 +158,8 @@ yumInstallCoreDeps () {
   # Enable and start redis
   sudo systemctl enable --now rh-redis32-redis.service
 
+  WWW_USER="apache"
+  SUDO_WWW="sudo -H -u $WWW_USER"
   RUN_PHP="/usr/bin/scl enable rh-php72"
   PHP_INI="/etc/opt/rh/rh-php72/php.ini"
   # Install PHP 7.2 from SCL, see https://www.softwarecollections.org/en/scls/rhscl/rh-php72/
@@ -169,10 +171,8 @@ yumInstallCoreDeps () {
                    rh-php72-php-opcache \
                    rh-php72-php-gd -y
 
-  # Install Python 3.6 from SCL, see
+  # Python 3.6 is now available in RHEL 7.7 base
-  # https://www.softwarecollections.org/en/scls/rhscl/rh-python36/
+  sudo yum install python3 python3-devel -y
-  RUN_PYTHON='/usr/bin/scl enable rh-python36'
-  sudo yum install rh-python36 -y
 
   sudo systemctl enable --now rh-php72-php-fpm.service
 }
@@ -197,10 +197,10 @@ sudo systemctl enable --now haveged.service
 ```bash
 # <snippet-begin 1_mispCoreInstall_RHEL.sh>
 installCoreRHEL () {
-  # Download MISP using git in the /var/www/ directory.
+  # Download MISP using git in the $PATH_TO_MISP directory.
-  sudo mkdir $PATH_TO_MISP
+  sudo mkdir -p $(dirname $PATH_TO_MISP)
-  sudo chown $WWW_USER:$WWW_USER $PATH_TO_MISP
+  sudo chown $WWW_USER:$WWW_USER $(dirname $PATH_TO_MISP)
-  cd /var/www
+  cd $(dirname $PATH_TO_MISP)
   $SUDO_WWW git clone https://github.com/MISP/MISP.git
   cd $PATH_TO_MISP
   ##$SUDO_WWW git checkout tags/$(git describe --tags `git rev-list --tags --max-count=1`)
@@ -217,7 +217,8 @@ installCoreRHEL () {
   $SUDO_WWW git config core.filemode false
 
   # Create a python3 virtualenv
-  $SUDO_WWW $RUN_PYTHON -- virtualenv -p python3 $PATH_TO_MISP/venv
+  sudo pip3 install virtualenv
+  $SUDO_WWW python3 -- virtualenv -p python3 $PATH_TO_MISP/venv
   sudo mkdir /usr/share/httpd/.cache
   sudo chown $WWW_USER:$WWW_USER /usr/share/httpd/.cache
   $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U pip setuptools
@@ -255,11 +256,10 @@ installCoreRHEL () {
   # lief needs manual compilation
   sudo yum install devtoolset-7 cmake3 cppcheck -y
 
-  # FIXME: This does not work!
   cd $PATH_TO_MISP/app/files/scripts/lief
   $SUDO_WWW mkdir build
   cd build
-  $SUDO_WWW scl enable devtoolset-7 rh-python36 "bash -c 'cmake3 \
+  $SUDO_WWW scl enable devtoolset-7 "bash -c 'cmake3 \
   -DLIEF_PYTHON_API=on \
   -DPYTHON_VERSION=3.6 \
   -DPYTHON_EXECUTABLE=$PATH_TO_MISP/venv/bin/python \
@@ -282,7 +282,7 @@ installCoreRHEL () {
   fi
 
   # The following adds a PYTHONPATH to where the pyLIEF module has been compiled
-  echo /var/www/MISP/app/files/scripts/lief/build/api/python |$SUDO_WWW tee /var/www/MISP/venv/lib/python3.6/site-packages/lief.pth
+  echo $PATH_TO_MISP/app/files/scripts/lief/build/api/python |$SUDO_WWW tee $PATH_TO_MISP/venv/lib/python3.6/site-packages/lief.pth
 
   # install magic, pydeep
   $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U python-magic git+https://github.com/kbandla/pydeep.git plyara
@@ -291,19 +291,13 @@ installCoreRHEL () {
   cd $PATH_TO_MISP/PyMISP
   $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U .
 
-  # Enable python3 for php-fpm
-  echo 'source scl_source enable rh-python36' | sudo tee -a /etc/opt/rh/rh-php72/sysconfig/php-fpm
-  sudo sed -i.org -e 's/^;\(clear_env = no\)/\1/' /etc/opt/rh/rh-php72/php-fpm.d/www.conf
-  sudo systemctl restart rh-php72-php-fpm.service
-
-  umask $UMASK
-
   # Enable dependencies detection in the diagnostics page
   # This allows MISP to detect GnuPG, the Python modules' versions and to read the PHP settings.
-  # The LD_LIBRARY_PATH setting is needed for rh-git218 to work, one might think to install httpd24 and not just httpd ...
+  # The LD_LIBRARY_PATH setting is needed for rh-git218 to work
-  echo "env[PATH] = /opt/rh/rh-git218/root/usr/bin:/opt/rh/rh-redis32/root/usr/bin:/opt/rh/rh-python36/root/usr/bin:/opt/rh/rh-php72/root/usr/bin:/usr/local/bin:/usr/bin:/bin" |sudo tee -a /etc/opt/rh/rh-php72/php-fpm.d/www.conf
+  echo "env[PATH] = /opt/rh/rh-git218/root/usr/bin:/opt/rh/rh-redis32/root/usr/bin:/opt/rh/rh-php72/root/usr/bin:/usr/local/bin:/usr/bin:/bin" |sudo tee -a /etc/opt/rh/rh-php72/php-fpm.d/www.conf
-  echo "env[LD_LIBRARY_PATH] = /opt/rh/httpd24/root/usr/lib64/" |sudo tee -a /etc/opt/rh/rh-php72/php-fpm.d/www.conf
+  sudo sed -i.org -e 's/^;\(clear_env = no\)/\1/' /etc/opt/rh/rh-php72/php-fpm.d/www.conf
   sudo systemctl restart rh-php72-php-fpm.service
+  umask $UMASK
 }
 # <snippet-end 1_mispCoreInstall_RHEL.sh>
 ```
@@ -332,19 +326,16 @@ installCake_RHEL ()
   ## sudo yum install php-redis -y
   sudo scl enable rh-php72 'pecl channel-update pecl.php.net'
   sudo scl enable rh-php72 'yes no|pecl install redis'
-  echo "extension=redis.so" |sudo tee /etc/opt/rh/rh-php72/php-fpm.d/redis.ini
+  echo "extension=redis.so" |sudo tee /etc/opt/rh/rh-php72/php.d/99-redis.ini
-  sudo ln -s /etc/opt/rh/rh-php72/php-fpm.d/redis.ini /etc/opt/rh/rh-php72/php.d/99-redis.ini
 
   # Install gnupg extension
   sudo yum install gpgme-devel -y
   sudo scl enable rh-php72 'pecl install gnupg'
-  echo "extension=gnupg.so" |sudo tee /etc/opt/rh/rh-php72/php-fpm.d/gnupg.ini
+  echo "extension=gnupg.so" |sudo tee /etc/opt/rh/rh-php72/php.d/99-gnupg.ini
-  sudo ln -s /etc/opt/rh/rh-php72/php-fpm.d/gnupg.ini /etc/opt/rh/rh-php72/php.d/99-gnupg.ini
   sudo systemctl restart rh-php72-php-fpm.service
 
   # If you have not yet set a timezone in php.ini
-  echo 'date.timezone = "Asia/Tokyo"' |sudo tee /etc/opt/rh/rh-php72/php-fpm.d/timezone.ini
+  echo 'date.timezone = "Asia/Tokyo"' |sudo tee /etc/opt/rh/rh-php72/php.d/timezone.ini
-  sudo ln -s ../php-fpm.d/timezone.ini /etc/opt/rh/rh-php72/php.d/99-timezone.ini
 
   # Recommended: Change some PHP settings in /etc/opt/rh/rh-php72/php.ini
   # max_execution_time = 300
@@ -369,10 +360,10 @@ installCake_RHEL ()
 # Main function to fix permissions to something sane
 permissions_RHEL () {
   sudo chown -R $WWW_USER:$WWW_USER $PATH_TO_MISP
-  ## ? chown -R root:apache /var/www/MISP
+  ## ? chown -R root:$WWW_USER $PATH_TO_MISP
   sudo find $PATH_TO_MISP -type d -exec chmod g=rx {} \;
   sudo chmod -R g+r,o= $PATH_TO_MISP
-  ## **Note :** For updates through the web interface to work, apache must own the /var/www/MISP folder and its subfolders as shown above, which can lead to security issues. If you do not require updates through the web interface to work, you can use the following more restrictive permissions :
+  ## **Note :** For updates through the web interface to work, apache must own the $PATH_TO_MISP folder and its subfolders as shown above, which can lead to security issues. If you do not require updates through the web interface to work, you can use the following more restrictive permissions :
   sudo chmod -R 750 $PATH_TO_MISP
   sudo chmod -R g+xws $PATH_TO_MISP/app/tmp
   sudo chmod -R g+ws $PATH_TO_MISP/app/files
@@ -495,12 +486,13 @@ apacheConfig_RHEL () {
   sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/files/scripts/tmp
   sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/Plugin/CakeResque/tmp
   sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/Console/cake
-  sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/Console/worker/start.sh
+  sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/Console/worker/*.sh
-  sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/mispzmq/mispzmq.py
+  sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/*.py
-  sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/mispzmq/mispzmqtest.py
+  sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/*/*.py
   sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/lief/build/api/python/lief.so
-  sudo chcon -t httpd_sys_rw_content_t /tmp
+  sudo chcon -R -t bin_t $PATH_TO_MISP/venv/bin/*
-  sudo chcon -R -t usr_t $PATH_TO_MISP/venv
+  find $PATH_TO_MISP/venv -type f -name "*.so*" -or -name "*.so.*" | xargs sudo chcon -t lib_t
+  # Only run these if you want to be able to update MISP from the web interface
   sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/.git
   sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/tmp
   sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/Lib
@@ -537,7 +529,7 @@ firewall_RHEL () {
 
 ### 8/ Log Rotation
 ## 8.01/ Enable log rotation
-MISP saves the stdout and stderr of its workers in /var/www/MISP/app/tmp/logs
+MISP saves the stdout and stderr of its workers in $PATH_TO_MISP/app/tmp/logs
 To rotate these logs install the supplied logrotate script:
 
 FIXME: The below does not work
@@ -553,12 +545,11 @@ logRotation_RHEL () {
 
   # Now make logrotate work under SELinux as well
   # Allow logrotate to modify the log files
-  sudo semanage fcontext -a -t httpd_sys_rw_content_t "/var/www/MISP(/.*)?"
+  sudo semanage fcontext -a -t httpd_sys_rw_content_t "$PATH_TO_MISP(/.*)?"
   sudo semanage fcontext -a -t httpd_log_t "$PATH_TO_MISP/app/tmp/logs(/.*)?"
   sudo chcon -R -t httpd_log_t $PATH_TO_MISP/app/tmp/logs
-  sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/tmp/logs
   # Impact of the following: ?!?!?!!?111
-  ##sudo restorecon -R /var/www/MISP/
+  ##sudo restorecon -R $PATH_TO_MISP
 
   # Allow logrotate to read /var/www
   sudo checkmodule -M -m -o /tmp/misplogrotate.mod $PATH_TO_MISP/INSTALL/misplogrotate.te
@@ -675,16 +666,16 @@ configWorkersRHEL () {
 
   [Service]
   Type=forking
-  User=apache
+  User=$WWW_USER
-  Group=apache
+  Group=$WWW_USER
-  ExecStart=/usr/bin/scl enable rh-php72 rh-redis32 rh-mariadb102 /var/www/MISP/app/Console/worker/start.sh
+  ExecStart=/usr/bin/scl enable rh-php72 rh-redis32 rh-mariadb102 $PATH_TO_MISP/app/Console/worker/start.sh
   Restart=always
   RestartSec=10
 
   [Install]
   WantedBy=multi-user.target" |sudo tee /etc/systemd/system/misp-workers.service
 
-  sudo chmod +x /var/www/MISP/app/Console/worker/start.sh
+  sudo chmod +x $PATH_TO_MISP/app/Console/worker/start.sh
   sudo systemctl daemon-reload
 
   sudo systemctl enable --now misp-workers.service

docs/INSTALL.rhel8.md

@@ -29,7 +29,9 @@ Make sure you are reading the parsed version of this Document. When in doubt [cl
     The core MISP team cannot verify if this guide is working or not. Please help us in keeping it up to date and accurate.
     Thus we also have difficulties in supporting RHEL issues but will do a best effort on a similar yet slightly different setup.
 
-This document details the steps to install MISP on Red Hat Enterprise Linux 8.x (RHEL 8.x).
+!!! notice
+    Maintenance for CentOS 8 will end on: May 31st, 2029 [Source[0]](https://wiki.centos.org/About/Product) [Source[1]](https://linuxlifecycle.com/)
+    CentOS 8 [NetInstallURL](http://mirrorlist.centos.org/?release=8&arch=x86_64&repo=BaseOS)
 
 This document details the steps to install MISP on Red Hat Enterprise Linux 8.x (RHEL 8.x) and CentOS 8.x.
 At time of this writing it was tested on versions 8.0 for RHEL.
@@ -104,12 +106,9 @@ sudo yum update -y
 
 ## 1.6/ **[RHEL]** Install the EPEL repo
 
-!!! note
-    There is no epel-releas-latest-8 yet, but the RHEL 7 seems to work for testing.
-
 ```bash
 # <snippet-begin 0_RHEL_EPEL.sh>
-sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm -y
+sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm -y
 # <snippet-end 0_RHEL_EPEL.sh>
 ```
 
@@ -142,7 +141,7 @@ yumInstallCoreDeps () {
   sudo systemctl enable --now redis.service
 
   PHP_INI=/etc/php.ini
-  sudo yum install php php-fpm php-devel \
+  sudo yum install php php-fpm php-devel php-pear \
        php-mysqlnd \
        php-mbstring \
        php-xml \
@@ -238,7 +237,7 @@ installCoreRHEL () {
 
   # lief needs manual compilation
   sudo yum groupinstall "Development Tools" -y
-  sudo yum install cmake3 cppcheck -y
+  sudo yum install cmake3 -y
 
   cd $PATH_TO_MISP/app/files/scripts/lief
   $SUDO_WWW mkdir build
@@ -252,14 +251,18 @@ installCoreRHEL () {
   ..
   $SUDO_WWW make -j3 pyLIEF
 
-  # In case you get "internal compiler error: Killed (program cc1plus)"
+  if [ $? == 2 ]; then
-  # You ran out of memory.
+    # In case you get "internal compiler error: Killed (program cc1plus)"
-  # Create some swap
+    # You ran out of memory.
-  # sudo dd if=/dev/zero of=/var/swap.img bs=1024k count=4000
+    # Create some swap
-  # sudo mkswap /var/swap.img
+    sudo dd if=/dev/zero of=/var/swap.img bs=1024k count=4000
-  # sudo swapon /var/swap.img
+    sudo mkswap /var/swap.img
-  # And compile again
+    sudo swapon /var/swap.img
-  # $SUDO_WWW make -j3 pyLIEF
+    # And compile again
+    $SUDO_WWW make -j3 pyLIEF
+    sudo swapoff /var/swap.img
+    sudo rm /var/swap.img
+  fi
 
   # The following adds a PYTHONPATH to where the pyLIEF module has been compiled
   echo /var/www/MISP/app/files/scripts/lief/build/api/python |$SUDO_WWW tee /var/www/MISP/venv/lib/python3.6/site-packages/lief.pth
@@ -420,7 +423,7 @@ EOF
   sudo systemctl restart mariadb
 
   mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "CREATE DATABASE $DBNAME;"
-  mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "GRANT USAGE on *.* to $DBUSER@localhost IDENTIFIED by '$DBPASSWORD_MISP';"
+  mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "GRANT USAGE on *.* to $DBUSER_MISP@localhost IDENTIFIED by '$DBPASSWORD_MISP';"
   mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "GRANT ALL PRIVILEGES on $DBNAME.* to '$DBUSER_MISP'@'localhost';"
   mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e 'FLUSH PRIVILEGES;'
 
@@ -676,6 +679,9 @@ sudo systemctl enable --now misp-workers.service
 ```
 
 ## 9.07/ misp-modules (Broken on RHEL8)
+
+Here are CentOS 8 packages of openjpeg2-devel: https://centos.pkgs.org/8/centos-powertools-x86_64/openjpeg2-devel-2.3.0-8.el8.x86_64.rpm.html
+
 ```bash
 # some misp-modules dependencies
 sudo yum install openjpeg2-devel -y

docs/generic/supportFunctions.md

@@ -830,8 +830,8 @@ genRCLOCAL () {
 
 # Run PyMISP tests
 runTests () {
-  echo "url = ${MISP_BASEURL}
+  echo "url = '${MISP_BASEURL}'
-key = ${AUTH_KEY}" |sudo tee ${PATH_TO_MISP}/PyMISP/tests/keys.py
+key = '${AUTH_KEY}'" |sudo tee ${PATH_TO_MISP}/PyMISP/tests/keys.py
   sudo chown -R $WWW_USER:$WWW_USER $PATH_TO_MISP/PyMISP/
 
   sudo -H -u $WWW_USER sh -c "cd $PATH_TO_MISP/PyMISP && git submodule foreach git pull origin master"

docs/xINSTALL.OpenBSD.md

@@ -250,18 +250,8 @@ doas virtualenv -ppython3 /usr/local/virtualenvs/MISP
 ```
 
 #### Install ssdeep
-```
+```bash
-doas mkdir /usr/local/src
+doas pkg_add -v ssdeep
-doas chown misp:misp /usr/local/src
-cd /usr/local/src
-doas -u misp git clone https://github.com/ssdeep-project/ssdeep.git
-cd ssdeep
-export AUTOMAKE_VERSION=1.16
-export AUTOCONF_VERSION=2.69
-doas -u misp ./bootstrap
-doas -u misp ./configure --prefix=/usr
-doas -u misp make
-doas make install
 ```
 
 #### Apache2 only

docs/xINSTALL.centos7.md

@@ -42,11 +42,10 @@ Make sure you are reading the parsed version of this Document. When in doubt [cl
 ```bash
 # <snippet-begin 0_RHEL_PHP_INI.sh>
 # RHEL/CentOS Specific
-RUN_PHP='/usr/bin/scl enable rh-php72'
+WWW_USER="apache"
-RUN_PYTHON='/usr/bin/scl enable rh-python36'
+SUDO_WWW="sudo -H -u $WWW_USER"
-SUDO_WWW='sudo -H -u apache'
-WWW_USER='apache'
 
+RUN_PHP='/usr/bin/scl enable rh-php72'
 PHP_INI=/etc/opt/rh/rh-php72/php.ini
 # <snippet-end 0_RHEL_PHP_INI.sh>
 ```
@@ -96,9 +95,8 @@ sudo yum install gcc git zip \
 # Install PHP 7.2 from SCL, see https://www.softwarecollections.org/en/scls/rhscl/rh-php72/
 sudo yum install rh-php72 rh-php72-php-fpm rh-php72-php-devel rh-php72-php-mysqlnd rh-php72-php-mbstring rh-php72-php-xml rh-php72-php-bcmath rh-php72-php-opcache rh-php72-php-gd -y
 
-# Install Python 3.6 from SCL, see
+# Python 3.6 in now available in CentOS 7.7 base
-# https://www.softwarecollections.org/en/scls/rhscl/rh-python36/
+sudo yum install python3 python3-devel -y
-sudo yum install rh-python36 -y
 
 sudo systemctl enable --now rh-php72-php-fpm.service
 ```
@@ -119,9 +117,10 @@ sudo systemctl enable --now redis.service
 ------------
 ```bash
 # Download MISP using git in the /var/www/ directory.
-sudo mkdir $PATH_TO_MISP
+PATH_TO_MISP="/var/www/MISP"
-sudo chown ${WWW_USER}:${WWW_USER} $PATH_TO_MISP
+sudo mkdir -p $(dirname $PATH_TO_MISP)
-cd /var/www
+sudo chown ${WWW_USER}:${WWW_USER} ($dirname $PATH_TO_MISP)
+cd $(dirname $PATH_TO_MISP)
 $SUDO_WWW git clone https://github.com/MISP/MISP.git
 cd $PATH_TO_MISP
 ##$SUDO_WWW git checkout tags/$(git describe --tags `git rev-list --tags --max-count=1`)
@@ -138,7 +137,8 @@ $SUDO_WWW git submodule foreach --recursive git config core.filemode false
 $SUDO_WWW git config core.filemode false
 
 # Create a python3 virtualenv
-$SUDO_WWW $RUN_PYTHON "virtualenv -p python3 $PATH_TO_MISP/venv"
+sudo pip3 install virtualenv
+$SUDO_WWW python3 "virtualenv -p python3 $PATH_TO_MISP/venv"
 sudo mkdir /usr/share/httpd/.cache
 sudo chown ${WWW_USER}:${WWW_USER} /usr/share/httpd/.cache
 $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U pip setuptools
@@ -176,13 +176,13 @@ sudo yum install devtoolset-7 cmake3 -y
 cd $PATH_TO_MISP/app/files/scripts/lief
 $SUDO_WWW mkdir build
 cd build
-$SUDO_WWW scl enable devtoolset-7 rh-python36 'bash -c "cmake3 \
+$SUDO_WWW scl enable devtoolset-7 'bash -c "cmake3 \
 -DLIEF_PYTHON_API=on \
 -DLIEF_DOC=off \
 -DCMAKE_INSTALL_PREFIX=$LIEF_INSTALL \
 -DCMAKE_BUILD_TYPE=Release \
 -DPYTHON_VERSION=3.6 \
--DPYTHON_EXECUTABLE=/var/www/MISP/venv/bin/python \
+-DPYTHON_EXECUTABLE=$PATH_TO_MISP/venv/bin/python \
 .."'
 $SUDO_WWW make -j3
 sudo make install
@@ -204,17 +204,13 @@ $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
 cd $PATH_TO_MISP/PyMISP
 $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
 
-# Enable python3 for php-fpm
+# Enable dependencies detection in the diagnostics page
-echo 'source scl_source enable rh-python36' | sudo tee -a /etc/opt/rh/rh-php72/sysconfig/php-fpm
+# This allows MISP to detect GnuPG, the Python modules' versions and to read the PHP settings.
+echo "env[PATH] =/opt/rh/rh-php72/root/usr/bin:/usr/local/bin:/usr/bin:/bin" |sudo tee -a /etc/opt/rh/rh-php72/php-fpm.d/www.conf
 sudo sed -i.org -e 's/^;\(clear_env = no\)/\1/' /etc/opt/rh/rh-php72/php-fpm.d/www.conf
 sudo systemctl restart rh-php72-php-fpm.service
 
 umask $UMASK
-
-# Enable dependencies detection in the diagnostics page
-# This allows MISP to detect GnuPG, the Python modules' versions and to read the PHP settings.
-echo "env[PATH] =/opt/rh/rh-python36/root/usr/bin:/opt/rh/rh-php72/root/usr/bin:/usr/local/bin:/usr/bin:/bin" |sudo tee -a /etc/opt/rh/rh-php72/php-fpm.d/www.conf
-sudo systemctl restart rh-php72-php-fpm.service
 ```
 
 ### 4/ CakePHP
@@ -237,8 +233,7 @@ sudo yum install php-redis -y
 sudo systemctl restart rh-php72-php-fpm.service
 
 # If you have not yet set a timezone in php.ini
-echo 'date.timezone = "Europe/Luxembourg"' |sudo tee /etc/opt/rh/rh-php72/php-fpm.d/timezone.ini
+echo 'date.timezone = "Europe/Luxembourg"' |sudo tee /etc/opt/rh/rh-php72/php.d/99-timezone.ini
-sudo ln -s ../php-fpm.d/timezone.ini /etc/opt/rh/rh-php72/php.d/99-timezone.ini
 
 # Recommended: Change some PHP settings in /etc/opt/rh/rh-php72/php.ini
 # max_execution_time = 300
@@ -384,20 +379,18 @@ cat /etc/pki/tls/certs/dhparam.pem |sudo tee -a /etc/pki/tls/certs/misp.local.cr
 sudo systemctl restart httpd.service
 
 # Since SELinux is enabled, we need to allow httpd to write to certain directories
-sudo chcon -t usr_t $PATH_TO_MISP/venv
+sudo chcon -t bin_t $PATH_TO_MISP/venv/bin/*
+find $PATH_TO_MISP/venv -type f -name "*.so*" -or -name "*.so.*" | xargs sudo chcon -t lib_t
 sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/files
 sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/files/terms
 sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/files/scripts/tmp
 sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/Plugin/CakeResque/tmp
 sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/Console/cake
-sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/Console/worker/start.sh
+sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/Console/worker/*.sh
-sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/mispzmq/mispzmq.py
+sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/*.py
-sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/mispzmq/mispzmqtest.py
+sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/*/*.py
-sudo chcon -t httpd_sys_script_exec_t /usr/bin/ps
+sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/lief/build/api/python/lief.so
-sudo chcon -t httpd_sys_script_exec_t /usr/bin/grep
+# Only run these if you want to be able to update MISP from the web interface
-sudo chcon -t httpd_sys_script_exec_t /usr/bin/awk
-sudo chcon -t httpd_sys_script_exec_t /usr/bin/gpg
-sudo chcon -R -t usr_t $PATH_TO_MISP/venv
 sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/.git
 sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/tmp
 sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/Lib
@@ -444,7 +437,6 @@ sudo chmod 0640 /etc/logrotate.d/misp
 # Allow logrotate to modify the log files
 sudo semanage fcontext -a -t httpd_log_t "$PATH_TO_MISP/app/tmp/logs(/.*)?"
 sudo chcon -R -t httpd_log_t $PATH_TO_MISP/app/tmp/logs
-sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/tmp/logs
 
 # Allow logrotate to read /var/www
 sudo checkmodule -M -m -o /tmp/misplogrotate.mod $PATH_TO_MISP/INSTALL/misplogrotate.te
@@ -543,7 +535,7 @@ then
 fi
 
 # TODO: Fix static path with PATH_TO_MISP
-sudo sed -i -e '$i \su -s /bin/bash apache -c "scl enable rh-php72 /var/www/MISP/app/Console/worker/start.sh" > /tmp/worker_start_rc.local.log\n' /etc/rc.local
+sudo sed -i -e '$i \su -s /bin/bash apache -c "scl enable rh-php72 $PATH_TO_MISP/app/Console/worker/start.sh" > /tmp/worker_start_rc.local.log\n' /etc/rc.local
 # Make sure it will execute
 sudo chmod +x /etc/rc.local
 
@@ -574,7 +566,7 @@ $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/py
 $SUDO_WWW ${PATH_TO_MISP}/venv/bin/misp-modules -l 0.0.0.0 -s &
 
 # TODO: Fix static path with PATH_TO_MISP
-sudo sed -i -e '$i \sudo -u apache /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s &\n' /etc/rc.local
+sudo sed -i -e '$i \sudo -u apache $PATH_TO_MISP/venv/bin/misp-modules -l 127.0.0.1 -s &\n' /etc/rc.local
 ```
 
 {!generic/misp-dashboard-centos.md!}

docs/xINSTALL.debian10.md

@@ -1,12 +1,12 @@
 # INSTALLATION INSTRUCTIONS
-## for Debian 10 "buster"
+## for Debian 10.1 "buster"
 
 ### 0/ MISP debian stable install - Status
 ------------------------------------
 
 !!! notice
     This is mostly the install [@SteveClement](https://twitter.com/SteveClement) uses for testing, qc and random development.
-    Maintained and tested by @SteveClement on 20190707
+    Maintained and tested by @SteveClement on 20191016
 
 !!! warning
     PHP 7.3.4-2 is not working at the moment with the packaged composer.phar<br />

mkdocs.yml

@@ -71,7 +71,7 @@ nav:
       - 'Ubuntu 18.04': 'INSTALL.ubuntu1804.md'
       - 'Kali Linux': 'INSTALL.kali.md'
       - 'RHEL7/CentOS7': 'INSTALL.rhel7.md'
-      - 'RHEL8': 'INSTALL.rhel8.md'
+      - 'RHEL8/CentOS8': 'INSTALL.rhel8.md'
     - xInstall Guides: 
       - 'Warning': 'xINSTALL.md'
       - 'Centos 6': 'xINSTALL.centos6.md'

tools/misp_retention.py

@@ -1,13 +1,13 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # This script requires the MISP retention taxonomy is installed and enabled
 # See https://github.com/MISP/misp-taxonomies/tree/master/retention/retention
 
-from pymisp import PyMISP, MISPEvent
+from pymisp import ExpandedPyMISP, MISPEvent
 from datetime import datetime
 from dateutil.relativedelta import relativedelta
 import re
-from keys import misp_url, misp_key, misp_verifycert
+from keys import misp_url, misp_key
 
 # pip install pymisp python-dateutil
 
@@ -18,16 +18,15 @@ class misphelper(object):
     expiredTag = "retention:expired"
 
     def __init__(self):
-        self.misp = PyMISP(url=misp_url,
+        self.misp = ExpandedPyMISP(url=misp_url,
                            key=misp_key,
-                           ssl=misp_verifycert,
+                           ssl=True)
-                           out_type="json")
         self.taxonomyId = self.searchTaxonomy()
 
     def searchTaxonomy(self):
-        res = self.misp.get_taxonomies_list()
+        res = self.misp.taxonomies()
 
-        for tax in res["response"]:
+        for tax in res:
             if (tax["Taxonomy"]["namespace"] == "retention" and tax["Taxonomy"]["enabled"]):
                 return tax["Taxonomy"]["id"]
 
@@ -44,12 +43,12 @@ class misphelper(object):
                 changed = True
                 attr["to_ids"] = False
 
+        self.misp.tag(mevent, self.expiredTag, True)
         if changed:
-            mevent.add_tag(self.expiredTag)
             res = self.misp.update_event(mevent.id, mevent)
 
     def findEventsAfterRetention(self, events, retention):
-        for event in events["response"]:
+        for event in events:
             ts = datetime.strptime(event["Event"]["date"], "%Y-%m-%d")
             now = datetime.utcnow()
 
@@ -66,12 +65,13 @@ class misphelper(object):
                 self.processEvent(event["Event"])
 
     def queryRetentionTags(self):
-        res = self.misp.get_taxonomy_tags_list(self.taxonomyId)
+        res = self.misp.get_taxonomy(self.taxonomyId)
 
-        for tag in res:
+        for tag in res['entries']:
             m = re.match(r"^retention:([0-9]+)([d,w,m,y])$", tag["tag"])
             if m:
-                events = self.misp.search(published=True, tags=tag["tag"], not_tags=self.expiredTag)
+                tagSearch = self.misp.build_complex_query(and_parameters = tag["tag"], not_parameters = self.expiredTag)
+                events = self.misp.search(published=True, tags=tagSearch)
                 self.findEventsAfterRetention(events, (m.group(1), m.group(2)))
 
             else: