diff --git a/INSTALL/MYSQL.sql b/INSTALL/MYSQL.sql index 0c26a8045..86f42f950 100644 --- a/INSTALL/MYSQL.sql +++ b/INSTALL/MYSQL.sql @@ -1358,7 +1358,7 @@ INSERT INTO `admin_settings` (`id`, `setting`, `value`) VALUES INSERT INTO `feeds` (`id`, `provider`, `name`, `url`, `distribution`, `default`, `enabled`) VALUES (1, 'CIRCL', 'CIRCL OSINT Feed', 'https://www.circl.lu/doc/misp/feed-osint', 3, 1, 0), -(2, 'Botvrij.eu', 'The Botvrij.eu Data', 'http://www.botvrij.eu/data/feed-osint', 3, 1, 0); +(2, 'Botvrij.eu', 'The Botvrij.eu Data', 'https://www.botvrij.eu/data/feed-osint', 3, 1, 0); INSERT INTO `regexp` (`id`, `regexp`, `replacement`, `type`) VALUES (1, '/.:.ProgramData./i', '%ALLUSERSPROFILE%\\\\', 'ALL'), diff --git a/INSTALL/POSTGRESQL-data-initial.sql b/INSTALL/POSTGRESQL-data-initial.sql index ebb35f845..ca95d6d33 100644 --- a/INSTALL/POSTGRESQL-data-initial.sql +++ b/INSTALL/POSTGRESQL-data-initial.sql @@ -118,7 +118,7 @@ COPY public.favourite_tags (id, tag_id, user_id) FROM stdin; COPY public.feeds (id, name, provider, url, rules, enabled, distribution, sharing_group_id, tag_id, "default", source_format, fixed_event, delta_merge, event_id, publish, override_ids, settings, input_source, delete_local_file, lookup_visible, headers, caching_enabled) FROM stdin; 1 CIRCL OSINT Feed CIRCL https://www.circl.lu/doc/misp/feed-osint \N f 3 0 0 t misp f f 0 f f \N network f f \N f -2 The Botvrij.eu Data Botvrij.eu http://www.botvrij.eu/data/feed-osint \N f 3 0 0 t misp f f 0 f f \N network f f \N f +2 The Botvrij.eu Data Botvrij.eu https://www.botvrij.eu/data/feed-osint \N f 3 0 0 t misp f f 0 f f \N network f f \N f \. diff --git a/INSTALL/misplogrotate.te b/INSTALL/misplogrotate.te index 4e3f35a7b..921989772 100644 --- a/INSTALL/misplogrotate.te +++ b/INSTALL/misplogrotate.te @@ -1,8 +1,9 @@ -module misplogrotate 1.1; +module misplogrotate 1.2; require { type httpd_t; type logrotate_t; type httpd_log_t; + type httpd_sys_script_t; type httpd_sys_content_t; type httpd_sys_rw_content_t; class dir { ioctl read getattr lock search open remove_name }; @@ -12,4 +13,4 @@ require { allow logrotate_t httpd_sys_content_t:dir { ioctl read getattr lock search open }; allow logrotate_t httpd_sys_rw_content_t:dir { ioctl read getattr lock search open }; allow httpd_t httpd_log_t:dir remove_name; -allow httpd_t httpd_log_t:file { unlink write }; +allow { httpd_t httpd_sys_script_t } httpd_log_t:file { unlink write }; diff --git a/PyMISP b/PyMISP index 21c16c8c7..204fd6ba8 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit 21c16c8c75c61a00a0f01d1848687f7fe6455325 +Subproject commit 204fd6ba8cc916844156c1819c8375f6bbbca995 diff --git a/app/Config/config.default.php b/app/Config/config.default.php index 9f190e3f5..dc55fb86e 100644 --- a/app/Config/config.default.php +++ b/app/Config/config.default.php @@ -17,7 +17,7 @@ $config = array( 'org' => 'ORGNAME', 'showorg' => true, 'threatlevel_in_email_subject' => true, - 'email_subject_TLP_string' => 'TLP Amber', + 'email_subject_TLP_string' => 'tlp:amber', 'email_subject_tag' => 'tlp', 'email_subject_include_tag_name' => true, 'background_jobs' => true, @@ -141,7 +141,7 @@ $config = array( /* 'ApacheSecureAuth' => // Configuration for kerberos authentication array( - 'apacheEnv' => 'REMOTE_USER', // If proxy variable = HTTP_REMOTE_USER + 'apacheEnv' => 'REMOTE_USER', // If proxy variable = HTTP_REMOTE_USER, If BasicAuth ldap = PHP_AUTH_USER 'ldapServer' => 'ldap://example.com', // FQDN or IP 'ldapProtocol' => 3, 'ldapNetworkTimeout' => -1, // use -1 for unlimited network timeout diff --git a/app/Console/Command/ServerShell.php b/app/Console/Command/ServerShell.php index 215404032..52da69563 100644 --- a/app/Console/Command/ServerShell.php +++ b/app/Console/Command/ServerShell.php @@ -6,6 +6,32 @@ class ServerShell extends AppShell { public $uses = array('Server', 'Task', 'Job', 'User', 'Feed'); + public function list() { + $res = ['servers'=>[]]; + + $servers = $this->Server->find('all', [ + 'fields' => ['Server.id', 'Server.name', 'Server.url'], + 'recursive' => 0 + ]); + foreach ($servers as $server) + $res['servers'][] = $server['Server']; + + echo json_encode($res) . PHP_EOL; + } + + public function test() { + if (empty($this->args[0])) { + die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Test'] . PHP_EOL); + } + + $serverId = intval($this->args[0]); + $res = @$this->Server->runConnectionTest($serverId); + if (!empty($res['message'])) + $res['message'] = json_decode($res['message']); + + echo json_encode($res) . PHP_EOL; + } + public function pull() { if (empty($this->args[0]) || empty($this->args[1])) { die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['pull'] . PHP_EOL); diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 114908215..91ecf4da3 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -2959,6 +2959,7 @@ class AttributesController extends AppController public function addTag($id = false, $tag_id = false) { + $this->Taxonomy = $log = ClassRegistry::init('Taxonomy'); $rearrangeRules = array( 'request' => false, 'Attribute' => false, @@ -3099,6 +3100,20 @@ class AttributesController extends AppController $fails++; continue; } + $tagsOnAttribute = $this->Attribute->AttributeTag->find('all', array( + 'conditions' => array( + 'AttributeTag.attribute_id' => $id, + 'AttributeTag.local' => $local + ), + 'contain' => 'Tag', + 'fields' => array('Tag.name'), + 'recursive' => -1 + )); + $exclusiveTestPassed = $this->Taxonomy->checkIfNewTagIsAllowedByTaxonomy($tag['Tag']['name'], Hash::extract($tagsOnAttribute, '{n}.Tag.name')); + if (!$exclusiveTestPassed) { + $fails++; + continue; + } $this->Attribute->AttributeTag->create(); if ($this->Attribute->AttributeTag->save(array('attribute_id' => $id, 'tag_id' => $tag_id, 'event_id' => $eventId, 'local' => $local))) { if (!$local) { diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 511268933..e58853906 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -1153,6 +1153,7 @@ class EventsController extends AppController // remove galaxies tags $this->loadModel('GalaxyCluster'); + $this->loadModel('Taxonomy'); $cluster_names = $this->GalaxyCluster->find('list', array('fields' => array('GalaxyCluster.tag_name'), 'group' => array('GalaxyCluster.tag_name', 'GalaxyCluster.id'))); foreach ($event['Object'] as $k => $object) { if (isset($object['Attribute'])) { @@ -1162,6 +1163,14 @@ class EventsController extends AppController unset($event['Object'][$k]['Attribute'][$k2]['AttributeTag'][$k3]); } } + $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); + foreach ($tagConflicts['global'] as $tagConflict) { + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } + foreach ($tagConflicts['local'] as $tagConflict) { + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } + $event['Object'][$k]['Attribute'][$k2]['tagConflicts'] = $tagConflicts; } } } @@ -1171,6 +1180,14 @@ class EventsController extends AppController unset($event['Attribute'][$k]['AttributeTag'][$k2]); } } + $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); + foreach ($tagConflicts['global'] as $tagConflict) { + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } + foreach ($tagConflicts['local'] as $tagConflict) { + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } + $event['Attribute'][$k]['tagConflicts'] = $tagConflicts; } if (empty($this->passedArgs['sort'])) { $filters['sort'] = 'timestamp'; @@ -1269,12 +1286,14 @@ class EventsController extends AppController private function __viewUI($event, $continue, $fromEvent) { + $this->loadModel('Taxonomy'); $filterData = array( 'request' => $this->request, 'paramArray' => $this->acceptedFilteringNamedParams, 'named_params' => $this->params['named'] ); $exception = false; + $warningTagConflicts = array(); $filters = $this->_harvestParameters($filterData, $exception); $this->loadModel('GalaxyCluster'); @@ -1375,6 +1394,16 @@ class EventsController extends AppController unset($event['EventTag'][$k]); } } + + $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($event['EventTag']); + foreach ($tagConflicts['global'] as $tagConflict) { + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } + foreach ($tagConflicts['local'] as $tagConflict) { + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } + $this->set('tagConflicts', $tagConflicts); + $startDate = null; $modificationMap = array(); foreach ($event['Attribute'] as $k => $attribute) { @@ -1391,6 +1420,14 @@ class EventsController extends AppController unset($event['Attribute'][$k]['AttributeTag'][$k2]); } } + $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); + foreach ($tagConflicts['global'] as $tagConflict) { + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } + foreach ($tagConflicts['local'] as $tagConflict) { + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } + $event['Attribute'][$k]['tagConflicts'] = $tagConflicts; } $attributeTagsName = $this->Event->Attribute->AttributeTag->extractAttributeTagsNameFromEvent($event, 'both'); $this->set('attributeTags', array_values($attributeTagsName['tags'])); @@ -1416,9 +1453,18 @@ class EventsController extends AppController unset($event['Object'][$k]['Attribute'][$k2]['AttributeTag'][$k3]); } } + $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); + foreach ($tagConflicts['global'] as $tagConflict) { + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } + foreach ($tagConflicts['local'] as $tagConflict) { + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } + $event['Object'][$k]['Attribute'][$k2]['tagConflicts'] = $tagConflicts; } } } + $this->set('warningTagConflicts', $warningTagConflicts); $filters['sort'] = 'timestamp'; $filters['direction'] = 'desc'; if (isset($filters['distribution'])) { @@ -3734,6 +3780,7 @@ class EventsController extends AppController public function addTag($id = false, $tag_id = false) { + $this->loadModel('Taxonomy'); $rearrangeRules = array( 'request' => false, 'Event' => false, @@ -3848,6 +3895,20 @@ class EventsController extends AppController $error = __('Tag is already attached to this event.'); continue; } + $tagsOnEvent = $this->Event->EventTag->find('all', array( + 'conditions' => array( + 'EventTag.event_id' => $id, + 'EventTag.local' => $local + ), + 'contain' => 'Tag', + 'fields' => array('Tag.name'), + 'recursive' => -1 + )); + $exclusiveTestPassed = $this->Taxonomy->checkIfNewTagIsAllowedByTaxonomy($tag['Tag']['name'], Hash::extract($tagsOnEvent, '{n}.Tag.name')); + if (!$exclusiveTestPassed) { + $fail = __('Tag is not allowed due to taxonomy exclusivity settings'); + continue; + } $this->Event->EventTag->create(); if ($this->Event->EventTag->save(array('event_id' => $id, 'tag_id' => $tag_id, 'local' => $local))) { if (!$local) { diff --git a/app/Controller/TagsController.php b/app/Controller/TagsController.php index e75f36d0c..c5c1e9778 100644 --- a/app/Controller/TagsController.php +++ b/app/Controller/TagsController.php @@ -459,6 +459,7 @@ class TagsController extends AppController public function showEventTag($id) { $this->loadModel('EventTag'); + $this->loadModel('Taxonomy'); if (!$this->EventTag->Event->checkIfAuthorised($this->Auth->user(), $id)) { throw new MethodNotAllowedException('Invalid event.'); } @@ -487,6 +488,8 @@ class TagsController extends AppController 'conditions' => array('Event.id' => $id) )); $this->set('required_taxonomies', $this->EventTag->Event->getRequiredTaxonomies()); + $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($tags); + $this->set('tagConflicts', $tagConflicts); $this->set('event', $event); $this->layout = 'ajax'; $this->render('/Events/ajax/ajaxTags'); @@ -496,6 +499,7 @@ class TagsController extends AppController { $this->helpers[] = 'TextColour'; $this->loadModel('AttributeTag'); + $this->loadModel('Taxonomy'); $this->Tag->AttributeTag->Attribute->id = $id; if (!$this->Tag->AttributeTag->Attribute->exists()) { @@ -528,6 +532,8 @@ class TagsController extends AppController $this->set('event', $event); $this->set('attributeTags', $attributeTags); $this->set('attributeId', $id); + $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attributeTags); + $this->set('tagConflicts', $tagConflicts); $this->layout = 'ajax'; $this->render('/Attributes/ajax/ajaxAttributeTags'); } diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 13c660694..55da06a37 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -122,7 +122,7 @@ class AppModel extends Model break; case '2.4.27': $newFeeds = array( - array('provider' => 'Botvrij.eu', 'name' => 'The Botvrij.eu Data','url' => 'http://www.botvrij.eu/data/feed-osint', 'enabled' => 0) + array('provider' => 'Botvrij.eu', 'name' => 'The Botvrij.eu Data','url' => 'https://www.botvrij.eu/data/feed-osint', 'enabled' => 0) ); $this->__addNewFeeds($newFeeds); break; diff --git a/app/Model/Event.php b/app/Model/Event.php index 39d630bbe..0e180a477 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -3178,7 +3178,7 @@ class Event extends AppModel $bodyevent = $temp[0]; $body = $temp[1]; $result = true; - $tplColorString = !empty(Configure::read('MISP.email_subject_TLP_string')) ? Configure::read('MISP.email_subject_TLP_string') : "TLP Amber"; + $tplColorString = !empty(Configure::read('MISP.email_subject_TLP_string')) ? Configure::read('MISP.email_subject_TLP_string') : "tlp:amber"; $subject = "[" . Configure::read('MISP.org') . " MISP] Need info about event " . $id . " - ".$tplColorString; $result = $this->User->sendEmail($reporter, $bodyevent, $body, $subject, $user) && $result; } diff --git a/app/Model/Post.php b/app/Model/Post.php index eac2a6b14..199450eec 100644 --- a/app/Model/Post.php +++ b/app/Model/Post.php @@ -118,7 +118,7 @@ class Post extends AppModel $bodyDetail .= "The following message was added: \n"; $bodyDetail .= "\n"; $bodyDetail .= $message . "\n"; - $tplColorString = !empty(Configure::read('MISP.email_subject_TLP_string')) ? Configure::read('MISP.email_subject_TLP_string') : "TLP Amber"; + $tplColorString = !empty(Configure::read('MISP.email_subject_TLP_string')) ? Configure::read('MISP.email_subject_TLP_string') : "tlp:amber"; $subject = "[" . Configure::read('MISP.org') . " MISP] New post in discussion " . $post['Post']['thread_id'] . " - ".$tplColorString; foreach ($orgMembers as $recipient) { $this->User->sendEmail($recipient, $bodyDetail, $body, $subject); diff --git a/app/Model/Server.php b/app/Model/Server.php index d792259c4..654e8529d 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -134,7 +134,9 @@ class Server extends AppModel 'Push' => 'MISP/app/Console/cake Server push [user_id] [server_id]', 'Cache feeds for quick lookups' => 'MISP/app/Console/cake Server cacheFeed [user_id] [feed_id|all|csv|text|misp]', 'Fetch feeds as local data' => 'MISP/app/Console/cake Server fetchFeed [user_id] [feed_id|all|csv|text|misp]', - 'Run enrichment' => 'MISP/app/Console/cake Event enrichEvent [user_id] [event_id] [json_encoded_module_list]' + 'Run enrichment' => 'MISP/app/Console/cake Event enrichEvent [user_id] [event_id] [json_encoded_module_list]', + 'Test' => 'MISP/app/Console/cake Server test [server_id]', + 'List' => 'MISP/app/Console/cake Server list' ), 'description' => __('If you would like to automate tasks such as caching feeds or pulling from server instances, you can do it using the following command line tools. Simply execute the given commands via the command line / create cron jobs easily out of them.'), 'header' => __('Automating certain console tasks') @@ -428,7 +430,7 @@ class Server extends AppModel 'email_subject_TLP_string' => array( 'level' => 2, 'description' => __('This is the TLP string for e-mails when email_subject_tag is not found.'), - 'value' => 'TLP Amber', + 'value' => 'tlp:amber', 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', @@ -4344,7 +4346,7 @@ class Server extends AppModel public function stixDiagnostics(&$diagnostic_errors, &$stixVersion, &$cyboxVersion, &$mixboxVersion, &$maecVersion, &$stix2Version, &$pymispVersion) { $result = array(); - $expected = array('stix' => '>1.2.0.6', 'cybox' => '>2.1.0.18.dev0', 'mixbox' => '1.0.3', 'maec' => '>4.1.0.14', 'stix2' => '1.2.0', 'pymisp' => '>2.4.93'); + $expected = array('stix' => '>1.2.0.6', 'cybox' => '>2.1.0.18.dev0', 'mixbox' => '1.0.3', 'maec' => '>4.1.0.14', 'stix2' => '>1.2.0', 'pymisp' => '>2.4.93'); // check if the STIX and Cybox libraries are working using the test script stixtest.py $scriptResult = shell_exec($this->getPythonVersion() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'stixtest.py'); $scriptResult = json_decode($scriptResult, true); diff --git a/app/Model/Taxonomy.php b/app/Model/Taxonomy.php index 5bc21a528..c987b176a 100644 --- a/app/Model/Taxonomy.php +++ b/app/Model/Taxonomy.php @@ -101,7 +101,7 @@ class Taxonomy extends AppModel } $this->deleteAll(array('Taxonomy.namespace' => $current['Taxonomy']['namespace'])); } - $taxonomy['Taxonomy'] = array('namespace' => $vocab['namespace'], 'description' => $vocab['description'], 'version' => $vocab['version'], 'enabled' => $enabled); + $taxonomy['Taxonomy'] = array('namespace' => $vocab['namespace'], 'description' => $vocab['description'], 'version' => $vocab['version'], 'exclusive' => $vocab['exclusive'], 'enabled' => $enabled); $predicateLookup = array(); foreach ($vocab['predicates'] as $k => $predicate) { $taxonomy['Taxonomy']['TaxonomyPredicate'][$k] = $predicate; @@ -489,27 +489,29 @@ class Taxonomy extends AppModel return $taxonomies; } - public function getTaxonomyForTag($tagName, $metaOnly = false) + public function getTaxonomyForTag($tagName, $metaOnly = false, $fullTaxonomy = False) { if (preg_match('/^[^:="]+:[^:="]+="[^:="]+"$/i', $tagName)) { $temp = explode(':', $tagName); $pieces = array_merge(array($temp[0]), explode('=', $temp[1])); $pieces[2] = trim($pieces[2], '"'); + $contain = array( + 'TaxonomyPredicate' => array( + 'TaxonomyEntry' => array() + ) + ); + if (!$fullTaxonomy) { + $contain['TaxonomyPredicate']['conditions'] = array( + 'LOWER(TaxonomyPredicate.value)' => strtolower($pieces[1]) + ); + $contain['TaxonomyPredicate']['TaxonomyEntry']['conditions'] = array( + 'LOWER(TaxonomyEntry.value)' => strtolower($pieces[2]) + ); + } $taxonomy = $this->find('first', array( 'recursive' => -1, 'conditions' => array('LOWER(Taxonomy.namespace)' => strtolower($pieces[0])), - 'contain' => array( - 'TaxonomyPredicate' => array( - 'conditions' => array( - 'LOWER(TaxonomyPredicate.value)' => strtolower($pieces[1]) - ), - 'TaxonomyEntry' => array( - 'conditions' => array( - 'LOWER(TaxonomyEntry.value)' => strtolower($pieces[2]) - ) - ) - ) - ) + 'contain' => $contain )); if ($metaOnly && !empty($taxonomy)) { return array('Taxonomy' => $taxonomy['Taxonomy']); @@ -517,16 +519,16 @@ class Taxonomy extends AppModel return $taxonomy; } elseif (preg_match('/^[^:="]+:[^:="]+$/i', $tagName)) { $pieces = explode(':', $tagName); + $contain = array('TaxonomyPredicate' => array()); + if (!$fullTaxonomy) { + $contain['TaxonomyPredicate']['conditions'] = array( + 'LOWER(TaxonomyPredicate.value)' => strtolower($pieces[1]) + ); + } $taxonomy = $this->find('first', array( 'recursive' => -1, 'conditions' => array('LOWER(Taxonomy.namespace)' => strtolower($pieces[0])), - 'contain' => array( - 'TaxonomyPredicate' => array( - 'conditions' => array( - 'LOWER(TaxonomyPredicate.value)' => strtolower($pieces[1]) - ) - ) - ) + 'contain' => $contain )); if ($metaOnly && !empty($taxonomy)) { return array('Taxonomy' => $taxonomy['Taxonomy']); @@ -536,4 +538,97 @@ class Taxonomy extends AppModel return false; } } + + // Remove the value for triple component tags or the predicate for double components tags + public function stripLastTagComponent($tagName) + { + $shortenedTag = ''; + if (preg_match('/^[^:="]+:[^:="]+="[^:="]+"$/i', $tagName)) { + $shortenedTag = explode('=', $tagName)[0]; + } elseif (preg_match('/^[^:="]+:[^:="]+$/i', $tagName)) { + $shortenedTag = explode(':', $tagName)[0]; + } + return $shortenedTag; + } + + public function checkIfNewTagIsAllowedByTaxonomy($newTagName, $tagNameList=array()) + { + $newTagShortened = $this->stripLastTagComponent($newTagName); + $prefixIsFree = true; + foreach ($tagNameList as $tagName) { + $tagShortened = $this->stripLastTagComponent($tagName); + if ($newTagShortened == $tagShortened) { + $prefixIsFree = false; + } + } + if (!$prefixIsFree) { + // at this point, we have a duplicated namespace(-predicate) + $taxonomy = $this->getTaxonomyForTag($newTagName); + if (!empty($taxonomy['Taxonomy']['exclusive'])) { + return false; // only one tag of this taxonomy is allowed + } elseif (!empty($taxonomy['TaxonomyPredicate'][0]['exclusive'])) { + return false; // only one tag belonging to this predicate is allowed + } + } + return true; + } + + public function checkIfTagInconsistencies($tagList) + { + $eventTags = array(); + $localEventTags = array(); + foreach($tagList as $tag) { + if ($tag['local'] == 0) { + $eventTags[] = $tag['Tag']['name']; + } else { + $localEventTags[] = $tag['Tag']['name']; + } + } + $tagConflicts = $this->getTagConflicts($eventTags); + $localTagConflicts = $this->getTagConflicts($localEventTags); + return array( + 'global' => $tagConflicts, + 'local' => $localTagConflicts + ); + } + + public function getTagConflicts($tagNameList) + { + $potentiallyConflictingTaxonomy = array(); + $conflictingTaxonomy = array(); + foreach ($tagNameList as $tagName) { + $tagShortened = $this->stripLastTagComponent($tagName); + if (isset($potentiallyConflictingTaxonomy[$tagShortened])) { + $potentiallyConflictingTaxonomy[$tagShortened]['taxonomy'] = $this->getTaxonomyForTag($tagName); + $potentiallyConflictingTaxonomy[$tagShortened]['count']++; + } else { + $potentiallyConflictingTaxonomy[$tagShortened] = array( + 'count' => 1 + ); + } + $potentiallyConflictingTaxonomy[$tagShortened]['tagNames'][] = $tagName; + } + foreach ($potentiallyConflictingTaxonomy as $potTaxonomy) { + if ($potTaxonomy['count'] > 1) { + $taxonomy = $potTaxonomy['taxonomy']; + if (isset($taxonomy['Taxonomy']['exclusive']) && $taxonomy['Taxonomy']['exclusive']) { + $conflictingTaxonomy[] = array( + 'tags' => $potTaxonomy['tagNames'], + 'taxonomy' => $taxonomy, + 'conflict' => sprintf(__('Taxonomy `%s` is an exclusive Taxonomy'), $taxonomy['Taxonomy']['namespace']) + ); + } elseif (isset($taxonomy['TaxonomyPredicate'][0]['exclusive']) && $taxonomy['TaxonomyPredicate'][0]['exclusive']) { + $conflictingTaxonomy[] = array( + 'tags' => $potTaxonomy['tagNames'], + 'taxonomy' => $taxonomy, + 'conflict' => sprintf( + __('Predicate `%s` is exclusive'), + $taxonomy['TaxonomyPredicate'][0]['value'] + ) + ); + } + } + } + return $conflictingTaxonomy; + } } diff --git a/app/View/Elements/Events/View/row_attribute.ctp b/app/View/Elements/Events/View/row_attribute.ctp index 7fcb78487..a252b713b 100644 --- a/app/View/Elements/Events/View/row_attribute.ctp +++ b/app/View/Elements/Events/View/row_attribute.ctp @@ -156,7 +156,7 @@
- element('ajaxTags', array('attributeId' => $object['id'], 'tags' => $object['AttributeTag'], 'tagAccess' => ($isSiteAdmin || $mayModify || $me['org_id'] == $event['Event']['org_id']), 'context' => $context, 'scope' => 'attribute')); ?> + element('ajaxTags', array('attributeId' => $object['id'], 'tags' => $object['AttributeTag'], 'tagAccess' => ($isSiteAdmin || $mayModify || $me['org_id'] == $event['Event']['org_id']), 'context' => $context, 'scope' => 'attribute', 'tagConflicts' => isset($object['tagConflicts']) ? $object['tagConflicts'] : array())); ?>
'attributes', 'action' => 'download', $object['id']); + $controller = isset($object['objectType']) && $object['objectType'] === 'proposal' ? 'shadow_attributes' : 'attributes'; + $url = array('controller' => $controller, 'action' => 'download', $object['id']); echo $this->Html->link($filename, $url, array('class' => $linkClass)); if (isset($filenameHash[1])) { echo '
' . $filenameHash[1]; diff --git a/app/View/Elements/ajaxTags.ctp b/app/View/Elements/ajaxTags.ctp index 3a4174c43..6a0c15255 100644 --- a/app/View/Elements/ajaxTags.ctp +++ b/app/View/Elements/ajaxTags.ctp @@ -162,4 +162,36 @@ '%s', $tagData ); + $tagConflictData = ''; + if (!empty($tagConflicts['global'])) { + $tagConflictData .= '
'; + $tagConflictData .= ''; + $tagConflictData .= '
'; + foreach ($tagConflicts['global'] as $tagConflict) { + $tagConflictData .= sprintf( + '%s
', + h($tagConflict['conflict']) + ); + foreach ($tagConflict['tags'] as $tag) { + $tagConflictData .= sprintf('%s
', h($tag)); + } + } + $tagConflictData .= '
'; + } + if (!empty($tagConflicts['local'])) { + $tagConflictData .= '
'; + $tagConflictData .= ''; + $tagConflictData .= '
'; + foreach ($tagConflicts['local'] as $tagConflict) { + $tagConflictData .= sprintf( + '%s
', + h($tagConflict['conflict']) + ); + foreach ($tagConflict['tags'] as $tag) { + $tagConflictData .= sprintf('%s
', h($tag)); + } + } + $tagConflictData .= '
'; + } + echo $tagConflictData; ?> diff --git a/app/View/Events/ajax/ajaxTags.ctp b/app/View/Events/ajax/ajaxTags.ctp index 6001997d0..e56be1370 100644 --- a/app/View/Events/ajax/ajaxTags.ctp +++ b/app/View/Events/ajax/ajaxTags.ctp @@ -6,6 +6,7 @@ echo $this->element('ajaxTags', array( 'event' => $event, 'tags' => $tags, - 'tagAccess' => ($isSiteAdmin || $mayModify) + 'tagAccess' => ($isSiteAdmin || $mayModify), + 'tagConflicts' => $tagConflicts )); ?> diff --git a/app/View/Events/view.ctp b/app/View/Events/view.ctp index d3c4615f7..fbd8ecb9d 100644 --- a/app/View/Events/view.ctp +++ b/app/View/Events/view.ctp @@ -133,7 +133,8 @@ 'event' => $event, 'tags' => $event['EventTag'], 'tagAccess' => ($isSiteAdmin || $mayModify || $me['org_id'] == $event['Event']['orgc_id']), - 'required_taxonomies' => $required_taxonomies + 'required_taxonomies' => $required_taxonomies, + 'tagConflicts' => $tagConflicts ) ) ) @@ -330,6 +331,41 @@ element('genericElements/viewMetaTable', array('table_data' => $table_data)); ?>
+ + +
+

+ '; ?> + %s', $baseurl, h($taxonomy['Taxonomy']['id']), h($taxonomy['Taxonomy']['namespace']), h($taxonomy['Taxonomy']['description'])); + echo '
    '; + if ($taxonomy['Taxonomy']['exclusive']) { + echo sprintf( + '
  • %s
    • ', + sprintf( + ('%s is an exclusive taxonomy. Only one Tag of this taxonomy is allowed on an element.'), + sprintf('%s', h($taxonomy['Taxonomy']['namespace'])) + ) + ); + } else { + foreach ($taxonomy['TaxonomyPredicate'] as $predicate) { + echo sprintf( + '
  • %s
    • ', + sprintf( + ('%s is an exclusive taxonomy predicate. Only one Tag of this predicate is allowed on an element'), + sprintf('%s', h($predicate['value'])) + ) + ); + } + } + echo '
'; + } + ?> + ' ?> +
+ + diff --git a/app/files/feed-metadata/defaults.json b/app/files/feed-metadata/defaults.json index f8256808c..c9497e4b2 100644 --- a/app/files/feed-metadata/defaults.json +++ b/app/files/feed-metadata/defaults.json @@ -29,7 +29,7 @@ "id": "2", "name": "The Botvrij.eu Data", "provider": "Botvrij.eu", - "url": "http://www.botvrij.eu/data/feed-osint", + "url": "https://www.botvrij.eu/data/feed-osint", "rules": "{\"tags\":{\"OR\":[],\"NOT\":[]},\"orgs\":{\"OR\":[],\"NOT\":[]}}", "enabled": true, "distribution": "3", diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy index 078a9f576..64a356980 160000 --- a/app/files/misp-galaxy +++ b/app/files/misp-galaxy @@ -1 +1 @@ -Subproject commit 078a9f5763ec65aab6acbe6b51eafc153947ef00 +Subproject commit 64a35698033c471f0aebb375fe4d08addede8227 diff --git a/app/files/scripts/stix2/stix2misp.py b/app/files/scripts/stix2/stix2misp.py index 6b1e9f19b..c5103126c 100644 --- a/app/files/scripts/stix2/stix2misp.py +++ b/app/files/scripts/stix2/stix2misp.py @@ -1133,7 +1133,8 @@ class ExternalStixParser(StixParser): ('url',): self.parse_url_observable, ('user-account',): self.parse_user_account_observable, ('windows-registry-key',): self.parse_regkey_observable} - self.pattern_mapping = {('directory', 'file'): self.parse_file_pattern, + self.pattern_mapping = {('directory',): self.parse_file_pattern, + ('directory', 'file'): self.parse_file_pattern, ('domain-name',): self.parse_domain_ip_port_pattern, ('domain-name', 'ipv4-addr', 'url'): self.parse_domain_ip_port_pattern, ('domain-name', 'ipv6-addr', 'url'): self.parse_domain_ip_port_pattern, @@ -1365,7 +1366,10 @@ class ExternalStixParser(StixParser): def parse_file_pattern(self, pattern, marking=None, uuid=None): pattern_types, pattern_values = self.get_types_and_values_from_pattern(pattern) attributes = self.attributes_from_file_pattern(pattern_types, pattern_values) - self.handle_import_case(attributes, 'file', marking, uuid) + if any((attribute['object_relation'] == 'path' for attribute in attributes)): + self.object_case_import(attributes, 'file', uuid) + else: + self.handle_import_case(attributes, 'file', marking, uuid) def parse_file_object_observable(self, objects, marking, uuid): file, data = self.split_file_observable(objects) diff --git a/app/files/taxonomies b/app/files/taxonomies index 9b31f7d02..4cef05f41 160000 --- a/app/files/taxonomies +++ b/app/files/taxonomies @@ -1 +1 @@ -Subproject commit 9b31f7d02d21f7557435e28c2296e62c52f97e2f +Subproject commit 4cef05f4185db1d8fbad0d19272ee5dadf43f355 diff --git a/app/webroot/css/main.css b/app/webroot/css/main.css index 3a3d0e3fe..31eadfe95 100644 --- a/app/webroot/css/main.css +++ b/app/webroot/css/main.css @@ -2398,3 +2398,33 @@ table tr:hover .down-expand-button { #notice_message { margin: 10px; } + +.tag-conflict-notice { + margin: 5px 0px; + padding-right: 14px; + position: relative; +} + +.tag-conflict-notice div.text-container { + overflow: auto; +} + +.attributeTagContainer .tag-conflict-notice { + max-width: 350px; + overflow: auto; +} + +.tag-conflict-notice .apply_css_arrow { + display: inline-block; + padding: 0px 6px; +} + +.tag-conflict-notice i.icon { + position: absolute; + padding: 3px; + right: 0px; + top: 0px; + background-color: #3b3b3b; + color: white; + border-radius: 3px; +} \ No newline at end of file diff --git a/cti-python-stix2 b/cti-python-stix2 index 4f1d68065..e2a4129ad 160000 --- a/cti-python-stix2 +++ b/cti-python-stix2 @@ -1 +1 @@ -Subproject commit 4f1d68065a866e3c2984bb5a1384a05972dc127d +Subproject commit e2a4129ad343eb8d98bd8bda36411486fe3a8d63 diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index ebf78094c..463917a5f 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -198,7 +198,6 @@ sudo systemctl enable --now haveged.service # installCoreRHEL () { # Download MISP using git in the $PATH_TO_MISP directory. - PATH_TO_MISP="/var/www/MISP" sudo mkdir -p $(dirname $PATH_TO_MISP) sudo chown $WWW_USER:$WWW_USER $(dirname $PATH_TO_MISP) cd $(dirname $PATH_TO_MISP) @@ -257,7 +256,6 @@ installCoreRHEL () { # lief needs manual compilation sudo yum install devtoolset-7 cmake3 cppcheck -y - # FIXME: This does not work! cd $PATH_TO_MISP/app/files/scripts/lief $SUDO_WWW mkdir build cd build diff --git a/docs/INSTALL.rhel8.md b/docs/INSTALL.rhel8.md index f5193c303..885380f1f 100644 --- a/docs/INSTALL.rhel8.md +++ b/docs/INSTALL.rhel8.md @@ -29,7 +29,9 @@ Make sure you are reading the parsed version of this Document. When in doubt [cl The core MISP team cannot verify if this guide is working or not. Please help us in keeping it up to date and accurate. Thus we also have difficulties in supporting RHEL issues but will do a best effort on a similar yet slightly different setup. -This document details the steps to install MISP on Red Hat Enterprise Linux 8.x (RHEL 8.x). +!!! notice + Maintenance for CentOS 8 will end on: May 31st, 2029 [Source[0]](https://wiki.centos.org/About/Product) [Source[1]](https://linuxlifecycle.com/) + CentOS 8 [NetInstallURL](http://mirrorlist.centos.org/?release=8&arch=x86_64&repo=BaseOS) This document details the steps to install MISP on Red Hat Enterprise Linux 8.x (RHEL 8.x) and CentOS 8.x. At time of this writing it was tested on versions 8.0 for RHEL. @@ -104,12 +106,9 @@ sudo yum update -y ## 1.6/ **[RHEL]** Install the EPEL repo -!!! note - There is no epel-releas-latest-8 yet, but the RHEL 7 seems to work for testing. - ```bash # -sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm -y +sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm -y # ``` @@ -142,7 +141,7 @@ yumInstallCoreDeps () { sudo systemctl enable --now redis.service PHP_INI=/etc/php.ini - sudo yum install php php-fpm php-devel \ + sudo yum install php php-fpm php-devel php-pear \ php-mysqlnd \ php-mbstring \ php-xml \ @@ -238,7 +237,7 @@ installCoreRHEL () { # lief needs manual compilation sudo yum groupinstall "Development Tools" -y - sudo yum install cmake3 cppcheck -y + sudo yum install cmake3 -y cd $PATH_TO_MISP/app/files/scripts/lief $SUDO_WWW mkdir build @@ -252,14 +251,18 @@ installCoreRHEL () { .. $SUDO_WWW make -j3 pyLIEF - # In case you get "internal compiler error: Killed (program cc1plus)" - # You ran out of memory. - # Create some swap - # sudo dd if=/dev/zero of=/var/swap.img bs=1024k count=4000 - # sudo mkswap /var/swap.img - # sudo swapon /var/swap.img - # And compile again - # $SUDO_WWW make -j3 pyLIEF + if [ $? == 2 ]; then + # In case you get "internal compiler error: Killed (program cc1plus)" + # You ran out of memory. + # Create some swap + sudo dd if=/dev/zero of=/var/swap.img bs=1024k count=4000 + sudo mkswap /var/swap.img + sudo swapon /var/swap.img + # And compile again + $SUDO_WWW make -j3 pyLIEF + sudo swapoff /var/swap.img + sudo rm /var/swap.img + fi # The following adds a PYTHONPATH to where the pyLIEF module has been compiled echo /var/www/MISP/app/files/scripts/lief/build/api/python |$SUDO_WWW tee /var/www/MISP/venv/lib/python3.6/site-packages/lief.pth @@ -420,7 +423,7 @@ EOF sudo systemctl restart mariadb mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "CREATE DATABASE $DBNAME;" - mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "GRANT USAGE on *.* to $DBUSER@localhost IDENTIFIED by '$DBPASSWORD_MISP';" + mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "GRANT USAGE on *.* to $DBUSER_MISP@localhost IDENTIFIED by '$DBPASSWORD_MISP';" mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "GRANT ALL PRIVILEGES on $DBNAME.* to '$DBUSER_MISP'@'localhost';" mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e 'FLUSH PRIVILEGES;' @@ -676,6 +679,9 @@ sudo systemctl enable --now misp-workers.service ``` ## 9.07/ misp-modules (Broken on RHEL8) + +Here are CentOS 8 packages of openjpeg2-devel: https://centos.pkgs.org/8/centos-powertools-x86_64/openjpeg2-devel-2.3.0-8.el8.x86_64.rpm.html + ```bash # some misp-modules dependencies sudo yum install openjpeg2-devel -y diff --git a/docs/generic/supportFunctions.md b/docs/generic/supportFunctions.md index 94d3e38b7..561147a9c 100644 --- a/docs/generic/supportFunctions.md +++ b/docs/generic/supportFunctions.md @@ -830,8 +830,8 @@ genRCLOCAL () { # Run PyMISP tests runTests () { - echo "url = ${MISP_BASEURL} -key = ${AUTH_KEY}" |sudo tee ${PATH_TO_MISP}/PyMISP/tests/keys.py + echo "url = '${MISP_BASEURL}' +key = '${AUTH_KEY}'" |sudo tee ${PATH_TO_MISP}/PyMISP/tests/keys.py sudo chown -R $WWW_USER:$WWW_USER $PATH_TO_MISP/PyMISP/ sudo -H -u $WWW_USER sh -c "cd $PATH_TO_MISP/PyMISP && git submodule foreach git pull origin master" diff --git a/docs/xINSTALL.debian10.md b/docs/xINSTALL.debian10.md index e9bd8932c..6fa719b7f 100644 --- a/docs/xINSTALL.debian10.md +++ b/docs/xINSTALL.debian10.md @@ -1,12 +1,12 @@ # INSTALLATION INSTRUCTIONS -## for Debian 10 "buster" +## for Debian 10.1 "buster" ### 0/ MISP debian stable install - Status ------------------------------------ !!! notice This is mostly the install [@SteveClement](https://twitter.com/SteveClement) uses for testing, qc and random development. - Maintained and tested by @SteveClement on 20190707 + Maintained and tested by @SteveClement on 20191016 !!! warning PHP 7.3.4-2 is not working at the moment with the packaged composer.phar