MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'sync_filter' into develop

pull/8039/head
iglocska 2021-12-17 16:09:54 +01:00
parent 880709b91b 1c5d7d2f2f
commit 975a15ab1c
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
12 changed files with 447 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/Config/config.default.php
View File

@ -104,6 +104,7 @@ $config = array(
'user_email_notification_ban_amount_threshold' => 10,
'user_email_notification_ban_refresh_on_retry' => true,
'warning_for_all' => true,
'enable_synchronisation_filtering_on_type' => false,
),
'GnuPG' => array(
'onlyencrypted' => false,

43
app/Controller/ServersController.php
View File

@ -311,7 +311,7 @@ class ServersController extends AppController
if (!$fail) {
if ($this->_isRest()) {
$defaultPushRules = json_encode(["tags" => ["OR" => [], "NOT" => []], "orgs" => ["OR" => [], "NOT" => []]]);
$defaultPullRules = json_encode(["tags" => ["OR" => [], "NOT" => []], "orgs" => ["OR" => [], "NOT" => []], "url_params" => ""]);
$defaultPullRules = json_encode(["tags" => ["OR" => [], "NOT" => []], "orgs" => ["OR" => [], "NOT" => []], "type_attributes" => ["NOT" => []], "type_objects" => ["NOT" => []], "url_params" => ""]);
$defaults = array(
'push' => 0,
'pull' => 0,
@ -442,11 +442,25 @@ class ServersController extends AppController
$allOrgs[] = array('id' => $o['Organisation']['id'], 'name' => $o['Organisation']['name']);
}
$allTypes = [];
$this->loadModel('Attribute');
$this->loadModel('ObjectTemplate');
$objects = $this->ObjectTemplate->find('list', [
'fields' => ['name'],
'group' => ['name', 'id'],
]);
$allTypes = [
'attribute' => array_unique(Hash::extract(Hash::extract($this->Attribute->categoryDefinitions, '{s}.types'), '{n}.{n}')),
'object' => $objects
];
$this->set('host_org_id', Configure::read('MISP.host_org_id'));
$this->set('organisationOptions', $organisationOptions);
$this->set('localOrganisations', $localOrganisations);
$this->set('externalOrganisations', $externalOrganisations);
$this->set('allOrganisations', $allOrgs);
$this->set('allAttributeTypes', $allTypes['attribute']);
$this->set('allObjectTypes', $allTypes['object']);
$this->set('allTags', $this->__getTags());
$this->set('host_org_id', Configure::read('MISP.host_org_id'));
@ -497,6 +511,16 @@ class ServersController extends AppController
$this->Flash->error($error_msg);
}
}
$pushRules = $this->Server->jsonDecode($this->request->data['Server']['push_rules']);
$this->loadModel('Tag');
foreach ($pushRules['tags'] as $operator => $list) {
foreach ($list as $i => $tagName) {
if (!is_numeric($tagName)) { // tag added from freetext
$tag_id = $this->Tag->captureTag(['name' => $tagName], $this->Auth->user());
$list[$i] = $tag_id;
}
}
}
if (!$fail) {
// say what fields are to be updated
$fieldList = array('id', 'url', 'push', 'pull', 'push_sightings', 'push_galaxy_clusters', 'pull_galaxy_clusters', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
@ -613,6 +637,21 @@ class ServersController extends AppController
$allOrgs[] = array('id' => $o['Organisation']['id'], 'name' => $o['Organisation']['name']);
}
$allTypes = [];
$this->loadModel('Attribute');
$this->loadModel('ObjectTemplate');
$objects = $this->ObjectTemplate->find('all', [
'recursive' => -1,
'fields' => ['uuid', 'name'],
'group' => ['uuid', 'name'],
]);
$allTypes = [
'attribute' => array_unique(Hash::extract(Hash::extract($this->Attribute->categoryDefinitions, '{s}.types'), '{n}.{n}')),
'object' => Hash::map($objects, '{n}.ObjectTemplate', function ($item) {
return ['id' => $item['uuid'], 'name' => sprintf('%s (%s)', $item['name'], $item['uuid'])];
})
];
$oldRemoteSetting = 0;
if (!$this->Server->data['RemoteOrg']['local']) {
$oldRemoteSetting = 1;
@ -627,6 +666,8 @@ class ServersController extends AppController
$this->set('allOrganisations', $allOrgs);
$this->set('allTags', $this->__getTags());
$this->set('allAttributeTypes', $allTypes['attribute']);
$this->set('allObjectTypes', $allTypes['object']);
$this->set('server', $s);
$this->set('id', $id);
$this->set('host_org_id', Configure::read('MISP.host_org_id'));

28
app/Lib/Tools/ServerSyncTool.php
View File

@ -219,6 +219,34 @@ class ServerSyncTool
return $this->server['Server']['id'];
}
/**
* @param string $key
* @return array
*/
public function decodeRule($key)
{
$rules = $this->server['Server'][$key];
return json_decode($rules, true);
}
/**
* @return array
*/
public function pullRules()
{
return $this->decodeRule('pull_rules');
}
/**
* @return array
*/
public function pushRules()
{
return $this->decodeRule('push_rules');
}
/**
* @param string $flag
* @return bool

23
app/Model/Event.php
View File

@ -1117,11 +1117,15 @@ class Event extends AppModel
return $data;
}
private function __prepareAttributesForSync($data, $server)
private function __prepareAttributesForSync($data,$server, $pushRules)
{
// prepare attribute for sync
if (!empty($data['Attribute'])) {
foreach ($data['Attribute'] as $key => $attribute) {
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && in_array($attribute['type'], $pushRules['type_attributes']['NOT'])) {
unset($data['Attribute'][$key]);
continue;
}
$data['Attribute'][$key] = $this->__updateAttributeForSync($attribute, $server);
if (empty($data['Attribute'][$key])) {
unset($data['Attribute'][$key]);
@ -1134,16 +1138,20 @@ class Event extends AppModel
return $data;
}
private function __prepareObjectsForSync($data, $server)
private function __prepareObjectsForSync($data,$server, $pushRules)
{
// prepare Object for sync
if (!empty($data['Object'])) {
foreach ($data['Object'] as $key => $object) {
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && in_array($object['template_uuid'], $pushRules['type_objects']['NOT'])) {
unset($data['Object'][$key]);
continue;
}
$data['Object'][$key] = $this->__updateObjectForSync($object, $server);
if (empty($data['Object'][$key])) {
unset($data['Object'][$key]);
} else {
$data['Object'][$key] = $this->__prepareAttributesForSync($data['Object'][$key], $server);
$data['Object'][$key] = $this->__prepareAttributesForSync($data['Object'][$key], $server, $pushRules);
}
}
$data['Object'] = array_values($data['Object']);
@ -1184,14 +1192,17 @@ class Event extends AppModel
}
}
}
$event['Event'] = $this->__prepareAttributesForSync($event['Event'], $server);
$event['Event'] = $this->__prepareObjectsForSync($event['Event'], $server);
$event['Event'] = $this->__prepareEventReportForSync($event['Event'], $server);
$pushRules = $this->jsonDecode($server['Server']['push_rules']);
$event['Event'] = $this->__prepareAttributesForSync($event['Event'], $server, $pushRules);
$event['Event'] = $this->__prepareObjectsForSync($event['Event'], $server, $pushRules);
$event['Event'] = $this->__prepareEventReportForSync($event['Event'], $server, $pushRules);
// Downgrade the event from connected communities to community only
if (!$server['Server']['internal'] && $event['Event']['distribution'] == 2) {
$event['Event']['distribution'] = 1;
}
return $event;
}

44
app/Model/Server.php
View File

@ -240,7 +240,7 @@ class Server extends AppModel
* @param array $server
* @param array $user
*/
private function __updatePulledEventBeforeInsert(array &$event, array $server, array $user)
private function __updatePulledEventBeforeInsert(array &$event, array $server, array $user, array $pullRules, bool &$pullRulesEmptiedEvent=false)
{
// we have an Event array
// The event came from a pull, so it should be locked.
@ -268,8 +268,14 @@ class Server extends AppModel
}
}
}
if (isset($event['Event']['Attribute'])) {
$originalCount = count($event['Event']['Attribute']);
foreach ($event['Event']['Attribute'] as $key => $attribute) {
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && in_array($attribute['type'], $pullRules['type_attributes']['NOT'])) {
unset($event['Event']['Attribute'][$key]);
continue;
}
switch ($attribute['distribution']) {
case '1':
$event['Event']['Attribute'][$key]['distribution'] = '0';
@ -287,9 +293,17 @@ class Server extends AppModel
}
}
}
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalCount > 0 && count($event['Event']['Attribute']) == 0) {
$pullRulesEmptiedEvent = true;
}
}
if (isset($event['Event']['Object'])) {
$originalObjectCount = count($event['Event']['Object']);
foreach ($event['Event']['Object'] as $i => $object) {
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && in_array($object['template_uuid'], $pullRules['type_objects']['NOT'])) {
unset($event['Event']['Object'][$i]);
continue;
}
switch ($object['distribution']) {
case '1':
$event['Event']['Object'][$i]['distribution'] = '0';
@ -299,7 +313,12 @@ class Server extends AppModel
break;
}
if (isset($object['Attribute'])) {
$originalAttributeCount = count($object['Attribute']);
foreach ($object['Attribute'] as $j => $a) {
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && in_array($a['type'], $pullRules['type_attributes']['NOT'])) {
unset($event['Event']['Object'][$i]['Attribute'][$j]);
continue;
}
switch ($a['distribution']) {
case '1':
$event['Event']['Object'][$i]['Attribute'][$j]['distribution'] = '0';
@ -317,8 +336,15 @@ class Server extends AppModel
}
}
}
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalAttributeCount > 0 && empty($event['Event']['Object'][$i]['Attribute'])) {
unset($event['Event']['Object'][$i]); // Object is empty, get rid of it
$pullRulesEmptiedEvent = true;
}
}
}
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalObjectCount > 0 && count($event['Event']['Object']) == 0) {
$pullRulesEmptiedEvent = true;
}
}
if (isset($event['Event']['EventReport'])) {
foreach ($event['Event']['EventReport'] as $key => $r) {
@ -441,9 +467,13 @@ class Server extends AppModel
if ($this->__checkIfEventIsBlockedBeforePull($event)) {
return false;
}
$this->__updatePulledEventBeforeInsert($event, $serverSync->server(), $user);
$pullRulesEmptiedEvent = false;
$this->__updatePulledEventBeforeInsert($event, $serverSync->server(), $user, $serverSync->pullRules(), $pullRulesEmptiedEvent);
if (!$this->__checkIfEventSaveAble($event)) {
$fails[$eventId] = __('Empty event detected.');
if (!$pullRulesEmptiedEvent) { // The event is empty because of the filtering rule. This is not considered a failure
$fails[$eventId] = __('Empty event detected.');
}
} else {
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force);
}
@ -5658,6 +5688,14 @@ class Server extends AppModel
'type' => 'string',
'null' => true,
],
'enable_synchronisation_filtering_on_type' => [
'level' => self::SETTING_OPTIONAL,
'description' => __('Allows server synchronisation connections to be filtered on Attribute type or Object name. Warning: This feature has can potentially cause your synchronisation partners to receive incomplete versions of the events you are propagating on behalf of others. This means that even if they would be receiving the unfiltered version through another instance, your filtered version might be the one they receive on a first-come-first-serve basis.'),
'value' => false,
'test' => 'testBoolFalse',
'type' => 'boolean',
'null' => true,
],
),
'GnuPG' => array(
'branch' => 1,

291
app/View/Elements/serverRuleElements/pull.ctp
View File

@ -1,5 +1,5 @@
<div style="display: flex; flex-direction: column;" class="server-rule-container-pull">
<?php if ($context == 'servers'): ?>
<?php if ($context == 'servers') : ?>
<div class="alert alert-primary notice-pull-rule-fetched">
<button type="button" class="close" data-dismiss="alert">&times;</button>
<i class="<?= $this->FontAwesome->getClass('spinner') ?> fa-spin"></i>
@ -13,68 +13,86 @@
<button type="button" class="close" data-dismiss="alert">&times;</button>
<?= __('Issues while trying to fetch Organisations and Tags from the remote server.') ?>
<div><strong><?= __('Reason:') ?></strong></div>
<div><pre class="reason" style="margin-bottom: 0;"></pre></div>
<div>
<pre class="reason" style="margin-bottom: 0;"></pre>
</div>
</div>
<?php endif; ?>
<?php
$tagAllowRules = [];
$tagBlockRules = [];
$orgAllowRules = [];
$orgBlockRules = [];
$ruleUrlParams = [];
if (!empty($ruleObject)) {
$tagAllowRules = $ruleObject['tags']['OR'];
$tagBlockRules = $ruleObject['tags']['NOT'];
$orgAllowRules = $ruleObject['orgs']['OR'];
$orgBlockRules = $ruleObject['orgs']['NOT'];
$ruleUrlParams = $ruleObject['url_params'];
}
$tagAllowRules = [];
$tagBlockRules = [];
$orgAllowRules = [];
$orgBlockRules = [];
$ruleUrlParams = [];
$attributeTypeBlockRules = [];
$objectTypeBlockRules = [];
if (!empty($ruleObject)) {
$tagAllowRules = $ruleObject['tags']['OR'];
$tagBlockRules = $ruleObject['tags']['NOT'];
$orgAllowRules = $ruleObject['orgs']['OR'];
$orgBlockRules = $ruleObject['orgs']['NOT'];
$ruleUrlParams = $ruleObject['url_params'];
$attributeTypeBlockRules = !empty($ruleObject['type_attributes']['NOT']) ? $ruleObject['type_attributes']['NOT'] : [];
$objectTypeBlockRules = !empty($ruleObject['type_objects']['NOT']) ? $ruleObject['type_objects']['NOT'] : [];
}
?>
<?php
echo $this->element('serverRuleElements/rules_widget', [
'scope' => 'tag',
'scopeI18n' => __('tag'),
'technique' => 'pull',
'allowEmptyOptions' => true,
'options' => $allTags,
'optionNoValue' => true,
'initAllowOptions' => $tagAllowRules,
'initBlockOptions' => $tagBlockRules
]);
echo $this->element('serverRuleElements/rules_widget', [
'scope' => 'tag',
'scopeI18n' => __('tag'),
'technique' => 'pull',
'allowEmptyOptions' => true,
'options' => $allTags,
'optionNoValue' => true,
'initAllowOptions' => $tagAllowRules,
'initBlockOptions' => $tagBlockRules
]);
?>
<div style="display: flex;">
<h4 class="bold green" style=""><?= __('AND');?></h4>
<h4 class="bold red" style="margin-left: auto;"><?= __('AND NOT');?></h4>
<h4 class="bold green" style=""><?= __('AND'); ?></h4>
<h4 class="bold red" style="margin-left: auto;"><?= __('AND NOT'); ?></h4>
</div>
<?php
echo $this->element('serverRuleElements/rules_widget', [
'scope' => 'org',
'scopeI18n' => __('org'),
'technique' => 'pull',
'allowEmptyOptions' => true,
'options' => $allOrganisations,
'optionNoValue' => true,
'initAllowOptions' => $orgAllowRules,
'initBlockOptions' => $orgBlockRules
]);
echo $this->element('serverRuleElements/rules_widget', [
'scope' => 'org',
'scopeI18n' => __('org'),
'technique' => 'pull',
'allowEmptyOptions' => true,
'options' => $allOrganisations,
'optionNoValue' => true,
'initAllowOptions' => $orgAllowRules,
'initBlockOptions' => $orgBlockRules
]);
?>
<div style="display: flex;">
<h4 class="bold green" style=""><?= __('AND');?></h4>
<h4 class="bold green" style=""><?= __('AND'); ?></h4>
</div>
<div style="display: flex; flex-direction: column;">
<div class="bold green">
<?= __('Additional sync parameters (based on the event index filters)');?>
<?= __('Additional sync parameters (based on the event index filters)'); ?>
</div>
<div style="display: flex;">
<textarea style="width:100%;" placeholder='{"timestamp": "30d"}' type="text" value="" id="urlParams" data-original-title="" title="" rows="3"
><?= !empty($ruleUrlParams) ? json_encode(h($ruleUrlParams), JSON_PRETTY_PRINT) : '' ?></textarea>
<textarea style="width:100%;" placeholder='{"timestamp": "30d"}' type="text" value="" id="urlParams" data-original-title="" title="" rows="3"><?= !empty($ruleUrlParams) ? json_encode(h($ruleUrlParams), JSON_PRETTY_PRINT) : '' ?></textarea>
</div>
</div>
</div>
<?php
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type'))) {
echo $this->element('serverRuleElements/rules_filtering_type', [
'technique' => 'pull',
'allowEmptyOptions' => true,
'allAttributeTypes' => $allAttributeTypes,
'attributeTypeBlockRules' => $attributeTypeBlockRules,
'allObjectTypes' => $allObjectTypes,
'objectTypeBlockRules' => $objectTypeBlockRules,
]);
}
?>
</div>
<?php
echo $this->element('genericElements/assetLoader', array(
@ -95,104 +113,111 @@ echo $this->element('genericElements/assetLoader', array(
?>
<script>
var pullRemoteRules404Error = '<?= __('Connection error or the remote version is not supporting remote filter lookups (v2.4.142+). Make sure that the remote instance is accessible and that it is up to date.') ?>'
var cm;
$(function() {
var serverID = "<?= isset($id) ? $id : '' ?>"
<?php if ($context == 'servers'): ?>
addPullFilteringRulesToPicker()
<?php endif; ?>
setupCodeMirror()
var pullRemoteRules404Error = '<?= __('Connection error or the remote version is not supporting remote filter lookups (v2.4.142+). Make sure that the remote instance is accessible and that it is up to date.') ?>'
var cm;
$(function() {
var serverID = "<?= isset($id) ? $id : '' ?>"
<?php if ($context == 'servers') : ?>
addPullFilteringRulesToPicker()
<?php endif; ?>
setupCodeMirror()
<?php if (empty($attributeTypeBlockRules) && empty($objectTypeBlockRules)) : ?>
$('div.server-rule-container-pull .type-filtering-subcontainer').hide()
<?php else : ?>
$('div.server-rule-container-pull #type-filtering-cb').prop('checked', true)
$('div.server-rule-container-pull #type-filtering-notice-cb').prop('checked', true)
$('div.server-rule-container-pull .type-filtering-container').show()
<?php endif; ?>
function addPullFilteringRulesToPicker() {
var $rootContainer = $('div.server-rule-container-pull')
var $pickerTags = $rootContainer.find('select.rules-select-picker-tag')
var $pickerOrgs = $rootContainer.find('select.rules-select-picker-org')
if (serverID !== "") {
$pickerOrgs.parent().children().prop('disabled', true)
$pickerTags.parent().children().prop('disabled', true)
getPullFilteringRules(
function(data) {
addOptions($pickerTags, data.tags)
addOptions($pickerOrgs, data.organisations)
$('div.notice-pull-rule-fetched.alert-success').show()
},
function(errorMessage) {
var regex = /Reponse was not OK\. \(HTTP code: (?<code>\d+)\)/m
var matches = errorMessage.match(regex)
if (matches !== null) {
if (matches.groups !== undefined && matches.groups.code !== undefined) {
errorMessage += '\n ↳ ' + pullRemoteRules404Error
function addPullFilteringRulesToPicker() {
var $rootContainer = $('div.server-rule-container-pull')
var $pickerTags = $rootContainer.find('select.rules-select-picker-tag')
var $pickerOrgs = $rootContainer.find('select.rules-select-picker-org')
if (serverID !== "") {
$pickerOrgs.parent().children().prop('disabled', true)
$pickerTags.parent().children().prop('disabled', true)
getPullFilteringRules(
function(data) {
addOptions($pickerTags, data.tags)
addOptions($pickerOrgs, data.organisations)
$('div.notice-pull-rule-fetched.alert-success').show()
},
function(errorMessage) {
var regex = /Reponse was not OK\. \(HTTP code: (?<code>\d+)\)/m
var matches = errorMessage.match(regex)
if (matches !== null) {
if (matches.groups !== undefined && matches.groups.code !== undefined) {
errorMessage += '\n ↳ ' + pullRemoteRules404Error
}
}
}
$('div.notice-pull-rule-fetched.alert-warning').show().find('.reason').text(errorMessage)
$pickerTags.parent().remove()
$pickerOrgs.parent().remove()
$rootContainer.find('.freetext-button-toggle-tag').collapse('show').remove()
$rootContainer.find('.freetext-button-toggle-org').collapse('show').remove()
$rootContainer.find('.collapse-freetext-tag').removeClass('collapse')
$rootContainer.find('.collapse-freetext-org').removeClass('collapse')
},
function() {
$('div.notice-pull-rule-fetched.alert-primary').hide()
$pickerTags.parent().children().prop('disabled', false).trigger('chosen:updated')
$pickerOrgs.parent().children().prop('disabled', false).trigger('chosen:updated')
},
)
} else {
$('div.notice-pull-rule-fetched.alert-warning').show().find('.reason').text('<?= __('The server must first be saved in order to fetch remote synchronisation rules.') ?>')
$pickerTags.parent().remove()
$pickerOrgs.parent().remove()
$('div.notice-pull-rule-fetched.alert-primary').hide()
}
}
function getPullFilteringRules(callback, failCallback, alwaysCallback) {
$.getJSON('/servers/queryAvailableSyncFilteringRules/' + serverID, function(availableRules) {
if (availableRules.error.length == 0) {
callback(availableRules.data)
$('div.notice-pull-rule-fetched.alert-warning').show().find('.reason').text(errorMessage)
$pickerTags.parent().remove()
$pickerOrgs.parent().remove()
$rootContainer.find('.freetext-button-toggle-tag').collapse('show').remove()
$rootContainer.find('.freetext-button-toggle-org').collapse('show').remove()
$rootContainer.find('.collapse-freetext-tag').removeClass('collapse')
$rootContainer.find('.collapse-freetext-org').removeClass('collapse')
},
function() {
$('div.notice-pull-rule-fetched.alert-primary').hide()
$pickerTags.parent().children().prop('disabled', false).trigger('chosen:updated')
$pickerOrgs.parent().children().prop('disabled', false).trigger('chosen:updated')
},
)
} else {
failCallback(availableRules.error)
$('div.notice-pull-rule-fetched.alert-warning').show().find('.reason').text('<?= __('The server must first be saved in order to fetch remote synchronisation rules.') ?>')
$pickerTags.parent().remove()
$pickerOrgs.parent().remove()
$('div.notice-pull-rule-fetched.alert-primary').hide()
}
})
.always(function() {
alwaysCallback()
})
}
function addOptions($select, data) {
data.forEach(function(entry) {
$select.append($('<option/>', {
value: entry,
text : entry
}));
});
}
function setupCodeMirror() {
var cmOptions = {
mode: "application/json",
theme:'default',
gutters: ["CodeMirror-lint-markers"],
lint: true,
lineNumbers: true,
indentUnit: 4,
showCursorWhenSelecting: true,
lineWrapping: true,
autoCloseBrackets: true,
}
cm = CodeMirror.fromTextArea(document.getElementById('urlParams'), cmOptions);
cm.on("keyup", function (cm, event) {
$('#urlParams').val(cm.getValue())
});
}
})
function getPullFilteringRules(callback, failCallback, alwaysCallback) {
$.getJSON('/servers/queryAvailableSyncFilteringRules/' + serverID, function(availableRules) {
if (availableRules.error.length == 0) {
callback(availableRules.data)
} else {
failCallback(availableRules.error)
}
})
.always(function() {
alwaysCallback()
})
}
function addOptions($select, data) {
data.forEach(function(entry) {
$select.append($('<option/>', {
value: entry,
text: entry
}));
});
}
function setupCodeMirror() {
var cmOptions = {
mode: "application/json",
theme: 'default',
gutters: ["CodeMirror-lint-markers"],
lint: true,
lineNumbers: true,
indentUnit: 4,
showCursorWhenSelecting: true,
lineWrapping: true,
autoCloseBrackets: true,
}
cm = CodeMirror.fromTextArea(document.getElementById('urlParams'), cmOptions);
cm.on("keyup", function(cm, event) {
$('#urlParams').val(cm.getValue())
});
}
})
</script>
<style>
div.server-rule-container-pull .CodeMirror {
height: 100px;
width: 100%;
border: 1px solid #ddd;
}
div.server-rule-container-pull .CodeMirror {
height: 100px;
width: 100%;
border: 1px solid #ddd;
}
</style>

32
app/View/Elements/serverRuleElements/push.ctp
View File

@ -4,11 +4,15 @@
$tagBlockRules = [];
$orgAllowRules = [];
$orgBlockRules = [];
$attributeTypeBlockRules = [];
$objectTypeBlockRules = [];
if (!empty($ruleObject)) {
$tagAllowRules = mapIDsToObject($allTags, $ruleObject['tags']['OR']);
$tagBlockRules = mapIDsToObject($allTags, $ruleObject['tags']['NOT']);
$orgAllowRules = mapIDsToObject($allOrganisations, $ruleObject['orgs']['OR']);
$orgBlockRules = mapIDsToObject($allOrganisations, $ruleObject['orgs']['NOT']);
$attributeTypeBlockRules = !empty($ruleObject['type_attributes']['NOT']) ? $ruleObject['type_attributes']['NOT'] : [];
$objectTypeBlockRules = !empty($ruleObject['type_objects']['NOT']) ? $ruleObject['type_objects']['NOT'] : [];
}
function mapIDsToObject($data, $ids) {
$result = [];
@ -58,5 +62,33 @@
'disableFreeText' => true
]);
?>
<?php
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type'))) {
echo $this->element('serverRuleElements/rules_filtering_type', [
'technique' => 'push',
'allowEmptyOptions' => true,
'allAttributeTypes' => $allAttributeTypes,
'attributeTypeBlockRules' => $attributeTypeBlockRules,
'allObjectTypes' => $allObjectTypes,
'objectTypeBlockRules' => $objectTypeBlockRules,
]);
}
?>
<div style="height: 50px;"></div>
</div>
<script>
var pullRemoteRules404Error = '<?= __('Connection error or the remote version is not supporting remote filter lookups (v2.4.142+). Make sure that the remote instance is accessible and that it is up to date.') ?>'
var cm;
$(function() {
var serverID = "<?= isset($id) ? $id : '' ?>"
<?php if (empty($attributeTypeBlockRules) && empty($objectTypeBlockRules)) : ?>
$('div.server-rule-container-push .type-filtering-subcontainer').hide()
<?php else : ?>
$('div.server-rule-container-push #type-filtering-cb').prop('checked', true)
$('div.server-rule-container-push #type-filtering-notice-cb').prop('checked', true)
$('div.server-rule-container-push .type-filtering-container').show()
<?php endif; ?>
})
</script>

57
app/View/Elements/serverRuleElements/rules_filtering_type.ctp Normal file
View File

@ -0,0 +1,57 @@
<div id="eventreport_div">
<span class="report-title-section">
<label class="checkbox">
<input id="type-filtering-cb" type="checkbox" onclick="$('div.server-rule-container-<?= $technique ?> .type-filtering-container').toggle()"><?= __('Type filtering') ?>
</label>
</span>
<div class="type-filtering-container hidden">
<div class="alert alert-error">
<button type="button" class="close" data-dismiss="alert">&times;</button>
<strong><?= __('Warning!') ?></strong>
<?= __('Use this feature only if you know exactly what you are doing as it might introduce unwanted behaviour:') ?>
<ul>
<li><?= __('This instance will potentially receive incomplete events (missing the filtered-out types)') ?></li>
<li><?= __('If later on you were to decide to have the previously filtered types included, the only way for this instance to receive them is to completely delete the affected events, as a full sync is needed') ?></li>
<li><?= __('Any instances synchronising with this instances will also receive incomplete events') ?></li>
</ul>
<strong><?= __('Any instance being synchronised with this one will also be affected by these shortcomings!') ?></strong>
<label class="checkbox">
<input id="type-filtering-notice-cb" type="checkbox" onclick="$('div.server-rule-container-<?= $technique ?> .type-filtering-subcontainer').toggle()"><?= __('I understand the caveats mentioned above resulting from the use of these filters') ?>
</label>
</div>
<div class="type-filtering-subcontainer" style="display: flex; flex-direction: column;">
<div style="display: flex;">
<h4 class="bold green" style=""></h4>
<h4 class="bold red" style="margin-left: auto;"><?= __('AND NOT'); ?></h4>
</div>
<?php
echo $this->element('serverRuleElements/rules_widget', [
'scope' => 'type_attributes',
'scopeI18n' => __('Attribute Types'),
'technique' => $technique,
'allowEmptyOptions' => true,
'options' => $allAttributeTypes,
'optionNoValue' => true,
'initAllowOptions' => [],
'initBlockOptions' => $attributeTypeBlockRules,
'disableAllow' => true,
'disableFreeText' => true,
]);
?>
<?php
echo $this->element('serverRuleElements/rules_widget', [
'scope' => 'type_objects',
'scopeI18n' => __('Object Types'),
'technique' => $technique,
'allowEmptyOptions' => true,
'options' => $allObjectTypes,
'optionNoValue' => false,
'initAllowOptions' => [],
'initBlockOptions' => $objectTypeBlockRules,
'disableAllow' => true,
'disableFreeText' => true,
]);
?>
</div>
</div>
</div>

30
app/View/Elements/serverRuleElements/rules_widget.ctp
View File

@ -4,6 +4,7 @@ $pickerDisplayed = false;
?>
<div>
<div style="display: flex;" class="rules-widget-container container-seed-<?= $seed ?> scope-<?= Inflector::pluralize(h($scope)) ?>" data-funname="initRuleWidgetPicker<?= $seed ?>">
<?php if (empty($disableAllow)): ?>
<div style="flex-grow: 1;">
<div class="bold green" style="display: flex; align-items: center;">
<?= __('Allowed %s (OR)', Inflector::pluralize(h($scopeI18n)));?>
@ -28,11 +29,13 @@ $pickerDisplayed = false;
<?php endforeach; ?>
</select>
</div>
<?php endif; ?>
<div style="display: flex; margin: 0 0.5em; flex-shrink: 1; padding-top: 20px;">
<div style="display: flex; flex-direction: column;">
<?php if(!empty($options) || $allowEmptyOptions): ?>
<?php $pickerDisplayed = true; ?>
<div class="input-prepend input-append">
<?php if (empty($disableAllow)): ?>
<button
class="btn"
type="button"
@ -43,19 +46,31 @@ $pickerDisplayed = false;
>
<i class="<?= $this->FontAwesome->getClass('caret-left') ?>"></i>
</button>
<?php endif; ?>
<select
class="rules-select-picker rules-select-picker-<?= h($scope) ?>"
multiple
placeholder="<?= sprintf('%s name', h($scopeI18n)) ?>"
>
<?php foreach($options as $option): ?>
<?php if(is_array($option)): ?>
<?php foreach($options as $optGroup => $option): ?>
<?php if(!is_numeric($optGroup) && is_array($option)): ?>
<optgroup label="<?= h($optGroup) ?>">
<?php foreach($option as $subOption): ?>
<?php if(is_array($subOption)): ?>
<option value="<?= !empty($optionNoValue) ? h($subOption['name']) : h($subOption['id']) ?>"><?= h($subOption['name']) ?></option>
<?php else: ?>
<option value="<?= h($subOption) ?>"><?= h($subOption) ?></option>
<?php endif; ?>
<?php endforeach; ?>
</optgroup>
<?php elseif(is_array($option)): ?>
<option value="<?= !empty($optionNoValue) ? h($option['name']) : h($option['id']) ?>"><?= h($option['name']) ?></option>
<?php else: ?>
<option value="<?= h($option) ?>"><?= h($option) ?></option>
<?php endif; ?>
<?php endforeach; ?>
</select>
<?php if (empty($disableBlock)): ?>
<button
class="btn"
type="button"
@ -66,6 +81,7 @@ $pickerDisplayed = false;
>
<i class="<?= $this->FontAwesome->getClass('caret-right') ?>"></i>
</button>
<?php endif; ?>
</div>
<?php endif; ?>
<?php if(!isset($disableFreeText) || !$disableFreeText): ?>
@ -110,6 +126,7 @@ $pickerDisplayed = false;
<?php endif; ?>
</div>
</div>
<?php if (empty($disableBlock)): ?>
<div style="flex-grow: 1;">
<div class="bold red" style="display: flex; align-items: center;">
<?php echo __('Blocked %s (AND NOT)', Inflector::pluralize(h($scopeI18n)));?>
@ -134,14 +151,17 @@ $pickerDisplayed = false;
<?php endforeach; ?>
</select>
</div>
<?php endif; ?>
</div>
</div>
<script>
function initRuleWidgetPicker<?= $seed ?>() {
var $baseContainer = $('.container-seed-<?= $seed ?>');
$baseContainer.find('select.rules-select-picker').chosen({
placeholder_text_multiple: "<?= __('Select some %s', Inflector::humanize(Inflector::pluralize(h($scopeI18n)))); ?>"
var $select = $baseContainer.find('select.rules-select-picker')
$select.chosen({
placeholder_text_multiple: "<?= __('Select some %s', Inflector::humanize(Inflector::pluralize(h($scopeI18n)))); ?>",
width: $select.is(":visible") ? undefined : 220,
})
$baseContainer.find('select.rules-select-data').keydown(function(evt) {
var $select = $(this)
@ -194,7 +214,7 @@ function handleFreetextButtonClick(targetClass, clicked) {
function handlePickerButtonClick(targetClass, clicked) {
var $select = $(clicked).parent().find('select');
var values = $select.val()
$select.children().each(function() {
$select.find('option').each(function() {
if (values.includes($(this).val())) {
var $target = $select.closest('.rules-widget-container').find('select.' + targetClass)
moveItemToSelect($target, $(this))

17
app/View/Servers/edit.ctp
View File

@ -157,12 +157,20 @@
<span id="push_tags_NOT" style="display:none;"><?php echo __('Events with the following tags blocked: ');?><span id="push_tags_NOT_text" style="color:red;"></span><br /></span>
<span id="push_orgs_OR" style="display:none;"><?php echo __('Events with the following organisations allowed: ');?><span id="push_orgs_OR_text" style="color:green;"></span><br /></span>
<span id="push_orgs_NOT" style="display:none;"><?php echo __('Events with the following organisations blocked: ');?><span id="push_orgs_NOT_text" style="color:red;"></span><br /></span>
<?php if(!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type'))): ?>
<span id="push_type_attributes_NOT" style="display:none;"><?php echo __('Attributes of the following types blocked: ');?><span id="push_type_attributes_NOT_text" style="color:red;"></span><br /></span>
<span id="push_type_objects_NOT" style="display:none;"><?php echo __('Objects of the following uuids blocked: ');?><span id="push_type_objects_NOT_text" style="color:red;"></span><br /></span>
<?php endif; ?>
<span id="push_modify" class="btn btn-inverse" style="line-height:10px; padding: 4px 4px;"><?php echo __('Modify');?></span><br /><br />
<b><?php echo __('Pull rules:');?></b><br />
<span id="pull_tags_OR" style="display:none;"><?php echo __('Events with the following tags allowed: ');?><span id="pull_tags_OR_text" style="color:green;"></span><br /></span>
<span id="pull_tags_NOT" style="display:none;"><?php echo __('Events with the following tags blocked: ');?><span id="pull_tags_NOT_text" style="color:red;"></span><br /></span>
<span id="pull_orgs_OR" style="display:none;"><?php echo __('Events with the following organisations allowed: ');?><span id="pull_orgs_OR_text" style="color:green;"></span><br /></span>
<span id="pull_orgs_NOT" style="display:none;"><?php echo __('Events with the following organisations blocked: ');?><span id="pull_orgs_NOT_text" style="color:red;"></span><br /></span>
<?php if(!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type'))): ?>
<span id="pull_type_attributes_NOT" style="display:none;"><?php echo __('Attributes of the following types blocked: ');?><span id="pull_type_attributes_NOT_text" style="color:red;"></span><br /></span>
<span id="pull_type_objects_NOT" style="display:none;"><?php echo __('Objects of the following uuids blocked: ');?><span id="pull_type_objects_NOT_text" style="color:red;"></span><br /></span>
<?php endif; ?>
<span id="pull_url_params" style="display:none;"><?php echo __('Additional parameters: ');?><span id="pull_url_params_text" style="color:green;"></span><br /></span>
<span id="pull_modify" class="btn btn-inverse" style="line-height:10px; padding: 4px 4px;"><?php echo __('Modify');?></span><br /><br />
<?php
@ -245,18 +253,23 @@ var formInfoValues = {
var rules = {
"push": {
"tags": {"OR":[], "NOT":[]},
"orgs": {"OR":[], "NOT":[]}
"orgs": {"OR":[], "NOT":[]},
"type_attributes": {"NOT":[]},
"type_objects": {"NOT":[]},
},
"pull": {
"tags": {"OR":[], "NOT":[]},
"orgs": {"OR":[], "NOT":[]},
"type_attributes": {"NOT":[]},
"type_objects": {"NOT":[]},
"url_params": ""
}
};
var validOptions = ['pull', 'push'];
var validFields = ['tags', 'orgs'];
var validFields = ['tags', 'orgs', 'type_attributes', 'type_objects'];
var tags = <?php echo json_encode($allTags); ?>;
var orgs = <?php echo json_encode($allOrganisations); ?>;
var type_objects = <?php echo json_encode($allObjectTypes); ?>;
var delete_cert = false;
var delete_client_cert = false;
var host_org_id = "<?php echo h($host_org_id); ?>";

3
app/View/Servers/index.ctp
View File

@ -42,6 +42,9 @@ foreach ($servers as $server):
$rules['pull'] = json_decode($server['Server']['pull_rules'], true);
$syncOptions = array('pull', 'push');
$fieldOptions = array('tags', 'orgs');
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type'))) {
$fieldOptions = array_merge($fieldOptions, ['type_attributes', 'type_objects']);
}
$typeOptions = array('OR' => array('colour' => 'green', 'text' => 'allowed'), 'NOT' => array('colour' => 'red', 'text' => 'blocked'));
$ruleDescription = array('pull' => '', 'push' => '');
foreach ($syncOptions as $syncOption) {

47
app/webroot/js/misp.js
View File

@ -3549,7 +3549,7 @@ function convertServerFilterRules(rules) {
rules[type] = JSON.parse($(container).val());
} else {
if (type === 'pull') {
rules[type] = {"tags": {"OR": [], "NOT": []}, "orgs": {"OR": [], "NOT": []}, "url_params": ""}
rules[type] = {"tags": {"OR": [], "NOT": []}, "orgs": {"OR": [], "NOT": []}, "type_attributes": {"NOT": []}, "type_objects": {"NOT": []}, "url_params": ""}
} else {
rules[type] = {"tags": {"OR": [], "NOT": []}, "orgs": {"OR": [], "NOT": []}}
}
@ -3563,26 +3563,35 @@ function serverRuleUpdate() {
var statusOptions = ["OR", "NOT"];
validOptions.forEach(function(type) {
validFields.forEach(function(field) {
if (type === 'push') {
var indexedList = {};
window[field].forEach(function(item) {
indexedList[item.id] = item.name;
});
var indexedList = {};
if (type === 'push' || field == 'type_objects') {
if (window[field] !== undefined) {
window[field].forEach(function(item) {
indexedList[item.id] = item.name;
});
}
}
statusOptions.forEach(function(status) {
if (rules[type][field][status].length > 0) {
$('#' + type + '_' + field + '_' + status).show();
var t = '';
rules[type][field][status].forEach(function(item) {
if (t.length > 0) t += ', ';
if (type === 'pull') t += item;
else {
t += indexedList[item] !== undefined ? indexedList[item] : item;
}
});
$('#' + type + '_' + field + '_' + status + '_text').text(t);
} else {
$('#' + type + '_' + field + '_' + status).hide();
if (rules[type][field] !== undefined && rules[type][field][status] !== undefined) {
if (rules[type][field][status].length > 0) {
$('#' + type + '_' + field + '_' + status).show();
var t = '';
rules[type][field][status].forEach(function(item) {
if (t.length > 0) t += ', ';
if (type === 'pull') {
if (indexedList[item] !== undefined) {
t += indexedList[item];
} else {
t += item;
}
} else {
t += indexedList[item] !== undefined ? indexedList[item] : item;
}
});
$('#' + type + '_' + field + '_' + status + '_text').text(t);
} else {
$('#' + type + '_' + field + '_' + status).hide();
}
}
});
});