new: [sighting sync] blocklisting added

- block organisations' sightings from being created / pulled
- Added a new option to the restsearch of sightings too which this feature uses if available
  - if it isn't, the system will block the insertion on the beforeValidate() level

- Outcome of the JTAN hackathon on 04.04.2024 in Luxembourg
pull/9665/head
iglocska 2024-04-04 12:08:22 +02:00
parent 31a2507fb4
commit ef39b8959e
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
9 changed files with 421 additions and 9 deletions

View File

@ -0,0 +1,47 @@
<?php
App::uses('AppController', 'Controller');
class SightingBlocklistsController extends AppController
{
public $components = array('Session', 'RequestHandler', 'BlockList');
public function beforeFilter()
{
parent::beforeFilter();
if (!$this->_isSiteAdmin()) {
$this->redirect('/');
}
if (Configure::check('MISP.enableSightingBlocklisting') && !Configure::read('MISP.enableSightingBlocklisting') !== false) {
$this->Flash->info(__('Sighting BlockListing is not currently enabled on this instance.'));
$this->redirect('/');
}
}
public $paginate = array(
'limit' => 60,
'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page.
'order' => array(
'SightingBlocklist.created' => 'DESC'
),
);
public function index()
{
return $this->BlockList->index($this->_isRest());
}
public function add()
{
return $this->BlockList->add($this->_isRest());
}
public function edit($id)
{
return $this->BlockList->edit($this->_isRest(), $id);
}
public function delete($id)
{
return $this->BlockList->delete($this->_isRest(), $id);
}
}

View File

@ -304,12 +304,24 @@ class ServerSyncTool
*/
public function fetchSightingsForEvents(array $eventUuids)
{
return $this->post('/sightings/restSearch/event', [
$SightingBlocklist = ClassRegistry::init('SightingBlocklist');
$blocked_sightings = $SightingBlocklist->find('column', [
'recursive' => -1,
'fields' => ['org_uuid']
]);
foreach ($blocked_sightings as $k => $uuid) {
$blocked_sightings[$k] = '!' . $uuid;
}
$postParams = [
'returnFormat' => 'json',
'last' => 0, // fetch all
'includeUuid' => true,
'uuid' => $eventUuids,
])->json()['response'];
];
if (!empty($blocked_sightings)) {
$postParams['org_id'] = $blocked_sightings;
}
return $this->post('/sightings/restSearch/event', $postParams)->json()['response'];
}
/**

View File

@ -91,7 +91,7 @@ class AppModel extends Model
105 => false, 106 => false, 107 => false, 108 => false, 109 => false, 110 => false,
111 => false, 112 => false, 113 => true, 114 => false, 115 => false, 116 => false,
117 => false, 118 => false, 119 => false, 120 => false, 121 => false, 122 => false,
123 => false,
123 => false, 124 => false,
);
const ADVANCED_UPDATES_DESCRIPTION = array(
@ -2164,6 +2164,18 @@ class AppModel extends Model
$sqlArray[] = 'ALTER TABLE `opinions` MODIFY `modified` datetime NOT NULL;';
$sqlArray[] = 'ALTER TABLE `relationships` MODIFY `modified` datetime NOT NULL;';
break;
case 124:
$sqlArray[] = 'CREATE TABLE IF NOT EXISTS `sighting_blocklists` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`org_uuid` varchar(40) COLLATE utf8_bin NOT NULL,
`created` datetime NOT NULL,
`org_name` varchar(255) COLLATE utf8_bin NOT NULL,
`comment` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci,
PRIMARY KEY (`id`),
INDEX `org_uuid` (`org_uuid`),
INDEX `org_name` (`org_name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;';
break;
case 'fixNonEmptySharingGroupID':
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';

View File

@ -25,6 +25,8 @@ class Sighting extends AppModel
public $recursive = -1;
private $__blockedOrgs = null;
public $actsAs = array(
'Containable',
);
@ -72,6 +74,16 @@ class Sighting extends AppModel
} else {
$this->data['Sighting']['uuid'] = strtolower($this->data['Sighting']['uuid']);
}
if ($this->__blockedOrgs === null) {
$SightingBlocklist = ClassRegistry::init('SightingBlocklist');
$this->__blockedOrgs = $SightingBlocklist->find('column', [
'recursive' => -1,
'fields' => ['org_uuid']
]);
}
if (!empty($this->data['Sighting']['org_uuid']) && in_array($this->data['Sighting']['org_uuid'], $this->__blockedOrgs)) {
return false;
}
return true;
}
@ -1413,20 +1425,17 @@ class Sighting extends AppModel
$this->logException("Could not fetch event IDs from server {$serverSync->server()['Server']['name']}", $e);
return 0;
}
// Remove events from list that do not have published sightings.
foreach ($remoteEvents as $k => $remoteEvent) {
if ($remoteEvent['sighting_timestamp'] == 0) {
unset($remoteEvents[$k]);
}
}
// Downloads sightings just from events that exists locally and remote sighting_timestamp is newer than local.
$localEvents = $this->Event->find('list', [
'fields' => ['Event.uuid', 'Event.sighting_timestamp'],
'conditions' => (count($remoteEvents) > 10000) ? [] : ['Event.uuid' => array_column($remoteEvents, 'uuid')],
]);
$eventUuids = [];
foreach ($remoteEvents as $remoteEvent) {
if (isset($localEvents[$remoteEvent['uuid']]) && $localEvents[$remoteEvent['uuid']] < $remoteEvent['sighting_timestamp']) {
@ -1434,12 +1443,11 @@ class Sighting extends AppModel
}
}
unset($remoteEvents, $localEvents);
if (empty($eventUuids)) {
return 0;
}
$this->removeFetched($serverSync->serverId(), $eventUuids);
//$this->removeFetched($serverSync->serverId(), $eventUuids);
if (empty($eventUuids)) {
return 0;
}
@ -1472,7 +1480,6 @@ class Sighting extends AppModel
$this->logException("Failed to download sightings from remote server {$serverSync->server()['Server']['name']}.", $e);
continue;
}
$sightingsToSave = [];
foreach ($sightings as $sighting) {
$sighting = $sighting['Sighting'];

View File

@ -0,0 +1,167 @@
<?php
App::uses('AppModel', 'Model');
class SightingBlocklist extends AppModel
{
public $useTable = 'sighting_blocklists';
public $recursive = -1;
public $actsAs = [
'AuditLog',
'SysLogLogable.SysLogLogable' => array( // TODO Audit, logable
'userModel' => 'User',
'userKey' => 'user_id',
'change' => 'full'),
'Containable',
];
public $blocklistFields = ['org_uuid', 'comment', 'org_name'];
public $blocklistTarget = 'org';
private $blockedCache = [];
public $validate = array(
'org_uuid' => array(
'unique' => array(
'rule' => 'isUnique',
'message' => 'Organisation already blocklisted.'
),
'uuid' => array(
'rule' => 'uuid',
'message' => 'Please provide a valid RFC 4122 UUID'
),
)
);
public function beforeValidate($options = array())
{
parent::beforeValidate();
if (empty($this->data['OrgBlocklist']['id'])) {
$this->data['OrgBlocklist']['date_created'] = date('Y-m-d H:i:s');
}
return true;
}
public function afterDelete()
{
parent::afterDelete();
if (!empty($this->data['OrgBlocklist']['org_uuid'])) {
$this->cleanupBlockedCount($this->data['OrgBlocklist']['org_uuid']);
}
}
public function afterFind($results, $primary = false)
{
foreach ($results as $k => $result) {
if (isset($result['OrgBlocklist']['org_uuid'])) {
$results[$k]['OrgBlocklist']['blocked_data'] = $this->getBlockedData($result['OrgBlocklist']['org_uuid']);
}
}
return $results;
}
/**
* @param array $eventArray
*/
public function removeBlockedEvents(array &$eventArray)
{
$blocklistHits = $this->find('column', array(
'conditions' => array('OrgBlocklist.org_uuid' => array_unique(array_column($eventArray, 'orgc_uuid'))),
'fields' => array('OrgBlocklist.org_uuid'),
));
if (empty($blocklistHits)) {
return;
}
$blocklistHits = array_flip($blocklistHits);
foreach ($eventArray as $k => $event) {
if (isset($blocklistHits[$event['orgc_uuid']])) {
unset($eventArray[$k]);
}
}
}
/**
* @param int|string $orgIdOrUuid Organisation ID or UUID
* @return bool
*/
public function isBlocked($orgIdOrUuid)
{
if (isset($this->blockedCache[$orgIdOrUuid])) {
return $this->blockedCache[$orgIdOrUuid];
}
if (is_numeric($orgIdOrUuid)) {
$orgUuid = $this->getUUIDFromID($orgIdOrUuid);
} else {
$orgUuid = $orgIdOrUuid;
}
$isBlocked = $this->hasAny(['OrgBlocklist.org_uuid' => $orgUuid]);
$this->blockedCache[$orgIdOrUuid] = $isBlocked;
return $isBlocked;
}
private function getUUIDFromID($orgID)
{
$this->Organisation = ClassRegistry::init('Organisation');
$orgUuid = $this->Organisation->find('first', [
'conditions' => ['Organisation.id' => $orgID],
'fields' => ['Organisation.uuid'],
'recursive' => -1,
]);
if (empty($orgUuid)) {
return false; // org not found by ID, so it is not blocked
}
$orgUuid = $orgUuid['Organisation']['uuid'];
return $orgUuid;
}
public function saveEventBlocked($orgIdOrUUID)
{
if (is_numeric($orgIdOrUUID)) {
$orgcUUID = $this->getUUIDFromID($orgIdOrUUID);
} else {
$orgcUUID = $orgIdOrUUID;
}
$lastBlockTime = time();
$redisKeyBlockAmount = "misp:blocklist_blocked_amount:{$orgcUUID}";
$redisKeyBlockLastTime = "misp:blocklist_blocked_last_time:{$orgcUUID}";
$redis = RedisTool::init();
if ($redis !== false) {
$pipe = $redis->multi(Redis::PIPELINE)
->incr($redisKeyBlockAmount)
->set($redisKeyBlockLastTime, $lastBlockTime);
$pipe->exec();
}
}
private function cleanupBlockedCount($orgcUUID)
{
$redisKeyBlockAmount = "misp:blocklist_blocked_amount:{$orgcUUID}";
$redisKeyBlockLastTime = "misp:blocklist_blocked_last_time:{$orgcUUID}";
$redis = RedisTool::init();
if ($redis !== false) {
$pipe = $redis->multi(Redis::PIPELINE)
->del($redisKeyBlockAmount)
->del($redisKeyBlockLastTime);
$pipe->exec();
}
}
public function getBlockedData($orgcUUID)
{
$redisKeyBlockAmount = "misp:blocklist_blocked_amount:{$orgcUUID}";
$redisKeyBlockLastTime = "misp:blocklist_blocked_last_time:{$orgcUUID}";
$blockData = [
'blocked_amount' => false,
'blocked_last_time' => false,
];
$redis = RedisTool::init();
if ($redis !== false) {
$blockData['blocked_amount'] = $redis->get($redisKeyBlockAmount);
$blockData['blocked_last_time'] = $redis->get($redisKeyBlockLastTime);
}
return $blockData;
}
}

View File

@ -1119,6 +1119,7 @@ $divider = '<li class="divider"></li>';
'url' => $baseurl . '/servers/eventBlockRule',
'text' => __('Event Block Rules')
));
echo $divider;
if (Configure::read('MISP.enableEventBlocklisting') !== false) {
echo $this->element('/genericElements/SideMenu/side_menu_link', array(
'element_id' => 'eventBlocklistsAdd',
@ -1130,6 +1131,7 @@ $divider = '<li class="divider"></li>';
'url' => $baseurl . '/eventBlocklists',
'text' => __('Manage Event Blocklists')
));
echo $divider;
}
if (!Configure::check('MISP.enableOrgBlocklisting') || Configure::read('MISP.enableOrgBlocklisting') !== false) {
echo $this->element('/genericElements/SideMenu/side_menu_link', array(
@ -1142,6 +1144,20 @@ $divider = '<li class="divider"></li>';
'url' => $baseurl . '/orgBlocklists',
'text' => __('Manage Org Blocklists')
));
echo $divider;
}
if (!Configure::check('MISP.enableSightingBlocklisting') || Configure::read('MISP.enableSightingBlocklisting') !== false) {
echo $this->element('/genericElements/SideMenu/side_menu_link', array(
'element_id' => 'sightingBlocklistsAdd',
'url' => $baseurl . '/sightingBlocklists/add',
'text' => __('Blocklists Sightings')
));
echo $this->element('/genericElements/SideMenu/side_menu_link', array(
'element_id' => 'sightingBlocklists',
'url' => $baseurl . '/sightingBlocklists',
'text' => __('Manage Sighting Blocklists')
));
echo $divider;
}
}
break;

View File

@ -0,0 +1,40 @@
<?php
echo $this->element(
'/genericElements/SideMenu/side_menu',
['menuList' => 'admin', 'menuItem' => 'sightingBlocklistsAdd']
);
echo $this->element('genericElements/Form/genericForm', [
'data' => [
'title' => __('Add Sighting Blocklist Entries'),
'description' => __('Blocklisting an organisation prevents the creation of any sighting by that organisation on this instance as well as syncing of that organisation\'s sightings to this instance. It does not prevent a local user of the blocklisted organisation from logging in and editing or viewing data. <br/>Paste a list of all the organisation UUIDs that you want to add to the blocklist below (one per line).'),
'fields' => [
[
'field' => 'uuids',
'label' => __('UUIDs'),
'div' => 'input clear',
'class' => 'input-xxlarge',
'type' => 'textarea',
'placeholder' => __('Enter a single or a list of UUIDs'),
],
[
'field' => 'org_name',
'label' => __('Organisation name'),
'class' => 'input-xxlarge',
'placeholder' => __('(Optional) The organisation name that the organisation is associated with')
],
[
'field' => 'comment',
'label' => __('Comment'),
'type' => 'textarea',
'div' => 'input clear',
'class' => 'input-xxlarge',
'placeholder' => __('(Optional) Any comments you would like to add regarding this (or these) entries.')
],
],
'submit' => [
'action' => $this->request->params['action'],
'ajaxSubmit' => 'submitGenericFormInPlace();'
]
]
]);

View File

@ -0,0 +1,41 @@
<?php
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'admin', 'menuItem' => 'sightingBlocklistsAdd'));
echo $this->element('genericElements/Form/genericForm', [
'data' => [
'title' => __('Edit Sighting Blocklist Entries'),
'description' => __('Blocklisting an organisation prevents the creation of any sighting by that organisation on this instance as well as syncing of that organisation\'s sightings to this instance. It does not prevent a local user of the blocklisted organisation from logging in and editing or viewing data. <br/>Paste a list of all the organisation UUIDs that you want to add to the blocklist below (one per line).'),
'fields' => [
[
'field' => 'org_uuid',
'label' => __('UUIDs'),
'default' => $blockEntry['SightingBlocklist']['org_uuid'],
'div' => 'input clear',
'class' => 'input-xxlarge',
'type' => 'textarea',
'disabled' => true,
'placeholder' => __('Enter a single or a list of UUIDs')
],
[
'field' => 'org_name',
'label' => __('Organisation name'),
'default' => $blockEntry['SightingBlocklist']['org_name'],
'class' => 'input-xxlarge',
'placeholder' => __('(Optional) The organisation name that the organisation is associated with')
],
[
'field' => 'comment',
'label' => __('Comment'),
'default' => $blockEntry['SightingBlocklist']['comment'],
'type' => 'textarea',
'div' => 'input clear',
'class' => 'input-xxlarge',
'placeholder' => __('(Optional) Any comments you would like to add regarding this (or these) entries.')
],
],
'submit' => [
'action' => $this->request->params['action'],
'ajaxSubmit' => 'submitGenericFormInPlace();'
]
]
]);

View File

@ -0,0 +1,70 @@
<?php
$this->set('menuData', ['menuList' => 'admin', 'menuItem' => 'sightingBlocklists']);
echo $this->element('genericElements/IndexTable/scaffold', [
'scaffold_data' => [
'data' => [
'data' => $response,
'fields' => [
[
'name' => 'Id',
'sort' => 'SightingBlocklist.id',
'data_path' => 'SightingBlocklist.id'
],
[
'name' => 'Organisation name',
'sort' => 'SightingBlocklist.org_name',
'data_path' => 'SightingBlocklist.org_name'
],
[
'name' => 'UUID',
'sort' => 'SightingBlocklist.org_uuid',
'data_path' => 'SightingBlocklist.org_uuid'
],
[
'name' => 'Created',
'sort' => 'SightingBlocklist.created',
'data_path' => 'SightingBlocklist.created',
'element' => 'datetime'
],
[
'name' => 'Comment',
'sort' => 'SightingBlocklist.comment',
'data_path' => 'SightingBlocklist.comment',
'class' => 'bitwider'
],
[
'name' => 'Blocked amount',
'sort' => 'SightingBlocklist.blocked_data.blocked_amount',
'data_path' => 'SightingBlocklist.blocked_data.blocked_amount',
],
[
'name' => 'Blocked last time ',
'sort' => 'SightingBlocklist.blocked_data.blocked_last_time',
'data_path' => 'SightingBlocklist.blocked_data.blocked_last_time',
'element' => 'datetime'
],
],
'title' => empty($ajax) ? __('Sighting Blocklists') : false,
'pull' => 'right',
'actions' => [
[
'url' => $baseurl . '/org_blocklists/edit',
'url_params_data_paths' => array(
'SightingBlocklist.id'
),
'icon' => 'edit',
'title' => 'Edit Blocklist',
],
[
'url' => $baseurl . '/org_blocklists/delete',
'url_params_data_paths' => array(
'SightingBlocklist.id'
),
'icon' => 'trash',
'title' => 'Delete Blocklist',
]
]
]
]
]);