mirror of https://github.com/MISP/MISP
new: [sighting sync] blocklisting added
- block organisations' sightings from being created / pulled - Added a new option to the restsearch of sightings too which this feature uses if available - if it isn't, the system will block the insertion on the beforeValidate() level - Outcome of the JTAN hackathon on 04.04.2024 in Luxembourgpull/9665/head
parent
31a2507fb4
commit
ef39b8959e
|
@ -0,0 +1,47 @@
|
|||
<?php
|
||||
App::uses('AppController', 'Controller');
|
||||
|
||||
class SightingBlocklistsController extends AppController
|
||||
{
|
||||
public $components = array('Session', 'RequestHandler', 'BlockList');
|
||||
|
||||
public function beforeFilter()
|
||||
{
|
||||
parent::beforeFilter();
|
||||
if (!$this->_isSiteAdmin()) {
|
||||
$this->redirect('/');
|
||||
}
|
||||
if (Configure::check('MISP.enableSightingBlocklisting') && !Configure::read('MISP.enableSightingBlocklisting') !== false) {
|
||||
$this->Flash->info(__('Sighting BlockListing is not currently enabled on this instance.'));
|
||||
$this->redirect('/');
|
||||
}
|
||||
}
|
||||
|
||||
public $paginate = array(
|
||||
'limit' => 60,
|
||||
'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page.
|
||||
'order' => array(
|
||||
'SightingBlocklist.created' => 'DESC'
|
||||
),
|
||||
);
|
||||
|
||||
public function index()
|
||||
{
|
||||
return $this->BlockList->index($this->_isRest());
|
||||
}
|
||||
|
||||
public function add()
|
||||
{
|
||||
return $this->BlockList->add($this->_isRest());
|
||||
}
|
||||
|
||||
public function edit($id)
|
||||
{
|
||||
return $this->BlockList->edit($this->_isRest(), $id);
|
||||
}
|
||||
|
||||
public function delete($id)
|
||||
{
|
||||
return $this->BlockList->delete($this->_isRest(), $id);
|
||||
}
|
||||
}
|
|
@ -304,12 +304,24 @@ class ServerSyncTool
|
|||
*/
|
||||
public function fetchSightingsForEvents(array $eventUuids)
|
||||
{
|
||||
return $this->post('/sightings/restSearch/event', [
|
||||
$SightingBlocklist = ClassRegistry::init('SightingBlocklist');
|
||||
$blocked_sightings = $SightingBlocklist->find('column', [
|
||||
'recursive' => -1,
|
||||
'fields' => ['org_uuid']
|
||||
]);
|
||||
foreach ($blocked_sightings as $k => $uuid) {
|
||||
$blocked_sightings[$k] = '!' . $uuid;
|
||||
}
|
||||
$postParams = [
|
||||
'returnFormat' => 'json',
|
||||
'last' => 0, // fetch all
|
||||
'includeUuid' => true,
|
||||
'uuid' => $eventUuids,
|
||||
])->json()['response'];
|
||||
];
|
||||
if (!empty($blocked_sightings)) {
|
||||
$postParams['org_id'] = $blocked_sightings;
|
||||
}
|
||||
return $this->post('/sightings/restSearch/event', $postParams)->json()['response'];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -91,7 +91,7 @@ class AppModel extends Model
|
|||
105 => false, 106 => false, 107 => false, 108 => false, 109 => false, 110 => false,
|
||||
111 => false, 112 => false, 113 => true, 114 => false, 115 => false, 116 => false,
|
||||
117 => false, 118 => false, 119 => false, 120 => false, 121 => false, 122 => false,
|
||||
123 => false,
|
||||
123 => false, 124 => false,
|
||||
);
|
||||
|
||||
const ADVANCED_UPDATES_DESCRIPTION = array(
|
||||
|
@ -2164,6 +2164,18 @@ class AppModel extends Model
|
|||
$sqlArray[] = 'ALTER TABLE `opinions` MODIFY `modified` datetime NOT NULL;';
|
||||
$sqlArray[] = 'ALTER TABLE `relationships` MODIFY `modified` datetime NOT NULL;';
|
||||
break;
|
||||
case 124:
|
||||
$sqlArray[] = 'CREATE TABLE IF NOT EXISTS `sighting_blocklists` (
|
||||
`id` int(11) NOT NULL AUTO_INCREMENT,
|
||||
`org_uuid` varchar(40) COLLATE utf8_bin NOT NULL,
|
||||
`created` datetime NOT NULL,
|
||||
`org_name` varchar(255) COLLATE utf8_bin NOT NULL,
|
||||
`comment` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci,
|
||||
PRIMARY KEY (`id`),
|
||||
INDEX `org_uuid` (`org_uuid`),
|
||||
INDEX `org_name` (`org_name`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;';
|
||||
break;
|
||||
case 'fixNonEmptySharingGroupID':
|
||||
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
|
||||
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
|
||||
|
|
|
@ -25,6 +25,8 @@ class Sighting extends AppModel
|
|||
|
||||
public $recursive = -1;
|
||||
|
||||
private $__blockedOrgs = null;
|
||||
|
||||
public $actsAs = array(
|
||||
'Containable',
|
||||
);
|
||||
|
@ -72,6 +74,16 @@ class Sighting extends AppModel
|
|||
} else {
|
||||
$this->data['Sighting']['uuid'] = strtolower($this->data['Sighting']['uuid']);
|
||||
}
|
||||
if ($this->__blockedOrgs === null) {
|
||||
$SightingBlocklist = ClassRegistry::init('SightingBlocklist');
|
||||
$this->__blockedOrgs = $SightingBlocklist->find('column', [
|
||||
'recursive' => -1,
|
||||
'fields' => ['org_uuid']
|
||||
]);
|
||||
}
|
||||
if (!empty($this->data['Sighting']['org_uuid']) && in_array($this->data['Sighting']['org_uuid'], $this->__blockedOrgs)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
@ -1413,20 +1425,17 @@ class Sighting extends AppModel
|
|||
$this->logException("Could not fetch event IDs from server {$serverSync->server()['Server']['name']}", $e);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Remove events from list that do not have published sightings.
|
||||
foreach ($remoteEvents as $k => $remoteEvent) {
|
||||
if ($remoteEvent['sighting_timestamp'] == 0) {
|
||||
unset($remoteEvents[$k]);
|
||||
}
|
||||
}
|
||||
|
||||
// Downloads sightings just from events that exists locally and remote sighting_timestamp is newer than local.
|
||||
$localEvents = $this->Event->find('list', [
|
||||
'fields' => ['Event.uuid', 'Event.sighting_timestamp'],
|
||||
'conditions' => (count($remoteEvents) > 10000) ? [] : ['Event.uuid' => array_column($remoteEvents, 'uuid')],
|
||||
]);
|
||||
|
||||
$eventUuids = [];
|
||||
foreach ($remoteEvents as $remoteEvent) {
|
||||
if (isset($localEvents[$remoteEvent['uuid']]) && $localEvents[$remoteEvent['uuid']] < $remoteEvent['sighting_timestamp']) {
|
||||
|
@ -1434,12 +1443,11 @@ class Sighting extends AppModel
|
|||
}
|
||||
}
|
||||
unset($remoteEvents, $localEvents);
|
||||
|
||||
if (empty($eventUuids)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
$this->removeFetched($serverSync->serverId(), $eventUuids);
|
||||
//$this->removeFetched($serverSync->serverId(), $eventUuids);
|
||||
if (empty($eventUuids)) {
|
||||
return 0;
|
||||
}
|
||||
|
@ -1472,7 +1480,6 @@ class Sighting extends AppModel
|
|||
$this->logException("Failed to download sightings from remote server {$serverSync->server()['Server']['name']}.", $e);
|
||||
continue;
|
||||
}
|
||||
|
||||
$sightingsToSave = [];
|
||||
foreach ($sightings as $sighting) {
|
||||
$sighting = $sighting['Sighting'];
|
||||
|
|
|
@ -0,0 +1,167 @@
|
|||
<?php
|
||||
App::uses('AppModel', 'Model');
|
||||
|
||||
class SightingBlocklist extends AppModel
|
||||
{
|
||||
public $useTable = 'sighting_blocklists';
|
||||
|
||||
public $recursive = -1;
|
||||
|
||||
public $actsAs = [
|
||||
'AuditLog',
|
||||
'SysLogLogable.SysLogLogable' => array( // TODO Audit, logable
|
||||
'userModel' => 'User',
|
||||
'userKey' => 'user_id',
|
||||
'change' => 'full'),
|
||||
'Containable',
|
||||
];
|
||||
public $blocklistFields = ['org_uuid', 'comment', 'org_name'];
|
||||
|
||||
public $blocklistTarget = 'org';
|
||||
|
||||
private $blockedCache = [];
|
||||
|
||||
public $validate = array(
|
||||
'org_uuid' => array(
|
||||
'unique' => array(
|
||||
'rule' => 'isUnique',
|
||||
'message' => 'Organisation already blocklisted.'
|
||||
),
|
||||
'uuid' => array(
|
||||
'rule' => 'uuid',
|
||||
'message' => 'Please provide a valid RFC 4122 UUID'
|
||||
),
|
||||
)
|
||||
);
|
||||
|
||||
public function beforeValidate($options = array())
|
||||
{
|
||||
parent::beforeValidate();
|
||||
if (empty($this->data['OrgBlocklist']['id'])) {
|
||||
$this->data['OrgBlocklist']['date_created'] = date('Y-m-d H:i:s');
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public function afterDelete()
|
||||
{
|
||||
parent::afterDelete();
|
||||
if (!empty($this->data['OrgBlocklist']['org_uuid'])) {
|
||||
$this->cleanupBlockedCount($this->data['OrgBlocklist']['org_uuid']);
|
||||
}
|
||||
}
|
||||
|
||||
public function afterFind($results, $primary = false)
|
||||
{
|
||||
foreach ($results as $k => $result) {
|
||||
if (isset($result['OrgBlocklist']['org_uuid'])) {
|
||||
$results[$k]['OrgBlocklist']['blocked_data'] = $this->getBlockedData($result['OrgBlocklist']['org_uuid']);
|
||||
}
|
||||
}
|
||||
return $results;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $eventArray
|
||||
*/
|
||||
public function removeBlockedEvents(array &$eventArray)
|
||||
{
|
||||
$blocklistHits = $this->find('column', array(
|
||||
'conditions' => array('OrgBlocklist.org_uuid' => array_unique(array_column($eventArray, 'orgc_uuid'))),
|
||||
'fields' => array('OrgBlocklist.org_uuid'),
|
||||
));
|
||||
if (empty($blocklistHits)) {
|
||||
return;
|
||||
}
|
||||
$blocklistHits = array_flip($blocklistHits);
|
||||
foreach ($eventArray as $k => $event) {
|
||||
if (isset($blocklistHits[$event['orgc_uuid']])) {
|
||||
unset($eventArray[$k]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param int|string $orgIdOrUuid Organisation ID or UUID
|
||||
* @return bool
|
||||
*/
|
||||
public function isBlocked($orgIdOrUuid)
|
||||
{
|
||||
if (isset($this->blockedCache[$orgIdOrUuid])) {
|
||||
return $this->blockedCache[$orgIdOrUuid];
|
||||
}
|
||||
|
||||
if (is_numeric($orgIdOrUuid)) {
|
||||
$orgUuid = $this->getUUIDFromID($orgIdOrUuid);
|
||||
} else {
|
||||
$orgUuid = $orgIdOrUuid;
|
||||
}
|
||||
|
||||
$isBlocked = $this->hasAny(['OrgBlocklist.org_uuid' => $orgUuid]);
|
||||
$this->blockedCache[$orgIdOrUuid] = $isBlocked;
|
||||
return $isBlocked;
|
||||
}
|
||||
|
||||
private function getUUIDFromID($orgID)
|
||||
{
|
||||
$this->Organisation = ClassRegistry::init('Organisation');
|
||||
$orgUuid = $this->Organisation->find('first', [
|
||||
'conditions' => ['Organisation.id' => $orgID],
|
||||
'fields' => ['Organisation.uuid'],
|
||||
'recursive' => -1,
|
||||
]);
|
||||
if (empty($orgUuid)) {
|
||||
return false; // org not found by ID, so it is not blocked
|
||||
}
|
||||
$orgUuid = $orgUuid['Organisation']['uuid'];
|
||||
return $orgUuid;
|
||||
}
|
||||
|
||||
public function saveEventBlocked($orgIdOrUUID)
|
||||
{
|
||||
if (is_numeric($orgIdOrUUID)) {
|
||||
$orgcUUID = $this->getUUIDFromID($orgIdOrUUID);
|
||||
} else {
|
||||
$orgcUUID = $orgIdOrUUID;
|
||||
}
|
||||
$lastBlockTime = time();
|
||||
$redisKeyBlockAmount = "misp:blocklist_blocked_amount:{$orgcUUID}";
|
||||
$redisKeyBlockLastTime = "misp:blocklist_blocked_last_time:{$orgcUUID}";
|
||||
$redis = RedisTool::init();
|
||||
if ($redis !== false) {
|
||||
$pipe = $redis->multi(Redis::PIPELINE)
|
||||
->incr($redisKeyBlockAmount)
|
||||
->set($redisKeyBlockLastTime, $lastBlockTime);
|
||||
$pipe->exec();
|
||||
}
|
||||
}
|
||||
|
||||
private function cleanupBlockedCount($orgcUUID)
|
||||
{
|
||||
$redisKeyBlockAmount = "misp:blocklist_blocked_amount:{$orgcUUID}";
|
||||
$redisKeyBlockLastTime = "misp:blocklist_blocked_last_time:{$orgcUUID}";
|
||||
$redis = RedisTool::init();
|
||||
if ($redis !== false) {
|
||||
$pipe = $redis->multi(Redis::PIPELINE)
|
||||
->del($redisKeyBlockAmount)
|
||||
->del($redisKeyBlockLastTime);
|
||||
$pipe->exec();
|
||||
}
|
||||
}
|
||||
|
||||
public function getBlockedData($orgcUUID)
|
||||
{
|
||||
$redisKeyBlockAmount = "misp:blocklist_blocked_amount:{$orgcUUID}";
|
||||
$redisKeyBlockLastTime = "misp:blocklist_blocked_last_time:{$orgcUUID}";
|
||||
$blockData = [
|
||||
'blocked_amount' => false,
|
||||
'blocked_last_time' => false,
|
||||
];
|
||||
$redis = RedisTool::init();
|
||||
if ($redis !== false) {
|
||||
$blockData['blocked_amount'] = $redis->get($redisKeyBlockAmount);
|
||||
$blockData['blocked_last_time'] = $redis->get($redisKeyBlockLastTime);
|
||||
}
|
||||
return $blockData;
|
||||
}
|
||||
}
|
|
@ -1119,6 +1119,7 @@ $divider = '<li class="divider"></li>';
|
|||
'url' => $baseurl . '/servers/eventBlockRule',
|
||||
'text' => __('Event Block Rules')
|
||||
));
|
||||
echo $divider;
|
||||
if (Configure::read('MISP.enableEventBlocklisting') !== false) {
|
||||
echo $this->element('/genericElements/SideMenu/side_menu_link', array(
|
||||
'element_id' => 'eventBlocklistsAdd',
|
||||
|
@ -1130,6 +1131,7 @@ $divider = '<li class="divider"></li>';
|
|||
'url' => $baseurl . '/eventBlocklists',
|
||||
'text' => __('Manage Event Blocklists')
|
||||
));
|
||||
echo $divider;
|
||||
}
|
||||
if (!Configure::check('MISP.enableOrgBlocklisting') || Configure::read('MISP.enableOrgBlocklisting') !== false) {
|
||||
echo $this->element('/genericElements/SideMenu/side_menu_link', array(
|
||||
|
@ -1142,6 +1144,20 @@ $divider = '<li class="divider"></li>';
|
|||
'url' => $baseurl . '/orgBlocklists',
|
||||
'text' => __('Manage Org Blocklists')
|
||||
));
|
||||
echo $divider;
|
||||
}
|
||||
if (!Configure::check('MISP.enableSightingBlocklisting') || Configure::read('MISP.enableSightingBlocklisting') !== false) {
|
||||
echo $this->element('/genericElements/SideMenu/side_menu_link', array(
|
||||
'element_id' => 'sightingBlocklistsAdd',
|
||||
'url' => $baseurl . '/sightingBlocklists/add',
|
||||
'text' => __('Blocklists Sightings')
|
||||
));
|
||||
echo $this->element('/genericElements/SideMenu/side_menu_link', array(
|
||||
'element_id' => 'sightingBlocklists',
|
||||
'url' => $baseurl . '/sightingBlocklists',
|
||||
'text' => __('Manage Sighting Blocklists')
|
||||
));
|
||||
echo $divider;
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
|
|
@ -0,0 +1,40 @@
|
|||
<?php
|
||||
echo $this->element(
|
||||
'/genericElements/SideMenu/side_menu',
|
||||
['menuList' => 'admin', 'menuItem' => 'sightingBlocklistsAdd']
|
||||
);
|
||||
|
||||
echo $this->element('genericElements/Form/genericForm', [
|
||||
'data' => [
|
||||
'title' => __('Add Sighting Blocklist Entries'),
|
||||
'description' => __('Blocklisting an organisation prevents the creation of any sighting by that organisation on this instance as well as syncing of that organisation\'s sightings to this instance. It does not prevent a local user of the blocklisted organisation from logging in and editing or viewing data. <br/>Paste a list of all the organisation UUIDs that you want to add to the blocklist below (one per line).'),
|
||||
'fields' => [
|
||||
[
|
||||
'field' => 'uuids',
|
||||
'label' => __('UUIDs'),
|
||||
'div' => 'input clear',
|
||||
'class' => 'input-xxlarge',
|
||||
'type' => 'textarea',
|
||||
'placeholder' => __('Enter a single or a list of UUIDs'),
|
||||
],
|
||||
[
|
||||
'field' => 'org_name',
|
||||
'label' => __('Organisation name'),
|
||||
'class' => 'input-xxlarge',
|
||||
'placeholder' => __('(Optional) The organisation name that the organisation is associated with')
|
||||
],
|
||||
[
|
||||
'field' => 'comment',
|
||||
'label' => __('Comment'),
|
||||
'type' => 'textarea',
|
||||
'div' => 'input clear',
|
||||
'class' => 'input-xxlarge',
|
||||
'placeholder' => __('(Optional) Any comments you would like to add regarding this (or these) entries.')
|
||||
],
|
||||
],
|
||||
'submit' => [
|
||||
'action' => $this->request->params['action'],
|
||||
'ajaxSubmit' => 'submitGenericFormInPlace();'
|
||||
]
|
||||
]
|
||||
]);
|
|
@ -0,0 +1,41 @@
|
|||
<?php
|
||||
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'admin', 'menuItem' => 'sightingBlocklistsAdd'));
|
||||
|
||||
echo $this->element('genericElements/Form/genericForm', [
|
||||
'data' => [
|
||||
'title' => __('Edit Sighting Blocklist Entries'),
|
||||
'description' => __('Blocklisting an organisation prevents the creation of any sighting by that organisation on this instance as well as syncing of that organisation\'s sightings to this instance. It does not prevent a local user of the blocklisted organisation from logging in and editing or viewing data. <br/>Paste a list of all the organisation UUIDs that you want to add to the blocklist below (one per line).'),
|
||||
'fields' => [
|
||||
[
|
||||
'field' => 'org_uuid',
|
||||
'label' => __('UUIDs'),
|
||||
'default' => $blockEntry['SightingBlocklist']['org_uuid'],
|
||||
'div' => 'input clear',
|
||||
'class' => 'input-xxlarge',
|
||||
'type' => 'textarea',
|
||||
'disabled' => true,
|
||||
'placeholder' => __('Enter a single or a list of UUIDs')
|
||||
],
|
||||
[
|
||||
'field' => 'org_name',
|
||||
'label' => __('Organisation name'),
|
||||
'default' => $blockEntry['SightingBlocklist']['org_name'],
|
||||
'class' => 'input-xxlarge',
|
||||
'placeholder' => __('(Optional) The organisation name that the organisation is associated with')
|
||||
],
|
||||
[
|
||||
'field' => 'comment',
|
||||
'label' => __('Comment'),
|
||||
'default' => $blockEntry['SightingBlocklist']['comment'],
|
||||
'type' => 'textarea',
|
||||
'div' => 'input clear',
|
||||
'class' => 'input-xxlarge',
|
||||
'placeholder' => __('(Optional) Any comments you would like to add regarding this (or these) entries.')
|
||||
],
|
||||
],
|
||||
'submit' => [
|
||||
'action' => $this->request->params['action'],
|
||||
'ajaxSubmit' => 'submitGenericFormInPlace();'
|
||||
]
|
||||
]
|
||||
]);
|
|
@ -0,0 +1,70 @@
|
|||
<?php
|
||||
$this->set('menuData', ['menuList' => 'admin', 'menuItem' => 'sightingBlocklists']);
|
||||
echo $this->element('genericElements/IndexTable/scaffold', [
|
||||
'scaffold_data' => [
|
||||
'data' => [
|
||||
'data' => $response,
|
||||
'fields' => [
|
||||
[
|
||||
'name' => 'Id',
|
||||
'sort' => 'SightingBlocklist.id',
|
||||
'data_path' => 'SightingBlocklist.id'
|
||||
],
|
||||
[
|
||||
'name' => 'Organisation name',
|
||||
'sort' => 'SightingBlocklist.org_name',
|
||||
'data_path' => 'SightingBlocklist.org_name'
|
||||
],
|
||||
[
|
||||
'name' => 'UUID',
|
||||
'sort' => 'SightingBlocklist.org_uuid',
|
||||
'data_path' => 'SightingBlocklist.org_uuid'
|
||||
],
|
||||
[
|
||||
'name' => 'Created',
|
||||
'sort' => 'SightingBlocklist.created',
|
||||
'data_path' => 'SightingBlocklist.created',
|
||||
'element' => 'datetime'
|
||||
],
|
||||
[
|
||||
'name' => 'Comment',
|
||||
'sort' => 'SightingBlocklist.comment',
|
||||
'data_path' => 'SightingBlocklist.comment',
|
||||
'class' => 'bitwider'
|
||||
],
|
||||
[
|
||||
'name' => 'Blocked amount',
|
||||
'sort' => 'SightingBlocklist.blocked_data.blocked_amount',
|
||||
'data_path' => 'SightingBlocklist.blocked_data.blocked_amount',
|
||||
],
|
||||
[
|
||||
'name' => 'Blocked last time ',
|
||||
'sort' => 'SightingBlocklist.blocked_data.blocked_last_time',
|
||||
'data_path' => 'SightingBlocklist.blocked_data.blocked_last_time',
|
||||
'element' => 'datetime'
|
||||
],
|
||||
|
||||
],
|
||||
'title' => empty($ajax) ? __('Sighting Blocklists') : false,
|
||||
'pull' => 'right',
|
||||
'actions' => [
|
||||
[
|
||||
'url' => $baseurl . '/org_blocklists/edit',
|
||||
'url_params_data_paths' => array(
|
||||
'SightingBlocklist.id'
|
||||
),
|
||||
'icon' => 'edit',
|
||||
'title' => 'Edit Blocklist',
|
||||
],
|
||||
[
|
||||
'url' => $baseurl . '/org_blocklists/delete',
|
||||
'url_params_data_paths' => array(
|
||||
'SightingBlocklist.id'
|
||||
),
|
||||
'icon' => 'trash',
|
||||
'title' => 'Delete Blocklist',
|
||||
]
|
||||
]
|
||||
]
|
||||
]
|
||||
]);
|
Loading…
Reference in New Issue