MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [analyst-data] Added `locked` flag, support of orgc/org, analyst-data-blocklist and most implementation of push synchronisation - WiP

notes
Sami Mokaddem 2024-02-01 14:24:41 +01:00
parent 8cef82f1ea
commit eaf8a2b98a
No known key found for this signature in database
GPG Key ID: 164C473F627A06FA
19 changed files with 879 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
app/Controller/AnalystDataBlocklistsController.php Normal file
View File

@ -0,0 +1,103 @@
<?php
App::uses('AppController', 'Controller');
class AnalystDataBlocklistsController extends AppController
{
public $components = array('Session', 'RequestHandler', 'BlockList');
public $paginate = array(
'limit' => 60,
'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 entries <- no we won't, this is the max a user van view/page.
'order' => array(
'AnalystDataBlocklist.created' => 'DESC'
),
);
public function index()
{
$passedArgsArray = array();
$passedArgs = $this->passedArgs;
$params = array();
$validParams = array('analyst_data_uuid', 'comment', 'analyst_data_info', 'analyst_data_orgc');
foreach ($validParams as $validParam) {
if (!empty($this->params['named'][$validParam])) {
$params[$validParam] = $this->params['named'][$validParam];
}
}
if (!empty($this->params['named']['searchall'])) {
$params['AND']['OR'] = array(
'analyst_data_uuid' => $this->params['named']['searchall'],
'comment' => $this->params['named']['searchall'],
'analyst_data_info' => $this->params['named']['searchall'],
'analyst_data_orgc' => $this->params['named']['searchall']
);
}
$this->set('passedArgs', json_encode($passedArgs));
$this->set('passedArgsArray', $passedArgsArray);
return $this->BlockList->index($this->_isRest(), $params);
}
public function add()
{
$this->set('action', 'add');
return $this->BlockList->add($this->_isRest());
}
public function edit($id)
{
$this->set('action', 'edit');
return $this->BlockList->edit($this->_isRest(), $id);
}
public function delete($id)
{
if (Validation::uuid($id)) {
$entry = $this->AnalystDataBlocklist->find('first', array(
'conditions' => array('analyst_data_uuid' => $id)
));
if (empty($entry)) {
throw new NotFoundException(__('Invalid blocklist entry'));
}
$id = $entry['AnalystDataBlocklist']['id'];
}
return $this->BlockList->delete($this->_isRest(), $id);
}
public function massDelete()
{
if ($this->request->is('post') || $this->request->is('put')) {
if (!isset($this->request->data['AnalystDataBlocklist'])) {
$this->request->data = array('AnalystDataBlocklist' => $this->request->data);
}
$ids = $this->request->data['AnalystDataBlocklist']['ids'];
$analyst_data_ids = json_decode($ids, true);
if (empty($analyst_data_ids)) {
throw new NotFoundException(__('Invalid Analyst Data IDs.'));
}
$result = $this->AnalystDataBlocklist->deleteAll(array('AnalystDataBlocklist.id' => $analyst_data_ids));
if ($result) {
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('AnalystDataBlocklist', 'Deleted', $ids, $this->response->type());
} else {
$this->Flash->success('Blocklist entry removed');
$this->redirect(array('controller' => 'AnalystDataBlocklist', 'action' => 'index'));
}
} else {
$error = __('Failed to delete Analyst Data from AnalystDataBlocklist. Error: ') . PHP_EOL . h($result);
if ($this->_isRest()) {
return $this->RestResponse->saveFailResponse('AnalystDataBlocklist', 'Deleted', false, $error, $this->response->type());
} else {
$this->Flash->error($error);
$this->redirect(array('controller' => 'AnalystDataBlocklists', 'action' => 'index'));
}
}
} else {
$ids = json_decode($this->request->query('ids'), true);
if (empty($ids)) {
throw new NotFoundException(__('Invalid analyst data IDs.'));
}
$this->set('analyst_data_ids', $ids);
}
}
}

71
app/Controller/AnalystDataController.php
View File

@ -90,7 +90,7 @@ class AnalystDataController extends AppController
$this->render('add');
}
public function delete($type = 'Note', $id)
public function delete($type = 'Note', $id, $hard=false)
{
$this->__typeSelector($type);
$params = [
@ -99,6 +99,36 @@ class AnalystDataController extends AppController
if (!$canEdit) {
throw new MethodNotAllowedException(__('You are not authorised to do that.'));
}
},
'afterDelete' => function($deletedAnalystData) use ($hard) {
if (empty($hard)) {
return;
}
$type = $this->AnalystData->deduceAnalystDataType($deletedAnalystData);
$info = '- Unsupported analyst type -';
if ($type === 'Note') {
$info = $deletedAnalystData[$type]['note'];
} else if ($type === 'Opinion') {
$info = sprintf('%s/100 :: %s', $deletedAnalystData[$type]['opinion'], $deletedAnalystData[$type]['comment']);
} else if ($type === 'Relationship') {
$info = sprintf('-- %s --> %s :: %s', $deletedAnalystData[$type]['relationship_type'] ?? '[undefined]', $deletedAnalystData[$type]['related_object_type'], $deletedAnalystData[$type]['related_object_uuid']);
}
$this->AnalystDataBlocklist = ClassRegistry::init('AnalystDataBlocklist');
$this->AnalystDataBlocklist->create();
if (!empty($deletedAnalystData[$type]['orgc_uuid'])) {
if (!empty($deletedAnalystData[$type]['Orgc'])) {
$orgc = $deletedAnalystData[$type];
} else {
$orgc = $this->Orgc->find('first', array(
'conditions' => ['Orgc.uuid' => $deletedAnalystData[$type]['orgc_uuid']],
'recursive' => -1,
'fields' => ['Orgc.name'],
));
}
} else {
$orgc = ['Orgc' => ['name' => 'MISP']];
}
$this->AnalystDataBlocklist->save(['analyst_data_uuid' => $deletedAnalystData[$type]['uuid'], 'analyst_data_info' => $info, 'analyst_data_orgc' => $orgc['Orgc']['name']]);
}
];
$this->CRUD->delete($id, $params);
@ -112,9 +142,11 @@ class AnalystDataController extends AppController
{
$this->__typeSelector($type);
$this->AnalystData->fetchRecursive = true;
$conditions = $this->AnalystData->buildConditions($this->Auth->user());
$this->CRUD->view($id, [
'conditions' => $conditions,
'contain' => ['Org', 'Orgc'],
'afterFind' => function(array $analystData) {
$canEdit = $this->ACL->canEditAnalystData($this->Auth->user(), $analystData, $this->modelSelection);
if (!$this->IndexFilter->isRest()) {
@ -169,6 +201,43 @@ class AnalystDataController extends AppController
return $this->RestResponse->viewData($data, 'json');
}
public function filterAnalystDataForPush()
{
if (!$this->request->is('post')) {
throw new MethodNotAllowedException(__('This function is only accessible via POST requests.'));
}
$this->loadModel('AnalystData');
$allIncomingAnalystData = $this->request->data;
$allData = $this->AnalystData->filterAnalystDataForPush($allIncomingAnalystData);
return $this->RestResponse->viewData($allData, $this->response->type());
}
public function pushAnalystData()
{
if (!$this->Auth->user()['Role']['perm_sync'] || !$this->Auth->user()['Role']['perm_analyst_data']) {
throw new MethodNotAllowedException(__('You do not have the permission to do that.'));
}
if (!$this->_isRest()) {
throw new MethodNotAllowedException(__('This action is only accessible via a REST request.'));
}
if ($this->request->is('post')) {
$this->loadModel('AnalystData');
$analystData = $this->request->data;
$saveResult = $this->AnalystData->captureAnalystData($this->Auth->user(), $analystData);
$messageInfo = __('%s imported, %s ignored, %s failed. %s', $saveResult['imported'], $saveResult['ignored'], $saveResult['failed'], !empty($saveResult['errors']) ? implode(', ', $saveResult['errors']) : '');
if ($saveResult['success']) {
$message = __('Analyst Data imported. ') . $messageInfo;
return $this->RestResponse->saveSuccessResponse('AnalystData', 'pushAnalystData', false, $this->response->type(), $message);
} else {
$message = __('Could not import analyst data. ') . $messageInfo;
return $this->RestResponse->saveFailResponse('AnalystData', 'pushAnalystData', false, $message);
}
}
}
private function __typeSelector($type) {
foreach ($this->__valid_types as $vt) {
if ($type === $vt) {

7
app/Controller/Component/ACLComponent.php
View File

@ -27,6 +27,13 @@ class ACLComponent extends Component
'index' => ['*'],
'view' => ['*'],
],
'analystDataBlocklists' => array(
'add' => array(),
'delete' => array(),
'edit' => array(),
'index' => array(),
'massDelete' => array(),
),
'api' => [
'rest' => ['perm_auth'],
'viewDeprecatedFunctionUse' => [],

3
app/Controller/Component/CRUDComponent.php
View File

@ -301,6 +301,9 @@ class CRUDComponent extends Component
$result = $this->Controller->{$modelName}->delete($id);
}
if ($result) {
if (isset($params['afterDelete']) && is_callable($params['afterDelete'])) {
$params['afterDelete']($data);
}
$message = __('%s deleted.', $modelName);
if ($this->Controller->IndexFilter->isRest()) {
$this->Controller->restResponsePayload = $this->Controller->RestResponse->saveSuccessResponse($modelName, 'delete', $id, 'json', $message);

2
app/Controller/ServersController.php
View File

@ -529,7 +529,7 @@ class ServersController extends AppController
if (!$fail) {
// say what fields are to be updated
$fieldList = array('id', 'url', 'push', 'pull', 'push_sightings', 'push_galaxy_clusters', 'pull_galaxy_clusters', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'remove_missing_tags', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
$fieldList = array('id', 'url', 'push', 'pull', 'push_sightings', 'push_galaxy_clusters', 'pull_galaxy_clusters', 'push_analyst_data', 'pull_analyst_data', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'remove_missing_tags', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
$this->request->data['Server']['id'] = $id;
if (isset($this->request->data['Server']['authkey']) && "" != $this->request->data['Server']['authkey']) {
$fieldList[] = 'authkey';

26
app/Lib/Tools/ServerSyncTool.php
View File

@ -14,6 +14,7 @@ class ServerSyncTool
FEATURE_EDIT_OF_GALAXY_CLUSTER = 'edit_of_galaxy_cluster',
PERM_SYNC = 'perm_sync',
PERM_GALAXY_EDITOR = 'perm_galaxy_editor',
PERM_ANALYST_DATA = 'perm_analyst_data',
FEATURE_SIGHTING_REST_SEARCH = 'sighting_rest';
/** @var array */
@ -215,6 +216,29 @@ class ServerSyncTool
return $this->post('/galaxies/pushCluster', [$cluster], $logMessage);
}
/**
* @param array $rules
* @return HttpSocketResponseExtended
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
*/
public function analystDataSearch(array $rules)
{
return $this->post('/analyst_data/filterAnalystDataForPush', $rules);
}
/**
* @param array $analystData
* @return HttpSocketResponseExtended
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
*/
public function pushAnalystData($type, array $analystData)
{
$logMessage = "Pushing Analyst Data #{$analystData[$type]['uuid']} to Server #{$this->serverId()}";
return $this->post('/analyst_data/pushAnalystData', $analystData, $logMessage);
}
/**
* @param array $params
* @return HttpSocketResponseExtended
@ -406,6 +430,8 @@ class ServerSyncTool
return isset($info['perm_sync']) && $info['perm_sync'];
case self::PERM_GALAXY_EDITOR:
return isset($info['perm_galaxy_editor']) && $info['perm_galaxy_editor'];
case self::PERM_ANALYST_DATA:
return isset($info['perm_analyst_data']) && $info['perm_analyst_data'];
case self::FEATURE_SIGHTING_REST_SEARCH:
$version = explode('.', $info['version']);
return $version[0] == 2 && $version[1] == 4 && $version[2] > 164;

390
app/Model/AnalystData.php
View File

@ -1,5 +1,6 @@
<?php
App::uses('AppModel', 'Model');
App::uses('ServerSyncTool', 'Tools');
class AnalystData extends AppModel
{
@ -28,16 +29,26 @@ class AnalystData extends AppModel
OPINION = 1,
RELATIONSHIP = 2;
const ANALYST_DATA_TYPES = [
'Note',
'Opinion',
'Relationship',
];
/** @var object|null */
protected $Note;
/** @var object|null */
protected $Opinion;
/** @var object|null */
protected $Relationship;
/** @var object|null */
protected $ObjectRelationship;
/** @var object|null */
protected $User;
/** @var object|null */
public $Organisation;
public $Org;
/** @var object|null */
public $Orgc;
/** @var object|null */
public $SharingGroup;
@ -47,7 +58,7 @@ class AnalystData extends AppModel
'SharingGroup' => [
'className' => 'SharingGroup',
'foreignKey' => 'sharing_group_id'
]
],
];
public function __construct($id = false, $table = null, $ds = null)
@ -55,11 +66,18 @@ class AnalystData extends AppModel
parent::__construct($id, $table, $ds);
$this->bindModel([
'belongsTo' => [
'Organisation' => [
'Org' => [
'className' => 'Organisation',
'foreignKey' => false,
'conditions' => [
sprintf('%s.orgc_uuid = Organisation.uuid', $this->alias)
sprintf('%s.org_uuid = Org.uuid', $this->alias)
],
],
'Orgc' => [
'className' => 'Organisation',
'foreignKey' => false,
'conditions' => [
sprintf('%s.orgc_uuid = Orgc.uuid', $this->alias)
],
],
'SharingGroup' => [
@ -71,6 +89,8 @@ class AnalystData extends AppModel
],
]
]);
$this->Org = ClassRegistry::init('Organisation');
$this->Orgc = ClassRegistry::init('Organisation');
}
public function afterFind($results, $primary = false)
@ -88,8 +108,8 @@ class AnalystData extends AppModel
$results[$i][$this->alias]['_canEdit'] = $this->canEditAnalystData($this->current_user, $v, $this->alias);
if (!empty($results[$i][$this->alias]['uuid'])) {
$results[$i][$this->alias] = $this->fetchChildNotesAndOpinions($results[$i][$this->alias]);
if (!empty($this->fetchRecursive) && !empty($results[$i][$this->alias]['uuid'])) {
$results[$i][$this->alias] = $this->fetchChildNotesAndOpinions($this->current_user, $results[$i][$this->alias]);
}
}
return $results;
@ -126,7 +146,7 @@ class AnalystData extends AppModel
if ($user['Role']['perm_site_admin']) {
return true;
}
if ($analystData[$modelType]['orgc_uuid'] == $user['Organisation']['uuid']) {
if (isset($analystData[$modelType]['orgc_uuid']) && $analystData[$modelType]['orgc_uuid'] == $user['Organisation']['uuid']) {
return true;
}
return false;
@ -171,13 +191,20 @@ class AnalystData extends AppModel
private function rearrangeOrganisation(array $analystData): array
{
if (!empty($analystData[$this->alias]['orgc_uuid'])) {
if (!isset($analystData['Organisation'])) {
$this->Organisation = ClassRegistry::init('Organisation');
$analystData[$this->alias]['Organisation'] = $this->Organisation->find('first', ['conditions' => ['uuid' => $analystData[$this->alias]['orgc_uuid']]])['Organisation'];
if (!isset($analystData['Orgc'])) {
$analystData[$this->alias]['Orgc'] = $this->Orgc->find('first', ['conditions' => ['uuid' => $analystData[$this->alias]['orgc_uuid']]])['Organisation'];
} else {
$analystData[$this->alias]['Organisation'] = $analystData['Organisation'];
$analystData[$this->alias]['Orgc'] = $analystData['Orgc'];
}
unset($analystData['Organisation']);
unset($analystData['Orgc']);
}
if (!empty($analystData[$this->alias]['org_uuid'])) {
if (!isset($analystData['Org'])) {
$analystData[$this->alias]['Org'] = $this->Org->find('first', ['conditions' => ['uuid' => $analystData[$this->alias]['org_uuid']]])['Organisation'];
} else {
$analystData[$this->alias]['Org'] = $analystData['Org'];
}
unset($analystData['Org']);
}
return $analystData;
}
@ -215,34 +242,50 @@ class AnalystData extends AppModel
throw new NotFoundException(__('Invalid UUID'));
}
public function fetchChildNotesAndOpinions(array $analystData): array
public function deduceAnalystDataType(array $analystData)
{
foreach (self::ANALYST_DATA_TYPES as $type) {
if (isset($analystData[$type])) {
return $type;
}
}
throw new NotFoundException(__('Invalid or could not deduce analyst data type'));
}
public function fetchChildNotesAndOpinions(array $user, array $analystData): array
{
$this->Note = ClassRegistry::init('Note');
$this->Opinion = ClassRegistry::init('Opinion');
$paramsNote = [
'recursive' => -1,
'contain' => ['Organisation'],
'contain' => ['Org', 'Orgc'],
'conditions' => [
'AND' => [
$this->buildConditions($user)
],
'object_type' => $this->current_type,
'object_uuid' => $analystData['uuid'],
]
];
$paramsOpinion = [
'recursive' => -1,
'contain' => ['Organisation'],
'contain' => ['Org', 'Orgc'],
'conditions' => [
'AND' => [
$this->buildConditions($user)
],
'object_type' => $this->current_type,
'object_uuid' => $analystData['uuid'],
]
];
// recursively fetch and include nested notes and opinions
$childNotes = array_map(function ($item) {
$expandedNotes = $this->fetchChildNotesAndOpinions($item[$this->Note->current_type]);
$childNotes = array_map(function ($item) use ($user) {
$expandedNotes = $this->fetchChildNotesAndOpinions($user, $item[$this->Note->current_type]);
return $expandedNotes;
}, $this->Note->find('all', $paramsNote));
$childOpinions = array_map(function ($item) {
$expandedNotes = $this->fetchChildNotesAndOpinions($item[$this->Opinion->current_type]);
$childOpinions = array_map(function ($item) use ($user) {
$expandedNotes = $this->fetchChildNotesAndOpinions($user, $item[$this->Opinion->current_type]);
return $expandedNotes;
}, $this->Opinion->find('all', $paramsOpinion));
@ -272,20 +315,323 @@ class AnalystData extends AppModel
}
/**
* Push sightings to remote server.
* Gets a cluster then save it.
*
* @param array $user
* @param array $analystData Analyst data to be saved
* @param bool $fromPull If the current capture is performed from a PULL sync
* @param int $orgId The organisation id that should own the analyst data
* @param array $server The server for which to capture is ongoing
* @return array Result of the capture including successes, fails and errors
*/
public function captureAnalystData(array $user, array $analystData, $fromPull=false, $orgUUId=false, $server=false): array
{
$results = ['success' => false, 'imported' => 0, 'ignored' => 0, 'failed' => 0, 'errors' => []];
$type = $this->deduceAnalystDataType($analystData);
$analystModel = ClassRegistry::init($type);
if ($fromPull && !empty($orgUUId)) {
$analystData[$type]['org_uuid'] = $orgUUId;
} else {
$analystData[$type]['org_uuid'] = $user['Organisation']['uuid'];
}
$this->AnalystDataBlocklist = ClassRegistry::init('AnalystDataBlocklist');
if ($this->AnalystDataBlocklist->checkIfBlocked($analystData[$type]['uuid'])) {
$results['errors'][] = __('Blocked by blocklist');
$results['ignored']++;
return $results;
}
if (!isset($analystData[$type]['orgc_uuid']) && !isset($cluster['Orgc'])) {
$analystData[$type]['orgc_uuid'] = $analystData[$type]['org_uuid'];
} else {
if (!isset($analystData[$type]['Orgc'])) {
if (isset($analystData[$type]['orgc_uuid']) && $analystData[$type]['orgc_uuid'] != $user['Organisation']['uuid'] && !$user['Role']['perm_sync'] && !$user['Role']['perm_site_admin']) {
$analystData[$type]['orgc_uuid'] = $analystData[$type]['org_uuid']; // Only sync user can create analyst data on behalf of other users
}
} else {
if ($analystData[$type]['Orgc']['uuid'] != $user['Organisation']['uuid'] && !$user['Role']['perm_sync'] && !$user['Role']['perm_site_admin']) {
$analystData[$type]['orgc_uuid'] = $analystData[$type]['org_uuid']; // Only sync user can create analyst data on behalf of other users
}
}
if (isset($analystData[$type]['orgc_uuid']) && $analystData[$type]['orgc_uuid'] != $user['Organisation']['uuid'] && !$user['Role']['perm_sync'] && !$user['Role']['perm_site_admin']) {
$analystData[$type]['orgc_uuid'] = $analystData[$type]['org_uuid']; // Only sync user can create analyst data on behalf of other users
}
}
if (!Configure::check('MISP.enableOrgBlocklisting') || Configure::read('MISP.enableOrgBlocklisting') !== false) {
$analystModel->OrgBlocklist = ClassRegistry::init('OrgBlocklist');
if (!isset($analystData[$type]['Orgc']['uuid'])) {
$orgc = $analystModel->Orgc->find('first', ['conditions' => ['Orgc.uuid' => $analystData[$type]['orgc_uuid']], 'fields' => ['Orgc.uuid'], 'recursive' => -1]);
} else {
$orgc = ['Orgc' => ['uuid' => $analystData[$type]['Orgc']['uuid']]];
}
if ($analystData[$type]['orgc_uuid'] != 0 && $analystModel->OrgBlocklist->hasAny(array('OrgBlocklist.org_uuid' => $orgc['Orgc']['uuid']))) {
$results['errors'][] = __('Organisation blocklisted (%s)', $orgc['Orgc']['uuid']);
$results['ignored']++;
return $results;
}
}
$analystData = $analystModel->captureOrganisationAndSG($analystData, $type, $user);
$existingAnalystData = $analystModel->find('first', [
'conditions' => ["{$type}.uuid" => $analystData[$type]['uuid'],],
]);
if (!isset($analystData[$type]['distribution'])) {
$analystData[$type]['distribution'] = Configure::read('MISP.default_event_distribution'); // use default event distribution
}
if ($analystData[$type]['distribution'] != 4) {
$analystData[$type]['sharing_group_id'] = null;
}
if (empty($existingAnalystData)) {
unset($analystData[$type]['id']);
$analystModel->create();
$saveSuccess = $analystModel->save($analystData);
} else {
if (!$existingAnalystData[$type]['locked'] && empty($server['Server']['internal'])) {
$results['errors'][] = __('Blocked an edit to an analyst data that was created locally. This can happen if a synchronised analyst data that was created on this instance was modified by an administrator on the remote side.');
$results['failed']++;
return $results;
}
if ($analystData[$type]['modified'] > $existingAnalystData[$type]['modified']) {
$analystData[$type]['id'] = $existingAnalystData[$type]['id'];
$saveSuccess = $analystModel->save($analystData);
} else {
$results['errors'][] = __('Remote version is not newer than local one for analyst data (%s)', $analystData[$type]['uuid']);
$results['ignored']++;
return $results;
}
}
if ($saveSuccess) {
$results['imported']++;
$analystModel->find('first', [
'conditions' => ['uuid' => $analystData[$type]['uuid']],
'recursive' => -1
]);
} else {
$results['failed']++;
foreach ($analystModel->validationErrors as $validationError) {
$results['errors'][] = $validationError[0];
}
}
$results['success'] = $results['imported'] > 0;
return $results;
}
public function captureOrganisationAndSG($element, $model, $user)
{
$this->Event = ClassRegistry::init('Event');
if (isset($element[$model]['distribution']) && $element[$model]['distribution'] == 4) {
$element[$model] = $this->Event->captureSGForElement($element[$model], $user);
}
// first we want to see how the creator organisation is encoded
// The options here are either by passing an organisation object along or simply passing a string along
if (isset($element[$model]['Orgc'])) {
$element[$model]['orgc_uuid'] = $this->Orgc->captureOrg($element[$model]['Orgc'], $user, false, true);
unset($element[$model]['Orgc']);
} else {
// Can't capture the Orgc, default to the current user
$element[$model]['orgc_uuid'] = $user['Organisation']['uuid'];
}
return $element;
}
/**
* Push Analyst Data to remote server.
* @param array $user
* @param ServerSyncTool $serverSync
* @return array
* @throws Exception
*/
public function pushAnalystData(array $user, array $serverSync): array
public function pushAnalystData(array $user, ServerSyncTool $serverSync): array
{
$server = $serverSync->server();
if (!$server['Server']['push_analyst_data']) {
return [];
}
$this->Server = ClassRegistry::init('Server');
$this->AnalystData = ClassRegistry::init('AnalystData');
return [];
$this->log("Starting Analyst Data sync with server #{$server['Server']['id']}", LOG_INFO);
$analystData = $this->getElligibleDataToPush($user);
$keyedAnalystData = [];
foreach ($analystData as $type => $entries) {
foreach ($entries as $entry) {
$entry = $entry[$type];
$keyedAnalystData[$type][$entry['uuid']] = $entry['modified'];
}
}
if (empty($analystData)) {
return [];
}
try {
$conditions = [];
foreach ($keyedAnalystData as $model => $entry) {
$conditions[$model] = array_keys($entry);
}
$analystDataToPush = $this->Server->getElligibleDataIdsFromServerForPush($serverSync, $analystData, $conditions);
} catch (Exception $e) {
$this->logException("Could not get eligible Analyst Data IDs from server #{$server['Server']['id']} for push.", $e);
return [];
}
$successes = [];
foreach ($analystDataToPush as $model => $entries) {
foreach ($entries as $entry) {
$result = $this->AnalystData->uploadEntryToServer($model, $entry, $server, $serverSync, $user);
if ($result === 'Success') {
$successes[] = __('AnalystData %s', $entry['GalaxyCluster']['uuid']);
}
}
}
return $successes;
}
/**
* Collect elligible data to be pushed on a server
*
* @param array $user
* @return array
*/
public function getElligibleDataToPush(array $user): array
{
$options = [
'recursive' => -1,
'conditions' => [
$this->buildConditions($user),
],
];
return $this->getAllAnalystData('all', $options);
}
public function filterAnalystDataForPush($allIncomingAnalystData): array
{
$validModels = [
'Note' => ClassRegistry::init('Note'),
'Opinion' => ClassRegistry::init('Opinion'),
'Relationship' => ClassRegistry::init('Relationship'),
];
$allData = ['Note' => [], 'Opinion' => [], 'Relationship' => []];
foreach ($allIncomingAnalystData as $model => $entries) {
$incomingAnalystData = $entries;
$incomingUuids = array_keys($entries);
$options = [
'conditions' => ["{$model}.uuid" => $incomingUuids],
'recursive' => -1,
'fields' => ['uuid', 'modified', 'locked']
];
$analystData = $validModels[$model]->find('all', $options);
foreach ($analystData as $entry) {
if (strtotime($entry[$model]['modified']) >= strtotime($incomingAnalystData[$entry[$model]['uuid']])) {
unset($incomingAnalystData[$entry[$model]['uuid']]);
continue;
}
if ($entry[$model]['locked'] == 0) {
unset($incomingAnalystData[$entry[$model]['uuid']]);
}
}
$allData[$model] = $incomingAnalystData;
}
return $allData;
}
/**
* getAllAnalystData Collect all analyst data regardless if they are notes, opinions or relationships
*
* @param array $user
* @return array
*/
public function getAllAnalystData($findType='all', array $findOptions=[]): array
{
$allData = [];
$this->Note = ClassRegistry::init('Note');
$this->Opinion = ClassRegistry::init('Opinion');
$this->Relationship = ClassRegistry::init('Relationship');
$validModels = [$this->Note, $this->Opinion, $this->Relationship];
foreach ($validModels as $model) {
$result = $model->find($findType, $findOptions);
$allData[$model->alias] = $result;
}
return $allData;
}
public function uploadEntryToServer($type, array $analystData, array $server, ServerSyncTool $serverSync, array $user)
{
$analystDataID = $analystData[$type]['id'];
$analystData = $this->prepareForPushToServer($type, $analystData, $server);
if (is_numeric($analystData)) {
return $analystData;
}
try {
if (!$serverSync->isSupported(ServerSyncTool::PERM_SYNC) || !$serverSync->isSupported(ServerSyncTool::PERM_ANALYST_DATA)) {
return __('The remote user does not have the permission to manipulate analyst data, the upload of the analyst data has been blocked.');
}
$serverSync->pushAnalystData($type, $analystData)->json();
} catch (Exception $e) {
$title = __('Uploading AnalystData (%s::%s) to Server (%s)', $type, $analystDataID, $server['Server']['id']);
$this->loadLog()->createLogEntry($user, 'push', 'AnalystData', $analystDataID, $title, $e->getMessage());
$this->logException("Could not push analyst data to remote server {$serverSync->serverId()}", $e);
return $e->getMessage();
}
return 'Success';
}
private function prepareForPushToServer($type, array $analystData, array $server)
{
if ($analystData[$type]['distribution'] == 4) {
if (!empty($analystData[$type]['SharingGroup']['SharingGroupServer'])) {
$found = false;
foreach ($analystData[$type]['SharingGroup']['SharingGroupServer'] as $sgs) {
if ($sgs['server_id'] == $server['Server']['id']) {
$found = true;
}
}
if (!$found) {
return 403;
}
} elseif (empty($analystData[$type]['SharingGroup']['roaming'])) {
return 403;
}
}
$this->Event = ClassRegistry::init('Event');
if ($this->Event->checkDistributionForPush($analystData, $server, $type)) {
return $this->updateAnalystDataForSync($type, $analystData, $server);
}
return 403;
}
private function updateAnalystDataForSync($type, array $analystData, array $server): array
{
$this->Event = ClassRegistry::init('Event');
// cleanup the array from things we do not want to expose
foreach (['id'] as $field) {
unset($analystData[$type][$field]);
}
// Add the local server to the list of instances in the SG
if (isset($analystData[$type]['SharingGroup']) && isset($analystData[$type]['SharingGroup']['SharingGroupServer'])) {
foreach ($analystData[$type]['SharingGroup']['SharingGroupServer'] as &$s) {
if ($s['server_id'] == 0) {
$s['Server'] = array(
'id' => 0,
'url' => $this->Event->__getAnnounceBaseurl(),
'name' => $this->Event->__getAnnounceBaseurl()
);
}
}
}
// Downgrade the event from connected communities to community only
if (!$server['Server']['internal'] && $analystData[$type]['distribution'] == 2) {
$analystData[$type]['distribution'] = 1;
}
return $analystData;
}
}

58
app/Model/AnalystDataBlocklist.php Normal file
View File

@ -0,0 +1,58 @@
<?php
App::uses('AppModel', 'Model');
class AnalystDataBlocklist extends AppModel
{
public $useTable = 'analyst_data_blocklists';
public $recursive = -1;
public $actsAs = array(
'AuditLog',
'SysLogLogable.SysLogLogable' => array( // TODO Audit, logable
'userModel' => 'User',
'userKey' => 'user_id',
'change' => 'full'
),
'Containable',
);
public $blocklistFields = array('analyst_data_uuid', 'comment', 'analyst_data_info', 'analyst_data_orgc');
public $blocklistTarget = 'analyst_data';
public $validate = array(
'analyst_data_uuid' => array(
'unique' => array(
'rule' => 'isUnique',
'message' => 'Analyst Data already blocklisted.'
),
'uuid' => array(
'rule' => array('uuid'),
'message' => 'Please provide a valid UUID'
),
)
);
public function beforeValidate($options = array())
{
parent::beforeValidate();
if (empty($this->data['AnalystDataBlocklist']['id'])) {
$this->data['AnalystDataBlocklist']['date_created'] = date('Y-m-d H:i:s');
}
if (empty($this->data['AnalystDataBlocklist']['comment'])) {
$this->data['AnalystDataBlocklist']['comment'] = '';
}
return true;
}
/**
* @param string $analystDataUUID
* @return bool
*/
public function checkIfBlocked($analystDataUUID)
{
return $this->hasAny([
'analyst_data_uuid' => $analystDataUUID,
]);
}
}

15
app/Model/AppModel.php
View File

@ -2021,6 +2021,7 @@ class AppModel extends Model
`modified` datetime ON UPDATE CURRENT_TIMESTAMP,
`distribution` tinyint(4) NOT NULL,
`sharing_group_id` int(10) unsigned,
`locked` tinyint(1) NOT NULL DEFAULT 0;
`note` mediumtext,
`language` varchar(16) DEFAULT 'en',
PRIMARY KEY (`id`),
@ -2045,6 +2046,7 @@ class AppModel extends Model
`modified` datetime ON UPDATE CURRENT_TIMESTAMP,
`distribution` tinyint(4) NOT NULL,
`sharing_group_id` int(10) unsigned,
`locked` tinyint(1) NOT NULL DEFAULT 0;
`opinion` int(10) unsigned,
`comment` text,
PRIMARY KEY (`id`),
@ -2070,6 +2072,7 @@ class AppModel extends Model
`modified` datetime ON UPDATE CURRENT_TIMESTAMP,
`distribution` tinyint(4) NOT NULL,
`sharing_group_id` int(10) unsigned,
`locked` tinyint(1) NOT NULL DEFAULT 0;
`relationship_type` varchar(255) CHARACTER SET utf8 COLLATE utf8_unicode_ci,
`related_object_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
`related_object_type` varchar(80) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
@ -2086,6 +2089,18 @@ class AppModel extends Model
KEY `related_object_type` (`related_object_type`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;";
$sqlArray[] = "CREATE TABLE IF NOT EXISTS `analyst_data_blocklists` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`analyst_data_uuid` varchar(40) COLLATE utf8_bin NOT NULL,
`created` datetime NOT NULL,
`analyst_data_info` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL,
`comment` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci,
`analyst_data_orgc` VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,
PRIMARY KEY (`id`),
KEY `analyst_data_uuid` (`analyst_data_uuid`),
KEY `analyst_data_orgc` (`analyst_data_orgc`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;";
$sqlArray[] = "ALTER TABLE `roles` ADD `perm_analyst_data` tinyint(1) NOT NULL DEFAULT 0;";
$sqlArray[] = "UPDATE `roles` SET `perm_analyst_data`=1 WHERE `perm_add` = 1;";

2
app/Model/Behavior/AnalystDataBehavior.php
View File

@ -40,7 +40,7 @@ class AnalystDataBehavior extends ModelBehavior
return $Model->find('all', [
'recursive' => -1,
'conditions' => $conditions,
'contain' => ['Organisation', 'SharingGroup'],
'contain' => ['Org', 'Orgc', 'SharingGroup'],
]);
}

1
app/Model/Behavior/AnalystDataParentBehavior.php
View File

@ -27,6 +27,7 @@ class AnalystDataParentBehavior extends ModelBehavior
$data = [];
foreach ($types as $type) {
$this->{$type} = ClassRegistry::init($type);
$this->{$type}->fetchRecursive = true;
$temp = $this->{$type}->fetchForUuid($object['uuid'], $this->__currentUser);
if (!empty($temp)) {
foreach ($temp as $k => $temp_element) {

6
app/Model/Organisation.php
View File

@ -182,7 +182,7 @@ class Organisation extends AppModel
* @return int Organisation ID
* @throws Exception
*/
public function captureOrg($org, array $user, $force = false)
public function captureOrg($org, array $user, $force = false, $returnUUID = false)
{
$fieldsToFetch = $force ?
['id', 'uuid', 'type', 'date_created', 'date_modified', 'nationality', 'sector', 'contacts'] :
@ -224,7 +224,7 @@ class Organisation extends AppModel
}
$this->create();
$this->save($organisation);
return $this->id;
return $returnUUID ? $organisation['uuid'] : $this->id;
} else {
$changed = false;
if (isset($org['uuid']) && empty($existingOrg[$this->alias]['uuid'])) {
@ -248,7 +248,7 @@ class Organisation extends AppModel
$this->save($existingOrg);
}
}
return $existingOrg[$this->alias]['id'];
return $returnUUID ? $existingOrg[$this->alias]['uuid']: $existingOrg[$this->alias]['id'];
}
/**

54
app/Model/Server.php
View File

@ -745,6 +745,24 @@ class Server extends AppModel
return $clusterArray;
}
/**
* fetchAnalystDataIdsFromServer Fetch remote analyst datas' UUIDs and versions
*
* @param ServerSyncTool $serverSync
* @param array $conditions
* @return array The list of analyst data
* @throws JsonException|HttpSocketHttpException|HttpSocketJsonException
*/
private function fetchAnalystDataIdsFromServer(ServerSyncTool $serverSync, array $conditions = [])
{
$filterRules = $conditions;
$dataArray = $serverSync->analystDataSearch($filterRules)->json();
if (isset($dataArray['response'])) {
$dataArray = $dataArray['response'];
}
return $dataArray;
}
/**
* Get a list of cluster IDs that are present on the remote server and returns clusters that should be pulled
*
@ -831,6 +849,42 @@ class Server extends AppModel
return $localClusters;
}
/**
* Get an array of analyst data that the remote is willing to get and returns analyst data that should be pushed.
* @param ServerSyncTool $serverSync
* @param array $localAnalystData
* @param array $conditions
* @return array
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
* @throws JsonException
*/
public function getElligibleDataIdsFromServerForPush(ServerSyncTool $serverSync, array $localAnalystData=[], array $conditions=[]): array
{
$this->log("Fetching eligible analyst data from server #{$serverSync->serverId()} for push: " . JsonTool::encode($conditions), LOG_INFO);
$localAnalystDataMinimal = [];
foreach ($localAnalystData as $type => $entries) {
foreach ($entries as $entry) {
$entry = $entry[$type];
$localAnalystDataMinimal[$type][$entry['uuid']] = $entry['modified'];
}
}
$remoteDataArray = $this->fetchAnalystDataIdsFromServer($serverSync, $localAnalystDataMinimal);
foreach ($localAnalystData as $type => $entries) {
foreach ($entries as $i => $entry) {
$entry = $entry[$type];
if (!isset($remoteDataArray[$type][$entry['uuid']])) {
unset($localAnalystData[$type][$i]);
// $remoteVersion = $remoteDataArray[$type][$entry['uuid']];
// if (strtotime($entry['modified']) <= strtotime($remoteVersion)) {
// unset($localAnalystData[$type][$entry['uuid']]);
// }
}
}
}
return $localAnalystData;
}
/**
* @param ServerSyncTool $serverSync
* @param bool $ignoreFilterRules Ignore defined server pull rules

4
app/View/AnalystData/index.ctp
View File

@ -6,9 +6,9 @@
'data_path' => $modelSelection . '.id'
],
[
'name' => __('Org'),
'name' => __('OrgC'),
'element' => 'org',
'data_path' => $modelSelection . '.Organisation'
'data_path' => $modelSelection . '.Orgc'
],
[
'name' => __('UUID'),

11
app/View/AnalystData/view.ctp
View File

@ -35,16 +35,9 @@ $fields = [
],
[
'key' => __('Creator org'),
'path' => $modelSelection . '.orgc_uuid',
'path' => $modelSelection . '.Orgc',
'pathName' => $modelSelection . '.orgc_uuid',
'type' => 'model',
'model' => 'organisations'
],
[
'key' => __('Owner org'),
'path' => $modelSelection . '.org_uuid',
'pathName' => $modelSelection . '.org_uuid',
'type' => 'model',
'type' => 'org',
'model' => 'organisations'
],
[

52
app/View/AnalystDataBlocklists/add.ctp Normal file
View File

@ -0,0 +1,52 @@
<?php
$fieldDesc = array();
$fieldDesc['uuids'] = __('Enter a single or a list of UUIDs');
$fieldDesc['analyst_data_orgc'] = __('(Optional) The organisation that the event is associated with');
$fieldDesc['analyst_data_info'] = __('(Optional) The analyst data value that you would like to block');
$fieldDesc['comment'] = __('(Optional) Any comments you would like to add regarding this (or these) entries');
echo $this->element('genericElements/Form/genericForm', array(
'form' => $this->Form,
'data' => array(
'model' => 'AnalystDataBlocklist',
'title' => $action == 'add' ? __('Add block entry for Analyst Data') : __('Edit block entry for Analyst Data'),
'fields' => array(
array(
'disabled' => $action != 'add' ? 'disabled' : '',
'field' => 'uuids',
'class' => 'span6',
'label' => __('Analyst Data UUID'),
'type' => 'textarea',
'default' => isset($blockEntry['AnalystDataBlocklist']['analyst_data_uuid']) ? $blockEntry['AnalystDataBlocklist']['analyst_data_uuid'] : '',
),
array(
'field' => 'analyst_data_orgc',
'label' => __('Creating organisation'),
'class' => 'span6',
'type' => 'text',
'default' => isset($blockEntry['AnalystDataBlocklist']['analyst_data_orgc']) ? $blockEntry['AnalystDataBlocklist']['analyst_data_orgc'] : ''
),
array(
'field' => 'analyst_data_info',
'label' => __('Analyst Data value'),
'class' => 'span6',
'type' => 'text',
'default' => isset($blockEntry['AnalystDataBlocklist']['analyst_data_info']) ? $blockEntry['AnalystDataBlocklist']['analyst_data_info'] : ''
),
array(
'field' => 'comment',
'label' => __('Comment'),
'class' => 'span6',
'type' => 'text',
'default' => isset($blockEntry['AnalystDataBlocklist']['comment']) ? $blockEntry['AnalystDataBlocklist']['comment'] : ''
),
),
'submit' => array(
'ajaxSubmit' => ''
)
),
'fieldDesc' => $fieldDesc
));
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'analyst_data', 'menuItem' => 'index_blocklist'));
?>
<?php echo $this->Js->writeBuffer(); // Write cached scripts

104
app/View/AnalystDataBlocklists/index.ctp Normal file
View File

@ -0,0 +1,104 @@
<?php
echo '<div class="index">';
echo $this->element('/genericElements/IndexTable/index_table', array(
'data' => array(
'data' => $response,
'top_bar' => array(
'children' => array(
array(
'type' => 'simple',
'children' => array(
array(
'url' => sprintf('%s/analyst_data_blocklists/add/', $baseurl),
'text' => __('+ Add entry to blocklist'),
),
)
),
array(
'type' => 'search',
'button' => __('Filter'),
'placeholder' => __('Enter value to search'),
'data' => '',
'searchKey' => 'searchall'
)
)
),
'fields' => array(
array(
'name' => __('Id'),
'sort' => 'id',
'class' => 'short',
'data_path' => 'AnalystDataBlocklist.id',
),
array(
'name' => __('Org'),
'class' => 'short',
'data_path' => 'AnalystDataBlocklist.analyst_data_orgc',
),
array(
'name' => __('Analyst Data UUID'),
'class' => 'short',
'data_path' => 'AnalystDataBlocklist.analyst_data_uuid',
),
array(
'name' => __('Created'),
'sort' => 'created',
'class' => 'short',
'data_path' => 'AnalystDataBlocklist.created',
),
array(
'name' => __('Analyst Data value'),
'sort' => 'value',
'class' => 'short',
'data_path' => 'AnalystDataBlocklist.analyst_data_info',
),
array(
'name' => __('Comment'),
'sort' => 'comment',
'class' => 'short',
'data_path' => 'AnalystDataBlocklist.comment',
),
),
'title' => __('Analyst Data Blocklist Index'),
'description' => __('List all analyst data that will be prevented to be created (also via synchronization) on this instance'),
'actions' => array(
array(
'title' => 'Edit',
'url' => '/analyst_data_blocklists/edit',
'url_params_data_paths' => array(
'AnalystDataBlocklist.id'
),
'icon' => 'edit',
),
array(
'title' => 'Delete',
'url' => $baseurl . '/analyst_data_blocklists/delete',
'url_params_data_paths' => array(
'AnalystDataBlocklist.id'
),
'postLink' => true,
'postLinkConfirm' => __('Are you sure you want to delete the entry?'),
'icon' => 'trash'
),
)
)
));
echo '</div>';
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'analyst_data', 'menuItem' => 'index_blocklist'));
?>
<script type="text/javascript">
var passedArgsArray = <?php echo $passedArgs; ?>;
if (passedArgsArray['context'] === undefined) {
passedArgsArray['context'] = 'pending';
}
$(document).ready(function() {
$('#quickFilterButton').click(function() {
runIndexQuickFilter('/context:' + passedArgsArray['context']);
});
$('#quickFilterField').on('keypress', function (e) {
if(e.which === 13) {
runIndexQuickFilter('/context:' + passedArgsArray['context']);
}
});
});
</script>

4
app/View/Elements/genericElements/Analyst_data/thread.ctp
View File

@ -166,9 +166,9 @@ var baseNoteTemplate = doT.template('\
<div style="flex-grow: 1;"> \
<div style="display: flex; flex-direction: column;"> \
<div style="display: flex; min-width: 250px; gap: 0.5rem;"> \
<img src="<?= $baseurl ?>/img/orgs/{{=it.Organisation.id}}.png" width="20" height="20" class="orgImg" style="width: 20px; height: 20px;" onerror="this.remove()" alt="Organisation logo"></object> \
<img src="<?= $baseurl ?>/img/orgs/{{=it.Orgc.id}}.png" width="20" height="20" class="orgImg" style="width: 20px; height: 20px;" onerror="this.remove()" alt="Organisation logo"></object> \
<span style="margin-left: 0rem; margin-right: 0.5rem;"> \
<span>{{=it.Organisation.name}}</span> \
<span>{{=it.Orgc.name}}</span> \
<i class="<?= $this->FontAwesome->getClass('angle-right') ?>" style="color: #999; margin: 0 0.25rem;"></i> \
<b>{{=it.authors}}</b> \
</span> \

7
app/View/Elements/genericElements/SideMenu/side_menu.ctp
View File

@ -1808,6 +1808,13 @@ $divider = '<li class="divider"></li>';
'url' => '/analystData/index',
'text' => __('List Analyst Data')
));
if ($this->Acl->canAccess('analyst_data_blocklists', 'index')) {
echo $this->element('/genericElements/SideMenu/side_menu_link', array(
'element_id' => 'index_blocklist',
'url' => $baseurl . '/analyst_data_blocklists/index',
'text' => __('List Analyst-Data Blocklists')
));
}
if ($this->Acl->canAccess('analyst_data', 'add')) {
echo $divider;
echo $this->element('/genericElements/SideMenu/side_menu_link', array(