MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [server:push] Drafty version of galaxyCluster push

pull/6120/head
mokaddem 2020-05-26 15:08:24 +02:00
parent 51391f8e57
commit 176e29c94f
No known key found for this signature in database
GPG Key ID: 164C473F627A06FA
10 changed files with 251 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/Controller/Component/ACLComponent.php
View File

@ -245,6 +245,7 @@ class ACLComponent extends Component
'detach' => array('perm_tagger'),
'edit' => array('perm_galaxy_editor'),
'index' => array('*'),
'pushCluster' => array('perm_sync'),
'view' => array('*'),
'viewGalaxyMatrix' => array('*')
),

22
app/Controller/GalaxiesController.php
View File

@ -144,9 +144,6 @@ class GalaxiesController extends AppController
}
}
// TODO: revise import strategy. Instead of asking in which galaxy to import data
// Based the decision on data contained in the clusters
// If Galaxy do not exist, add possibility to create it on the fly
public function import()
{
if ($this->request->is('post') || $this->request->is('put')) {
@ -194,6 +191,25 @@ class GalaxiesController extends AppController
$this->set('action', 'import');
}
public function pushCluster()
{
if (!$this->Auth->user()['Role']['perm_sync'] || !$this->Auth->user()['Role']['perm_galaxy_editor'] ) {
throw new MethodNotAllowedException(__('You do not have the permission to do that.'));
}
if ($this->request->is('post')) {
$clusters = $this->request->data;
$saveResult = $this->Galaxy->importGalaxyAndClusters($this->Auth->user(), $clusters);
$messageInfo = __('%s imported, %s ignored, %s failed. %s', $saveResult['imported'], $saveResult['ignored'], $saveResult['failed'], !empty($saveResult['errors']) ? implode(', ', $saveResult['errors']) : '');
if ($saveResult['success']) {
$message = __('Galaxy clusters imported. ') . $messageInfo;
return $this->RestResponse->saveSuccessResponse('Galaxy', 'pushCluster', false, $this->response->type(), $message);
} else {
$message = __('Could not import galaxy clusters. ') . $messageInfo;
return $this->RestResponse->saveFailResponse('Galaxy', 'pushCluster', false, $message);
}
}
}
public function export($galaxyId)
{
$galaxy = $this->Galaxy->find('first', array(

3
app/Controller/ServersController.php
View File

@ -257,6 +257,7 @@ class ServersController extends AppController
'push' => 0,
'pull' => 0,
'push_sightings' => 0,
'push_galaxy_clusters' => 0,
'caching_enabled' => 0,
'json' => '[]',
'push_rules' => '[]',
@ -452,7 +453,7 @@ class ServersController extends AppController
}
if (!$fail) {
// say what fields are to be updated
$fieldList = array('id', 'url', 'push', 'pull', 'push_sightings', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
$fieldList = array('id', 'url', 'push', 'pull', 'push_sightings', 'push_galaxy_clusters', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
$this->request->data['Server']['id'] = $id;
if (isset($this->request->data['Server']['authkey']) && "" != $this->request->data['Server']['authkey']) {
$fieldList[] = 'authkey';

4
app/Model/AppModel.php
View File

@ -1391,9 +1391,6 @@ class AppModel extends Model
$sqlArray[] = "ALTER TABLE `galaxy_cluster_relations` ADD `distribution` tinyint(4) NOT NULL DEFAULT 0;";
$sqlArray[] = "ALTER TABLE `galaxy_cluster_relations` ADD `sharing_group_id` int(11);";
$sqlArray[] = "ALTER TABLE `galaxy_cluster_relations` ADD `default` tinyint(1) NOT NULL DEFAULT 0;";
// $sqlArray[] = "ALTER TABLE `galaxy_cluster_relations` ADD `org_id` int(11) NOT NULL;";
// $sqlArray[] = "ALTER TABLE `galaxy_cluster_relations` ADD `orgc_id` int(11) NOT NULL;";
// $sqlArray[] = "ALTER TABLE `galaxy_cluster_relations` ADD `locked` tinyint(1) NOT NULL DEFAULT 0;";
$sqlArray[] = "UPDATE `galaxy_cluster_relations` SET `distribution`=3, `default`=1 WHERE `org_id`=0;";
$sqlArray[] = "CREATE TABLE IF NOT EXISTS `galaxy_cluster_relation_tags` (
`id` int(11) NOT NULL AUTO_INCREMENT,
@ -1401,6 +1398,7 @@ class AppModel extends Model
`tag_id` int(11) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;";
$sqlArray[] = "ALTER TABLE `servers` ADD `push_galaxy_clusters` tinyint(1) NOT NULL DEFAULT 0 AFTER `push_sightings`;";
// TODO: ADD INDEXES
break;
case 'fixNonEmptySharingGroupID':

2
app/Model/Galaxy.php
View File

@ -285,7 +285,7 @@ class Galaxy extends AppModel
$results['failed'] += $saveResult['failed'];
$results['errors'] = array_merge($results['errors'], $saveResult['errors']);
}
$results['success'] = $results['imported'] > 0;
$results['success'] = !($results['failed'] > 0 && $results['imported'] == 0 && $results['ignored'] == 0);
return $results;
}

145
app/Model/GalaxyCluster.php
View File

@ -251,7 +251,7 @@ class GalaxyCluster extends AppModel
public function captureClusters($user, $galaxy, $clusters, $forceUpdate=false, $orgId=0)
{
$importResult = array('success' => false, 'imported' => 0, 'ignored' => 0, 'failed' => 0,'errors' => array());
$importResult = array('success' => true, 'imported' => 0, 'ignored' => 0, 'failed' => 0,'errors' => array());
foreach ($clusters as $k => $cluster) {
$cluster['GalaxyCluster']['galaxy_id'] = $galaxy['Galaxy']['id'];
$saveResult = $this->captureCluster($user, $cluster, $fromPull=true, $orgId=$orgId);
@ -260,8 +260,8 @@ class GalaxyCluster extends AppModel
$importResult['failed'] += $saveResult['failed'];
$importResult['errors'] = array_merge($importResult['errors'], $saveResult['errors']);
}
if ($importResult['imported'] > 0) {
$importResult['success'] = true;
if ($importResult['failed'] > 0 && $importResult['imported'] == 0 && $importResult['ignored'] == 0) {
$importResult['success'] = false;
}
return $importResult;
}
@ -322,8 +322,27 @@ class GalaxyCluster extends AppModel
}
$cluster = $this->captureOrganisationAndSG($cluster, 'GalaxyCluster', $user);
$this->create();
$saveSuccess = $this->save($cluster);
$existingGalaxyCluster = $this->find('first', array('conditions' => array(
'GalaxyCluster.uuid' => $cluster['GalaxyCluster']['uuid']
)));
if (empty($existingGalaxyCluster)) {
$this->create();
$saveSuccess = $this->save($cluster);
} else {
if ($cluster['GalaxyCluster']['default']) {
$results['errors'][] = __('Can only save non default clusters');
$results['failed']++;
return $results;
}
if ($cluster['GalaxyCluster']['version'] > $existingGalaxyCluster['GalaxyCluster']['version']) {
$cluster['GalaxyCluster']['id'] = $existingGalaxyCluster['GalaxyCluster']['id'];
$saveSuccess = $this->save($cluster);
} else {
$results['errors'][] = __('Remote version is not newer than local one');
$results['ignored']++;
return $results;
}
}
if ($saveSuccess) {
$results['imported']++;
$savedCluster = $this->find('first', array(
@ -871,6 +890,122 @@ class GalaxyCluster extends AppModel
return array_values($clusterTags);
}
public function uploadClusterToServer($cluster, $server, $HttpSocket, $user)
{
$this->Server = ClassRegistry::init('Server');
$this->Log = ClassRegistry::init('Log');
$push = $this->Server->checkVersionCompatibility($server['Server']['id'], false, $HttpSocket);
if (empty($push['canPush']) && empty($push['canPushGalaxyCluster'])) {
return 'The remote user is not a sightings user - the upload of the galaxy clusters has been blocked.';
}
$updated = null;
$newLocation = $newTextBody = '';
$result = $this->__executeRestfulGalaxyClusterToServer($cluster, $server, null, $newLocation, $newTextBody, $HttpSocket, $user);
if ($result !== true) {
return $result;
}
if (strlen($newLocation)) { // HTTP/1.1 302 Found and Location: http://<newLocation>
$result = $this->__executeRestfulGalaxyClusterToServer($cluster, $server, $newLocation, $newLocation, $newTextBody, $HttpSocket, $user);
if ($result !== true) {
return $result;
}
}
$uploadFailed = false;
try {
$json = json_decode($newTextBody, true);
} catch (Exception $e) {
$uploadFailed = true;
}
if (!is_array($json) || $uploadFailed) {
$this->Log->createLogEntry($user, 'push', 'GalaxyCluster', $cluster['GalaxyCluster']['id'], 'push', $newTextBody);
}
return 'Success';
}
private function __executeRestfulGalaxyClusterToServer($cluster, $server, $resourceId, &$newLocation, &$newTextBody, $HttpSocket, $user)
{
$result = $this->restfulGalaxyClusterToServer($cluster, $server, $resourceId, $newLocation, $newTextBody, $HttpSocket);
if (is_numeric($result)) {
$error = $this->__resolveErrorCode($result, $cluster, $server, $user);
if ($error) {
return $error . ' Error code: ' . $result;
}
}
return true;
}
public function restfulGalaxyClusterToServer($cluster, $server, $urlPath, &$newLocation, &$newTextBody, $HttpSocket = null)
{
$url = $server['Server']['url'];
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
$scope = 'galaxies/pushCluster';
$uri = $url . '/' . $scope;
$clusters = array($cluster);
$data = json_encode($clusters);
if (!empty(Configure::read('Security.sync_audit'))) {
$pushLogEntry = sprintf(
"==============================================================\n\n[%s] Pushing Galaxy Cluster #%d to Server #%d:\n\n%s\n\n",
date("Y-m-d H:i:s"),
$cluster['GalaxyCluster']['id'],
$server['Server']['id'],
$data
);
file_put_contents(APP . 'files/scripts/tmp/debug_server_' . $server['Server']['id'] . '.log', $pushLogEntry, FILE_APPEND);
}
$response = $HttpSocket->post($uri, $data, $request);
return $this->__handleRestfulGalaxyClusterToServerResponse($response, $newLocation, $newTextBody);
}
private function __handleRestfulGalaxyClusterToServerResponse($response, &$newLocation, &$newTextBody)
{
switch ($response->code) {
case '200': // 200 (OK) + entity-action-result
if ($response->isOk()) {
$newTextBody = $response->body();
return true;
} else {
try {
$jsonArray = json_decode($response->body, true);
} catch (Exception $e) {
return true;
}
return $jsonArray['name'];
}
// no break
case '302': // Found
$newLocation = $response->headers['Location'];
$newTextBody = $response->body();
return true;
case '404': // Not Found
$newLocation = $response->headers['Location'];
$newTextBody = $response->body();
return 404;
case '405':
return 405;
case '403': // Not authorised
return 403;
}
}
private function __resolveErrorCode($code, &$cluster, &$server, $user)
{
$this->Log = ClassRegistry::init('Log');
$error = false;
switch ($code) {
case 403:
return 'The distribution level of the cluster blocks it from being pushed.';
case 405:
$error = 'The sync user on the remote instance does not have the required privileges to handle this cluster.';
break;
}
if ($error) {
$newTextBody = 'Uploading GalaxyCluster (' . $cluster['GalaxyCluster']['id'] . ') to Server (' . $server['Server']['id'] . ')';
$this->Log->createLogEntry($user, 'push', 'GalaxyCluster', $cluster['GalaxyCluster']['id'], 'push', $newTextBody);
}
return $error;
}
public function attachClusterToRelations($user, $cluster)
{
if (!empty($cluster['GalaxyClusterRelation'])) {

84
app/Model/Server.php
View File

@ -2682,6 +2682,51 @@ class Server extends AppModel
return $filter_rules;
}
// Get an array of cluster_ids that are present on the remote server
public function getClusterIdsFromServer($server, $HttpSocket=null)
{
$url = $server['Server']['url'];
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
$uri = $url . '/galaxy_clusters/restSearch';
$filter_rules['minimal'] = 1;
$filter_rules['custom'] = 1;
try {
$response = $HttpSocket->post($uri, json_encode($filter_rules), $request);
if ($response->isOk()) {
$clusterArray = json_decode($response->body, true);
// correct $eventArray if just one event
$clusterIds = array();
if (isset($clusterArray['response'])) {
$clusterArray = $clusterArray['response'];
}
if (!empty($clusterArray)) {
foreach ($clusterArray as $cluster) {
$localCluster = $this->GalaxyCluster->find('first', array(
'recursive' => -1,
'fields' => array('GalaxyCluster.uuid', 'GalaxyCluster.version'),
'conditions' => array('GalaxyCluster.uuid' => $cluster['GalaxyCluster']['uuid'])
));
if (!empty($localCluster) && $localCluster['GalaxyCluster']['version'] > $cluster['GalaxyCluster']['version']) { // FIXME: TO UNCOMMENT
$clusterIds[] = $localCluster['GalaxyCluster']['uuid'];
}
}
}
return $clusterIds;
}
if ($response->code == '403') {
return 403;
}
} catch (SocketException $e) {
return $e->getMessage();
}
// error, so return error message, since that is handled and everything is expecting an array
return "Error: got response code " . $response->code;
}
// Get an array of event_ids that are present on the remote server
public function getEventIdsFromServer($server, $all = false, $HttpSocket=null, $force_uuid=false, $ignoreFilterRules = false, $scope = 'events')
{
@ -2948,6 +2993,14 @@ class Server extends AppModel
if (!isset($fails)) {
$fails = array();
}
if ($push['canPush'] || $push['canEditGalaxyCluster']) {
$clustersSuccesses = $this->syncGalaxyClusters($HttpSocket, $this->data, $user);
} else {
$clustersSuccesses = array();
}
$successes = array_merge($successes, $clustersSuccesses);
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
@ -2996,6 +3049,34 @@ class Server extends AppModel
return $uuidList;
}
public function syncGalaxyClusters($HttpSocket, $server, $user)
{
$successes = array();
if (!$server['Server']['push_galaxy_clusters']) {
return $successes;
}
$this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$clusterIds = $this->getClusterIdsFromServer($server, $HttpSocket);
if (!empty($clusterIds)) {
// check each cluster push it when needed
foreach ($clusterIds as $k => $clusterId) {
$options = array('conditions' => array(
'GalaxyCluster.uuid' => $clusterId
));
$cluster = $this->GalaxyCluster->fetchGalaxyClusters($user, $options, $full=true);
if (!empty($cluster)) {
$cluster = $cluster[0];
$result = $this->GalaxyCluster->uploadClusterToServer($cluster, $server, $HttpSocket, $user);
if ($result === 'Success') {
$successes[] = __('GalaxyCluster %s', $cluster['GalaxyCluster']['uuid']);
}
}
}
}
return $successes;
}
public function syncSightings($HttpSocket, $server, $user, $eventModel)
{
$successes = array();
@ -4319,6 +4400,7 @@ class Server extends AppModel
$remoteVersion = json_decode($response->body, true);
$canPush = isset($remoteVersion['perm_sync']) ? $remoteVersion['perm_sync'] : false;
$canSight = isset($remoteVersion['perm_sighting']) ? $remoteVersion['perm_sighting'] : false;
$canEditGalaxyCluster = isset($remoteVersion['perm_galaxy_editor']) ? $remoteVersion['perm_galaxy_editor'] : false;
$remoteVersion = explode('.', $remoteVersion['version']);
if (!isset($remoteVersion[0])) {
$this->Log = ClassRegistry::init('Log');
@ -4380,7 +4462,7 @@ class Server extends AppModel
'title' => ucfirst($issueLevel) . ': ' . $response,
));
}
return array('success' => $success, 'response' => $response, 'canPush' => $canPush, 'canSight' => $canSight, 'version' => $remoteVersion);
return array('success' => $success, 'response' => $response, 'canPush' => $canPush, 'canSight' => $canSight, 'canEditGalaxyCluster' => $canEditGalaxyCluster, 'version' => $remoteVersion);
}
public function isJson($string)

1
app/View/Servers/add.ctp
View File

@ -81,6 +81,7 @@
echo $this->Form->input('push', array());
echo $this->Form->input('pull', array());
echo $this->Form->input('push_sightings', array());
echo $this->Form->input('push_galaxy_clusters', array());
echo $this->Form->input('caching_enabled', array());
echo '<div class = "input clear" style="width:100%;"><hr /></div>';
echo $this->Form->input('unpublish_event', array(

1
app/View/Servers/edit.ctp
View File

@ -87,6 +87,7 @@
echo $this->Form->input('pull', array());
echo $this->Form->input('push_sightings', array());
echo $this->Form->input('caching_enabled', array());
echo $this->Form->input('push_galaxy_clusters', array());
echo '<div class = "input clear" style="width:100%;"><hr /><h4>' . __('Misc settings') . '</h4></div>';
echo $this->Form->input('unpublish_event', array(
'type' => 'checkbox',

2
app/View/Servers/index.ctp
View File

@ -28,6 +28,7 @@
<th><?php echo $this->Paginator->sort('push');?></th>
<th><?php echo $this->Paginator->sort('pull');?></th>
<th><?php echo $this->Paginator->sort('push_sightings', 'Push Sightings');?></th>
<th><?php echo $this->Paginator->sort('push_galaxy_clusters', 'Push Clusters');?></th>
<th><?php echo $this->Paginator->sort('caching_enabled', 'Cache');?></th>
<th><?php echo $this->Paginator->sort('unpublish_event (push event)');?></th>
<th><?php echo $this->Paginator->sort('publish_without_email (pull event)');?></th>
@ -120,6 +121,7 @@ foreach ($servers as $row_pos => $server):
<td><span class="<?php echo ($server['Server']['push']? 'icon-ok' : 'icon-remove'); ?>" role="img" aria-label="<?php echo ($server['Server']['push']? __('Yes') : __('No')); ?>"></span><span class="short <?php if (!$server['Server']['push'] || empty($ruleDescription['push'])) echo "hidden"; ?>" data-toggle="popover" title="Distribution List" data-content="<?php echo $ruleDescription['push']; ?>"> (<?php echo __('Rules');?>)</span></td>
<td><span class="<?php echo ($server['Server']['pull']? 'icon-ok' : 'icon-remove'); ?>" role="img" aria-label="<?php echo ($server['Server']['pull']? __('Yes') : __('No')); ?>"></span><span class="short <?php if (!$server['Server']['pull'] || empty($ruleDescription['pull'])) echo "hidden"; ?>" data-toggle="popover" title="Distribution List" data-content="<?php echo $ruleDescription['pull']; ?>"> (<?php echo __('Rules');?>)</span></td>
<td class="short"><span class="<?php echo ($server['Server']['push_sightings'] ? 'icon-ok' : 'icon-remove'); ?>" role="img" aria-label="<?php echo ($server['Server']['push_sightings'] ? __('Yes') : __('No')); ?>"></span></td>
<td class="short"><span class="<?php echo ($server['Server']['push_galaxy_clusters'] ? 'icon-ok' : 'icon-remove'); ?>" role="img" aria-label="<?php echo ($server['Server']['push_galaxy_clusters'] ? __('Yes') : __('No')); ?>"></span></td>
<td>
<?php
if ($server['Server']['caching_enabled']) {