mirror of https://github.com/MISP/MISP
chg: [sync] Simplify galaxy cluster pushing
Jakub Onderka
parent
0326d35387
commit
d8fd3e937a
|
|
@ -1,5 +1,6 @@
|
|||
<?php
|
||||
App::uses('SyncTool', 'Tools');
|
||||
App::uses('JsonTool', 'Tools');
|
||||
|
||||
class ServerSyncTool
|
||||
{
|
||||
|
|
@ -10,7 +11,9 @@ class ServerSyncTool
|
|||
FEATURE_PROPOSALS = 'proposals',
|
||||
FEATURE_PROTECTED_EVENT = 'protected_event',
|
||||
FEATURE_POST_TEST = 'post_test',
|
||||
FEATURE_EDIT_OF_GALAXY_CLUSTER = 'edit_of_galaxy_cluster';
|
||||
FEATURE_EDIT_OF_GALAXY_CLUSTER = 'edit_of_galaxy_cluster',
|
||||
PERM_SYNC = 'perm_sync',
|
||||
PERM_GALAXY_EDITOR = 'perm_galaxy_editor';
|
||||
|
||||
/** @var array */
|
||||
private $server;
|
||||
|
|
@ -197,6 +200,18 @@ class ServerSyncTool
|
|||
return $this->get('/galaxy_clusters/view/' . $galaxyClusterId);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $cluster
|
||||
* @return HttpSocketResponseExtended
|
||||
* @throws HttpSocketHttpException
|
||||
* @throws HttpSocketJsonException
|
||||
*/
|
||||
public function pushGalaxyCluster(array $cluster)
|
||||
{
|
||||
$logMessage = "Pushing Galaxy Cluster #{$cluster['GalaxyCluster']['id']} to Server #{$this->serverId()}";
|
||||
return $this->post('/galaxies/pushCluster', [$cluster], $logMessage);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $params
|
||||
* @return HttpSocketResponseExtended
|
||||
|
|
@ -367,6 +382,10 @@ class ServerSyncTool
|
|||
return $version[0] == 2 && $version[1] == 4 && $version[2] > 155;
|
||||
case self::FEATURE_EDIT_OF_GALAXY_CLUSTER:
|
||||
return isset($info['perm_galaxy_editor']);
|
||||
case self::PERM_SYNC:
|
||||
return isset($info['perm_sync']) && $info['perm_sync'];
|
||||
case self::PERM_GALAXY_EDITOR:
|
||||
return isset($info['perm_galaxy_editor']) && $info['perm_galaxy_editor'];
|
||||
default:
|
||||
throw new InvalidArgumentException("Invalid flag `$flag` provided");
|
||||
}
|
||||
|
|
@ -408,7 +427,7 @@ class ServerSyncTool
|
|||
private function post($url, $data, $logMessage = null)
|
||||
{
|
||||
$protectedMode = !empty($data['Event']['protected']);
|
||||
$data = json_encode($data, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
|
||||
$data = JsonTool::encode($data);
|
||||
|
||||
if ($logMessage && !empty(Configure::read('Security.sync_audit'))) {
|
||||
$pushLogEntry = sprintf(
|
||||
|
|
|
|||
|
|
@ -628,13 +628,11 @@ class GalaxyCluster extends AppModel
|
|||
$elevatedUser = array(
|
||||
'Role' => array(
|
||||
'perm_site_admin' => 1,
|
||||
'perm_sync' => 1
|
||||
'perm_sync' => 1,
|
||||
'perm_audit' => 0,
|
||||
),
|
||||
'org_id' => $clusterOrgcId['GalaxyCluster']['orgc_id']
|
||||
);
|
||||
$elevatedUser['Role']['perm_site_admin'] = 1;
|
||||
$elevatedUser['Role']['perm_sync'] = 1;
|
||||
$elevatedUser['Role']['perm_audit'] = 0;
|
||||
$cluster = $this->fetchGalaxyClusters($elevatedUser, array('minimal' => true, 'conditions' => array('id' => $clusterId)), $full=false);
|
||||
if (empty($cluster)) {
|
||||
return true;
|
||||
|
|
@ -655,14 +653,10 @@ class GalaxyCluster extends AppModel
|
|||
return true;
|
||||
}
|
||||
$uploaded = false;
|
||||
$failedServers = array();
|
||||
App::uses('SyncTool', 'Tools');
|
||||
foreach ($servers as &$server) {
|
||||
foreach ($servers as $server) {
|
||||
if ((!isset($server['Server']['internal']) || !$server['Server']['internal']) && $cluster['GalaxyCluster']['distribution'] < 2) {
|
||||
continue;
|
||||
}
|
||||
$syncTool = new SyncTool();
|
||||
$HttpSocket = $syncTool->setupHttpSocket($server);
|
||||
$fakeSyncUser = array(
|
||||
'id' => 0,
|
||||
'email' => 'fakeSyncUser@user.test',
|
||||
|
|
@ -678,14 +672,13 @@ class GalaxyCluster extends AppModel
|
|||
);
|
||||
$cluster = $this->fetchGalaxyClusters($fakeSyncUser, array('conditions' => array('GalaxyCluster.id' => $clusterId)), $full=true);
|
||||
if (empty($cluster)) {
|
||||
return true;
|
||||
continue;
|
||||
}
|
||||
$cluster = $cluster[0];
|
||||
$result = $this->uploadClusterToServer($cluster, $server, $HttpSocket, $fakeSyncUser);
|
||||
if ($result == 'Success') {
|
||||
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
|
||||
$result = $this->uploadClusterToServer($cluster, $server, $serverSync, $fakeSyncUser);
|
||||
if ($result === 'Success') {
|
||||
$uploaded = true;
|
||||
} else {
|
||||
$failedServers[] = $server;
|
||||
}
|
||||
}
|
||||
return $uploaded;
|
||||
|
|
@ -1672,79 +1665,29 @@ class GalaxyCluster extends AppModel
|
|||
|
||||
/**
|
||||
* @return string|bool The result of the upload. True if success, a string otherwise
|
||||
* @throws Exception
|
||||
*/
|
||||
public function uploadClusterToServer($cluster, $server, $HttpSocket, $user)
|
||||
{
|
||||
$this->Server = ClassRegistry::init('Server');
|
||||
$this->Log = ClassRegistry::init('Log');
|
||||
$push = $this->Server->checkVersionCompatibility($server, false);
|
||||
if (empty($push['canPush']) && empty($push['canPushGalaxyCluster'])) {
|
||||
return __('The remote user does not have the permission to manipulate galaxies - the upload of the galaxy clusters has been blocked.');
|
||||
}
|
||||
$updated = null;
|
||||
$newLocation = $newTextBody = '';
|
||||
$result = $this->__executeRestfulGalaxyClusterToServer($cluster, $server, null, $newLocation, $newTextBody, $HttpSocket, $user);
|
||||
if ($result !== true) {
|
||||
return $result;
|
||||
}
|
||||
if (strlen($newLocation)) { // HTTP/1.1 302 Found and Location: http://<newLocation>
|
||||
$result = $this->__executeRestfulGalaxyClusterToServer($cluster, $server, $newLocation, $newLocation, $newTextBody, $HttpSocket, $user);
|
||||
if ($result !== true) {
|
||||
return $result;
|
||||
}
|
||||
}
|
||||
$uploadFailed = false;
|
||||
try {
|
||||
$json = json_decode($newTextBody, true);
|
||||
} catch (Exception $e) {
|
||||
$uploadFailed = true;
|
||||
}
|
||||
if (!is_array($json) || $uploadFailed) {
|
||||
$this->Log->createLogEntry($user, 'push', 'GalaxyCluster', $cluster['GalaxyCluster']['id'], 'push', $newTextBody);
|
||||
}
|
||||
return 'Success';
|
||||
}
|
||||
|
||||
private function __executeRestfulGalaxyClusterToServer($cluster, $server, $resourceId, &$newLocation, &$newTextBody, $HttpSocket, $user)
|
||||
{
|
||||
$result = $this->restfulGalaxyClusterToServer($cluster, $server, $resourceId, $newLocation, $newTextBody, $HttpSocket);
|
||||
if (is_numeric($result)) {
|
||||
$error = $this->__resolveErrorCode($result, $cluster, $server, $user);
|
||||
if ($error) {
|
||||
return $error . ' Error code: ' . $result;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return string|bool|int The result of the upload.
|
||||
*/
|
||||
public function restfulGalaxyClusterToServer($cluster, $server, $urlPath, &$newLocation, &$newTextBody, $HttpSocket = null)
|
||||
public function uploadClusterToServer(array $cluster, array $server, ServerSyncTool $serverSync, array $user)
|
||||
{
|
||||
$cluster = $this->__prepareForPushToServer($cluster, $server);
|
||||
if (is_numeric($cluster)) {
|
||||
return $cluster;
|
||||
}
|
||||
$url = $server['Server']['url'];
|
||||
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
|
||||
$request = $this->setupSyncRequest($server);
|
||||
$scope = 'galaxies/pushCluster';
|
||||
$uri = $url . '/' . $scope;
|
||||
$clusters = array($cluster);
|
||||
$data = json_encode($clusters);
|
||||
if (!empty(Configure::read('Security.sync_audit'))) {
|
||||
$pushLogEntry = sprintf(
|
||||
"==============================================================\n\n[%s] Pushing Galaxy Cluster #%d to Server #%d:\n\n%s\n\n",
|
||||
date("Y-m-d H:i:s"),
|
||||
$cluster['GalaxyCluster']['id'],
|
||||
$server['Server']['id'],
|
||||
$data
|
||||
);
|
||||
file_put_contents(APP . 'files/scripts/tmp/debug_server_' . $server['Server']['id'] . '.log', $pushLogEntry, FILE_APPEND);
|
||||
|
||||
try {
|
||||
if (!$serverSync->isSupported(ServerSyncTool::PERM_SYNC) || $serverSync->isSupported(ServerSyncTool::PERM_GALAXY_EDITOR)) {
|
||||
return __('The remote user does not have the permission to manipulate galaxies - the upload of the galaxy clusters has been blocked.');
|
||||
}
|
||||
$serverSync->pushGalaxyCluster($cluster)->json();
|
||||
} catch (Exception $e) {
|
||||
$title = __('Uploading GalaxyCluster (%s) to Server (%s)', $cluster['GalaxyCluster']['id'], $server['Server']['id']);
|
||||
$this->loadLog()->createLogEntry($user, 'push', 'GalaxyCluster', $cluster['GalaxyCluster']['id'], $title, $e->getMessage());
|
||||
|
||||
$this->logException("Could not push galaxy cluster to remote server {$serverSync->serverId()}", $e);
|
||||
return $e->getMessage();
|
||||
}
|
||||
$response = $HttpSocket->post($uri, $data, $request);
|
||||
return $this->__handleRestfulGalaxyClusterToServerResponse($response, $newLocation, $newTextBody);
|
||||
|
||||
return 'Success';
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -1752,7 +1695,7 @@ class GalaxyCluster extends AppModel
|
|||
*
|
||||
* @param array $cluster
|
||||
* @param array $server
|
||||
* @return array The cluster ready to be pushed
|
||||
* @return array|int The cluster ready to be pushed
|
||||
*/
|
||||
private function __prepareForPushToServer(array $cluster, array $server)
|
||||
{
|
||||
|
|
@ -1773,11 +1716,9 @@ class GalaxyCluster extends AppModel
|
|||
}
|
||||
$this->Event = ClassRegistry::init('Event');
|
||||
if ($this->Event->checkDistributionForPush($cluster, $server, 'GalaxyCluster')) {
|
||||
$cluster = $this->__updateClusterForSync($cluster, $server);
|
||||
} else {
|
||||
return 403;
|
||||
return $this->__updateClusterForSync($cluster, $server);
|
||||
}
|
||||
return $cluster;
|
||||
return 403;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -1886,59 +1827,6 @@ class GalaxyCluster extends AppModel
|
|||
return $relation;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return string|bool|int The result of the upload.
|
||||
*/
|
||||
private function __handleRestfulGalaxyClusterToServerResponse($response, &$newLocation, &$newTextBody)
|
||||
{
|
||||
switch ($response->code) {
|
||||
case '200': // 200 (OK) + entity-action-result
|
||||
if ($response->isOk()) {
|
||||
$newTextBody = $response->body();
|
||||
return true;
|
||||
} else {
|
||||
try {
|
||||
$jsonArray = json_decode($response->body, true);
|
||||
} catch (Exception $e) {
|
||||
return true;
|
||||
}
|
||||
return $jsonArray['name'];
|
||||
}
|
||||
// no break
|
||||
case '302': // Found
|
||||
$newLocation = $response->headers['Location'];
|
||||
$newTextBody = $response->body();
|
||||
return true;
|
||||
case '404': // Not Found
|
||||
$newLocation = $response->headers['Location'];
|
||||
$newTextBody = $response->body();
|
||||
return 404;
|
||||
case '405':
|
||||
return 405;
|
||||
case '403': // Not authorised
|
||||
return 403;
|
||||
}
|
||||
}
|
||||
|
||||
private function __resolveErrorCode($code, &$cluster, &$server, $user)
|
||||
{
|
||||
$this->Log = ClassRegistry::init('Log');
|
||||
$error = false;
|
||||
switch ($code) {
|
||||
case 403:
|
||||
return __('The distribution level of the cluster blocks it from being pushed.');
|
||||
case 405:
|
||||
$error = __('The sync user on the remote instance does not have the required privileges to handle this cluster.');
|
||||
break;
|
||||
}
|
||||
if ($error) {
|
||||
$newTextBody = 'Uploading GalaxyCluster (' . $cluster['GalaxyCluster']['id'] . ') to Server (' . $server['Server']['id'] . ')';
|
||||
$newTextBody = __('Uploading GalaxyCluster (%s) to Server (%s)', $cluster['GalaxyCluster']['id'], $server['Server']['id']);
|
||||
$this->Log->createLogEntry($user, 'push', 'GalaxyCluster', $cluster['GalaxyCluster']['id'], 'push', $newTextBody);
|
||||
}
|
||||
return $error;
|
||||
}
|
||||
|
||||
/**
|
||||
* pullGalaxyClusters
|
||||
*
|
||||
|
|
|
|||
|
|
@ -1129,7 +1129,6 @@ class Server extends AppModel
|
|||
}
|
||||
$this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
|
||||
$this->Event = ClassRegistry::init('Event');
|
||||
$HttpSocket = $this->setupHttpSocket($server);
|
||||
$clusters = array();
|
||||
if ($technique == 'full') {
|
||||
$clusters = $this->GalaxyCluster->getElligibleClustersToPush($user, $conditions=array(), $full=true);
|
||||
|
|
@ -1150,7 +1149,7 @@ class Server extends AppModel
|
|||
return [];
|
||||
}
|
||||
foreach ($clustersToPush as $cluster) {
|
||||
$result = $this->GalaxyCluster->uploadClusterToServer($cluster, $server, $HttpSocket, $user);
|
||||
$result = $this->GalaxyCluster->uploadClusterToServer($cluster, $server, $serverSync, $user);
|
||||
if ($result === 'Success') {
|
||||
$successes[] = __('GalaxyCluster %s', $cluster['GalaxyCluster']['uuid']);
|
||||
}
|
||||
|
|
@ -3740,8 +3739,7 @@ class Server extends AppModel
|
|||
public function extensionDiagnostics()
|
||||
{
|
||||
try {
|
||||
$file = new File(APP . DS . 'composer.json');
|
||||
$composer = $this->jsonDecode($file->read());
|
||||
$composer = FileAccessTool::readJsonFromFile(APP . DS . 'composer.json');
|
||||
$extensions = [];
|
||||
foreach ($composer['require'] as $require => $foo) {
|
||||
if (substr($require, 0, 4) === 'ext-') {
|
||||
|
|
|
|||
Loading…
Reference in New Issue