MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

new: [API] exposed the server related functionalities to the API 

- server index
- server push
- server pull

- improved logging / error reporting of the sync functionalities
pull/3587/head
iglocska 2018-08-22 17:00:13 +02:00
parent 6caf0c6758
commit 113fa25471
3 changed files with 178 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
app/Console/Command/ServerShell.php
View File

@ -45,28 +45,13 @@ class ServerShell extends AppShell
'progress' => 100,
'status' => 4
));
if (is_numeric($result[0])) {
switch ($result[0]) {
case '1' :
$message = 'Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server.';
break;
case '2' :
$message = $result[1];
break;
case '3' :
$message = 'Sorry, incremental pushes are not yet implemented.';
break;
case '4' :
$message = 'Invalid technique chosen.';
break;
}
if (!empty($message)) {
echo $message . PHP_EOL;
$this->Job->saveField('message', $message);
}
if (is_array($result)) {
$message = sprintf(__('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled.', count($result[0]), count($result[1]), count($result[2])));
} else {
echo 'Job done.';
$message = sprintf(__('ERROR: %s'), $result);
}
$this->Job->saveField('message', $message);
echo $message . PHP_EOL;
}
public function push() {
@ -100,7 +85,7 @@ class ServerShell extends AppShell
$HttpSocket = $syncTool->setupHttpSocket($server);
$result = $this->Server->push($serverId, 'full', $jobId, $HttpSocket, $user);
$message = 'Job done.';
if ($result === false) $message = 'Job failed. The remote instance is too far outdated to initiate a push.';
if ($result !== true && !is_array($result)) $message = 'Job failed. Reason: ' . $result;
$this->Job->save(array(
'id' => $jobId,
'message' => $message,

170
app/Controller/ServersController.php
View File

@ -50,16 +50,28 @@ class ServersController extends AppController
}
$this->paginate['conditions'] = array('Server.org_id LIKE' => $this->Auth->user('org_id'));
}
$this->set('servers', $this->paginate());
$collection = array();
$collection['orgs'] = $this->Server->Organisation->find('list', array(
'fields' => array('id', 'name'),
));
$this->loadModel('Tag');
$collection['tags'] = $this->Tag->find('list', array(
'fields' => array('id', 'name'),
));
$this->set('collection', $collection);
if ($this->_isRest()) {
$params = array(
'recursive' => -1,
'contain' => array(
'Organisation' => array('Organisation.id', 'Organisation.name', 'Organisation.uuid', 'Organisation.nationality', 'Organisation.sector', 'Organisation.type'),
'RemoteOrg' => array('RemoteOrg.id', 'RemoteOrg.name', 'RemoteOrg.uuid', 'RemoteOrg.nationality', 'RemoteOrg.sector', 'RemoteOrg.type'),
)
);
$servers = $this->Server->find('all', $params);
return $this->RestResponse->viewData($servers, $this->response->type());
} else {
$this->set('servers', $this->paginate());
$collection = array();
$collection['orgs'] = $this->Server->Organisation->find('list', array(
'fields' => array('id', 'name'),
));
$this->loadModel('Tag');
$collection['tags'] = $this->Tag->find('list', array(
'fields' => array('id', 'name'),
));
$this->set('collection', $collection);
}
}
public function previewIndex($id)
@ -583,73 +595,74 @@ class ServersController extends AppController
* incremental - only new events
* <int> - specific id of the event to pull
*/
public function pull($id = null, $technique=false)
public function pull($id = null, $technique='full')
{
$this->Server->id = $id;
if (!$this->Server->exists()) {
throw new NotFoundException(__('Invalid server'));
}
$s = $this->Server->read(null, $id);
$error = false;
if (!$this->_isSiteAdmin() && !($s['Server']['org_id'] == $this->Auth->user('org_id') && $this->_isAdmin())) {
$this->redirect(array('controller' => 'servers', 'action' => 'index'));
throw new MethodNotAllowedException(__('You are not authorised to do that.'));
}
$this->Server->id = $id;
if (!$this->Server->exists()) {
throw new NotFoundException(__('Invalid server'));
}
if (false == $this->Server->data['Server']['pull'] && ($technique == 'full' || $technique == 'incremental')) {
$this->Flash->info(__('Pull setting not enabled for this server.'));
$this->redirect(array('action' => 'index'));
}
if (!Configure::read('MISP.background_jobs')) {
$result = $this->Server->pull($this->Auth->user(), $id, $technique, $s);
// error codes
if (isset($result[0]) && is_numeric($result[0])) {
switch ($result[0]) {
case '1':
$this->Flash->error(__('Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server. Another reason could be an incorrect sync server setting.'));
break;
case '2':
$this->Flash->error($result[1]);
break;
case '3':
throw new NotFoundException('Sorry, this is not yet implemented');
break;
case '4':
$this->redirect(array('action' => 'index'));
break;
}
$this->redirect($this->referer());
} else {
$this->set('successes', $result[0]);
$this->set('fails', $result[1]);
$this->set('pulledProposals', $result[2]);
$this->set('lastpulledid', $result[3]);
}
} else {
$this->loadModel('Job');
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'pull',
'job_input' => 'Server: ' . $id,
'status' => 0,
'retries' => 0,
'org' => $this->Auth->user('Organisation')['name'],
'message' => 'Pulling.',
);
$this->Job->save($data);
$jobId = $this->Job->id;
$process_id = CakeResque::enqueue(
'default',
'ServerShell',
array('pull', $this->Auth->user('id'), $id, $technique, $jobId)
);
$this->Job->saveField('process_id', $process_id);
$this->Flash->success('Pull queued for background execution.');
$this->redirect($this->referer());
$error = __('Pull setting not enabled for this server.');
}
if (empty($error)) {
if (!Configure::read('MISP.background_jobs')) {
$result = $this->Server->pull($this->Auth->user(), $id, $technique, $s);
if (is_array($result)) {
$success = sprintf(__('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled.', count($result[0]), count($result[1]), count($result[2])));
} else {
$error = $result;
}
$this->set('successes', $result[0]);
$this->set('fails', $result[1]);
$this->set('pulledProposals', $result[2]);
$this->set('lastpulledid', $result[3]);
} else {
$this->loadModel('Job');
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'pull',
'job_input' => 'Server: ' . $id,
'status' => 0,
'retries' => 0,
'org' => $this->Auth->user('Organisation')['name'],
'message' => 'Pulling.',
);
$this->Job->save($data);
$jobId = $this->Job->id;
$process_id = CakeResque::enqueue(
'default',
'ServerShell',
array('pull', $this->Auth->user('id'), $id, $technique, $jobId)
);
$this->Job->saveField('process_id', $process_id);
$success = sprintf(__('Pull queued for background execution. Job ID: %s'), $jobId);
}
}
if ($this->_isRest()) {
if (!empty($error)) {
return $this->RestResponse->saveFailResponse('Servers', 'pull', false, $error, $this->response->type());
} else {
return $this->RestResponse->saveSuccessResponse('Servers', 'pull', $success, $this->response->type());
}
} else {
if (!empty($error)) {
$this->Flash->error($error);
$this->redirect(array('action' => 'index'));
} else {
$this->Flash->success($success);
$this->redirect($this->referer());
}
}
}
public function push($id = null, $technique=false)
@ -660,7 +673,7 @@ class ServersController extends AppController
}
$s = $this->Server->read(null, $id);
if (!$this->_isSiteAdmin() && !($s['Server']['org_id'] == $this->Auth->user('org_id') && $this->_isAdmin())) {
$this->redirect(array('controller' => 'servers', 'action' => 'index'));
throw new MethodNotAllowedException(__('You are not authorised to do that.'));
}
if (!Configure::read('MISP.background_jobs')) {
$server = $this->Server->read(null, $id);
@ -669,11 +682,24 @@ class ServersController extends AppController
$HttpSocket = $syncTool->setupHttpSocket($server);
$result = $this->Server->push($id, $technique, false, $HttpSocket, $this->Auth->user());
if ($result === false) {
$this->Flash->info('The remote server is too outdated to initiate a push towards it. Please notify the hosting organisation of the remote instance.');
$this->redirect(array('action' => 'index'));
}
$this->set('successes', $result[0]);
$this->set('fails', $result[1]);
$error = __('The remote server is too outdated to initiate a push towards it. Please notify the hosting organisation of the remote instance.');
} else if (!is_array($result)) {
$error = $result;
}
if (!empty($error)) {
if ($this->_isRest()) {
return $this->RestResponse->saveFailResponse('Servers', 'push', false, $error, $this->response->type());
} else {
$this->Flash->info($error);
$this->redirect(array('action' => 'index'));
}
}
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('Servers', 'push', array(sprintf(__('Push complete. %s events pushed, %s events could not be pushed.', $result[0], $result[1]))), $this->response->type());
} else {
$this->set('successes', $result[0]);
$this->set('fails', $result[1]);
}
} else {
$this->loadModel('Job');
$this->Job->create();
@ -694,7 +720,11 @@ class ServersController extends AppController
array('push', $this->Auth->user('id'), $id, $jobId)
);
$this->Job->saveField('process_id', $process_id);
$this->Flash->success('Push queued for background execution.');
$message = sprintf(__('Push queued for background execution. Job ID: %s'), $jobId);
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('Servers', 'push', $message, $this->response->type());
}
$this->Flash->success($message);
$this->redirect(array('action' => 'index'));
}
}

106
app/Model/Server.php
View File

@ -1833,6 +1833,27 @@ class Server extends AppModel
$eventIds = array();
$conditions = array();
$eventIds = $this->__getEventIdListBasedOnPullTechnique($technique, $server);
if (!empty($eventIds['error'])) {
$errors = array(
'1' => __('Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server. Another reason could be an incorrect sync server setting.'),
'2' => $eventIds['error'][1],
'3' => __('Sorry, this is not yet implemented'),
'4' => __('Something went wrong while trying to pull')
);
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Server',
'model_id' => $id,
'email' => $user['email'],
'action' => 'error',
'user_id' => $user['id'],
'title' => 'Failed pull from ' . $server['Server']['url'] . ' initiated by ' . $email,
'change' => !empty($errors[$eventIds['error'][0]]) ? $errors[$eventIds['error'][0]] : __('Unknown issue.')
));
return !empty($errors[$eventIds['error'][0]]) ? $errors[$eventIds['error'][0]] : __('Unknown issue.');
}
$successes = array();
$fails = array();
$pulledProposals = array();
@ -2129,16 +2150,32 @@ class Server extends AppModel
$this->Event = ClassRegistry::init('Event');
$this->read(null, $id);
$url = $this->data['Server']['url'];
$push = $this->checkVersionCompatibility($id, $user)['canPush'];
if (!isset($push) || !$push) {
if ($jobId) {
$job->id = $jobId;
$job->saveField('progress', 100);
$job->saveField('message', 'Push to server ' . $id . ' failed. Remote instance is outdated.');
$job->saveField('status', 4);
}
return false;
}
$push = $this->checkVersionCompatibility($id, $user);
if (isset($push['canPush']) && !$push['canPush']) {
$push = 'Remote instance is outdated.';
}
if (!is_array($push)) {
$message = sprintf('Push to server %s failed. Reason: %s', $id, $push);
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Server',
'model_id' => $id,
'email' => $user['email'],
'action' => 'error',
'user_id' => $user['id'],
'title' => 'Failed: Push to ' . $url . ' initiated by ' . $user['email'],
'change' => $message
));
if ($jobId) {
$job->id = $jobId;
$job->saveField('progress', 100);
$job->saveField('message', $message);
$job->saveField('status', 4);
}
return $push;
}
if ("full" == $technique) {
$eventid_conditions_key = 'Event.id >';
$eventid_conditions_value = 0;
@ -2272,6 +2309,7 @@ class Server extends AppModel
} else {
return array($successes, $fails);
}
return true;
}
public function getEventIdsForPush($id, $HttpSocket, $eventIds, $user)
@ -3290,34 +3328,34 @@ class Server extends AppModel
try {
$response = $HttpSocket->get($uri, '', $request);
} catch (Exception $e) {
if (!isset($response) || $response->code != '200') {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
if (isset($response->code)) {
$title = 'Error: Connection to the server has failed.' . isset($response->code) ? ' Returned response code: ' . $response->code : '';
} else {
$title = 'Error: Connection to the server has failed. The returned exception\'s error message was: ' . $e->getMessage();
}
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Server',
'model_id' => $id,
'email' => $user['email'],
'action' => 'error',
'user_id' => $user['id'],
'title' => $title,
));
}
}
if (!isset($response) || $response->code != '200') {
return 1;
$error = $e->getMessage;
}
if (!isset($response) || $response->code != '200') {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
if (isset($response->code)) {
$title = 'Error: Connection to the server has failed.' . (isset($response->code) ? ' Returned response code: ' . $response->code : '');
} else {
$title = 'Error: Connection to the server has failed. The returned exception\'s error message was: ' . $e->getMessage();
}
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Server',
'model_id' => $id,
'email' => $user['email'],
'action' => 'error',
'user_id' => $user['id'],
'title' => $title
));
return $title;
}
$remoteVersion = json_decode($response->body, true);
$canPush = isset($remoteVersion['perm_sync']) ? $remoteVersion['perm_sync'] : false;
$remoteVersion = explode('.', $remoteVersion['version']);
if (!isset($remoteVersion[0])) {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$message = __('Error: Server didn\'t send the expected response. This may be because the remote server version is outdated.');
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Server',
@ -3325,9 +3363,9 @@ class Server extends AppModel
'email' => $user['email'],
'action' => 'error',
'user_id' => $user['id'],
'title' => 'Error: Server didn\'t send the expected response. This may be because the remote server version is outdated.',
'title' => $message,
));
return 2;
return $message;
}
$response = false;
$success = false;
@ -4030,7 +4068,7 @@ class Server extends AppModel
return APP . 'files';
}
public function getDefaultTmp_dir()
public function getDefaultTmp_dir()
{
return sys_get_temp_dir();
}