MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch '2.4' into feature/api_rework

pull/3608/head
iglocska 2018-08-22 17:39:56 +02:00
parent ce8ee288ab 113fa25471
commit 2a10276d07
7 changed files with 218 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
app/Console/Command/ServerShell.php
View File

@ -45,28 +45,13 @@ class ServerShell extends AppShell
'progress' => 100,
'status' => 4
));
if (is_numeric($result[0])) {
switch ($result[0]) {
case '1' :
$message = 'Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server.';
break;
case '2' :
$message = $result[1];
break;
case '3' :
$message = 'Sorry, incremental pushes are not yet implemented.';
break;
case '4' :
$message = 'Invalid technique chosen.';
break;
}
if (!empty($message)) {
echo $message . PHP_EOL;
$this->Job->saveField('message', $message);
}
if (is_array($result)) {
$message = sprintf(__('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled.', count($result[0]), count($result[1]), count($result[2])));
} else {
echo 'Job done.';
$message = sprintf(__('ERROR: %s'), $result);
}
$this->Job->saveField('message', $message);
echo $message . PHP_EOL;
}
public function push() {
@ -100,7 +85,7 @@ class ServerShell extends AppShell
$HttpSocket = $syncTool->setupHttpSocket($server);
$result = $this->Server->push($serverId, 'full', $jobId, $HttpSocket, $user);
$message = 'Job done.';
if ($result === false) $message = 'Job failed. The remote instance is too far outdated to initiate a push.';
if ($result !== true && !is_array($result)) $message = 'Job failed. Reason: ' . $result;
$this->Job->save(array(
'id' => $jobId,
'message' => $message,

4
app/Controller/AppController.php
View File

@ -463,9 +463,9 @@ class AppController extends Controller
public function blackhole($type)
{
if ($type === 'csrf') {
throw new BadRequestException(__d('cake_dev', $type));
throw new BadRequestException($type);
}
throw new BadRequestException(__d('cake_dev', 'The request has been black-holed'));
throw new BadRequestException('The request has been black-holed');
}
public $userRole = null;

124
app/Controller/ServersController.php
View File

@ -50,16 +50,28 @@ class ServersController extends AppController
}
$this->paginate['conditions'] = array('Server.org_id LIKE' => $this->Auth->user('org_id'));
}
$this->set('servers', $this->paginate());
$collection = array();
$collection['orgs'] = $this->Server->Organisation->find('list', array(
'fields' => array('id', 'name'),
));
$this->loadModel('Tag');
$collection['tags'] = $this->Tag->find('list', array(
'fields' => array('id', 'name'),
));
$this->set('collection', $collection);
if ($this->_isRest()) {
$params = array(
'recursive' => -1,
'contain' => array(
'Organisation' => array('Organisation.id', 'Organisation.name', 'Organisation.uuid', 'Organisation.nationality', 'Organisation.sector', 'Organisation.type'),
'RemoteOrg' => array('RemoteOrg.id', 'RemoteOrg.name', 'RemoteOrg.uuid', 'RemoteOrg.nationality', 'RemoteOrg.sector', 'RemoteOrg.type'),
)
);
$servers = $this->Server->find('all', $params);
return $this->RestResponse->viewData($servers, $this->response->type());
} else {
$this->set('servers', $this->paginate());
$collection = array();
$collection['orgs'] = $this->Server->Organisation->find('list', array(
'fields' => array('id', 'name'),
));
$this->loadModel('Tag');
$collection['tags'] = $this->Tag->find('list', array(
'fields' => array('id', 'name'),
));
$this->set('collection', $collection);
}
}
public function previewIndex($id)
@ -583,22 +595,23 @@ class ServersController extends AppController
* incremental - only new events
* <int> - specific id of the event to pull
*/
public function pull($id = null, $technique=false)
public function pull($id = null, $technique='full')
{
$this->Server->id = $id;
if (!$this->Server->exists()) {
throw new NotFoundException(__('Invalid server'));
}
$s = $this->Server->read(null, $id);
$error = false;
if (!$this->_isSiteAdmin() && !($s['Server']['org_id'] == $this->Auth->user('org_id') && $this->_isAdmin())) {
$this->redirect(array('controller' => 'servers', 'action' => 'index'));
throw new MethodNotAllowedException(__('You are not authorised to do that.'));
}
$this->Server->id = $id;
if (!$this->Server->exists()) {
throw new NotFoundException(__('Invalid server'));
}
if (false == $this->Server->data['Server']['pull'] && ($technique == 'full' || $technique == 'incremental')) {
<<<<<<< HEAD
$this->Flash->info(__('Pull setting not enabled for this server.'));
$this->redirect(array('action' => 'index'));
}
@ -649,6 +662,60 @@ class ServersController extends AppController
$this->Flash->success('Pull queued for background execution.');
$this->redirect($this->referer());
}
=======
$error = __('Pull setting not enabled for this server.');
}
if (empty($error)) {
if (!Configure::read('MISP.background_jobs')) {
$result = $this->Server->pull($this->Auth->user(), $id, $technique, $s);
if (is_array($result)) {
$success = sprintf(__('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled.', count($result[0]), count($result[1]), count($result[2])));
} else {
$error = $result;
}
$this->set('successes', $result[0]);
$this->set('fails', $result[1]);
$this->set('pulledProposals', $result[2]);
$this->set('lastpulledid', $result[3]);
} else {
$this->loadModel('Job');
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'pull',
'job_input' => 'Server: ' . $id,
'status' => 0,
'retries' => 0,
'org' => $this->Auth->user('Organisation')['name'],
'message' => 'Pulling.',
);
$this->Job->save($data);
$jobId = $this->Job->id;
$process_id = CakeResque::enqueue(
'default',
'ServerShell',
array('pull', $this->Auth->user('id'), $id, $technique, $jobId)
);
$this->Job->saveField('process_id', $process_id);
$success = sprintf(__('Pull queued for background execution. Job ID: %s'), $jobId);
}
}
if ($this->_isRest()) {
if (!empty($error)) {
return $this->RestResponse->saveFailResponse('Servers', 'pull', false, $error, $this->response->type());
} else {
return $this->RestResponse->saveSuccessResponse('Servers', 'pull', $success, $this->response->type());
}
} else {
if (!empty($error)) {
$this->Flash->error($error);
$this->redirect(array('action' => 'index'));
} else {
$this->Flash->success($success);
$this->redirect($this->referer());
}
}
>>>>>>> 2.4
}
public function push($id = null, $technique=false)
@ -659,7 +726,7 @@ class ServersController extends AppController
}
$s = $this->Server->read(null, $id);
if (!$this->_isSiteAdmin() && !($s['Server']['org_id'] == $this->Auth->user('org_id') && $this->_isAdmin())) {
$this->redirect(array('controller' => 'servers', 'action' => 'index'));
throw new MethodNotAllowedException(__('You are not authorised to do that.'));
}
if (!Configure::read('MISP.background_jobs')) {
$server = $this->Server->read(null, $id);
@ -668,11 +735,24 @@ class ServersController extends AppController
$HttpSocket = $syncTool->setupHttpSocket($server);
$result = $this->Server->push($id, $technique, false, $HttpSocket, $this->Auth->user());
if ($result === false) {
$this->Flash->info('The remote server is too outdated to initiate a push towards it. Please notify the hosting organisation of the remote instance.');
$this->redirect(array('action' => 'index'));
}
$this->set('successes', $result[0]);
$this->set('fails', $result[1]);
$error = __('The remote server is too outdated to initiate a push towards it. Please notify the hosting organisation of the remote instance.');
} else if (!is_array($result)) {
$error = $result;
}
if (!empty($error)) {
if ($this->_isRest()) {
return $this->RestResponse->saveFailResponse('Servers', 'push', false, $error, $this->response->type());
} else {
$this->Flash->info($error);
$this->redirect(array('action' => 'index'));
}
}
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('Servers', 'push', array(sprintf(__('Push complete. %s events pushed, %s events could not be pushed.', $result[0], $result[1]))), $this->response->type());
} else {
$this->set('successes', $result[0]);
$this->set('fails', $result[1]);
}
} else {
$this->loadModel('Job');
$this->Job->create();
@ -693,7 +773,11 @@ class ServersController extends AppController
array('push', $this->Auth->user('id'), $id, $jobId)
);
$this->Job->saveField('process_id', $process_id);
$this->Flash->success('Push queued for background execution.');
$message = sprintf(__('Push queued for background execution. Job ID: %s'), $jobId);
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('Servers', 'push', $message, $this->response->type());
}
$this->Flash->success($message);
$this->redirect(array('action' => 'index'));
}
}

53
app/Locale/default.pot
View File

@ -3190,6 +3190,11 @@ msgstr ""
msgid "Add"
msgstr ""
#: View/Elements/global_menu.ctp:20
#: View/Pages/doc/general.ctp:20
msgid "Event Actions"
msgstr ""
#: View/Elements/global_menu.ctp:24
#: View/Elements/side_menu.ctp:174;188
#: View/Events/export_alternate.ctp:87
@ -3231,6 +3236,12 @@ msgstr ""
msgid "View Proposals"
msgstr ""
#: View/Elements/global_menu.ctp:33
#: View/Elements/side_menu.ctp:242
#: View/Pages/doc/general.ctp:39
msgid "Events with proposals"
msgstr ""
#: View/Elements/global_menu.ctp:35
#: View/Elements/side_menu.ctp:446
#: View/Pages/doc/general.ctp:40
@ -3318,6 +3329,11 @@ msgstr ""
msgid "List Noticelists"
msgstr ""
#: View/Elements/global_menu.ctp:86
#: View/Pages/doc/general.ctp:22;54
msgid "Global Actions"
msgstr ""
#: View/Elements/global_menu.ctp:90
#: View/News/index.ctp:2
#: View/Pages/doc/general.ctp:56
@ -3406,6 +3422,11 @@ msgstr ""
msgid "Start Discussion"
msgstr ""
#: View/Elements/global_menu.ctp:121
#: View/Pages/doc/general.ctp:23;66
msgid "Sync Actions"
msgstr ""
#: View/Elements/global_menu.ctp:125
#: View/Elements/side_menu.ctp:354
#: View/Pages/doc/general.ctp:68
@ -3456,15 +3477,22 @@ msgid "Add Organisation"
msgstr ""
#: View/Elements/global_menu.ctp:149
#: View/Elements/side_menu.ctp:395
#: View/Pages/doc/general.ctp:76
msgid "List Roles"
msgstr ""
#: View/Elements/global_menu.ctp:151
#: View/Elements/side_menu.ctp:393
#: View/Roles/admin_add.ctp:4
msgid "Add Role"
msgstr ""
#: View/Elements/global_menu.ctp:155
#: View/Pages/doc/general.ctp:79
msgid "Server Settings"
msgstr ""
#: View/Elements/global_menu.ctp:155
msgid "Maintenance"
msgstr ""
@ -3484,6 +3512,10 @@ msgstr ""
msgid "Scheduled Tasks"
msgstr ""
#: View/Elements/global_menu.ctp:164
msgid "Blacklist Event"
msgstr ""
#: View/Elements/global_menu.ctp:165
#: View/Elements/side_menu.ctp:407
msgid "Manage Event Blacklists"
@ -3652,11 +3684,6 @@ msgstr ""
msgid "Download results as CSV"
msgstr ""
#: View/Elements/side_menu.ctp:242
#: View/Pages/doc/general.ctp:39
msgid "Events with proposals"
msgstr ""
#: View/Elements/side_menu.ctp:259
msgid "List Regexp"
msgstr ""
@ -9440,10 +9467,6 @@ msgstr ""
msgid "This button will return you to the start screen of the application, which is the event index page (more about this later)."
msgstr ""
#: View/Pages/doc/general.ctp:20
msgid "Event Actions"
msgstr ""
#: View/Pages/doc/general.ctp:20
msgid "All the malware data entered into MISP is made up of an event object that is described by its connected attributes. The Event actions menu gives access to all the functionality that has to do with the creation, modification, deletion, publishing, searching and listing of events and attributes."
msgstr ""
@ -9452,18 +9475,10 @@ msgstr ""
msgid "Input filters alter what and how data can be entered into this instance. Apart from the basic validation of attribute entry by type, it is possible for the site administrators to define regular expression replacements and blacklists for certain values in addition to blocking certain values from being exportable. Users can view these replacement and blacklist rules here whilst administrator can alter them."
msgstr ""
#: View/Pages/doc/general.ctp:22;54
msgid "Global Actions"
msgstr ""
#: View/Pages/doc/general.ctp:22
msgid "This menu gives you access to information about MISP and this instance. You can view and edit your own profile, view the manual, read the news or the terms of use again, see a list of the active organisations on this instance and a histogram of their contributions by attribute type."
msgstr ""
#: View/Pages/doc/general.ctp:23;66
msgid "Sync Actions"
msgstr ""
#: View/Pages/doc/general.ctp:23
msgid "With administrator access rights, shows a list of the connected instances and allows the initiation of a push and a pull (more about the synchronisation mechanisms later)."
msgstr ""
@ -9637,10 +9652,6 @@ msgstr ""
msgid "Various tools, upgrade scripts that can help a site-admin run the instance."
msgstr ""
#: View/Pages/doc/general.ctp:79
msgid "Server Settings"
msgstr ""
#: View/Pages/doc/general.ctp:79
msgid "Set up and diagnose your MISP installation."
msgstr ""

106
app/Model/Server.php
View File

@ -1994,6 +1994,27 @@ class Server extends AppModel
$eventIds = array();
$conditions = array();
$eventIds = $this->__getEventIdListBasedOnPullTechnique($technique, $server);
if (!empty($eventIds['error'])) {
$errors = array(
'1' => __('Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server. Another reason could be an incorrect sync server setting.'),
'2' => $eventIds['error'][1],
'3' => __('Sorry, this is not yet implemented'),
'4' => __('Something went wrong while trying to pull')
);
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Server',
'model_id' => $id,
'email' => $user['email'],
'action' => 'error',
'user_id' => $user['id'],
'title' => 'Failed pull from ' . $server['Server']['url'] . ' initiated by ' . $email,
'change' => !empty($errors[$eventIds['error'][0]]) ? $errors[$eventIds['error'][0]] : __('Unknown issue.')
));
return !empty($errors[$eventIds['error'][0]]) ? $errors[$eventIds['error'][0]] : __('Unknown issue.');
}
$successes = array();
$fails = array();
// now process the $eventIds to pull each of the events sequentially
@ -2149,16 +2170,32 @@ class Server extends AppModel
$this->Event = ClassRegistry::init('Event');
$this->read(null, $id);
$url = $this->data['Server']['url'];
$push = $this->checkVersionCompatibility($id, $user)['canPush'];
if (!isset($push) || !$push) {
if ($jobId) {
$job->id = $jobId;
$job->saveField('progress', 100);
$job->saveField('message', 'Push to server ' . $id . ' failed. Remote instance is outdated.');
$job->saveField('status', 4);
}
return false;
}
$push = $this->checkVersionCompatibility($id, $user);
if (isset($push['canPush']) && !$push['canPush']) {
$push = 'Remote instance is outdated.';
}
if (!is_array($push)) {
$message = sprintf('Push to server %s failed. Reason: %s', $id, $push);
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Server',
'model_id' => $id,
'email' => $user['email'],
'action' => 'error',
'user_id' => $user['id'],
'title' => 'Failed: Push to ' . $url . ' initiated by ' . $user['email'],
'change' => $message
));
if ($jobId) {
$job->id = $jobId;
$job->saveField('progress', 100);
$job->saveField('message', $message);
$job->saveField('status', 4);
}
return $push;
}
if ("full" == $technique) {
$eventid_conditions_key = 'Event.id >';
$eventid_conditions_value = 0;
@ -2292,6 +2329,7 @@ class Server extends AppModel
} else {
return array($successes, $fails);
}
return true;
}
public function getEventIdsForPush($id, $HttpSocket, $eventIds, $user)
@ -3310,34 +3348,34 @@ class Server extends AppModel
try {
$response = $HttpSocket->get($uri, '', $request);
} catch (Exception $e) {
if (!isset($response) || $response->code != '200') {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
if (isset($response->code)) {
$title = 'Error: Connection to the server has failed.' . isset($response->code) ? ' Returned response code: ' . $response->code : '';
} else {
$title = 'Error: Connection to the server has failed. The returned exception\'s error message was: ' . $e->getMessage();
}
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Server',
'model_id' => $id,
'email' => $user['email'],
'action' => 'error',
'user_id' => $user['id'],
'title' => $title,
));
}
}
if (!isset($response) || $response->code != '200') {
return 1;
$error = $e->getMessage;
}
if (!isset($response) || $response->code != '200') {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
if (isset($response->code)) {
$title = 'Error: Connection to the server has failed.' . (isset($response->code) ? ' Returned response code: ' . $response->code : '');
} else {
$title = 'Error: Connection to the server has failed. The returned exception\'s error message was: ' . $e->getMessage();
}
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Server',
'model_id' => $id,
'email' => $user['email'],
'action' => 'error',
'user_id' => $user['id'],
'title' => $title
));
return $title;
}
$remoteVersion = json_decode($response->body, true);
$canPush = isset($remoteVersion['perm_sync']) ? $remoteVersion['perm_sync'] : false;
$remoteVersion = explode('.', $remoteVersion['version']);
if (!isset($remoteVersion[0])) {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$message = __('Error: Server didn\'t send the expected response. This may be because the remote server version is outdated.');
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Server',
@ -3345,9 +3383,9 @@ class Server extends AppModel
'email' => $user['email'],
'action' => 'error',
'user_id' => $user['id'],
'title' => 'Error: Server didn\'t send the expected response. This may be because the remote server version is outdated.',
'title' => $message,
));
return 2;
return $message;
}
$response = false;
$success = false;
@ -3494,7 +3532,7 @@ class Server extends AppModel
public function stixDiagnostics(&$diagnostic_errors, &$stixVersion, &$cyboxVersion, &$mixboxVersion, &$maecVersion, &$pymispVersion)
{
$result = array();
$expected = array('stix' => '1.2.0.6', 'cybox' => '2.1.0.17', 'mixbox' => '1.0.3', 'maec' => '4.1.0.13', 'pymisp' => '>2.4.93');
$expected = array('stix' => '1.2.0.6', 'cybox' => '2.1.0.17', 'mixbox' => '1.0.3', 'maec' => '4.1.0.14', 'pymisp' => '>2.4.93');
// check if the STIX and Cybox libraries are working using the test script stixtest.py
$scriptResult = shell_exec('python3 ' . APP . 'files' . DS . 'scripts' . DS . 'stixtest.py');
$scriptResult = json_decode($scriptResult, true);

2
app/Plugin/magic_tools/models/behaviors/orphans_protectable.php
View File

@ -99,7 +99,7 @@ class OrphansProtectableBehavior extends ModelBehavior {
foreach ($possibleOrphans as $model => $ids) {
$count = count($ids);
$modelName = $count > 1 ? Inflector::pluralize($model) : $model;
$errorParts[] = $count.' '.__($modelName, true).' (ID: '.$Model->createDeletionErrorIds($model, $ids).')';
$errorParts[] = $count.' '.$modelName.' (ID: '.$Model->createDeletionErrorIds($model, $ids).')';
}
return __('it has the following dependent items', true).': '.implode($errorParts, ', ');
}

2
app/View/EventGraph/ajax/eventGraph_add_form.ctp
View File

@ -17,7 +17,7 @@
'type' => 'textarea'
));
echo $this->Form->button(__($action));
echo $this->Form->button($action);
echo $this->Form->end();
?>
</div>