MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [internal] Code cleanup for Server::pull method

pull/7797/head
Jakub Onderka 2021-09-30 23:19:29 +02:00
parent 0b54f6c557
commit 459afd6f67
3 changed files with 54 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/Console/Command/ServerShell.php
View File

@ -112,7 +112,7 @@ class ServerShell extends AppShell
$force = true;
}
try {
$result = $this->Server->pull($user, $serverId, $technique, $server, $jobId, $force);
$result = $this->Server->pull($user, $technique, $server, $jobId, $force);
if (is_array($result)) {
$message = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4]);
$this->Job->saveStatus($jobId, true, $message);
@ -399,7 +399,7 @@ class ServerShell extends AppShell
);
$this->Job->save($data);
$jobId = $this->Job->id;
$result = $this->Server->pull($user, $server['Server']['id'], 'full', $server, $jobId);
$result = $this->Server->pull($user, 'full', $server, $jobId);
$this->Job->save(array(
'id' => $jobId,
'message' => 'Job done.',

49
app/Controller/ServersController.php
View File

@ -704,32 +704,36 @@ class ServersController extends AppController
* incremental - only new events
* <int> - specific id of the event to pull
*/
public function pull($id = null, $technique='full')
public function pull($id = null, $technique = 'full')
{
if (!empty($id)) {
$this->Server->id = $id;
} else if (!empty($this->request->data['id'])) {
$this->Server->id = $this->request->data['id'];
} else {
if (empty($id)) {
if (!empty($this->request->data['id'])) {
$id = $this->request->data['id'];
} else {
throw new NotFoundException(__('Invalid server'));
}
}
$s = $this->Server->find('first', [
'conditions' => ['id' => $id],
'recursive' => -1,
]);
if (empty($s)) {
throw new NotFoundException(__('Invalid server'));
}
if (!$this->Server->exists()) {
throw new NotFoundException(__('Invalid server'));
}
$s = $this->Server->read(null, $id);
$error = false;
if (!$this->_isSiteAdmin() && !($s['Server']['org_id'] == $this->Auth->user('org_id') && $this->_isAdmin())) {
throw new MethodNotAllowedException(__('You are not authorised to do that.'));
}
if (false == $this->Server->data['Server']['pull'] && ($technique == 'full' || $technique == 'incremental')) {
if (false == $s['Server']['pull'] && ($technique === 'full' || $technique === 'incremental')) {
$error = __('Pull setting not enabled for this server.');
}
if (false == $this->Server->data['Server']['pull_galaxy_clusters'] && ($technique == 'pull_relevant_clusters')) {
if (false == $s['Server']['pull_galaxy_clusters'] && ($technique === 'pull_relevant_clusters')) {
$error = __('Pull setting not enabled for this server.');
}
if (empty($error)) {
if (!Configure::read('MISP.background_jobs')) {
$result = $this->Server->pull($this->Auth->user(), $id, $technique, $s);
$result = $this->Server->pull($this->Auth->user(), $technique, $s);
if (is_array($result)) {
$success = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4]);
} else {
@ -741,22 +745,11 @@ class ServersController extends AppController
$this->set('pulledSightings', $result[3]);
} else {
$this->loadModel('Job');
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'pull',
'job_input' => 'Server: ' . $id,
'status' => 0,
'retries' => 0,
'org' => $this->Auth->user('Organisation')['name'],
'message' => __('Pulling.'),
);
$this->Job->save($data);
$jobId = $this->Job->id;
$jobId = $this->Job->createJob($this->Auth->user(), Job::WORKER_DEFAULT, 'pull', 'Server: ' . $id, __('Pulling.'));
$process_id = CakeResque::enqueue(
'default',
'ServerShell',
array('pull', $this->Auth->user('id'), $id, $technique, $jobId)
Job::WORKER_DEFAULT,
'ServerShell',
array('pull', $this->Auth->user('id'), $id, $technique, $jobId)
);
$this->Job->saveField('process_id', $process_id);
$success = __('Pull queued for background execution. Job ID: %s', $jobId);

67
app/Model/Server.php
View File

@ -444,12 +444,22 @@ class Server extends AppModel
return true;
}
public function pull($user, $id = null, $technique=false, $server, $jobId = false, $force = false)
/**
* @param array $user
* @param string $technique
* @param array $server
* @param int|false $jobId
* @param bool $force
* @return array|string
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
* @throws JsonException
*/
public function pull(array $user, $technique, array $server, $jobId = false, $force = false)
{
if ($jobId) {
Configure::write('CurrentUserId', $user['id']);
$job = ClassRegistry::init('Job');
$job->read(null, $jobId);
$email = "Scheduled job";
} else {
$email = $user['email'];
@ -474,19 +484,17 @@ class Server extends AppModel
if (!empty($server['Server']['pull_galaxy_clusters'])) {
$this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
if ($jobId) {
$job->saveField('message', $technique == 'pull_relevant_clusters' ? __('Pulling relevant galaxy clusters.') : __('Pulling galaxy clusters.'));
$job->saveProgress($jobId, $technique === 'pull_relevant_clusters' ? __('Pulling relevant galaxy clusters.') : __('Pulling galaxy clusters.'));
}
$pulledClusters = $this->GalaxyCluster->pullGalaxyClusters($user, $server, $technique);
if ($technique == 'pull_relevant_clusters') {
if ($technique === 'pull_relevant_clusters') {
if ($jobId) {
$job->saveField('progress', 100);
$job->saveField('message', 'Pulling complete.');
$job->saveStatus($jobId, true, 'Pulling complete.');
}
return array(array(), array(), 0, 0, $pulledClusters);
}
if ($jobId) {
$job->saveField('progress', 10);
$job->saveField('message', 'Pulling events.');
$job->saveProgress($jobId, 'Pulling events.', 10);
}
}
@ -516,16 +524,12 @@ class Server extends AppModel
}
foreach ($eventIds as $k => $eventId) {
$this->__pullEvent($eventId, $successes, $fails, $eventModel, $serverSync, $user, $jobId, $force);
if ($jobId) {
if ($k % 10 == 0) {
$job->saveProgress($jobId, null, 10 + 40 * (($k + 1) / count($eventIds)));
}
if ($jobId && $k % 10 === 0) {
$job->saveProgress($jobId, null, 10 + 40 * (($k + 1) / count($eventIds)));
}
}
}
if (!empty($fails)) {
foreach ($fails as $eventid => $message) {
$this->loadLog()->createLogEntry($user, 'pull', 'Server', $id, "Failed to pull event #$eventid.", 'Reason: ' . $message);
$this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['Server']['id'], "Failed to pull event #$eventid.", 'Reason: ' . $message);
}
}
if ($jobId) {
@ -541,27 +545,18 @@ class Server extends AppModel
$pulledSightings = $eventModel->Sighting->pullSightings($user, $serverSync);
}
if ($jobId) {
$job->saveProgress($jobId, 'Pull completed.', 100);
$job->saveStatus($jobId, true, 'Pull completed.');
}
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Server',
'model_id' => $id,
'email' => $user['email'],
'action' => 'pull',
'user_id' => $user['id'],
'title' => 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $email,
'change' => sprintf(
'%s events, %s proposals, %s sightings and %s galaxyClusters pulled or updated. %s events failed or didn\'t need an update.',
count($successes),
$pulledProposals,
$pulledSightings,
$pulledClusters,
count($fails)
)
));
$change = sprintf(
'%s events, %s proposals, %s sightings and %s galaxyClusters pulled or updated. %s events failed or didn\'t need an update.',
count($successes),
$pulledProposals,
$pulledSightings,
$pulledClusters,
count($fails)
);
$this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['Server']['id'], 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $email, $change);
return array($successes, $fails, $pulledProposals, $pulledSightings, $pulledClusters);
}
@ -731,7 +726,7 @@ class Server extends AppModel
* @throws HttpSocketJsonException
* @throws InvalidArgumentException
*/
public function getEventIdsFromServer(ServerSyncTool $serverSync, $all = false, $ignoreFilterRules = false, $scope = 'events', $force = false)
private function getEventIdsFromServer(ServerSyncTool $serverSync, $all = false, $ignoreFilterRules = false, $scope = 'events', $force = false)
{
if (!in_array($scope, ['events', 'sightings'], true)) {
throw new InvalidArgumentException("Scope must be 'events' or 'sightings', '$scope' given.");