mirror of https://github.com/MISP/MISP
new: Feeds added to the scheduled jobs
parent
d1ee8dc956
commit
bcbee05e2f
|
@ -185,6 +185,85 @@ class ServerShell extends AppShell
|
|||
$this->Task->saveField('message', count($servers) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '. Failed jobs: ' . $failCount . '/' . $count);
|
||||
}
|
||||
|
||||
public function enqueueFeed() {
|
||||
$timestamp = $this->args[0];
|
||||
$userId = $this->args[1];
|
||||
$taskId = $this->args[2];
|
||||
// action options:
|
||||
// 0 = pull
|
||||
// 1 = cache
|
||||
$action = $this->args[3];
|
||||
$task = $this->Task->read(null, $taskId);
|
||||
if ($timestamp != $task['Task']['next_execution_time']) {
|
||||
return;
|
||||
}
|
||||
if ($task['Task']['timer'] > 0) $this->Task->reQueue($task, 'default', 'ServerShell', 'enqueueCachePull', $userId, $taskId, $action);
|
||||
$user = $this->User->getAuthUser($userId);
|
||||
$count = count($feeds);
|
||||
$failCount = 0;
|
||||
if ($action == 0) {
|
||||
$feeds = $this->Feed->find('all', array(
|
||||
'recursive' => -1,
|
||||
'conditions' => array('enabled' => 1)
|
||||
));
|
||||
foreach ($feeds as $k => $feed) {
|
||||
$this->Job->create();
|
||||
$data = array(
|
||||
'worker' => 'default',
|
||||
'job_type' => 'feed_pull',
|
||||
'job_input' => 'Feed: ' . $feed['Feed']['id'],
|
||||
'retries' => 0,
|
||||
'org' => $user['Organisation']['name'],
|
||||
'org_id' => $user['org_id'],
|
||||
'process_id' => 'Part of scheduled feed pull',
|
||||
'message' => 'Pulling.',
|
||||
);
|
||||
$this->Job->save($data);
|
||||
$jobId = $this->Job->id;
|
||||
App::uses('SyncTool', 'Tools');
|
||||
$result = $this->Feed->downloadFromFeedInitiator($feed['Feed']['id'], $user, $jobId);
|
||||
$this->Job->save(array(
|
||||
'id' => $jobId,
|
||||
'message' => 'Job done.',
|
||||
'progress' => 100,
|
||||
'status' => 4
|
||||
));
|
||||
if ($result !== true) {
|
||||
$this->Job->saveField('message', 'Could not pull feed.');
|
||||
$failCount++;
|
||||
}
|
||||
}
|
||||
$this->Task->id = $task['Task']['id'];
|
||||
$this->Task->saveField('message', count($feeds) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '. Failed jobs: ' . $failCount . '/' . count($feeds));
|
||||
} else {
|
||||
$this->Job->create();
|
||||
$data = array(
|
||||
'worker' => 'default',
|
||||
'job_type' => 'feed_cache',
|
||||
'job_input' => 'Feed: ' . $feed['Feed']['id'],
|
||||
'retries' => 0,
|
||||
'org' => $user['Organisation']['name'],
|
||||
'org_id' => $user['org_id'],
|
||||
'process_id' => 'Part of scheduled feed caching',
|
||||
'message' => 'Caching.',
|
||||
);
|
||||
$this->Job->save($data);
|
||||
$jobId = $this->Job->id;
|
||||
$result = $this->Feed->cacheFeedInitiator($user, $jobId, 'all');
|
||||
$this->Job->save(array(
|
||||
'id' => $jobId,
|
||||
'message' => 'Job done.',
|
||||
'progress' => 100,
|
||||
'status' => 4
|
||||
));
|
||||
$this->Task->id = $task['Task']['id'];
|
||||
$this->Task->saveField('message', 'Job completed at ' . date('d/m/Y - H:i:s'));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
public function enqueuePush() {
|
||||
$timestamp = $this->args[0];
|
||||
$taskId = $this->args[1];
|
||||
|
|
|
@ -81,6 +81,8 @@ class TasksController extends AppController {
|
|||
if ($type === 'cache_exports') $this->_cacheScheduler($timestamp, $id);
|
||||
if ($type === 'pull_all') $this->_pullScheduler($timestamp, $id);
|
||||
if ($type === 'push_all') $this->_pushScheduler($timestamp, $id);
|
||||
if ($type === 'cache_feeds') $this->_feedScheduler($timestamp, $id, 1);
|
||||
if ($type === 'pull_feeds') $this->_feedScheduler($timestamp, $id, 0);
|
||||
}
|
||||
|
||||
private function _cacheScheduler($timestamp, $id) {
|
||||
|
@ -119,4 +121,40 @@ class TasksController extends AppController {
|
|||
$this->Task->saveField('process_id', $process_id);
|
||||
}
|
||||
|
||||
private function _feedScheduler($timestamp, $id, $type) {
|
||||
$process_id = CakeResque::enqueueAt(
|
||||
$timestamp,
|
||||
'default',
|
||||
'ServerShell',
|
||||
array('enqueueFeed', $timestamp, $this->Auth->user('id'), $id, $type),
|
||||
true
|
||||
);
|
||||
$this->Task->id = $id;
|
||||
$this->Task->saveField('process_id', $process_id);
|
||||
}
|
||||
|
||||
private function _feedPullScheduler($timestamp, $id) {
|
||||
$process_id = CakeResque::enqueueAt(
|
||||
$timestamp,
|
||||
'default',
|
||||
'ServerShell',
|
||||
array('enqueuePull', $timestamp, $this->Auth->user('id'), $id),
|
||||
true
|
||||
);
|
||||
$this->Task->id = $id;
|
||||
$this->Task->saveField('process_id', $process_id);
|
||||
}
|
||||
|
||||
private function _feedCacheScheduler($timestamp, $id) {
|
||||
$process_id = CakeResque::enqueueAt(
|
||||
$timestamp,
|
||||
'default',
|
||||
'ServerShell',
|
||||
array('enqueuePull', $timestamp, $this->Auth->user('id'), $id),
|
||||
true
|
||||
);
|
||||
$this->Task->id = $id;
|
||||
$this->Task->saveField('process_id', $process_id);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -32,7 +32,25 @@ class Task extends AppModel {
|
|||
'description' => 'Initiates a full push for all eligible instances.',
|
||||
'next_execution_time' => 1391601600,
|
||||
'message' => 'Not scheduled yet.'
|
||||
)
|
||||
),
|
||||
'cache_feeds' => array(
|
||||
'type' => 'cache_feeds',
|
||||
'timer' => 0,
|
||||
'scheduled_time' => '12:00',
|
||||
'process_id' => '',
|
||||
'description' => 'Initiates the caching of all feeds.',
|
||||
'next_execution_time' => 1391601600,
|
||||
'message' => 'Not scheduled yet.'
|
||||
),
|
||||
'pull_feeds' => array(
|
||||
'type' => 'pull_feeds',
|
||||
'timer' => 0,
|
||||
'scheduled_time' => '12:00',
|
||||
'process_id' => '',
|
||||
'description' => 'Initiates the pull of all feeds.',
|
||||
'next_execution_time' => 1391601600,
|
||||
'message' => 'Not scheduled yet.'
|
||||
),
|
||||
);
|
||||
|
||||
// takes a time in the 24h format (13:49) and an integer representing the number of hours
|
||||
|
|
Loading…
Reference in New Issue