From bcbee05e2f39b07704cd7663237812ab5a724803 Mon Sep 17 00:00:00 2001 From: iglocska Date: Fri, 25 Aug 2017 14:38:32 +0200 Subject: [PATCH] new: Feeds added to the scheduled jobs --- app/Console/Command/ServerShell.php | 79 +++++++++++++++++++++++++++++ app/Controller/TasksController.php | 38 ++++++++++++++ app/Model/Task.php | 20 +++++++- 3 files changed, 136 insertions(+), 1 deletion(-) diff --git a/app/Console/Command/ServerShell.php b/app/Console/Command/ServerShell.php index 4f04fbbae..67dd3d468 100644 --- a/app/Console/Command/ServerShell.php +++ b/app/Console/Command/ServerShell.php @@ -185,6 +185,85 @@ class ServerShell extends AppShell $this->Task->saveField('message', count($servers) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '. Failed jobs: ' . $failCount . '/' . $count); } + public function enqueueFeed() { + $timestamp = $this->args[0]; + $userId = $this->args[1]; + $taskId = $this->args[2]; + // action options: + // 0 = pull + // 1 = cache + $action = $this->args[3]; + $task = $this->Task->read(null, $taskId); + if ($timestamp != $task['Task']['next_execution_time']) { + return; + } + if ($task['Task']['timer'] > 0) $this->Task->reQueue($task, 'default', 'ServerShell', 'enqueueCachePull', $userId, $taskId, $action); + $user = $this->User->getAuthUser($userId); + $count = count($feeds); + $failCount = 0; + if ($action == 0) { + $feeds = $this->Feed->find('all', array( + 'recursive' => -1, + 'conditions' => array('enabled' => 1) + )); + foreach ($feeds as $k => $feed) { + $this->Job->create(); + $data = array( + 'worker' => 'default', + 'job_type' => 'feed_pull', + 'job_input' => 'Feed: ' . $feed['Feed']['id'], + 'retries' => 0, + 'org' => $user['Organisation']['name'], + 'org_id' => $user['org_id'], + 'process_id' => 'Part of scheduled feed pull', + 'message' => 'Pulling.', + ); + $this->Job->save($data); + $jobId = $this->Job->id; + App::uses('SyncTool', 'Tools'); + $result = $this->Feed->downloadFromFeedInitiator($feed['Feed']['id'], $user, $jobId); + $this->Job->save(array( + 'id' => $jobId, + 'message' => 'Job done.', + 'progress' => 100, + 'status' => 4 + )); + if ($result !== true) { + $this->Job->saveField('message', 'Could not pull feed.'); + $failCount++; + } + } + $this->Task->id = $task['Task']['id']; + $this->Task->saveField('message', count($feeds) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '. Failed jobs: ' . $failCount . '/' . count($feeds)); + } else { + $this->Job->create(); + $data = array( + 'worker' => 'default', + 'job_type' => 'feed_cache', + 'job_input' => 'Feed: ' . $feed['Feed']['id'], + 'retries' => 0, + 'org' => $user['Organisation']['name'], + 'org_id' => $user['org_id'], + 'process_id' => 'Part of scheduled feed caching', + 'message' => 'Caching.', + ); + $this->Job->save($data); + $jobId = $this->Job->id; + $result = $this->Feed->cacheFeedInitiator($user, $jobId, 'all'); + $this->Job->save(array( + 'id' => $jobId, + 'message' => 'Job done.', + 'progress' => 100, + 'status' => 4 + )); + $this->Task->id = $task['Task']['id']; + $this->Task->saveField('message', 'Job completed at ' . date('d/m/Y - H:i:s')); + } + } + + + + public function enqueuePush() { $timestamp = $this->args[0]; $taskId = $this->args[1]; diff --git a/app/Controller/TasksController.php b/app/Controller/TasksController.php index b3a63aa3f..d1e5b1a2b 100644 --- a/app/Controller/TasksController.php +++ b/app/Controller/TasksController.php @@ -81,6 +81,8 @@ class TasksController extends AppController { if ($type === 'cache_exports') $this->_cacheScheduler($timestamp, $id); if ($type === 'pull_all') $this->_pullScheduler($timestamp, $id); if ($type === 'push_all') $this->_pushScheduler($timestamp, $id); + if ($type === 'cache_feeds') $this->_feedScheduler($timestamp, $id, 1); + if ($type === 'pull_feeds') $this->_feedScheduler($timestamp, $id, 0); } private function _cacheScheduler($timestamp, $id) { @@ -119,4 +121,40 @@ class TasksController extends AppController { $this->Task->saveField('process_id', $process_id); } + private function _feedScheduler($timestamp, $id, $type) { + $process_id = CakeResque::enqueueAt( + $timestamp, + 'default', + 'ServerShell', + array('enqueueFeed', $timestamp, $this->Auth->user('id'), $id, $type), + true + ); + $this->Task->id = $id; + $this->Task->saveField('process_id', $process_id); + } + + private function _feedPullScheduler($timestamp, $id) { + $process_id = CakeResque::enqueueAt( + $timestamp, + 'default', + 'ServerShell', + array('enqueuePull', $timestamp, $this->Auth->user('id'), $id), + true + ); + $this->Task->id = $id; + $this->Task->saveField('process_id', $process_id); + } + + private function _feedCacheScheduler($timestamp, $id) { + $process_id = CakeResque::enqueueAt( + $timestamp, + 'default', + 'ServerShell', + array('enqueuePull', $timestamp, $this->Auth->user('id'), $id), + true + ); + $this->Task->id = $id; + $this->Task->saveField('process_id', $process_id); + } + } diff --git a/app/Model/Task.php b/app/Model/Task.php index eff1b69a6..dbc643e4d 100644 --- a/app/Model/Task.php +++ b/app/Model/Task.php @@ -32,7 +32,25 @@ class Task extends AppModel { 'description' => 'Initiates a full push for all eligible instances.', 'next_execution_time' => 1391601600, 'message' => 'Not scheduled yet.' - ) + ), + 'cache_feeds' => array( + 'type' => 'cache_feeds', + 'timer' => 0, + 'scheduled_time' => '12:00', + 'process_id' => '', + 'description' => 'Initiates the caching of all feeds.', + 'next_execution_time' => 1391601600, + 'message' => 'Not scheduled yet.' + ), + 'pull_feeds' => array( + 'type' => 'pull_feeds', + 'timer' => 0, + 'scheduled_time' => '12:00', + 'process_id' => '', + 'description' => 'Initiates the pull of all feeds.', + 'next_execution_time' => 1391601600, + 'message' => 'Not scheduled yet.' + ), ); // takes a time in the 24h format (13:49) and an integer representing the number of hours