From 94b36c92597834f6db440a4070cd16fc75c54587 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Wed, 2 Oct 2019 18:55:37 +0200
Subject: [PATCH] chg: [feed] Provide more info when caching feeds about
 failures

---
 app/Console/Command/ServerShell.php | 73 ++++++++++++++++++-----------
 app/Controller/FeedsController.php  |  2 +-
 app/Model/Feed.php                  | 17 ++++---
 3 files changed, 55 insertions(+), 37 deletions(-)

diff --git a/app/Console/Command/ServerShell.php b/app/Console/Command/ServerShell.php
index 85a226721..9cd0ab26e 100644
--- a/app/Console/Command/ServerShell.php
+++ b/app/Console/Command/ServerShell.php
@@ -303,23 +303,21 @@ class ServerShell extends AppShell
             $result = false;
         }
 
-        $this->Job->id = $jobId;
-        if ($result !== true) {
-            $message = 'Job failed. See logs for more details.';
-            $this->Job->save(array(
-                    'id' => $jobId,
-                    'message' => $message,
-                    'progress' => 0,
-                    'status' => 3
-            ));
+        if ($result === false) {
+            $message = __('Job failed. See error logs for more details.');
+            $this->saveJobStatus($jobId, true, $message);
+
         } else {
-            $message = 'Job done.';
-            $this->Job->save(array(
-                    'id' => $jobId,
-                    'message' => $message,
-                    'progress' => 100,
-                    'status' => 4
-            ));
+            $total = $result['successes'] + $result['fails'];
+            $message = __n(
+                '%s feed from %s cached. Failed: %s',
+                '%s feeds from %s cached. Failed: %s',
+                $result['successes'], $total, $result['fails']
+            );
+            if ($result['fails'] > 0) {
+                $message .= ' ' . __('See error logs for more details.');
+            }
+            $this->saveJobStatus($jobId, false, $message);
         }
         echo $message . PHP_EOL;
     }
@@ -462,18 +460,22 @@ class ServerShell extends AppShell
             CakeLog::error($e->getMessage());
             $result = false;
         }
-        if ($result) {
-            $this->Job->save(array(
-                'message' => 'Job done.',
-                'progress' => 100,
-                'status' => 4
-            ));
+
+        if ($result === false) {
+            $message = __('Job failed. See error logs for more details.');
+            $this->saveJobStatus($jobId, true, $message);
+
         } else {
-            $this->Job->save(array(
-                'message' => 'Job failed. See logs for more details.',
-                'progress' => 100,
-                'status' => 3,
-            ));
+            $total = $result['successes'] + $result['fails'];
+            $message = __n(
+                '%s feed from %s cached. Failed: %s',
+                '%s feeds from %s cached. Failed: %s',
+                $result['successes'], $total, $result['fails']
+            );
+            if ($result['fails'] > 0) {
+                $message .= ' ' . __('See error logs for more details.');
+            }
+            $this->saveJobStatus($jobId, false, $message);
         }
 
         $this->Task->id = $task['Task']['id'];
@@ -518,4 +520,21 @@ class ServerShell extends AppShell
         $this->Task->id = $task['Task']['id'];
         $this->Task->saveField('message', count($servers) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '.');
     }
+
+    /**
+     * @param int $jobId
+     * @param bool $failed
+     * @param string $message
+     * @return mixed
+     */
+    private function saveJobStatus($jobId, $failed, $message)
+    {
+        $this->Job->id = $jobId;
+        return $this->Job->save(array(
+            'id' => $jobId,
+            'message' => $message,
+            'progress' => $failed ? 0 : 100,
+            'status' => $failed ? 3 : 4,
+        ));
+    }
 }
diff --git a/app/Controller/FeedsController.php b/app/Controller/FeedsController.php
index eab8d3de0..5b79d18f6 100644
--- a/app/Controller/FeedsController.php
+++ b/app/Controller/FeedsController.php
@@ -971,7 +971,7 @@ class FeedsController extends AppController
             $message = 'Feed caching job initiated.';
         } else {
             $result = $this->Feed->cacheFeedInitiator($this->Auth->user(), false, $scope);
-            if (!$result) {
+            if ($result['fails'] > 0) {
                 $this->Flash->error(__('Caching the feeds has failed.'));
                 $this->redirect(array('action' => 'index'));
             }
diff --git a/app/Model/Feed.php b/app/Model/Feed.php
index 610883682..25dfdfef1 100644
--- a/app/Model/Feed.php
+++ b/app/Model/Feed.php
@@ -1008,10 +1008,10 @@ class Feed extends AppModel
     }
 
     /**
-     * @param $user Not used
+     * @param $user - Not used
      * @param int|bool $jobId
      * @param string $scope
-     * @return bool Returns true if at least one feed was cached successfully.
+     * @return array
      * @throws Exception
      */
     public function cacheFeedInitiator($user, $jobId = false, $scope = 'freetext')
@@ -1021,10 +1021,7 @@ class Feed extends AppModel
             'recursive' => -1,
             'fields' => array('source_format', 'input_source', 'url', 'id', 'settings', 'headers')
         );
-        $redis = $this->setupRedis();
-        if ($redis === false) {
-            throw new Exception('Could not reach Redis.');
-        }
+        $redis = $this->setupRedisWithException();
         if ($scope !== 'all') {
             if (is_numeric($scope)) {
                 $params['conditions']['id'] = $scope;
@@ -1041,18 +1038,20 @@ class Feed extends AppModel
             $redis->del($redis->keys('misp:feed_cache:event_uuid_lookup:*'));
         }
         $feeds = $this->find('all', $params);
-        $atLeastOneSuccess = false;
+
+        $results = array('successes' => 0, 'fails' => 0);
         foreach ($feeds as $k => $feed) {
             if ($this->__cacheFeed($feed, $redis, $jobId)) {
                 $message = 'Feed ' . $feed['Feed']['id'] . ' cached.';
-                $atLeastOneSuccess = true;
+                $results['successes']++;
             } else {
                 $message = 'Failed to cache feed ' . $feed['Feed']['id'] . '. See logs for more details.';
+                $results['fails']++;
             }
 
             $this->jobProgress($jobId, $message, 100 * $k / count($feeds));
         }
-        return $atLeastOneSuccess;
+        return $results;
     }
 
     public function attachFeedCacheTimestamps($data)