Server: implement video views

server/controllers/api/remote/videos.js

@@ -38,6 +38,13 @@ router.post('/qadu',
   remoteVideosQadu
 )
 
+router.post('/events',
+  signatureValidators.signature,
+  secureMiddleware.checkSignature,
+  videosValidators.remoteEventsVideos,
+  remoteVideosEvents
+)
+
 // ---------------------------------------------------------------------------
 
 module.exports = router
@@ -84,6 +91,84 @@ function remoteVideosQadu (req, res, next) {
   return res.type('json').status(204).end()
 }
 
+function remoteVideosEvents (req, res, next) {
+  const requests = req.body.data
+  const fromPod = res.locals.secure.pod
+
+  eachSeries(requests, function (request, callbackEach) {
+    const eventData = request.data
+
+    processVideosEventsRetryWrapper(eventData, fromPod, callbackEach)
+  }, function (err) {
+    if (err) logger.error('Error managing remote videos.', { error: err })
+  })
+
+  return res.type('json').status(204).end()
+}
+
+function processVideosEventsRetryWrapper (eventData, fromPod, finalCallback) {
+  const options = {
+    arguments: [ eventData, fromPod ],
+    errorMessage: 'Cannot process videos events with many retries.'
+  }
+
+  databaseUtils.retryTransactionWrapper(processVideosEvents, options, finalCallback)
+}
+
+function processVideosEvents (eventData, fromPod, finalCallback) {
+  waterfall([
+    databaseUtils.startSerializableTransaction,
+
+    function findVideo (t, callback) {
+      fetchOwnedVideo(eventData.remoteId, function (err, videoInstance) {
+        return callback(err, t, videoInstance)
+      })
+    },
+
+    function updateVideoIntoDB (t, videoInstance, callback) {
+      const options = { transaction: t }
+
+      let columnToUpdate
+
+      switch (eventData.eventType) {
+        case constants.REQUEST_VIDEO_EVENT_TYPES.VIEWS:
+          columnToUpdate = 'views'
+          break
+
+        case constants.REQUEST_VIDEO_EVENT_TYPES.LIKES:
+          columnToUpdate = 'likes'
+          break
+
+        case constants.REQUEST_VIDEO_EVENT_TYPES.DISLIKES:
+          columnToUpdate = 'dislikes'
+          break
+
+        default:
+          return callback(new Error('Unknown video event type.'))
+      }
+
+      const query = {}
+      query[columnToUpdate] = eventData.count
+
+      videoInstance.increment(query, options).asCallback(function (err) {
+        return callback(err, t)
+      })
+    },
+
+    databaseUtils.commitTransaction
+
+  ], function (err, t) {
+    if (err) {
+      console.log(err)
+      logger.debug('Cannot process a video event.', { error: err })
+      return databaseUtils.rollbackTransaction(err, t, finalCallback)
+    }
+
+    logger.info('Remote video event processed for video %s.', eventData.remoteId)
+    return finalCallback(null)
+  })
+}
+
 function quickAndDirtyUpdateVideoRetryWrapper (videoData, fromPod, finalCallback) {
   const options = {
     arguments: [ videoData, fromPod ],
@@ -98,7 +183,7 @@ function quickAndDirtyUpdateVideo (videoData, fromPod, finalCallback) {
     databaseUtils.startSerializableTransaction,
 
     function findVideo (t, callback) {
-      fetchVideo(fromPod.host, videoData.remoteId, function (err, videoInstance) {
+      fetchRemoteVideo(fromPod.host, videoData.remoteId, function (err, videoInstance) {
         return callback(err, t, videoInstance)
       })
     },
@@ -264,7 +349,7 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) {
     databaseUtils.startSerializableTransaction,
 
     function findVideo (t, callback) {
-      fetchVideo(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) {
+      fetchRemoteVideo(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) {
         return callback(err, t, videoInstance)
       })
     },
@@ -317,7 +402,7 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) {
 
 function removeRemoteVideo (videoToRemoveData, fromPod, callback) {
   // We need the instance because we have to remove some other stuffs (thumbnail etc)
-  fetchVideo(fromPod.host, videoToRemoveData.remoteId, function (err, video) {
+  fetchRemoteVideo(fromPod.host, videoToRemoveData.remoteId, function (err, video) {
     // Do not return the error, continue the process
     if (err) return callback(null)
 
@@ -334,7 +419,7 @@ function removeRemoteVideo (videoToRemoveData, fromPod, callback) {
 }
 
 function reportAbuseRemoteVideo (reportData, fromPod, callback) {
-  db.Video.load(reportData.videoRemoteId, function (err, video) {
+  fetchOwnedVideo(reportData.videoRemoteId, function (err, video) {
     if (err || !video) {
       if (!err) err = new Error('video not found')
 
@@ -362,7 +447,20 @@ function reportAbuseRemoteVideo (reportData, fromPod, callback) {
   })
 }
 
-function fetchVideo (podHost, remoteId, callback) {
+function fetchOwnedVideo (id, callback) {
+  db.Video.load(id, function (err, video) {
+    if (err || !video) {
+      if (!err) err = new Error('video not found')
+
+      logger.error('Cannot load owned video from id.', { error: err, id })
+      return callback(err)
+    }
+
+    return callback(null, video)
+  })
+}
+
+function fetchRemoteVideo (podHost, remoteId, callback) {
   db.Video.loadByHostAndRemoteId(podHost, remoteId, function (err, video) {
     if (err || !video) {
       if (!err) err = new Error('video not found')

server/controllers/api/videos.js

@@ -333,6 +333,9 @@ function getVideo (req, res, next) {
       // For example, only add a view when a user watch a video during 30s etc
       friends.quickAndDirtyUpdateVideoToFriends(videoInstance.id, constants.REQUEST_VIDEO_QADU_TYPES.VIEWS)
     })
+  } else {
+    // Just send the event to our friends
+    friends.addEventToRemoteVideo(videoInstance.id, constants.REQUEST_VIDEO_EVENT_TYPES.VIEWS)
   }
 
   // Do not wait the view system

server/helpers/custom-validators/remote/videos.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const has = require('lodash/has')
+const values = require('lodash/values')
 
 const constants = require('../../../initializers/constants')
 const videosValidators = require('../videos')
@@ -10,13 +11,17 @@ const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_EN
 
 const remoteVideosValidators = {
   isEachRemoteRequestVideosValid,
-  isEachRemoteRequestVideosQaduValid
+  isEachRemoteRequestVideosQaduValid,
+  isEachRemoteRequestVideosEventsValid
 }
 
 function isEachRemoteRequestVideosValid (requests) {
   return miscValidators.isArray(requests) &&
     requests.every(function (request) {
       const video = request.data
+
+      if (!video) return false
+
       return (
         isRequestTypeAddValid(request.type) &&
         isCommonVideoAttributesValid(video) &&
@@ -45,6 +50,8 @@ function isEachRemoteRequestVideosQaduValid (requests) {
     requests.every(function (request) {
       const video = request.data
 
+      if (!video) return false
+
       return (
         videosValidators.isVideoRemoteIdValid(video.remoteId) &&
         (has(video, 'views') === false || videosValidators.isVideoViewsValid) &&
@@ -54,6 +61,21 @@ function isEachRemoteRequestVideosQaduValid (requests) {
     })
 }
 
+function isEachRemoteRequestVideosEventsValid (requests) {
+  return miscValidators.isArray(requests) &&
+    requests.every(function (request) {
+      const eventData = request.data
+
+      if (!eventData) return false
+
+      return (
+        videosValidators.isVideoRemoteIdValid(eventData.remoteId) &&
+        values(constants.REQUEST_VIDEO_EVENT_TYPES).indexOf(eventData.eventType) !== -1 &&
+        videosValidators.isVideoEventCountValid(eventData.count)
+      )
+    })
+}
+
 // ---------------------------------------------------------------------------
 
 module.exports = remoteVideosValidators

server/helpers/custom-validators/videos.js

@@ -7,6 +7,7 @@ const usersValidators = require('./users')
 const miscValidators = require('./misc')
 const VIDEOS_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEOS
 const VIDEO_ABUSES_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEO_ABUSES
+const VIDEO_EVENTS_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEO_EVENTS
 
 const videosValidators = {
   isVideoAuthorValid,
@@ -25,7 +26,8 @@ const videosValidators = {
   isVideoFile,
   isVideoViewsValid,
   isVideoLikesValid,
-  isVideoDislikesValid
+  isVideoDislikesValid,
+  isVideoEventCountValid
 }
 
 function isVideoAuthorValid (value) {
@@ -86,15 +88,19 @@ function isVideoAbuseReporterUsernameValid (value) {
 }
 
 function isVideoViewsValid (value) {
-  return validator.isInt(value, { min: 0 })
+  return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.VIEWS)
 }
 
 function isVideoLikesValid (value) {
-  return validator.isInt(value, { min: 0 })
+  return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.LIKES)
 }
 
 function isVideoDislikesValid (value) {
-  return validator.isInt(value, { min: 0 })
+  return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DISLIKES)
+}
+
+function isVideoEventCountValid (value) {
+  return validator.isInt(value + '', VIDEO_EVENTS_CONSTRAINTS_FIELDS.COUNT)
 }
 
 function isVideoFile (value, files) {

server/helpers/utils.js

@@ -6,6 +6,7 @@ const logger = require('./logger')
 
 const utils = {
   badRequest,
+  createEmptyCallback,
   cleanForExit,
   generateRandomString,
   isTestInstance,
@@ -29,6 +30,12 @@ function cleanForExit (webtorrentProcess) {
   process.kill(-webtorrentProcess.pid)
 }
 
+function createEmptyCallback () {
+  return function (err) {
+    if (err) logger.error('Error in empty callback.', { error: err })
+  }
+}
+
 function isTestInstance () {
   return (process.env.NODE_ENV === 'test')
 }

server/initializers/constants.js

@@ -85,7 +85,13 @@ const CONSTRAINTS_FIELDS = {
     TAGS: { min: 1, max: 3 }, // Number of total tags
     TAG: { min: 2, max: 10 }, // Length
     THUMBNAIL: { min: 2, max: 30 },
-    THUMBNAIL_DATA: { min: 0, max: 20000 } // Bytes
+    THUMBNAIL_DATA: { min: 0, max: 20000 }, // Bytes
+    VIEWS: { min: 0 },
+    LIKES: { min: 0 },
+    DISLIKES: { min: 0 }
+  },
+  VIDEO_EVENTS: {
+    COUNT: { min: 0 }
   }
 }
 
@@ -120,12 +126,17 @@ const REQUESTS_VIDEO_QADU_LIMIT_PODS = 10
 // The QADU requests are not big
 const REQUESTS_VIDEO_QADU_LIMIT_PER_POD = 50
 
+const REQUESTS_VIDEO_EVENT_LIMIT_PODS = 10
+// The EVENTS requests are not big
+const REQUESTS_VIDEO_EVENT_LIMIT_PER_POD = 50
+
 // Number of requests to retry for replay requests module
 const RETRY_REQUESTS = 5
 
 const REQUEST_ENDPOINTS = {
   VIDEOS: 'videos',
-  QADU: 'videos/qadu'
+  QADU: 'videos/qadu',
+  EVENT: 'videos/events'
 }
 const REQUEST_ENDPOINT_ACTIONS = {}
 REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] = {
@@ -141,6 +152,12 @@ const REQUEST_VIDEO_QADU_TYPES = {
   VIEWS: 'views'
 }
 
+const REQUEST_VIDEO_EVENT_TYPES = {
+  LIKES: 'likes',
+  DISLIKES: 'dislikes',
+  VIEWS: 'views'
+}
+
 const REMOTE_SCHEME = {
   HTTP: 'https',
   WS: 'wss'
@@ -210,6 +227,7 @@ module.exports = {
   REMOTE_SCHEME,
   REQUEST_ENDPOINT_ACTIONS,
   REQUEST_ENDPOINTS,
+  REQUEST_VIDEO_EVENT_TYPES,
   REQUEST_VIDEO_QADU_TYPES,
   REQUESTS_IN_PARALLEL,
   REQUESTS_INTERVAL,
@@ -217,6 +235,8 @@ module.exports = {
   REQUESTS_LIMIT_PODS,
   REQUESTS_VIDEO_QADU_LIMIT_PER_POD,
   REQUESTS_VIDEO_QADU_LIMIT_PODS,
+  REQUESTS_VIDEO_EVENT_LIMIT_PER_POD,
+  REQUESTS_VIDEO_EVENT_LIMIT_PODS,
   RETRY_REQUESTS,
   SEARCHABLE_COLUMNS,
   SIGNATURE_ALGORITHM,

server/lib/friends.js

@@ -11,13 +11,16 @@ const db = require('../initializers/database')
 const logger = require('../helpers/logger')
 const peertubeCrypto = require('../helpers/peertube-crypto')
 const requests = require('../helpers/requests')
+const utils = require('../helpers/utils')
 const RequestScheduler = require('./request-scheduler')
 const RequestVideoQaduScheduler = require('./request-video-qadu-scheduler')
+const RequestVideoEventScheduler = require('./request-video-event-scheduler')
 
 const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS]
 
 const requestScheduler = new RequestScheduler()
 const requestSchedulerVideoQadu = new RequestVideoQaduScheduler()
+const requestSchedulerVideoEvent = new RequestVideoEventScheduler()
 
 const friends = {
   activate,
@@ -25,6 +28,7 @@ const friends = {
   updateVideoToFriends,
   reportAbuseVideoToFriend,
   quickAndDirtyUpdateVideoToFriends,
+  addEventToRemoteVideo,
   hasFriends,
   makeFriends,
   quitFriends,
@@ -35,6 +39,7 @@ const friends = {
 function activate () {
   requestScheduler.activate()
   requestSchedulerVideoQadu.activate()
+  requestSchedulerVideoEvent.activate()
 }
 
 function addVideoToFriends (videoData, transaction, callback) {
@@ -85,6 +90,15 @@ function quickAndDirtyUpdateVideoToFriends (videoId, type, transaction, callback
   return createVideoQaduRequest(options, callback)
 }
 
+function addEventToRemoteVideo (videoId, type, transaction, callback) {
+  const options = {
+    videoId,
+    type,
+    transaction
+  }
+  createVideoEventRequest(options, callback)
+}
+
 function hasFriends (callback) {
   db.Pod.countAll(function (err, count) {
     if (err) return callback(err)
@@ -329,11 +343,17 @@ function createRequest (options, callback) {
 }
 
 function createVideoQaduRequest (options, callback) {
-  if (!callback) callback = function () {}
+  if (!callback) callback = utils.createEmptyCallback()
 
   requestSchedulerVideoQadu.createRequest(options, callback)
 }
 
+function createVideoEventRequest (options, callback) {
+  if (!callback) callback = utils.createEmptyCallback()
+
+  requestSchedulerVideoEvent.createRequest(options, callback)
+}
+
 function isMe (host) {
   return host === constants.CONFIG.WEBSERVER.HOST
 }

server/lib/request-video-event-scheduler.js

@@ -0,0 +1,109 @@
+'use strict'
+
+const BaseRequestScheduler = require('./base-request-scheduler')
+const constants = require('../initializers/constants')
+const db = require('../initializers/database')
+
+module.exports = class RequestVideoEventScheduler extends BaseRequestScheduler {
+
+  constructor () {
+    super()
+
+    // We limit the size of the requests
+    this.limitPods = constants.REQUESTS_VIDEO_EVENT_LIMIT_PODS
+    this.limitPerPod = constants.REQUESTS_VIDEO_EVENT_LIMIT_PER_POD
+
+    this.description = 'video event requests'
+  }
+
+  getRequestModel () {
+    return db.RequestVideoEvent
+  }
+
+  getRequestToPodModel () {
+    return db.RequestVideoEvent
+  }
+
+  buildRequestObjects (eventsToProcess) {
+    const requestsToMakeGrouped = {}
+
+    /* Example:
+        {
+          pod1: {
+            video1: { views: 4, likes: 5 },
+            video2: { likes: 5 }
+          }
+        }
+    */
+    const eventsPerVideoPerPod = {}
+
+    // We group video events per video and per pod
+    // We add the counts of the same event types
+    Object.keys(eventsToProcess).forEach(toPodId => {
+      eventsToProcess[toPodId].forEach(eventToProcess => {
+        if (!eventsPerVideoPerPod[toPodId]) eventsPerVideoPerPod[toPodId] = {}
+
+        if (!requestsToMakeGrouped[toPodId]) {
+          requestsToMakeGrouped[toPodId] = {
+            toPod: eventToProcess.pod,
+            endpoint: constants.REQUEST_ENDPOINTS.EVENT,
+            ids: [], // request ids, to delete them from the DB in the future
+            datas: [] // requests data
+          }
+        }
+        requestsToMakeGrouped[toPodId].ids.push(eventToProcess.id)
+
+        const eventsPerVideo = eventsPerVideoPerPod[toPodId]
+        const remoteId = eventToProcess.video.remoteId
+        if (!eventsPerVideo[remoteId]) eventsPerVideo[remoteId] = {}
+
+        const events = eventsPerVideo[remoteId]
+        if (!events[eventToProcess.type]) events[eventToProcess.type] = 0
+
+        events[eventToProcess.type] += eventToProcess.count
+      })
+    })
+
+    // Now we build our requests array per pod
+    Object.keys(eventsPerVideoPerPod).forEach(toPodId => {
+      const eventsForPod = eventsPerVideoPerPod[toPodId]
+
+      Object.keys(eventsForPod).forEach(remoteId => {
+        const eventsForVideo = eventsForPod[remoteId]
+
+        Object.keys(eventsForVideo).forEach(eventType => {
+          requestsToMakeGrouped[toPodId].datas.push({
+            data: {
+              remoteId,
+              eventType,
+              count: eventsForVideo[eventType]
+            }
+          })
+        })
+      })
+    })
+
+    return requestsToMakeGrouped
+  }
+
+  // { type, videoId, count?, transaction? }
+  createRequest (options, callback) {
+    const type = options.type
+    const videoId = options.videoId
+    const transaction = options.transaction
+    let count = options.count
+
+    if (count === undefined) count = 1
+
+    const dbRequestOptions = {}
+    if (transaction) dbRequestOptions.transaction = transaction
+
+    const createQuery = {
+      type,
+      count,
+      videoId
+    }
+
+    return db.RequestVideoEvent.create(createQuery, dbRequestOptions).asCallback(callback)
+  }
+}

server/lib/request-video-qadu-scheduler.js

@@ -12,7 +12,7 @@ module.exports = class RequestVideoQaduScheduler extends BaseRequestScheduler {
 
     // We limit the size of the requests
     this.limitPods = constants.REQUESTS_VIDEO_QADU_LIMIT_PODS
-    this.limitPerPod = constants.REQUESTS_VIDEO_QADU_LIMIT_PODS
+    this.limitPerPod = constants.REQUESTS_VIDEO_QADU_LIMIT_PER_POD
 
     this.description = 'video QADU requests'
   }

server/middlewares/validators/remote/videos.js

@@ -5,7 +5,8 @@ const logger = require('../../../helpers/logger')
 
 const validatorsRemoteVideos = {
   remoteVideos,
-  remoteQaduVideos
+  remoteQaduVideos,
+  remoteEventsVideos
 }
 
 function remoteVideos (req, res, next) {
@@ -19,11 +20,18 @@ function remoteVideos (req, res, next) {
 function remoteQaduVideos (req, res, next) {
   req.checkBody('data').isEachRemoteRequestVideosQaduValid()
 
-  logger.debug('Checking remoteVideosQadu parameters', { parameters: req.body })
+  logger.debug('Checking remoteQaduVideos parameters', { parameters: req.body })
 
   checkErrors(req, res, next)
 }
 
+function remoteEventsVideos (req, res, next) {
+  req.checkBody('data').isEachRemoteRequestVideosEventsValid()
+
+  logger.debug('Checking remoteEventsVideos parameters', { parameters: req.body })
+
+  checkErrors(req, res, next)
+}
 // ---------------------------------------------------------------------------
 
 module.exports = validatorsRemoteVideos

server/models/pod.js

@@ -148,7 +148,12 @@ function listAllIds (transaction, callback) {
   })
 }
 
-function listRandomPodIdsWithRequest (limit, tableRequestPod, callback) {
+function listRandomPodIdsWithRequest (limit, tableWithPods, tableWithPodsJoins, callback) {
+  if (!callback) {
+    callback = tableWithPodsJoins
+    tableWithPodsJoins = ''
+  }
+
   const self = this
 
   self.count().asCallback(function (err, count) {
@@ -170,7 +175,7 @@ function listRandomPodIdsWithRequest (limit, tableRequestPod, callback) {
       where: {
         id: {
           $in: [
-            this.sequelize.literal('SELECT "podId" FROM "' + tableRequestPod + '"')
+            this.sequelize.literal(`SELECT DISTINCT "${tableWithPods}"."podId" FROM "${tableWithPods}" ${tableWithPodsJoins}`)
           ]
         }
       }

server/models/request-video-event.js

@@ -0,0 +1,169 @@
+'use strict'
+
+/*
+  Request Video events (likes, dislikes, views...)
+*/
+
+const values = require('lodash/values')
+
+const constants = require('../initializers/constants')
+const customVideosValidators = require('../helpers/custom-validators').videos
+
+// ---------------------------------------------------------------------------
+
+module.exports = function (sequelize, DataTypes) {
+  const RequestVideoEvent = sequelize.define('RequestVideoEvent',
+    {
+      type: {
+        type: DataTypes.ENUM(values(constants.REQUEST_VIDEO_EVENT_TYPES)),
+        allowNull: false
+      },
+      count: {
+        type: DataTypes.INTEGER,
+        allowNull: false,
+        validate: {
+          countValid: function (value) {
+            const res = customVideosValidators.isVideoEventCountValid(value)
+            if (res === false) throw new Error('Video event count is not valid.')
+          }
+        }
+      }
+    },
+    {
+      updatedAt: false,
+      indexes: [
+        {
+          fields: [ 'videoId' ]
+        }
+      ],
+      classMethods: {
+        associate,
+
+        listWithLimitAndRandom,
+
+        countTotalRequests,
+        removeAll,
+        removeByRequestIdsAndPod
+      }
+    }
+  )
+
+  return RequestVideoEvent
+}
+
+// ------------------------------ STATICS ------------------------------
+
+function associate (models) {
+  this.belongsTo(models.Video, {
+    foreignKey: {
+      name: 'videoId',
+      allowNull: false
+    },
+    onDelete: 'CASCADE'
+  })
+}
+
+function countTotalRequests (callback) {
+  const query = {}
+  return this.count(query).asCallback(callback)
+}
+
+function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) {
+  const self = this
+  const Pod = this.sequelize.models.Pod
+
+  // We make a join between videos and authors to find the podId of our video event requests
+  const podJoins = 'INNER JOIN "Videos" ON "Videos"."authorId" = "Authors"."id" ' +
+                   'INNER JOIN "RequestVideoEvents" ON "RequestVideoEvents"."videoId" = "Videos"."id"'
+
+  Pod.listRandomPodIdsWithRequest(limitPods, 'Authors', podJoins, function (err, podIds) {
+    if (err) return callback(err)
+
+    // We don't have friends that have requests
+    if (podIds.length === 0) return callback(null, [])
+
+    const query = {
+      include: [
+        {
+          model: self.sequelize.models.Video,
+          include: [
+            {
+              model: self.sequelize.models.Author,
+              include: [
+                {
+                  model: self.sequelize.models.Pod,
+                  where: {
+                    id: {
+                      $in: podIds
+                    }
+                  }
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    }
+
+    self.findAll(query).asCallback(function (err, requests) {
+      if (err) return callback(err)
+
+      const requestsGrouped = groupAndTruncateRequests(requests, limitRequestsPerPod)
+      return callback(err, requestsGrouped)
+    })
+  })
+}
+
+function removeByRequestIdsAndPod (ids, podId, callback) {
+  const query = {
+    where: {
+      id: {
+        $in: ids
+      }
+    },
+    include: [
+      {
+        model: this.sequelize.models.Video,
+        include: [
+          {
+            model: this.sequelize.models.Author,
+            where: {
+              podId
+            }
+          }
+        ]
+      }
+    ]
+  }
+
+  this.destroy(query).asCallback(callback)
+}
+
+function removeAll (callback) {
+  // Delete all requests
+  this.truncate({ cascade: true }).asCallback(callback)
+}
+
+// ---------------------------------------------------------------------------
+
+function groupAndTruncateRequests (events, limitRequestsPerPod) {
+  const eventsGrouped = {}
+
+  events.forEach(function (event) {
+    const pod = event.Video.Author.Pod
+
+    if (!eventsGrouped[pod.id]) eventsGrouped[pod.id] = []
+
+    if (eventsGrouped[pod.id].length < limitRequestsPerPod) {
+      eventsGrouped[pod.id].push({
+        id: event.id,
+        type: event.type,
+        count: event.count,
+        video: event.Video,
+        pod
+      })
+    }
+  })
+
+  return eventsGrouped
+}

server/models/request-video-qadu.js

@@ -71,10 +71,7 @@ function associate (models) {
 }
 
 function countTotalRequests (callback) {
-  const query = {
-    include: [ this.sequelize.models.Pod ]
-  }
-
+  const query = {}
   return this.count(query).asCallback(callback)
 }
 

server/models/request.js

@@ -48,6 +48,8 @@ function associate (models) {
 }
 
 function countTotalRequests (callback) {
+  // We need to include Pod because there are no cascade delete when a pod is removed
+  // So we could count requests that do not have existing pod anymore
   const query = {
     include: [ this.sequelize.models.Pod ]
   }

server/tests/api/multiple-pods.js

@@ -377,19 +377,44 @@ describe('Test multiple pods', function () {
   })
 
   describe('Should update video views', function () {
-    let videoId1
-    let videoId2
+    let localVideosPod3 = []
+    let remoteVideosPod1 = []
+    let remoteVideosPod2 = []
+    let remoteVideosPod3 = []
 
     before(function (done) {
-      videosUtils.getVideosList(servers[2].url, function (err, res) {
-        if (err) throw err
+      parallel([
+        function (callback) {
+          videosUtils.getVideosList(servers[0].url, function (err, res) {
+            if (err) throw err
 
-        const videos = res.body.data.filter(video => video.isLocal === true)
-        videoId1 = videos[0].id
-        videoId2 = videos[1].id
+            remoteVideosPod1 = res.body.data.filter(video => video.isLocal === false).map(video => video.id)
 
-        done()
-      })
+            callback()
+          })
+        },
+
+        function (callback) {
+          videosUtils.getVideosList(servers[1].url, function (err, res) {
+            if (err) throw err
+
+            remoteVideosPod2 = res.body.data.filter(video => video.isLocal === false).map(video => video.id)
+
+            callback()
+          })
+        },
+
+        function (callback) {
+          videosUtils.getVideosList(servers[2].url, function (err, res) {
+            if (err) throw err
+
+            localVideosPod3 = res.body.data.filter(video => video.isLocal === true).map(video => video.id)
+            remoteVideosPod3 = res.body.data.filter(video => video.isLocal === false).map(video => video.id)
+
+            callback()
+          })
+        }
+      ], done)
     })
 
     it('Should views multiple videos on owned servers', function (done) {
@@ -397,42 +422,115 @@ describe('Test multiple pods', function () {
 
       parallel([
         function (callback) {
-          videosUtils.getVideo(servers[2].url, videoId1, callback)
+          videosUtils.getVideo(servers[2].url, localVideosPod3[0], callback)
         },
 
         function (callback) {
-          videosUtils.getVideo(servers[2].url, videoId1, callback)
+          videosUtils.getVideo(servers[2].url, localVideosPod3[0], callback)
         },
 
         function (callback) {
-          videosUtils.getVideo(servers[2].url, videoId1, callback)
+          videosUtils.getVideo(servers[2].url, localVideosPod3[0], callback)
         },
 
         function (callback) {
-          videosUtils.getVideo(servers[2].url, videoId2, callback)
+          videosUtils.getVideo(servers[2].url, localVideosPod3[1], callback)
+        },
+
+        function (callback) {
+          setTimeout(done, 22000)
         }
       ], function (err) {
         if (err) throw err
 
-        setTimeout(done, 22000)
+        each(servers, function (server, callback) {
+          videosUtils.getVideosList(server.url, function (err, res) {
+            if (err) throw err
+
+            const videos = res.body.data
+            expect(videos.find(video => video.views === 3)).to.be.exist
+            expect(videos.find(video => video.views === 1)).to.be.exist
+
+            callback()
+          })
+        }, done)
       })
     })
 
-    it('Should have views updated on each pod', function (done) {
-      each(servers, function (server, callback) {
-        videosUtils.getVideosList(server.url, function (err, res) {
-          if (err) throw err
+    it('Should views multiple videos on each servers', function (done) {
+      this.timeout(30000)
 
-          const videos = res.body.data
-          expect(videos.find(video => video.views === 3)).to.be.exist
-          expect(videos.find(video => video.views === 1)).to.be.exist
+      parallel([
+        function (callback) {
+          videosUtils.getVideo(servers[0].url, remoteVideosPod1[0], callback)
+        },
 
-          callback()
-        })
-      }, done)
+        function (callback) {
+          videosUtils.getVideo(servers[1].url, remoteVideosPod2[0], callback)
+        },
+
+        function (callback) {
+          videosUtils.getVideo(servers[1].url, remoteVideosPod2[0], callback)
+        },
+
+        function (callback) {
+          videosUtils.getVideo(servers[2].url, remoteVideosPod3[0], callback)
+        },
+
+        function (callback) {
+          videosUtils.getVideo(servers[2].url, remoteVideosPod3[1], callback)
+        },
+
+        function (callback) {
+          videosUtils.getVideo(servers[2].url, remoteVideosPod3[1], callback)
+        },
+
+        function (callback) {
+          videosUtils.getVideo(servers[2].url, remoteVideosPod3[1], callback)
+        },
+
+        function (callback) {
+          videosUtils.getVideo(servers[2].url, localVideosPod3[1], callback)
+        },
+
+        function (callback) {
+          videosUtils.getVideo(servers[2].url, localVideosPod3[1], callback)
+        },
+
+        function (callback) {
+          videosUtils.getVideo(servers[2].url, localVideosPod3[1], callback)
+        },
+
+        function (callback) {
+          setTimeout(done, 22000)
+        }
+      ], function (err) {
+        if (err) throw err
+
+        let baseVideos = null
+        each(servers, function (server, callback) {
+          videosUtils.getVideosList(server.url, function (err, res) {
+            if (err) throw err
+
+            const videos = res.body
+
+            // Initialize base videos for future comparisons
+            if (baseVideos === null) {
+              baseVideos = videos
+              return callback()
+            }
+
+            for (let i = 0; i < baseVideos.length; i++) {
+              expect(baseVideos[i].views).to.equal(videos[i].views)
+            }
+
+            callback()
+          })
+        }, done)
+      })
     })
   })
-/*
+
   describe('Should manipulate these videos', function () {
     it('Should update the video 3 by asking pod 3', function (done) {
       this.timeout(15000)
@@ -520,7 +618,7 @@ describe('Test multiple pods', function () {
       }, done)
     })
   })
-*/
+
   after(function (done) {
     servers.forEach(function (server) {
       process.kill(-server.app.pid)