From b211106695bb82f6c32e53306081b5262c3d109d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 24 Mar 2022 13:36:47 +0100 Subject: [PATCH] Support video views/viewers stats in server * Add "currentTime" and "event" body params to view endpoint * Merge watching and view endpoints * Introduce WatchAction AP activity * Add tables to store viewer information of local videos * Add endpoints to fetch video views/viewers stats of local videos * Refactor views/viewers handlers * Support "views" and "viewers" counters for both VOD and live videos --- config/default.yaml | 7 + config/production.yaml.example | 7 + config/test.yaml | 3 + package.json | 1 + scripts/benchmark.ts | 8 +- scripts/ci.sh | 3 +- server.ts | 6 +- server/controllers/activitypub/client.ts | 14 +- server/controllers/api/server/debug.ts | 10 +- server/controllers/api/videos/index.ts | 31 +- server/controllers/api/videos/stats.ts | 66 ++++ server/controllers/api/videos/view.ts | 68 ++++ server/controllers/api/videos/watching.ts | 44 --- .../custom-validators/activitypub/activity.ts | 2 + .../custom-validators/activitypub/misc.ts | 11 +- .../custom-validators/activitypub/videos.ts | 14 +- .../activitypub/watch-action.ts | 37 +++ .../helpers/custom-validators/video-stats.ts | 16 + .../helpers/custom-validators/video-view.ts | 12 + server/helpers/geo-ip.ts | 78 +++++ server/initializers/checker-before-init.ts | 1 + server/initializers/config.ts | 6 + server/initializers/constants.ts | 17 +- server/initializers/database.ts | 10 +- .../migrations/0705-local-video-viewers.ts | 52 ++++ server/lib/activitypub/activity.ts | 15 +- server/lib/activitypub/context.ts | 23 +- server/lib/activitypub/local-video-viewer.ts | 42 +++ .../lib/activitypub/process/process-create.ts | 21 +- .../lib/activitypub/process/process-view.ts | 4 +- server/lib/activitypub/send/send-create.ts | 17 +- server/lib/activitypub/send/send-view.ts | 63 ++-- server/lib/activitypub/url.ts | 6 + .../shared/object-to-model-attributes.ts | 4 +- server/lib/client-html.ts | 2 +- .../job-queue/handlers/video-views-stats.ts | 2 +- server/lib/redis.ts | 68 +++- .../lib/schedulers/geo-ip-update-scheduler.ts | 22 ++ .../schedulers/remove-old-views-scheduler.ts | 6 +- .../video-views-buffer-scheduler.ts | 4 +- server/lib/video-views.ts | 131 -------- server/lib/views/shared/index.ts | 2 + server/lib/views/shared/video-viewers.ts | 276 +++++++++++++++++ server/lib/views/shared/video-views.ts | 60 ++++ server/lib/views/video-views-manager.ts | 70 +++++ server/middlewares/cache/shared/api-cache.ts | 4 +- server/middlewares/validators/express.ts | 15 + server/middlewares/validators/index.ts | 25 +- server/middlewares/validators/videos/index.ts | 3 +- .../validators/videos/video-stats.ts | 73 +++++ .../validators/videos/video-view.ts | 74 +++++ .../validators/videos/video-watch.ts | 38 --- .../video/formatter/video-format-utils.ts | 31 +- server/models/video/video.ts | 2 +- .../view/local-video-viewer-watch-section.ts | 63 ++++ server/models/view/local-video-viewer.ts | 274 +++++++++++++++++ server/models/{video => view}/video-view.ts | 2 +- server/tests/api/activitypub/client.ts | 20 +- server/tests/api/check-params/index.ts | 1 + .../tests/api/check-params/videos-history.ts | 46 +-- server/tests/api/check-params/views.ts | 157 ++++++++++ server/tests/api/live/index.ts | 1 - server/tests/api/live/live-socket-messages.ts | 4 +- server/tests/api/live/live-views.ts | 132 -------- server/tests/api/redundancy/redundancy.ts | 10 +- server/tests/api/server/reverse-proxy.ts | 16 +- server/tests/api/server/stats.ts | 2 +- server/tests/api/videos/index.ts | 1 - server/tests/api/videos/multiple-servers.ts | 40 +-- server/tests/api/videos/single-server.ts | 17 +- server/tests/api/videos/video-channels.ts | 4 +- server/tests/api/videos/videos-history.ts | 73 ++--- server/tests/api/views/index.ts | 5 + server/tests/api/views/video-views-counter.ts | 155 ++++++++++ .../api/views/video-views-overall-stats.ts | 291 ++++++++++++++++++ .../api/views/video-views-retention-stats.ts | 56 ++++ .../api/views/video-views-timeserie-stats.ts | 109 +++++++ .../{videos => views}/videos-views-cleaner.ts | 8 +- server/tests/plugins/action-hooks.ts | 4 +- server/tests/plugins/plugin-helpers.ts | 2 +- server/tests/shared/index.ts | 1 + server/tests/shared/views.ts | 93 ++++++ server/types/express.d.ts | 2 + server/types/models/video/index.ts | 2 + .../video/local-video-viewer-watch-section.ts | 5 + .../types/models/video/local-video-viewer.ts | 19 ++ shared/models/activitypub/activity.ts | 8 +- shared/models/activitypub/context.ts | 3 +- shared/models/activitypub/objects/index.ts | 1 + .../objects/watch-action-object.ts | 22 ++ shared/models/server/debug.model.ts | 2 +- shared/models/users/index.ts | 1 - .../models/users/user-watching-video.model.ts | 3 - shared/models/videos/index.ts | 2 + shared/models/videos/stats/index.ts | 4 + .../videos/stats/video-stats-overall.model.ts | 17 + .../stats/video-stats-retention.model.ts | 6 + .../video-stats-timeserie-metric.type.ts | 1 + .../stats/video-stats-timeserie.model.ts | 6 + shared/models/videos/video-view.model.ts | 6 + shared/models/videos/video.model.ts | 3 +- shared/server-commands/server/server.ts | 8 +- .../server-commands/videos/history-command.ts | 19 -- shared/server-commands/videos/index.ts | 1 + .../videos/video-stats-command.ts | 48 +++ .../server-commands/videos/videos-command.ts | 17 - .../server-commands/videos/views-command.ts | 51 +++ yarn.lock | 18 ++ 108 files changed, 2834 insertions(+), 655 deletions(-) create mode 100644 server/controllers/api/videos/stats.ts create mode 100644 server/controllers/api/videos/view.ts delete mode 100644 server/controllers/api/videos/watching.ts create mode 100644 server/helpers/custom-validators/activitypub/watch-action.ts create mode 100644 server/helpers/custom-validators/video-stats.ts create mode 100644 server/helpers/custom-validators/video-view.ts create mode 100644 server/helpers/geo-ip.ts create mode 100644 server/initializers/migrations/0705-local-video-viewers.ts create mode 100644 server/lib/activitypub/local-video-viewer.ts create mode 100644 server/lib/schedulers/geo-ip-update-scheduler.ts delete mode 100644 server/lib/video-views.ts create mode 100644 server/lib/views/shared/index.ts create mode 100644 server/lib/views/shared/video-viewers.ts create mode 100644 server/lib/views/shared/video-views.ts create mode 100644 server/lib/views/video-views-manager.ts create mode 100644 server/middlewares/validators/express.ts create mode 100644 server/middlewares/validators/videos/video-stats.ts create mode 100644 server/middlewares/validators/videos/video-view.ts delete mode 100644 server/middlewares/validators/videos/video-watch.ts create mode 100644 server/models/view/local-video-viewer-watch-section.ts create mode 100644 server/models/view/local-video-viewer.ts rename server/models/{video => view}/video-view.ts (96%) create mode 100644 server/tests/api/check-params/views.ts delete mode 100644 server/tests/api/live/live-views.ts create mode 100644 server/tests/api/views/index.ts create mode 100644 server/tests/api/views/video-views-counter.ts create mode 100644 server/tests/api/views/video-views-overall-stats.ts create mode 100644 server/tests/api/views/video-views-retention-stats.ts create mode 100644 server/tests/api/views/video-views-timeserie-stats.ts rename server/tests/api/{videos => views}/videos-views-cleaner.ts (90%) create mode 100644 server/tests/shared/views.ts create mode 100644 server/types/models/video/local-video-viewer-watch-section.ts create mode 100644 server/types/models/video/local-video-viewer.ts create mode 100644 shared/models/activitypub/objects/watch-action-object.ts delete mode 100644 shared/models/users/user-watching-video.model.ts create mode 100644 shared/models/videos/stats/index.ts create mode 100644 shared/models/videos/stats/video-stats-overall.model.ts create mode 100644 shared/models/videos/stats/video-stats-retention.model.ts create mode 100644 shared/models/videos/stats/video-stats-timeserie-metric.type.ts create mode 100644 shared/models/videos/stats/video-stats-timeserie.model.ts create mode 100644 shared/models/videos/video-view.model.ts create mode 100644 shared/server-commands/videos/video-stats-command.ts create mode 100644 shared/server-commands/videos/views-command.ts diff --git a/config/default.yaml b/config/default.yaml index 009c9b6d4..5130afdce 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -261,6 +261,13 @@ views: ip_view_expiration: '1 hour' +# Used to get country location of views of local videos +geo_ip: + enabled: true + + country: + database_url: 'https://dbip.mirror.framasoft.org/files/dbip-country-lite-latest.mmdb' + plugins: # The website PeerTube will ask for available PeerTube plugins and themes # This is an unmoderated plugin index, so only install plugins/themes you trust diff --git a/config/production.yaml.example b/config/production.yaml.example index 8efe07c01..3a6813687 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -257,6 +257,13 @@ views: ip_view_expiration: '1 hour' +# Used to get country location of views of local videos +geo_ip: + enabled: true + + country: + database_url: 'https://dbip.mirror.framasoft.org/files/dbip-country-lite-latest.mmdb' + plugins: # The website PeerTube will ask for available PeerTube plugins and themes # This is an unmoderated plugin index, so only install plugins/themes you trust diff --git a/config/test.yaml b/config/test.yaml index 247100fdf..898bc0324 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -168,5 +168,8 @@ views: local_buffer_update_interval: '5 seconds' ip_view_expiration: '1 second' +geo_ip: + enabled: false + video_studio: enabled: true diff --git a/package.json b/package.json index 0b57a4321..a4beae7ac 100644 --- a/package.json +++ b/package.json @@ -121,6 +121,7 @@ "magnet-uri": "^6.1.0", "markdown-it": "^12.0.4", "markdown-it-emoji": "^2.0.0", + "maxmind": "^4.3.6", "memoizee": "^0.4.14", "morgan": "^1.5.3", "multer": "^1.4.4", diff --git a/scripts/benchmark.ts b/scripts/benchmark.ts index 623c11e27..4a414a2fa 100644 --- a/scripts/benchmark.ts +++ b/scripts/benchmark.ts @@ -153,21 +153,23 @@ async function run () { } }, { - title: 'API - watching', + title: 'API - views with token', method: 'PUT', headers: { ...buildAuthorizationHeader(), ...buildJSONHeader() }, body: JSON.stringify({ currentTime: 2 }), - path: '/api/v1/videos/' + video.uuid + '/watching', + path: '/api/v1/videos/' + video.uuid + '/views', expecter: (body, status) => { return status === 204 } }, { - title: 'API - views', + title: 'API - views without token', method: 'POST', + headers: buildJSONHeader(), + body: JSON.stringify({ currentTime: 2 }), path: '/api/v1/videos/' + video.uuid + '/views', expecter: (body, status) => { return status === 204 diff --git a/scripts/ci.sh b/scripts/ci.sh index a45f91a6b..2dd5e25ce 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -84,8 +84,9 @@ elif [ "$1" = "api-3" ]; then npm run build:server videosFiles=$(findTestFiles ./dist/server/tests/api/videos) + viewsFiles=$(findTestFiles ./dist/server/tests/api/views) - MOCHA_PARALLEL=true runTest "$1" $((3*$speedFactor)) $videosFiles + MOCHA_PARALLEL=true runTest "$1" $((3*$speedFactor)) $viewsFiles $videosFiles elif [ "$1" = "api-4" ]; then npm run build:server diff --git a/server.ts b/server.ts index bb7a0c210..ad162832b 100644 --- a/server.ts +++ b/server.ts @@ -112,6 +112,7 @@ import { RemoveOldHistoryScheduler } from './server/lib/schedulers/remove-old-hi import { AutoFollowIndexInstances } from './server/lib/schedulers/auto-follow-index-instances' import { RemoveDanglingResumableUploadsScheduler } from './server/lib/schedulers/remove-dangling-resumable-uploads-scheduler' import { VideoViewsBufferScheduler } from './server/lib/schedulers/video-views-buffer-scheduler' +import { GeoIPUpdateScheduler } from './server/lib/schedulers/geo-ip-update-scheduler' import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto' import { PeerTubeSocket } from './server/lib/peertube-socket' import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls' @@ -123,7 +124,7 @@ import { LiveManager } from './server/lib/live' import { HttpStatusCode } from './shared/models/http/http-error-codes' import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache' import { ServerConfigManager } from '@server/lib/server-config-manager' -import { VideoViews } from '@server/lib/video-views' +import { VideoViewsManager } from '@server/lib/views/video-views-manager' import { isTestInstance } from './server/helpers/core-utils' // ----------- Command line ----------- @@ -295,10 +296,11 @@ async function startApplication () { AutoFollowIndexInstances.Instance.enable() RemoveDanglingResumableUploadsScheduler.Instance.enable() VideoViewsBufferScheduler.Instance.enable() + GeoIPUpdateScheduler.Instance.enable() Redis.Instance.init() PeerTubeSocket.Instance.init(server) - VideoViews.Instance.init() + VideoViewsManager.Instance.init() updateStreamingPlaylistsInfohashesIfNeeded() .catch(err => logger.error('Cannot update streaming playlist infohashes.', { err })) diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts index d0f761009..8e064fb5b 100644 --- a/server/controllers/activitypub/client.ts +++ b/server/controllers/activitypub/client.ts @@ -27,7 +27,7 @@ import { videosShareValidator } from '../../middlewares' import { cacheRoute } from '../../middlewares/cache/cache' -import { getAccountVideoRateValidatorFactory, videoCommentGetValidator } from '../../middlewares/validators' +import { getAccountVideoRateValidatorFactory, getVideoLocalViewerValidator, videoCommentGetValidator } from '../../middlewares/validators' import { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy' import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '../../middlewares/validators/videos/video-playlists' import { AccountModel } from '../../models/account/account' @@ -175,6 +175,12 @@ activityPubClientRouter.get('/video-playlists/:playlistId/videos/:playlistElemen videoPlaylistElementController ) +activityPubClientRouter.get('/videos/local-viewer/:localViewerId', + executeIfActivityPub, + asyncMiddleware(getVideoLocalViewerValidator), + getVideoLocalViewerController +) + // --------------------------------------------------------------------------- export { @@ -399,6 +405,12 @@ function videoPlaylistElementController (req: express.Request, res: express.Resp return activityPubResponse(activityPubContextify(json, 'Playlist'), res) } +function getVideoLocalViewerController (req: express.Request, res: express.Response) { + const localViewer = res.locals.localViewerFull + + return activityPubResponse(activityPubContextify(localViewer.toActivityPubObject(), 'WatchAction'), res) +} + // --------------------------------------------------------------------------- function actorFollowing (req: express.Request, actor: MActorId) { diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts index 093e6a03c..6b6ff027c 100644 --- a/server/controllers/api/server/debug.ts +++ b/server/controllers/api/server/debug.ts @@ -1,6 +1,8 @@ import express from 'express' import { InboxManager } from '@server/lib/activitypub/inbox-manager' import { RemoveDanglingResumableUploadsScheduler } from '@server/lib/schedulers/remove-dangling-resumable-uploads-scheduler' +import { VideoViewsBufferScheduler } from '@server/lib/schedulers/video-views-buffer-scheduler' +import { VideoViewsManager } from '@server/lib/views/video-views-manager' import { Debug, SendDebugCommand } from '@shared/models' import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' import { UserRight } from '../../../../shared/models/users' @@ -38,9 +40,13 @@ function getDebug (req: express.Request, res: express.Response) { async function runCommand (req: express.Request, res: express.Response) { const body: SendDebugCommand = req.body - if (body.command === 'remove-dandling-resumable-uploads') { - await RemoveDanglingResumableUploadsScheduler.Instance.execute() + const processors: { [id in SendDebugCommand['command']]: () => Promise } = { + 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(), + 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(), + 'process-video-viewers': () => VideoViewsManager.Instance.processViewers() } + await processors[body.command]() + return res.status(HttpStatusCode.NO_CONTENT_204).end() } diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index c7617093c..be233722c 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -1,7 +1,6 @@ import express from 'express' import { pickCommonVideoQuery } from '@server/helpers/query' import { doJSONRequest } from '@server/helpers/requests' -import { VideoViews } from '@server/lib/video-views' import { openapiOperationDoc } from '@server/middlewares/doc' import { getServerActor } from '@server/models/application/application' import { guessAdditionalAttributesFromQuery } from '@server/models/video/formatter/video-format-utils' @@ -13,7 +12,6 @@ import { logger } from '../../../helpers/logger' import { getFormattedObjects } from '../../../helpers/utils' import { REMOTE_SCHEME, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../initializers/constants' import { sequelizeTypescript } from '../../../initializers/database' -import { sendView } from '../../../lib/activitypub/send/send-view' import { JobQueue } from '../../../lib/job-queue' import { Hooks } from '../../../lib/plugins/hooks' import { @@ -35,28 +33,30 @@ import { VideoModel } from '../../../models/video/video' import { blacklistRouter } from './blacklist' import { videoCaptionsRouter } from './captions' import { videoCommentRouter } from './comment' -import { studioRouter } from './studio' import { filesRouter } from './files' import { videoImportsRouter } from './import' import { liveRouter } from './live' import { ownershipVideoRouter } from './ownership' import { rateVideoRouter } from './rate' +import { statsRouter } from './stats' +import { studioRouter } from './studio' import { transcodingRouter } from './transcoding' import { updateRouter } from './update' import { uploadRouter } from './upload' -import { watchingRouter } from './watching' +import { viewRouter } from './view' const auditLogger = auditLoggerFactory('videos') const videosRouter = express.Router() videosRouter.use('/', blacklistRouter) +videosRouter.use('/', statsRouter) videosRouter.use('/', rateVideoRouter) videosRouter.use('/', videoCommentRouter) videosRouter.use('/', studioRouter) videosRouter.use('/', videoCaptionsRouter) videosRouter.use('/', videoImportsRouter) videosRouter.use('/', ownershipVideoRouter) -videosRouter.use('/', watchingRouter) +videosRouter.use('/', viewRouter) videosRouter.use('/', liveRouter) videosRouter.use('/', uploadRouter) videosRouter.use('/', updateRouter) @@ -103,11 +103,6 @@ videosRouter.get('/:id', asyncMiddleware(checkVideoFollowConstraints), getVideo ) -videosRouter.post('/:id/views', - openapiOperationDoc({ operationId: 'addView' }), - asyncMiddleware(videosCustomGetValidator('only-video')), - asyncMiddleware(viewVideo) -) videosRouter.delete('/:id', openapiOperationDoc({ operationId: 'delVideo' }), @@ -150,22 +145,6 @@ function getVideo (_req: express.Request, res: express.Response) { return res.json(video.toFormattedDetailsJSON()) } -async function viewVideo (req: express.Request, res: express.Response) { - const video = res.locals.onlyVideo - - const ip = req.ip - const success = await VideoViews.Instance.processView({ video, ip }) - - if (success) { - const serverActor = await getServerActor() - await sendView(serverActor, video, undefined) - - Hooks.runAction('action:api.video.viewed', { video: video, ip, req, res }) - } - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - async function getVideoDescription (req: express.Request, res: express.Response) { const videoInstance = res.locals.videoAll diff --git a/server/controllers/api/videos/stats.ts b/server/controllers/api/videos/stats.ts new file mode 100644 index 000000000..5f8513e9e --- /dev/null +++ b/server/controllers/api/videos/stats.ts @@ -0,0 +1,66 @@ +import express from 'express' +import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' +import { VideoStatsTimeserieMetric } from '@shared/models' +import { + asyncMiddleware, + authenticate, + videoOverallStatsValidator, + videoRetentionStatsValidator, + videoTimeserieStatsValidator +} from '../../../middlewares' + +const statsRouter = express.Router() + +statsRouter.get('/:videoId/stats/overall', + authenticate, + asyncMiddleware(videoOverallStatsValidator), + asyncMiddleware(getOverallStats) +) + +statsRouter.get('/:videoId/stats/timeseries/:metric', + authenticate, + asyncMiddleware(videoTimeserieStatsValidator), + asyncMiddleware(getTimeserieStats) +) + +statsRouter.get('/:videoId/stats/retention', + authenticate, + asyncMiddleware(videoRetentionStatsValidator), + asyncMiddleware(getRetentionStats) +) + +// --------------------------------------------------------------------------- + +export { + statsRouter +} + +// --------------------------------------------------------------------------- + +async function getOverallStats (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + + const stats = await LocalVideoViewerModel.getOverallStats(video) + + return res.json(stats) +} + +async function getRetentionStats (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + + const stats = await LocalVideoViewerModel.getRetentionStats(video) + + return res.json(stats) +} + +async function getTimeserieStats (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + const metric = req.params.metric as VideoStatsTimeserieMetric + + const stats = await LocalVideoViewerModel.getTimeserieStats({ + video, + metric + }) + + return res.json(stats) +} diff --git a/server/controllers/api/videos/view.ts b/server/controllers/api/videos/view.ts new file mode 100644 index 000000000..e28cf371a --- /dev/null +++ b/server/controllers/api/videos/view.ts @@ -0,0 +1,68 @@ +import express from 'express' +import { sendView } from '@server/lib/activitypub/send/send-view' +import { Hooks } from '@server/lib/plugins/hooks' +import { VideoViewsManager } from '@server/lib/views/video-views-manager' +import { getServerActor } from '@server/models/application/application' +import { MVideoId } from '@server/types/models' +import { HttpStatusCode, VideoView } from '@shared/models' +import { asyncMiddleware, methodsValidator, openapiOperationDoc, optionalAuthenticate, videoViewValidator } from '../../../middlewares' +import { UserVideoHistoryModel } from '../../../models/user/user-video-history' + +const viewRouter = express.Router() + +viewRouter.all( + [ '/:videoId/views', '/:videoId/watching' ], + openapiOperationDoc({ operationId: 'addView' }), + methodsValidator([ 'PUT', 'POST' ]), + optionalAuthenticate, + asyncMiddleware(videoViewValidator), + asyncMiddleware(viewVideo) +) + +// --------------------------------------------------------------------------- + +export { + viewRouter +} + +// --------------------------------------------------------------------------- + +async function viewVideo (req: express.Request, res: express.Response) { + const video = res.locals.onlyVideo + + const body = req.body as VideoView + + const ip = req.ip + const { successView, successViewer } = await VideoViewsManager.Instance.processLocalView({ + video, + ip, + currentTime: body.currentTime, + viewEvent: body.viewEvent + }) + + if (successView) { + await sendView({ byActor: await getServerActor(), video, type: 'view' }) + + Hooks.runAction('action:api.video.viewed', { video: video, ip, req, res }) + } + + if (successViewer) { + await sendView({ byActor: await getServerActor(), video, type: 'viewer' }) + } + + await updateUserHistoryIfNeeded(body, video, res) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function updateUserHistoryIfNeeded (body: VideoView, video: MVideoId, res: express.Response) { + const user = res.locals.oauth?.token.User + if (!user) return + if (user.videosHistoryEnabled !== true) return + + await UserVideoHistoryModel.upsert({ + videoId: video.id, + userId: user.id, + currentTime: body.currentTime + }) +} diff --git a/server/controllers/api/videos/watching.ts b/server/controllers/api/videos/watching.ts deleted file mode 100644 index 3fd22caac..000000000 --- a/server/controllers/api/videos/watching.ts +++ /dev/null @@ -1,44 +0,0 @@ -import express from 'express' -import { HttpStatusCode, UserWatchingVideo } from '@shared/models' -import { - asyncMiddleware, - asyncRetryTransactionMiddleware, - authenticate, - openapiOperationDoc, - videoWatchingValidator -} from '../../../middlewares' -import { UserVideoHistoryModel } from '../../../models/user/user-video-history' - -const watchingRouter = express.Router() - -watchingRouter.put('/:videoId/watching', - openapiOperationDoc({ operationId: 'setProgress' }), - authenticate, - asyncMiddleware(videoWatchingValidator), - asyncRetryTransactionMiddleware(userWatchVideo) -) - -// --------------------------------------------------------------------------- - -export { - watchingRouter -} - -// --------------------------------------------------------------------------- - -async function userWatchVideo (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.User - - const body: UserWatchingVideo = req.body - const { id: videoId } = res.locals.videoId - - await UserVideoHistoryModel.upsert({ - videoId, - userId: user.id, - currentTime: body.currentTime - }) - - return res.type('json') - .status(HttpStatusCode.NO_CONTENT_204) - .end() -} diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts index b5c96f6e7..90a918523 100644 --- a/server/helpers/custom-validators/activitypub/activity.ts +++ b/server/helpers/custom-validators/activitypub/activity.ts @@ -8,6 +8,7 @@ import { isActivityPubUrlValid, isBaseActivityValid, isObjectValid } from './mis import { isPlaylistObjectValid } from './playlist' import { sanitizeAndCheckVideoCommentObject } from './video-comments' import { sanitizeAndCheckVideoTorrentObject } from './videos' +import { isWatchActionObjectValid } from './watch-action' function isRootActivityValid (activity: any) { return isCollection(activity) || isActivity(activity) @@ -82,6 +83,7 @@ function isCreateActivityValid (activity: any) { isDislikeActivityValid(activity.object) || isFlagActivityValid(activity.object) || isPlaylistObjectValid(activity.object) || + isWatchActionObjectValid(activity.object) || isCacheFileObjectValid(activity.object) || sanitizeAndCheckVideoCommentObject(activity.object) || diff --git a/server/helpers/custom-validators/activitypub/misc.ts b/server/helpers/custom-validators/activitypub/misc.ts index 4ee8e6fee..9d823299f 100644 --- a/server/helpers/custom-validators/activitypub/misc.ts +++ b/server/helpers/custom-validators/activitypub/misc.ts @@ -57,10 +57,19 @@ function setValidAttributedTo (obj: any) { return true } +function isActivityPubVideoDurationValid (value: string) { + // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration + return exists(value) && + typeof value === 'string' && + value.startsWith('PT') && + value.endsWith('S') +} + export { isUrlValid, isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo, - isObjectValid + isObjectValid, + isActivityPubVideoDurationValid } diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index 80a321117..2a2f008b9 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -4,7 +4,7 @@ import { ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject } from '@s import { LiveVideoLatencyMode, VideoState } from '../../../../shared/models/videos' import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers/constants' import { peertubeTruncate } from '../../core-utils' -import { exists, isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc' +import { isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc' import { isLiveLatencyModeValid } from '../video-lives' import { isVideoDurationValid, @@ -14,22 +14,13 @@ import { isVideoTruncatedDescriptionValid, isVideoViewsValid } from '../videos' -import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc' +import { isActivityPubUrlValid, isActivityPubVideoDurationValid, isBaseActivityValid, setValidAttributedTo } from './misc' function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { return isBaseActivityValid(activity, 'Update') && sanitizeAndCheckVideoTorrentObject(activity.object) } -function isActivityPubVideoDurationValid (value: string) { - // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration - return exists(value) && - typeof value === 'string' && - value.startsWith('PT') && - value.endsWith('S') && - isVideoDurationValid(value.replace(/[^0-9]+/g, '')) -} - function sanitizeAndCheckVideoTorrentObject (video: any) { if (!video || video.type !== 'Video') return false @@ -71,6 +62,7 @@ function sanitizeAndCheckVideoTorrentObject (video: any) { return isActivityPubUrlValid(video.id) && isVideoNameValid(video.name) && isActivityPubVideoDurationValid(video.duration) && + isVideoDurationValid(video.duration.replace(/[^0-9]+/g, '')) && isUUIDValid(video.uuid) && (!video.category || isRemoteNumberIdentifierValid(video.category)) && (!video.licence || isRemoteNumberIdentifierValid(video.licence)) && diff --git a/server/helpers/custom-validators/activitypub/watch-action.ts b/server/helpers/custom-validators/activitypub/watch-action.ts new file mode 100644 index 000000000..b9ffa63f6 --- /dev/null +++ b/server/helpers/custom-validators/activitypub/watch-action.ts @@ -0,0 +1,37 @@ +import { WatchActionObject } from '@shared/models' +import { exists, isDateValid, isUUIDValid } from '../misc' +import { isVideoTimeValid } from '../video-view' +import { isActivityPubVideoDurationValid, isObjectValid } from './misc' + +function isWatchActionObjectValid (action: WatchActionObject) { + return exists(action) && + action.type === 'WatchAction' && + isObjectValid(action.id) && + isActivityPubVideoDurationValid(action.duration) && + isDateValid(action.startTime) && + isDateValid(action.endTime) && + isLocationValid(action.location) && + isUUIDValid(action.uuid) && + isObjectValid(action.object) && + isWatchSectionsValid(action.watchSections) +} + +// --------------------------------------------------------------------------- + +export { + isWatchActionObjectValid +} + +// --------------------------------------------------------------------------- + +function isLocationValid (location: any) { + if (!location) return true + + return typeof location === 'object' && typeof location.addressCountry === 'string' +} + +function isWatchSectionsValid (sections: WatchActionObject['watchSections']) { + return Array.isArray(sections) && sections.every(s => { + return isVideoTimeValid(s.startTimestamp) && isVideoTimeValid(s.endTimestamp) + }) +} diff --git a/server/helpers/custom-validators/video-stats.ts b/server/helpers/custom-validators/video-stats.ts new file mode 100644 index 000000000..1e22f0654 --- /dev/null +++ b/server/helpers/custom-validators/video-stats.ts @@ -0,0 +1,16 @@ +import { VideoStatsTimeserieMetric } from '@shared/models' + +const validMetrics = new Set([ + 'viewers', + 'aggregateWatchTime' +]) + +function isValidStatTimeserieMetric (value: VideoStatsTimeserieMetric) { + return validMetrics.has(value) +} + +// --------------------------------------------------------------------------- + +export { + isValidStatTimeserieMetric +} diff --git a/server/helpers/custom-validators/video-view.ts b/server/helpers/custom-validators/video-view.ts new file mode 100644 index 000000000..091c92083 --- /dev/null +++ b/server/helpers/custom-validators/video-view.ts @@ -0,0 +1,12 @@ +import { exists } from './misc' + +function isVideoTimeValid (value: number, videoDuration?: number) { + if (value < 0) return false + if (exists(videoDuration) && value > videoDuration) return false + + return true +} + +export { + isVideoTimeValid +} diff --git a/server/helpers/geo-ip.ts b/server/helpers/geo-ip.ts new file mode 100644 index 000000000..4ba7011c2 --- /dev/null +++ b/server/helpers/geo-ip.ts @@ -0,0 +1,78 @@ +import { pathExists, writeFile } from 'fs-extra' +import maxmind, { CountryResponse, Reader } from 'maxmind' +import { join } from 'path' +import { CONFIG } from '@server/initializers/config' +import { logger, loggerTagsFactory } from './logger' +import { isBinaryResponse, peertubeGot } from './requests' + +const lTags = loggerTagsFactory('geo-ip') + +const mmbdFilename = 'dbip-country-lite-latest.mmdb' +const mmdbPath = join(CONFIG.STORAGE.BIN_DIR, mmbdFilename) + +export class GeoIP { + private static instance: GeoIP + + private reader: Reader + + private constructor () { + } + + async safeCountryISOLookup (ip: string): Promise { + if (CONFIG.GEO_IP.ENABLED === false) return null + + await this.initReaderIfNeeded() + + try { + const result = this.reader.get(ip) + if (!result) return null + + return result.country.iso_code + } catch (err) { + logger.error('Cannot get country from IP.', { err }) + + return null + } + } + + async updateDatabase () { + if (CONFIG.GEO_IP.ENABLED === false) return + + const url = CONFIG.GEO_IP.COUNTRY.DATABASE_URL + + logger.info('Updating GeoIP database from %s.', url, lTags()) + + const gotOptions = { context: { bodyKBLimit: 200_000 }, responseType: 'buffer' as 'buffer' } + + try { + const gotResult = await peertubeGot(url, gotOptions) + + if (!isBinaryResponse(gotResult)) { + throw new Error('Not a binary response') + } + + await writeFile(mmdbPath, gotResult.body) + + // Reini reader + this.reader = undefined + + logger.info('GeoIP database updated %s.', mmdbPath, lTags()) + } catch (err) { + logger.error('Cannot update GeoIP database from %s.', url, { err, ...lTags() }) + } + } + + private async initReaderIfNeeded () { + if (!this.reader) { + if (!await pathExists(mmdbPath)) { + await this.updateDatabase() + } + + this.reader = await maxmind.open(mmdbPath) + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index 0f23a2d73..f2ef3d567 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts @@ -44,6 +44,7 @@ function checkMissedConfig () { 'history.videos.max_age', 'views.videos.remote.max_age', 'views.videos.local_buffer_update_interval', 'views.videos.ip_view_expiration', 'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max', 'theme.default', + 'geo_ip.enabled', 'geo_ip.country.database_url', 'remote_redundancy.videos.accept_from', 'federation.videos.federate_unlisted', 'federation.videos.cleanup_remote_interactions', 'peertube.check_latest_version.enabled', 'peertube.check_latest_version.url', diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 122cb9472..d8f5f3496 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -215,6 +215,12 @@ const CONFIG = { IP_VIEW_EXPIRATION: parseDurationToMs(config.get('views.videos.ip_view_expiration')) } }, + GEO_IP: { + ENABLED: config.get('geo_ip.enabled'), + COUNTRY: { + DATABASE_URL: config.get('geo_ip.country.database_url') + } + }, PLUGINS: { INDEX: { ENABLED: config.get('plugins.index.enabled'), diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 6bcefe0db..4929923dc 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 700 +const LAST_MIGRATION_VERSION = 705 // --------------------------------------------------------------------------- @@ -228,6 +228,7 @@ const SCHEDULER_INTERVALS_MS = { REMOVE_OLD_JOBS: 60000 * 60, // 1 hour UPDATE_VIDEOS: 60000, // 1 minute YOUTUBE_DL_UPDATE: 60000 * 60 * 24, // 1 day + GEO_IP_UPDATE: 60000 * 60 * 24, // 1 day VIDEO_VIEWS_BUFFER_UPDATE: CONFIG.VIEWS.VIDEOS.LOCAL_BUFFER_UPDATE_INTERVAL, CHECK_PLUGINS: CONFIG.PLUGINS.INDEX.CHECK_LATEST_VERSIONS_INTERVAL, CHECK_PEERTUBE_VERSION: 60000 * 60 * 24, // 1 day @@ -366,9 +367,12 @@ const CONSTRAINTS_FIELDS = { const VIEW_LIFETIME = { VIEW: CONFIG.VIEWS.VIDEOS.IP_VIEW_EXPIRATION, - VIEWER: 60000 * 5 // 5 minutes + VIEWER: 60000 * 5, // 5 minutes + VIEWER_STATS: 60000 * 60 // 1 hour } +const MAX_LOCAL_VIEWER_WATCH_SECTIONS = 10 + let CONTACT_FORM_LIFETIME = 60000 * 60 // 1 hour const VIDEO_TRANSCODING_FPS: VideoTranscodingFPS = { @@ -800,6 +804,12 @@ const SEARCH_INDEX = { // --------------------------------------------------------------------------- +const STATS_TIMESERIE = { + MAX_DAYS: 30 +} + +// --------------------------------------------------------------------------- + // Special constants for a test instance if (isTestInstance() === true) { PRIVATE_RSA_KEY_SIZE = 1024 @@ -836,6 +846,7 @@ if (isTestInstance() === true) { REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1 VIEW_LIFETIME.VIEWER = 1000 * 5 // 5 second + VIEW_LIFETIME.VIEWER_STATS = 1000 * 5 // 5 second CONTACT_FORM_LIFETIME = 1000 // 1 second JOB_ATTEMPTS['email'] = 1 @@ -907,6 +918,7 @@ export { LAST_MIGRATION_VERSION, OAUTH_LIFETIME, CUSTOM_HTML_TAG_COMMENTS, + STATS_TIMESERIE, BROADCAST_CONCURRENCY, AUDIT_LOG_FILENAME, PAGINATION, @@ -949,6 +961,7 @@ export { ABUSE_STATES, LRU_CACHE, REQUEST_TIMEOUTS, + MAX_LOCAL_VIEWER_WATCH_SECTIONS, USER_PASSWORD_RESET_LIFETIME, USER_PASSWORD_CREATE_LIFETIME, MEMOIZE_TTL, diff --git a/server/initializers/database.ts b/server/initializers/database.ts index 0e690f6ae..7a7ba61f4 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts @@ -1,10 +1,14 @@ import { QueryTypes, Transaction } from 'sequelize' import { Sequelize as SequelizeTypescript } from 'sequelize-typescript' +import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' import { TrackerModel } from '@server/models/server/tracker' import { VideoTrackerModel } from '@server/models/server/video-tracker' import { UserModel } from '@server/models/user/user' import { UserNotificationModel } from '@server/models/user/user-notification' import { UserVideoHistoryModel } from '@server/models/user/user-video-history' +import { VideoJobInfoModel } from '@server/models/video/video-job-info' +import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' +import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' import { isTestInstance } from '../helpers/core-utils' import { logger } from '../helpers/logger' import { AbuseModel } from '../models/abuse/abuse' @@ -42,10 +46,8 @@ import { VideoPlaylistElementModel } from '../models/video/video-playlist-elemen import { VideoShareModel } from '../models/video/video-share' import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' import { VideoTagModel } from '../models/video/video-tag' -import { VideoViewModel } from '../models/video/video-view' +import { VideoViewModel } from '../models/view/video-view' import { CONFIG } from './config' -import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string @@ -140,6 +142,8 @@ async function initDatabaseModels (silent: boolean) { VideoStreamingPlaylistModel, VideoPlaylistModel, VideoPlaylistElementModel, + LocalVideoViewerModel, + LocalVideoViewerWatchSectionModel, ThumbnailModel, TrackerModel, VideoTrackerModel, diff --git a/server/initializers/migrations/0705-local-video-viewers.ts b/server/initializers/migrations/0705-local-video-viewers.ts new file mode 100644 index 000000000..123402641 --- /dev/null +++ b/server/initializers/migrations/0705-local-video-viewers.ts @@ -0,0 +1,52 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize + db: any +}): Promise { + const { transaction } = utils + + { + const query = ` + CREATE TABLE IF NOT EXISTS "localVideoViewer" ( + "id" serial, + "startDate" timestamp with time zone NOT NULL, + "endDate" timestamp with time zone NOT NULL, + "watchTime" integer NOT NULL, + "country" varchar(255), + "uuid" uuid NOT NULL, + "url" varchar(255) NOT NULL, + "videoId" integer NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + "createdAt" timestamp with time zone NOT NULL, + PRIMARY KEY ("id") + ); + ` + await utils.sequelize.query(query, { transaction }) + } + + { + const query = ` + CREATE TABLE IF NOT EXISTS "localVideoViewerWatchSection" ( + "id" serial, + "watchStart" integer NOT NULL, + "watchEnd" integer NOT NULL, + "localVideoViewerId" integer NOT NULL REFERENCES "localVideoViewer" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + "createdAt" timestamp with time zone NOT NULL, + PRIMARY KEY ("id") + ); + ` + await utils.sequelize.query(query, { transaction }) + } + +} + +function down () { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/activitypub/activity.ts b/server/lib/activitypub/activity.ts index cccb7b1c1..e6cec1ba7 100644 --- a/server/lib/activitypub/activity.ts +++ b/server/lib/activitypub/activity.ts @@ -4,6 +4,17 @@ function getAPId (object: string | { id: string }) { return object.id } -export { - getAPId +function getActivityStreamDuration (duration: number) { + // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration + return 'PT' + duration + 'S' +} + +function getDurationFromActivityStream (duration: string) { + return parseInt(duration.replace(/[^\d]+/, '')) +} + +export { + getAPId, + getActivityStreamDuration, + getDurationFromActivityStream } diff --git a/server/lib/activitypub/context.ts b/server/lib/activitypub/context.ts index 3bc40e2aa..b452cf9b3 100644 --- a/server/lib/activitypub/context.ts +++ b/server/lib/activitypub/context.ts @@ -15,7 +15,7 @@ export { type ContextValue = { [ id: string ]: (string | { '@type': string, '@id': string }) } -const contextStore = { +const contextStore: { [ id in ContextType ]: (string | { [ id: string ]: string })[] } = { Video: buildContext({ Hashtag: 'as:Hashtag', uuid: 'sc:identifier', @@ -109,7 +109,8 @@ const contextStore = { stopTimestamp: { '@type': 'sc:Number', '@id': 'pt:stopTimestamp' - } + }, + uuid: 'sc:identifier' }), CacheFile: buildContext({ @@ -128,6 +129,24 @@ const contextStore = { } }), + WatchAction: buildContext({ + WatchAction: 'sc:WatchAction', + startTimestamp: { + '@type': 'sc:Number', + '@id': 'pt:startTimestamp' + }, + stopTimestamp: { + '@type': 'sc:Number', + '@id': 'pt:stopTimestamp' + }, + watchSection: { + '@type': 'sc:Number', + '@id': 'pt:stopTimestamp' + }, + uuid: 'sc:identifier' + }), + + Collection: buildContext(), Follow: buildContext(), Reject: buildContext(), Accept: buildContext(), diff --git a/server/lib/activitypub/local-video-viewer.ts b/server/lib/activitypub/local-video-viewer.ts new file mode 100644 index 000000000..738083adc --- /dev/null +++ b/server/lib/activitypub/local-video-viewer.ts @@ -0,0 +1,42 @@ +import { Transaction } from 'sequelize' +import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' +import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' +import { MVideo } from '@server/types/models' +import { WatchActionObject } from '@shared/models' +import { getDurationFromActivityStream } from './activity' + +async function createOrUpdateLocalVideoViewer (watchAction: WatchActionObject, video: MVideo, t: Transaction) { + const stats = await LocalVideoViewerModel.loadByUrl(watchAction.id) + if (stats) await stats.destroy({ transaction: t }) + + const localVideoViewer = await LocalVideoViewerModel.create({ + url: watchAction.id, + uuid: watchAction.uuid, + + watchTime: getDurationFromActivityStream(watchAction.duration), + + startDate: new Date(watchAction.startTime), + endDate: new Date(watchAction.endTime), + + country: watchAction.location + ? watchAction.location.addressCountry + : null, + + videoId: video.id + }) + + await LocalVideoViewerWatchSectionModel.bulkCreateSections({ + localVideoViewerId: localVideoViewer.id, + + watchSections: watchAction.watchSections.map(s => ({ + start: s.startTimestamp, + end: s.endTimestamp + })) + }) +} + +// --------------------------------------------------------------------------- + +export { + createOrUpdateLocalVideoViewer +} diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index b5b1a0feb..3e7931bb2 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -1,6 +1,7 @@ import { isBlockedByServerOrAccount } from '@server/lib/blocklist' import { isRedundancyAccepted } from '@server/lib/redundancy' -import { ActivityCreate, CacheFileObject, PlaylistObject, VideoCommentObject, VideoObject } from '@shared/models' +import { VideoModel } from '@server/models/video/video' +import { ActivityCreate, CacheFileObject, PlaylistObject, VideoCommentObject, VideoObject, WatchActionObject } from '@shared/models' import { retryTransactionWrapper } from '../../../helpers/database-utils' import { logger } from '../../../helpers/logger' import { sequelizeTypescript } from '../../../initializers/database' @@ -8,6 +9,7 @@ import { APProcessorOptions } from '../../../types/activitypub-processor.model' import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models' import { Notifier } from '../../notifier' import { createOrUpdateCacheFile } from '../cache-file' +import { createOrUpdateLocalVideoViewer } from '../local-video-viewer' import { createOrUpdateVideoPlaylist } from '../playlists' import { forwardVideoRelatedActivity } from '../send/shared/send-utils' import { resolveThread } from '../video-comments' @@ -32,6 +34,10 @@ async function processCreateActivity (options: APProcessorOptions { + return createOrUpdateLocalVideoViewer(watchAction, video, t) + }) +} + async function processCreateVideoComment (activity: ActivityCreate, byActor: MActorSignature, notify: boolean) { const commentObject = activity.object as VideoCommentObject const byAccount = byActor.Account diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts index c59940164..bad079843 100644 --- a/server/lib/activitypub/process/process-view.ts +++ b/server/lib/activitypub/process/process-view.ts @@ -1,4 +1,4 @@ -import { VideoViews } from '@server/lib/video-views' +import { VideoViewsManager } from '@server/lib/views/video-views-manager' import { ActivityView } from '../../../../shared/models/activitypub' import { APProcessorOptions } from '../../../types/activitypub-processor.model' import { MActorSignature } from '../../../types/models' @@ -32,7 +32,7 @@ async function processCreateView (activity: ActivityView, byActor: MActorSignatu ? new Date(activity.expires) : undefined - await VideoViews.Instance.processView({ video, ip: null, viewerExpires }) + await VideoViewsManager.Instance.processRemoteView({ video, viewerExpires }) if (video.isOwned()) { // Forward the view but don't resend the activity to the sender diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts index 5d8763495..7c3a6bdd0 100644 --- a/server/lib/activitypub/send/send-create.ts +++ b/server/lib/activitypub/send/send-create.ts @@ -6,6 +6,7 @@ import { VideoCommentModel } from '../../../models/video/video-comment' import { MActorLight, MCommentOwnerVideo, + MLocalVideoViewerWithWatchSections, MVideoAccountLight, MVideoAP, MVideoPlaylistFull, @@ -19,6 +20,7 @@ import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getVideoCommentAudience, + sendVideoActivityToOrigin, sendVideoRelatedActivity, unicastTo } from './shared' @@ -61,6 +63,18 @@ async function sendCreateCacheFile ( }) } +async function sendCreateWatchAction (stats: MLocalVideoViewerWithWatchSections, transaction: Transaction) { + logger.info('Creating job to send create watch action %s.', stats.url, lTags(stats.uuid)) + + const byActor = await getServerActor() + + const activityBuilder = (audience: ActivityAudience) => { + return buildCreateActivity(stats.url, byActor, stats.toActivityPubObject(), audience) + } + + return sendVideoActivityToOrigin(activityBuilder, { byActor, video: stats.Video, transaction, contextType: 'WatchAction' }) +} + async function sendCreateVideoPlaylist (playlist: MVideoPlaylistFull, transaction: Transaction) { if (playlist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined @@ -175,7 +189,8 @@ export { buildCreateActivity, sendCreateVideoComment, sendCreateVideoPlaylist, - sendCreateCacheFile + sendCreateCacheFile, + sendCreateWatchAction } // --------------------------------------------------------------------------- diff --git a/server/lib/activitypub/send/send-view.ts b/server/lib/activitypub/send/send-view.ts index 1f97307b9..1088bf258 100644 --- a/server/lib/activitypub/send/send-view.ts +++ b/server/lib/activitypub/send/send-view.ts @@ -1,38 +1,31 @@ import { Transaction } from 'sequelize' -import { VideoViews } from '@server/lib/video-views' -import { MActorAudience, MVideoImmutable, MVideoUrl } from '@server/types/models' +import { VideoViewsManager } from '@server/lib/views/video-views-manager' +import { MActorAudience, MActorLight, MVideoImmutable, MVideoUrl } from '@server/types/models' import { ActivityAudience, ActivityView } from '@shared/models' import { logger } from '../../../helpers/logger' -import { ActorModel } from '../../../models/actor/actor' import { audiencify, getAudience } from '../audience' import { getLocalVideoViewActivityPubUrl } from '../url' import { sendVideoRelatedActivity } from './shared/send-utils' -async function sendView (byActor: ActorModel, video: MVideoImmutable, t: Transaction) { - logger.info('Creating job to send view of %s.', video.url) +type ViewType = 'view' | 'viewer' + +async function sendView (options: { + byActor: MActorLight + type: ViewType + video: MVideoImmutable + transaction?: Transaction +}) { + const { byActor, type, video, transaction } = options + + logger.info('Creating job to send %s of %s.', type, video.url) const activityBuilder = (audience: ActivityAudience) => { const url = getLocalVideoViewActivityPubUrl(byActor, video) - return buildViewActivity(url, byActor, video, audience) + return buildViewActivity({ url, byActor, video, audience, type }) } - return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t, contextType: 'View' }) -} - -function buildViewActivity (url: string, byActor: MActorAudience, video: MVideoUrl, audience?: ActivityAudience): ActivityView { - if (!audience) audience = getAudience(byActor) - - return audiencify( - { - id: url, - type: 'View' as 'View', - actor: byActor.url, - object: video.url, - expires: new Date(VideoViews.Instance.buildViewerExpireTime()).toISOString() - }, - audience - ) + return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction, contextType: 'View' }) } // --------------------------------------------------------------------------- @@ -40,3 +33,29 @@ function buildViewActivity (url: string, byActor: MActorAudience, video: MVideoU export { sendView } + +// --------------------------------------------------------------------------- + +function buildViewActivity (options: { + url: string + byActor: MActorAudience + video: MVideoUrl + type: ViewType + audience?: ActivityAudience +}): ActivityView { + const { url, byActor, type, video, audience = getAudience(byActor) } = options + + return audiencify( + { + id: url, + type: 'View' as 'View', + actor: byActor.url, + object: video.url, + + expires: type === 'viewer' + ? new Date(VideoViewsManager.Instance.buildViewerExpireTime()).toISOString() + : undefined + }, + audience + ) +} diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts index 50be4fac9..8443fef4c 100644 --- a/server/lib/activitypub/url.ts +++ b/server/lib/activitypub/url.ts @@ -7,6 +7,7 @@ import { MActorId, MActorUrl, MCommentId, + MLocalVideoViewer, MVideoId, MVideoPlaylistElement, MVideoUrl, @@ -59,6 +60,10 @@ function getLocalVideoViewActivityPubUrl (byActor: MActorUrl, video: MVideoId) { return byActor.url + '/views/videos/' + video.id + '/' + new Date().toISOString() } +function getLocalVideoViewerActivityPubUrl (stats: MLocalVideoViewer) { + return WEBSERVER.URL + '/videos/local-viewer/' + stats.uuid +} + function getVideoLikeActivityPubUrlByLocalActor (byActor: MActorUrl, video: MVideoId) { return byActor.url + '/likes/' + video.id } @@ -167,6 +172,7 @@ export { getLocalVideoCommentsActivityPubUrl, getLocalVideoLikesActivityPubUrl, getLocalVideoDislikesActivityPubUrl, + getLocalVideoViewerActivityPubUrl, getAbuseTargetUrl, checkUrlsSameHost, diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts index c97217669..f02b9cba6 100644 --- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts +++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts @@ -24,6 +24,7 @@ import { VideoPrivacy, VideoStreamingPlaylistType } from '@shared/models' +import { getDurationFromActivityStream } from '../../activity' function getThumbnailFromIcons (videoObject: VideoObject) { let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth) @@ -170,7 +171,6 @@ function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: Vi ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED - const duration = videoObject.duration.replace(/[^\d]+/, '') const language = videoObject.language?.identifier const category = videoObject.category @@ -200,7 +200,7 @@ function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: Vi isLive: videoObject.isLiveBroadcast, state: videoObject.state, channelId: videoChannel.id, - duration: parseInt(duration, 10), + duration: getDurationFromActivityStream(videoObject.duration), createdAt: new Date(videoObject.published), publishedAt: new Date(videoObject.published), diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index a9c835fbf..337364ac9 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts @@ -23,11 +23,11 @@ import { WEBSERVER } from '../initializers/constants' import { AccountModel } from '../models/account/account' -import { getActivityStreamDuration } from '../models/video/formatter/video-format-utils' import { VideoModel } from '../models/video/video' import { VideoChannelModel } from '../models/video/video-channel' import { VideoPlaylistModel } from '../models/video/video-playlist' import { MAccountActor, MChannelActor } from '../types/models' +import { getActivityStreamDuration } from './activitypub/activity' import { getBiggestActorImage } from './actor-image' import { ServerConfigManager } from './server-config-manager' diff --git a/server/lib/job-queue/handlers/video-views-stats.ts b/server/lib/job-queue/handlers/video-views-stats.ts index caf5f6962..689a5a3b4 100644 --- a/server/lib/job-queue/handlers/video-views-stats.ts +++ b/server/lib/job-queue/handlers/video-views-stats.ts @@ -1,7 +1,7 @@ +import { VideoViewModel } from '@server/models/view/video-view' import { isTestInstance } from '../../../helpers/core-utils' import { logger } from '../../../helpers/logger' import { VideoModel } from '../../../models/video/video' -import { VideoViewModel } from '../../../models/video/video-view' import { Redis } from '../../redis' async function processVideosViewsStats () { diff --git a/server/lib/redis.ts b/server/lib/redis.ts index c4c1fa443..b86aefa0e 100644 --- a/server/lib/redis.ts +++ b/server/lib/redis.ts @@ -249,6 +249,45 @@ class Redis { ]) } + /* ************ Video viewers stats ************ */ + + getLocalVideoViewer (options: { + key?: string + // Or + ip?: string + videoId?: number + }) { + if (options.key) return this.getObject(options.key) + + const { viewerKey } = this.generateLocalVideoViewerKeys(options.ip, options.videoId) + + return this.getObject(viewerKey) + } + + setLocalVideoViewer (ip: string, videoId: number, object: any) { + const { setKey, viewerKey } = this.generateLocalVideoViewerKeys(ip, videoId) + + return Promise.all([ + this.addToSet(setKey, viewerKey), + this.setObject(viewerKey, object) + ]) + } + + listLocalVideoViewerKeys () { + const { setKey } = this.generateLocalVideoViewerKeys() + + return this.getSet(setKey) + } + + deleteLocalVideoViewersKeys (key: string) { + const { setKey } = this.generateLocalVideoViewerKeys() + + return Promise.all([ + this.deleteFromSet(setKey, key), + this.deleteKey(key) + ]) + } + /* ************ Resumable uploads final responses ************ */ setUploadSession (uploadId: string, response?: { video: { id: number, shortUUID: string, uuid: string } }) { @@ -290,10 +329,18 @@ class Redis { /* ************ Keys generation ************ */ - private generateLocalVideoViewsKeys (videoId?: Number) { + private generateLocalVideoViewsKeys (videoId: number): { setKey: string, videoKey: string } + private generateLocalVideoViewsKeys (): { setKey: string } + private generateLocalVideoViewsKeys (videoId?: number) { return { setKey: `local-video-views-buffer`, videoKey: `local-video-views-buffer-${videoId}` } } + private generateLocalVideoViewerKeys (ip: string, videoId: number): { setKey: string, viewerKey: string } + private generateLocalVideoViewerKeys (): { setKey: string } + private generateLocalVideoViewerKeys (ip?: string, videoId?: number) { + return { setKey: `local-video-viewer-stats-keys`, viewerKey: `local-video-viewer-stats-${ip}-${videoId}` } + } + private generateVideoViewStatsKeys (options: { videoId?: number, hour?: number }) { const hour = exists(options.hour) ? options.hour @@ -352,8 +399,23 @@ class Redis { return this.client.del(this.prefix + key) } - private async setValue (key: string, value: string, expirationMilliseconds: number) { - const result = await this.client.set(this.prefix + key, value, { PX: expirationMilliseconds }) + private async getObject (key: string) { + const value = await this.getValue(key) + if (!value) return null + + return JSON.parse(value) + } + + private setObject (key: string, value: { [ id: string ]: number | string }) { + return this.setValue(key, JSON.stringify(value)) + } + + private async setValue (key: string, value: string, expirationMilliseconds?: number) { + const options = expirationMilliseconds + ? { PX: expirationMilliseconds } + : {} + + const result = await this.client.set(this.prefix + key, value, options) if (result !== 'OK') throw new Error('Redis set result is not OK.') } diff --git a/server/lib/schedulers/geo-ip-update-scheduler.ts b/server/lib/schedulers/geo-ip-update-scheduler.ts new file mode 100644 index 000000000..9dda6d76c --- /dev/null +++ b/server/lib/schedulers/geo-ip-update-scheduler.ts @@ -0,0 +1,22 @@ +import { GeoIP } from '@server/helpers/geo-ip' +import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' +import { AbstractScheduler } from './abstract-scheduler' + +export class GeoIPUpdateScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.YOUTUBE_DL_UPDATE + + private constructor () { + super() + } + + protected internalExecute () { + return GeoIP.Instance.updateDatabase() + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/lib/schedulers/remove-old-views-scheduler.ts b/server/lib/schedulers/remove-old-views-scheduler.ts index 64bef97fe..8bc53a045 100644 --- a/server/lib/schedulers/remove-old-views-scheduler.ts +++ b/server/lib/schedulers/remove-old-views-scheduler.ts @@ -1,8 +1,8 @@ +import { VideoViewModel } from '@server/models/view/video-view' import { logger } from '../../helpers/logger' -import { AbstractScheduler } from './abstract-scheduler' -import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' import { CONFIG } from '../../initializers/config' -import { VideoViewModel } from '../../models/video/video-view' +import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' +import { AbstractScheduler } from './abstract-scheduler' export class RemoveOldViewsScheduler extends AbstractScheduler { diff --git a/server/lib/schedulers/video-views-buffer-scheduler.ts b/server/lib/schedulers/video-views-buffer-scheduler.ts index c0e72c461..937764155 100644 --- a/server/lib/schedulers/video-views-buffer-scheduler.ts +++ b/server/lib/schedulers/video-views-buffer-scheduler.ts @@ -21,8 +21,6 @@ export class VideoViewsBufferScheduler extends AbstractScheduler { const videoIds = await Redis.Instance.listLocalVideosViewed() if (videoIds.length === 0) return - logger.info('Processing local video views buffer.', { videoIds, ...lTags() }) - for (const videoId of videoIds) { try { const views = await Redis.Instance.getLocalVideoViews(videoId) @@ -34,6 +32,8 @@ export class VideoViewsBufferScheduler extends AbstractScheduler { continue } + logger.info('Processing local video %s views buffer.', video.uuid, lTags(video.uuid)) + // If this is a remote video, the origin instance will send us an update await VideoModel.incrementViews(videoId, views) diff --git a/server/lib/video-views.ts b/server/lib/video-views.ts deleted file mode 100644 index c024eb93c..000000000 --- a/server/lib/video-views.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { isTestInstance } from '@server/helpers/core-utils' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { VIEW_LIFETIME } from '@server/initializers/constants' -import { VideoModel } from '@server/models/video/video' -import { MVideo } from '@server/types/models' -import { PeerTubeSocket } from './peertube-socket' -import { Redis } from './redis' - -const lTags = loggerTagsFactory('views') - -export class VideoViews { - - // Values are Date().getTime() - private readonly viewersPerVideo = new Map() - - private static instance: VideoViews - - private constructor () { - } - - init () { - setInterval(() => this.cleanViewers(), VIEW_LIFETIME.VIEWER) - } - - async processView (options: { - video: MVideo - ip: string | null - viewerExpires?: Date - }) { - const { video, ip, viewerExpires } = options - - logger.debug('Processing view for %s and ip %s.', video.url, ip, lTags()) - - let success = await this.addView(video, ip) - - if (video.isLive) { - const successViewer = await this.addViewer(video, ip, viewerExpires) - success ||= successViewer - } - - return success - } - - getViewers (video: MVideo) { - const viewers = this.viewersPerVideo.get(video.id) - if (!viewers) return 0 - - return viewers.length - } - - buildViewerExpireTime () { - return new Date().getTime() + VIEW_LIFETIME.VIEWER - } - - private async addView (video: MVideo, ip: string | null) { - const promises: Promise[] = [] - - if (ip !== null) { - const viewExists = await Redis.Instance.doesVideoIPViewExist(ip, video.uuid) - if (viewExists) return false - - promises.push(Redis.Instance.setIPVideoView(ip, video.uuid)) - } - - if (video.isOwned()) { - promises.push(Redis.Instance.addLocalVideoView(video.id)) - } - - promises.push(Redis.Instance.addVideoViewStats(video.id)) - - await Promise.all(promises) - - return true - } - - private async addViewer (video: MVideo, ip: string | null, viewerExpires?: Date) { - if (ip !== null) { - const viewExists = await Redis.Instance.doesVideoIPViewerExist(ip, video.uuid) - if (viewExists) return false - - await Redis.Instance.setIPVideoViewer(ip, video.uuid) - } - - let watchers = this.viewersPerVideo.get(video.id) - - if (!watchers) { - watchers = [] - this.viewersPerVideo.set(video.id, watchers) - } - - const expiration = viewerExpires - ? viewerExpires.getTime() - : this.buildViewerExpireTime() - - watchers.push(expiration) - await this.notifyClients(video.id, watchers.length) - - return true - } - - private async cleanViewers () { - if (!isTestInstance()) logger.info('Cleaning video viewers.', lTags()) - - for (const videoId of this.viewersPerVideo.keys()) { - const notBefore = new Date().getTime() - - const viewers = this.viewersPerVideo.get(videoId) - - // Only keep not expired viewers - const newViewers = viewers.filter(w => w > notBefore) - - if (newViewers.length === 0) this.viewersPerVideo.delete(videoId) - else this.viewersPerVideo.set(videoId, newViewers) - - await this.notifyClients(videoId, newViewers.length) - } - } - - private async notifyClients (videoId: string | number, viewersLength: number) { - const video = await VideoModel.loadImmutableAttributes(videoId) - if (!video) return - - PeerTubeSocket.Instance.sendVideoViewsUpdate(video, viewersLength) - - logger.debug('Live video views update for %s is %d.', video.url, viewersLength, lTags()) - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/views/shared/index.ts b/server/lib/views/shared/index.ts new file mode 100644 index 000000000..dd510f4e2 --- /dev/null +++ b/server/lib/views/shared/index.ts @@ -0,0 +1,2 @@ +export * from './video-viewers' +export * from './video-views' diff --git a/server/lib/views/shared/video-viewers.ts b/server/lib/views/shared/video-viewers.ts new file mode 100644 index 000000000..5c26f8982 --- /dev/null +++ b/server/lib/views/shared/video-viewers.ts @@ -0,0 +1,276 @@ +import { Transaction } from 'sequelize/types' +import { isTestInstance } from '@server/helpers/core-utils' +import { GeoIP } from '@server/helpers/geo-ip' +import { logger, loggerTagsFactory } from '@server/helpers/logger' +import { MAX_LOCAL_VIEWER_WATCH_SECTIONS, VIEW_LIFETIME } from '@server/initializers/constants' +import { sequelizeTypescript } from '@server/initializers/database' +import { sendCreateWatchAction } from '@server/lib/activitypub/send' +import { getLocalVideoViewerActivityPubUrl } from '@server/lib/activitypub/url' +import { PeerTubeSocket } from '@server/lib/peertube-socket' +import { Redis } from '@server/lib/redis' +import { VideoModel } from '@server/models/video/video' +import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' +import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' +import { MVideo } from '@server/types/models' +import { VideoViewEvent } from '@shared/models' + +const lTags = loggerTagsFactory('views') + +type LocalViewerStats = { + firstUpdated: number // Date.getTime() + lastUpdated: number // Date.getTime() + + watchSections: { + start: number + end: number + }[] + + watchTime: number + + country: string + + videoId: number +} + +export class VideoViewers { + + // Values are Date().getTime() + private readonly viewersPerVideo = new Map() + + private processingViewerCounters = false + private processingViewerStats = false + + constructor () { + setInterval(() => this.cleanViewerCounters(), VIEW_LIFETIME.VIEWER) + + setInterval(() => this.processViewerStats(), VIEW_LIFETIME.VIEWER_STATS) + } + + // --------------------------------------------------------------------------- + + getViewers (video: MVideo) { + const viewers = this.viewersPerVideo.get(video.id) + if (!viewers) return 0 + + return viewers.length + } + + buildViewerExpireTime () { + return new Date().getTime() + VIEW_LIFETIME.VIEWER + } + + async getWatchTime (videoId: number, ip: string) { + const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId }) + + return stats?.watchTime || 0 + } + + async addLocalViewer (options: { + video: MVideo + currentTime: number + ip: string + viewEvent?: VideoViewEvent + }) { + const { video, ip, viewEvent, currentTime } = options + + logger.debug('Adding local viewer to video %s.', video.uuid, { currentTime, viewEvent, ...lTags(video.uuid) }) + + await this.updateLocalViewerStats({ video, viewEvent, currentTime, ip }) + + const viewExists = await Redis.Instance.doesVideoIPViewerExist(ip, video.uuid) + if (viewExists) return false + + await Redis.Instance.setIPVideoViewer(ip, video.uuid) + + return this.addViewerToVideo({ video }) + } + + async addRemoteViewer (options: { + video: MVideo + viewerExpires: Date + }) { + const { video, viewerExpires } = options + + logger.debug('Adding remote viewer to video %s.', video.uuid, { ...lTags(video.uuid) }) + + return this.addViewerToVideo({ video, viewerExpires }) + } + + private async addViewerToVideo (options: { + video: MVideo + viewerExpires?: Date + }) { + const { video, viewerExpires } = options + + let watchers = this.viewersPerVideo.get(video.id) + + if (!watchers) { + watchers = [] + this.viewersPerVideo.set(video.id, watchers) + } + + const expiration = viewerExpires + ? viewerExpires.getTime() + : this.buildViewerExpireTime() + + watchers.push(expiration) + await this.notifyClients(video.id, watchers.length) + + return true + } + + private async updateLocalViewerStats (options: { + video: MVideo + ip: string + currentTime: number + viewEvent?: VideoViewEvent + }) { + const { video, ip, viewEvent, currentTime } = options + const nowMs = new Date().getTime() + + let stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId: video.id }) + + if (stats && stats.watchSections.length >= MAX_LOCAL_VIEWER_WATCH_SECTIONS) { + logger.warn('Too much watch section to store for a viewer, skipping this one', { currentTime, viewEvent, ...lTags(video.uuid) }) + return + } + + if (!stats) { + const country = await GeoIP.Instance.safeCountryISOLookup(ip) + + stats = { + firstUpdated: nowMs, + lastUpdated: nowMs, + + watchSections: [], + + watchTime: 0, + + country, + videoId: video.id + } + } + + stats.lastUpdated = nowMs + + if (viewEvent === 'seek' || stats.watchSections.length === 0) { + stats.watchSections.push({ + start: currentTime, + end: currentTime + }) + } else { + const lastSection = stats.watchSections[stats.watchSections.length - 1] + lastSection.end = currentTime + } + + stats.watchTime = this.buildWatchTimeFromSections(stats.watchSections) + + logger.debug('Set local video viewer stats for video %s.', video.uuid, { stats, ...lTags(video.uuid) }) + + await Redis.Instance.setLocalVideoViewer(ip, video.id, stats) + } + + private async cleanViewerCounters () { + if (this.processingViewerCounters) return + this.processingViewerCounters = true + + if (!isTestInstance()) logger.info('Cleaning video viewers.', lTags()) + + try { + for (const videoId of this.viewersPerVideo.keys()) { + const notBefore = new Date().getTime() + + const viewers = this.viewersPerVideo.get(videoId) + + // Only keep not expired viewers + const newViewers = viewers.filter(w => w > notBefore) + + if (newViewers.length === 0) this.viewersPerVideo.delete(videoId) + else this.viewersPerVideo.set(videoId, newViewers) + + await this.notifyClients(videoId, newViewers.length) + } + } catch (err) { + logger.error('Error in video clean viewers scheduler.', { err, ...lTags() }) + } + + this.processingViewerCounters = false + } + + private async notifyClients (videoId: string | number, viewersLength: number) { + const video = await VideoModel.loadImmutableAttributes(videoId) + if (!video) return + + PeerTubeSocket.Instance.sendVideoViewsUpdate(video, viewersLength) + + logger.debug('Video viewers update for %s is %d.', video.url, viewersLength, lTags()) + } + + async processViewerStats () { + if (this.processingViewerStats) return + this.processingViewerStats = true + + if (!isTestInstance()) logger.info('Processing viewers.', lTags()) + + const now = new Date().getTime() + + try { + const allKeys = await Redis.Instance.listLocalVideoViewerKeys() + + for (const key of allKeys) { + const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ key }) + + if (stats.lastUpdated > now - VIEW_LIFETIME.VIEWER_STATS) { + continue + } + + try { + await sequelizeTypescript.transaction(async t => { + const video = await VideoModel.load(stats.videoId, t) + + const statsModel = await this.saveViewerStats(video, stats, t) + + if (video.remote) { + await sendCreateWatchAction(statsModel, t) + } + }) + + await Redis.Instance.deleteLocalVideoViewersKeys(key) + } catch (err) { + logger.error('Cannot process viewer stats for Redis key %s.', key, { err, ...lTags() }) + } + } + } catch (err) { + logger.error('Error in video save viewers stats scheduler.', { err, ...lTags() }) + } + + this.processingViewerStats = false + } + + private async saveViewerStats (video: MVideo, stats: LocalViewerStats, transaction: Transaction) { + const statsModel = new LocalVideoViewerModel({ + startDate: new Date(stats.firstUpdated), + endDate: new Date(stats.lastUpdated), + watchTime: stats.watchTime, + country: stats.country, + videoId: video.id + }) + + statsModel.url = getLocalVideoViewerActivityPubUrl(statsModel) + statsModel.Video = video as VideoModel + + await statsModel.save({ transaction }) + + statsModel.WatchSections = await LocalVideoViewerWatchSectionModel.bulkCreateSections({ + localVideoViewerId: statsModel.id, + watchSections: stats.watchSections, + transaction + }) + + return statsModel + } + + private buildWatchTimeFromSections (sections: { start: number, end: number }[]) { + return sections.reduce((p, current) => p + (current.end - current.start), 0) + } +} diff --git a/server/lib/views/shared/video-views.ts b/server/lib/views/shared/video-views.ts new file mode 100644 index 000000000..19250f993 --- /dev/null +++ b/server/lib/views/shared/video-views.ts @@ -0,0 +1,60 @@ +import { logger, loggerTagsFactory } from '@server/helpers/logger' +import { MVideo } from '@server/types/models' +import { Redis } from '../../redis' + +const lTags = loggerTagsFactory('views') + +export class VideoViews { + + async addLocalView (options: { + video: MVideo + ip: string + watchTime: number + }) { + const { video, ip, watchTime } = options + + logger.debug('Adding local view to video %s.', video.uuid, { watchTime, ...lTags(video.uuid) }) + + if (!this.hasEnoughWatchTime(video, watchTime)) return false + + const viewExists = await Redis.Instance.doesVideoIPViewExist(ip, video.uuid) + if (viewExists) return false + + await Redis.Instance.setIPVideoView(ip, video.uuid) + + await this.addView(video) + + return true + } + + async addRemoteView (options: { + video: MVideo + }) { + const { video } = options + + logger.debug('Adding remote view to video %s.', video.uuid, { ...lTags(video.uuid) }) + + await this.addView(video) + + return true + } + + private async addView (video: MVideo) { + const promises: Promise[] = [] + + if (video.isOwned()) { + promises.push(Redis.Instance.addLocalVideoView(video.id)) + } + + promises.push(Redis.Instance.addVideoViewStats(video.id)) + + await Promise.all(promises) + } + + private hasEnoughWatchTime (video: MVideo, watchTime: number) { + if (video.isLive || video.duration >= 30) return watchTime >= 30 + + // Check more than 50% of the video is watched + return video.duration / watchTime < 2 + } +} diff --git a/server/lib/views/video-views-manager.ts b/server/lib/views/video-views-manager.ts new file mode 100644 index 000000000..e07af1ca9 --- /dev/null +++ b/server/lib/views/video-views-manager.ts @@ -0,0 +1,70 @@ +import { logger, loggerTagsFactory } from '@server/helpers/logger' +import { MVideo } from '@server/types/models' +import { VideoViewEvent } from '@shared/models' +import { VideoViewers, VideoViews } from './shared' + +const lTags = loggerTagsFactory('views') + +export class VideoViewsManager { + + private static instance: VideoViewsManager + + private videoViewers: VideoViewers + private videoViews: VideoViews + + private constructor () { + } + + init () { + this.videoViewers = new VideoViewers() + this.videoViews = new VideoViews() + } + + async processLocalView (options: { + video: MVideo + currentTime: number + ip: string | null + viewEvent?: VideoViewEvent + }) { + const { video, ip, viewEvent, currentTime } = options + + logger.debug('Processing local view for %s and ip %s.', video.url, ip, lTags()) + + const successViewer = await this.videoViewers.addLocalViewer({ video, ip, viewEvent, currentTime }) + + // Do it after added local viewer to fetch updated information + const watchTime = await this.videoViewers.getWatchTime(video.id, ip) + + const successView = await this.videoViews.addLocalView({ video, watchTime, ip }) + + return { successView, successViewer } + } + + async processRemoteView (options: { + video: MVideo + viewerExpires?: Date + }) { + const { video, viewerExpires } = options + + logger.debug('Processing remote view for %s.', video.url, { viewerExpires, ...lTags() }) + + if (viewerExpires) await this.videoViewers.addRemoteViewer({ video, viewerExpires }) + else await this.videoViews.addRemoteView({ video }) + } + + getViewers (video: MVideo) { + return this.videoViewers.getViewers(video) + } + + buildViewerExpireTime () { + return this.videoViewers.buildViewerExpireTime() + } + + processViewers () { + return this.videoViewers.processViewerStats() + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/middlewares/cache/shared/api-cache.ts b/server/middlewares/cache/shared/api-cache.ts index 86c5095b5..abc919339 100644 --- a/server/middlewares/cache/shared/api-cache.ts +++ b/server/middlewares/cache/shared/api-cache.ts @@ -6,8 +6,8 @@ import { OutgoingHttpHeaders } from 'http' import { isTestInstance, parseDurationToMs } from '@server/helpers/core-utils' import { logger } from '@server/helpers/logger' import { Redis } from '@server/lib/redis' -import { HttpStatusCode } from '@shared/models' import { asyncMiddleware } from '@server/middlewares' +import { HttpStatusCode } from '@shared/models' export interface APICacheOptions { headerBlacklist?: string[] @@ -152,7 +152,7 @@ export class ApiCache { end: res.end, cacheable: true, content: undefined, - headers: {} + headers: undefined } // Patch express diff --git a/server/middlewares/validators/express.ts b/server/middlewares/validators/express.ts new file mode 100644 index 000000000..718aec55b --- /dev/null +++ b/server/middlewares/validators/express.ts @@ -0,0 +1,15 @@ +import * as express from 'express' + +const methodsValidator = (methods: string[]) => { + return (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (methods.includes(req.method) !== true) { + return res.sendStatus(405) + } + + return next() + } +} + +export { + methodsValidator +} diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts index 94a3c2dea..b0ad04819 100644 --- a/server/middlewares/validators/index.ts +++ b/server/middlewares/validators/index.ts @@ -1,17 +1,26 @@ +export * from './activitypub' +export * from './videos' export * from './abuse' export * from './account' export * from './actor-image' export * from './blocklist' -export * from './oembed' -export * from './activitypub' -export * from './pagination' -export * from './follows' +export * from './bulk' +export * from './config' +export * from './express' export * from './feeds' -export * from './sort' -export * from './users' -export * from './user-subscriptions' -export * from './videos' +export * from './follows' +export * from './jobs' +export * from './logs' +export * from './oembed' +export * from './pagination' +export * from './plugins' +export * from './redundancy' export * from './search' export * from './server' +export * from './sort' +export * from './themes' export * from './user-history' +export * from './user-notifications' +export * from './user-subscriptions' +export * from './users' export * from './webfinger' diff --git a/server/middlewares/validators/videos/index.ts b/server/middlewares/validators/videos/index.ts index c7dea4b3d..bd2590bc5 100644 --- a/server/middlewares/validators/videos/index.ts +++ b/server/middlewares/validators/videos/index.ts @@ -6,9 +6,10 @@ export * from './video-files' export * from './video-imports' export * from './video-live' export * from './video-ownership-changes' -export * from './video-watch' +export * from './video-view' export * from './video-rates' export * from './video-shares' +export * from './video-stats' export * from './video-studio' export * from './video-transcoding' export * from './videos' diff --git a/server/middlewares/validators/videos/video-stats.ts b/server/middlewares/validators/videos/video-stats.ts new file mode 100644 index 000000000..358b6b473 --- /dev/null +++ b/server/middlewares/validators/videos/video-stats.ts @@ -0,0 +1,73 @@ +import express from 'express' +import { param } from 'express-validator' +import { isValidStatTimeserieMetric } from '@server/helpers/custom-validators/video-stats' +import { HttpStatusCode, UserRight } from '@shared/models' +import { logger } from '../../../helpers/logger' +import { areValidationErrors, checkUserCanManageVideo, doesVideoExist, isValidVideoIdParam } from '../shared' + +const videoOverallStatsValidator = [ + isValidVideoIdParam('videoId'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking videoOverallStatsValidator parameters', { parameters: req.body }) + + if (areValidationErrors(req, res)) return + if (!await commonStatsCheck(req, res)) return + + return next() + } +] + +const videoRetentionStatsValidator = [ + isValidVideoIdParam('videoId'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking videoRetentionStatsValidator parameters', { parameters: req.body }) + + if (areValidationErrors(req, res)) return + if (!await commonStatsCheck(req, res)) return + + if (res.locals.videoAll.isLive) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Cannot get retention stats of live video' + }) + } + + return next() + } +] + +const videoTimeserieStatsValidator = [ + isValidVideoIdParam('videoId'), + + param('metric') + .custom(isValidStatTimeserieMetric) + .withMessage('Should have a valid timeserie metric'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking videoTimeserieStatsValidator parameters', { parameters: req.body }) + + if (areValidationErrors(req, res)) return + if (!await commonStatsCheck(req, res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + videoOverallStatsValidator, + videoTimeserieStatsValidator, + videoRetentionStatsValidator +} + +// --------------------------------------------------------------------------- + +async function commonStatsCheck (req: express.Request, res: express.Response) { + if (!await doesVideoExist(req.params.videoId, res, 'all')) return false + if (!checkUserCanManageVideo(res.locals.oauth.token.User, res.locals.videoAll, UserRight.SEE_ALL_VIDEOS, res)) return false + + return true +} diff --git a/server/middlewares/validators/videos/video-view.ts b/server/middlewares/validators/videos/video-view.ts new file mode 100644 index 000000000..7a4994e8a --- /dev/null +++ b/server/middlewares/validators/videos/video-view.ts @@ -0,0 +1,74 @@ +import express from 'express' +import { body, param } from 'express-validator' +import { isVideoTimeValid } from '@server/helpers/custom-validators/video-view' +import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' +import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' +import { exists, isIdValid, isIntOrNull, toIntOrNull } from '../../../helpers/custom-validators/misc' +import { logger } from '../../../helpers/logger' +import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared' + +const getVideoLocalViewerValidator = [ + param('localViewerId') + .custom(isIdValid).withMessage('Should have a valid local viewer id'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking getVideoLocalViewerValidator parameters', { parameters: req.params }) + + if (areValidationErrors(req, res)) return + + const localViewer = await LocalVideoViewerModel.loadFullById(+req.params.localViewerId) + if (!localViewer) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Local viewer not found' + }) + } + + res.locals.localViewerFull = localViewer + + return next() + } +] + +const videoViewValidator = [ + isValidVideoIdParam('videoId'), + + body('currentTime') + .optional() // TODO: remove optional in a few versions, introduced in 4.2 + .customSanitizer(toIntOrNull) + .custom(isIntOrNull).withMessage('Should have correct current time'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking videoView parameters', { parameters: req.body }) + + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res, 'only-video')) return + + const video = res.locals.onlyVideo + const videoDuration = video.isLive + ? undefined + : video.duration + + if (!exists(req.body.currentTime)) { // TODO: remove in a few versions, introduced in 4.2 + req.body.currentTime = Math.min(videoDuration ?? 0, 30) + } + + const currentTime: number = req.body.currentTime + + if (!isVideoTimeValid(currentTime, videoDuration)) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Current time is invalid' + }) + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + videoViewValidator, + getVideoLocalViewerValidator +} diff --git a/server/middlewares/validators/videos/video-watch.ts b/server/middlewares/validators/videos/video-watch.ts deleted file mode 100644 index d83710a64..000000000 --- a/server/middlewares/validators/videos/video-watch.ts +++ /dev/null @@ -1,38 +0,0 @@ -import express from 'express' -import { body } from 'express-validator' -import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { toIntOrNull } from '../../../helpers/custom-validators/misc' -import { logger } from '../../../helpers/logger' -import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared' - -const videoWatchingValidator = [ - isValidVideoIdParam('videoId'), - - body('currentTime') - .customSanitizer(toIntOrNull) - .isInt().withMessage('Should have correct current time'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - logger.debug('Checking videoWatching parameters', { parameters: req.body }) - - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res, 'id')) return - - const user = res.locals.oauth.token.User - if (user.videosHistoryEnabled === false) { - logger.warn('Cannot set videos to watch by user %d: videos history is disabled.', user.id) - return res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'Video history is disabled' - }) - } - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - videoWatchingValidator -} diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts index 611edf0b9..6222107d7 100644 --- a/server/models/video/formatter/video-format-utils.ts +++ b/server/models/video/formatter/video-format-utils.ts @@ -1,11 +1,19 @@ import { generateMagnetUri } from '@server/helpers/webtorrent' +import { getActivityStreamDuration } from '@server/lib/activitypub/activity' import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls' -import { VideoViews } from '@server/lib/video-views' +import { VideoViewsManager } from '@server/lib/views/video-views-manager' import { uuidToShort } from '@shared/extra-utils' -import { VideoFile, VideosCommonQueryAfterSanitize } from '@shared/models' -import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../../shared/models/activitypub/objects' -import { Video, VideoDetails, VideoInclude } from '../../../../shared/models/videos' -import { VideoStreamingPlaylist } from '../../../../shared/models/videos/video-streaming-playlist.model' +import { + ActivityTagObject, + ActivityUrlObject, + Video, + VideoDetails, + VideoFile, + VideoInclude, + VideoObject, + VideosCommonQueryAfterSanitize, + VideoStreamingPlaylist +} from '@shared/models' import { isArray } from '../../../helpers/custom-validators/misc' import { MIMETYPES, @@ -97,7 +105,10 @@ function videoModelToFormattedJSON (video: MVideoFormattable, options: VideoForm isLocal: video.isOwned(), duration: video.duration, + views: video.views, + viewers: VideoViewsManager.Instance.getViewers(video), + likes: video.likes, dislikes: video.dislikes, thumbnailPath: video.getMiniatureStaticPath(), @@ -121,10 +132,6 @@ function videoModelToFormattedJSON (video: MVideoFormattable, options: VideoForm pluginData: (video as any).pluginData } - if (video.isLive) { - videoObject.viewers = VideoViews.Instance.getViewers(video) - } - const add = options.additionalAttributes if (add?.state === true) { videoObject.state = { @@ -459,11 +466,6 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject { } } -function getActivityStreamDuration (duration: number) { - // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration - return 'PT' + duration + 'S' -} - function getCategoryLabel (id: number) { return VIDEO_CATEGORIES[id] || 'Misc' } @@ -489,7 +491,6 @@ export { videoModelToFormattedDetailsJSON, videoFilesModelToFormattedJSON, videoModelToActivityPubObject, - getActivityStreamDuration, guessAdditionalAttributesFromQuery, diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 8bad2a01e..13d81561a 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -106,6 +106,7 @@ import { setAsUpdated } from '../shared' import { UserModel } from '../user/user' import { UserVideoHistoryModel } from '../user/user-video-history' import { buildTrigramSearchIndex, buildWhereIdOrUUID, getVideoSort, isOutdated, throwIfNotValid } from '../utils' +import { VideoViewModel } from '../view/video-view' import { videoFilesModelToFormattedJSON, VideoFormattingJSONOptions, @@ -135,7 +136,6 @@ import { VideoPlaylistElementModel } from './video-playlist-element' import { VideoShareModel } from './video-share' import { VideoStreamingPlaylistModel } from './video-streaming-playlist' import { VideoTagModel } from './video-tag' -import { VideoViewModel } from './video-view' export enum ScopeNames { FOR_API = 'FOR_API', diff --git a/server/models/view/local-video-viewer-watch-section.ts b/server/models/view/local-video-viewer-watch-section.ts new file mode 100644 index 000000000..e29bb7847 --- /dev/null +++ b/server/models/view/local-video-viewer-watch-section.ts @@ -0,0 +1,63 @@ +import { Transaction } from 'sequelize' +import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, Model, Table } from 'sequelize-typescript' +import { MLocalVideoViewerWatchSection } from '@server/types/models' +import { AttributesOnly } from '@shared/typescript-utils' +import { LocalVideoViewerModel } from './local-video-viewer' + +@Table({ + tableName: 'localVideoViewerWatchSection', + updatedAt: false, + indexes: [ + { + fields: [ 'localVideoViewerId' ] + } + ] +}) +export class LocalVideoViewerWatchSectionModel extends Model>> { + @CreatedAt + createdAt: Date + + @AllowNull(false) + @Column + watchStart: number + + @AllowNull(false) + @Column + watchEnd: number + + @ForeignKey(() => LocalVideoViewerModel) + @Column + localVideoViewerId: number + + @BelongsTo(() => LocalVideoViewerModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + LocalVideoViewer: LocalVideoViewerModel + + static async bulkCreateSections (options: { + localVideoViewerId: number + watchSections: { + start: number + end: number + }[] + transaction?: Transaction + }) { + const { localVideoViewerId, watchSections, transaction } = options + const models: MLocalVideoViewerWatchSection[] = [] + + for (const section of watchSections) { + const model = await this.create({ + watchStart: section.start, + watchEnd: section.end, + localVideoViewerId + }, { transaction }) + + models.push(model) + } + + return models + } +} diff --git a/server/models/view/local-video-viewer.ts b/server/models/view/local-video-viewer.ts new file mode 100644 index 000000000..6f8de53cd --- /dev/null +++ b/server/models/view/local-video-viewer.ts @@ -0,0 +1,274 @@ +import { QueryTypes } from 'sequelize' +import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, HasMany, IsUUID, Model, Table } from 'sequelize-typescript' +import { STATS_TIMESERIE } from '@server/initializers/constants' +import { getActivityStreamDuration } from '@server/lib/activitypub/activity' +import { MLocalVideoViewer, MLocalVideoViewerWithWatchSections, MVideo } from '@server/types/models' +import { VideoStatsOverall, VideoStatsRetention, VideoStatsTimeserie, VideoStatsTimeserieMetric, WatchActionObject } from '@shared/models' +import { AttributesOnly } from '@shared/typescript-utils' +import { VideoModel } from '../video/video' +import { LocalVideoViewerWatchSectionModel } from './local-video-viewer-watch-section' + +@Table({ + tableName: 'localVideoViewer', + updatedAt: false, + indexes: [ + { + fields: [ 'videoId' ] + } + ] +}) +export class LocalVideoViewerModel extends Model>> { + @CreatedAt + createdAt: Date + + @AllowNull(false) + @Column(DataType.DATE) + startDate: Date + + @AllowNull(false) + @Column(DataType.DATE) + endDate: Date + + @AllowNull(false) + @Column + watchTime: number + + @AllowNull(true) + @Column + country: string + + @AllowNull(false) + @Default(DataType.UUIDV4) + @IsUUID(4) + @Column(DataType.UUID) + uuid: string + + @AllowNull(false) + @Column + url: string + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + Video: VideoModel + + @HasMany(() => LocalVideoViewerWatchSectionModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + WatchSections: LocalVideoViewerWatchSectionModel[] + + static loadByUrl (url: string): Promise { + return this.findOne({ + where: { + url + } + }) + } + + static loadFullById (id: number): Promise { + return this.findOne({ + include: [ + { + model: VideoModel.unscoped(), + required: true + }, + { + model: LocalVideoViewerWatchSectionModel.unscoped(), + required: true + } + ], + where: { + id + } + }) + } + + static async getOverallStats (video: MVideo): Promise { + const options = { + type: QueryTypes.SELECT as QueryTypes.SELECT, + replacements: { videoId: video.id } + } + + const watchTimeQuery = `SELECT ` + + `SUM("localVideoViewer"."watchTime") AS "totalWatchTime", ` + + `AVG("localVideoViewer"."watchTime") AS "averageWatchTime" ` + + `FROM "localVideoViewer" ` + + `INNER JOIN "video" ON "video"."id" = "localVideoViewer"."videoId" ` + + `WHERE "videoId" = :videoId` + + const watchTimePromise = LocalVideoViewerModel.sequelize.query(watchTimeQuery, options) + + const watchPeakQuery = `WITH "watchPeakValues" AS ( + SELECT "startDate" AS "dateBreakpoint", 1 AS "inc" + FROM "localVideoViewer" + WHERE "videoId" = :videoId + UNION ALL + SELECT "endDate" AS "dateBreakpoint", -1 AS "inc" + FROM "localVideoViewer" + WHERE "videoId" = :videoId + ) + SELECT "dateBreakpoint", "concurrent" + FROM ( + SELECT "dateBreakpoint", SUM(SUM("inc")) OVER (ORDER BY "dateBreakpoint") AS "concurrent" + FROM "watchPeakValues" + GROUP BY "dateBreakpoint" + ) tmp + ORDER BY "concurrent" DESC + FETCH FIRST 1 ROW ONLY` + const watchPeakPromise = LocalVideoViewerModel.sequelize.query(watchPeakQuery, options) + + const commentsQuery = `SELECT COUNT(*) AS comments FROM "videoComment" WHERE "videoId" = :videoId` + const commentsPromise = LocalVideoViewerModel.sequelize.query(commentsQuery, options) + + const countriesQuery = `SELECT country, COUNT(country) as viewers ` + + `FROM "localVideoViewer" ` + + `WHERE "videoId" = :videoId AND country IS NOT NULL ` + + `GROUP BY country ` + + `ORDER BY viewers DESC` + const countriesPromise = LocalVideoViewerModel.sequelize.query(countriesQuery, options) + + const [ rowsWatchTime, rowsWatchPeak, rowsComment, rowsCountries ] = await Promise.all([ + watchTimePromise, + watchPeakPromise, + commentsPromise, + countriesPromise + ]) + + return { + totalWatchTime: rowsWatchTime.length !== 0 + ? Math.round(rowsWatchTime[0].totalWatchTime) || 0 + : 0, + averageWatchTime: rowsWatchTime.length !== 0 + ? Math.round(rowsWatchTime[0].averageWatchTime) || 0 + : 0, + + viewersPeak: rowsWatchPeak.length !== 0 + ? parseInt(rowsWatchPeak[0].concurrent) || 0 + : 0, + viewersPeakDate: rowsWatchPeak.length !== 0 + ? rowsWatchPeak[0].dateBreakpoint || null + : null, + + views: video.views, + likes: video.likes, + dislikes: video.dislikes, + + comments: rowsComment.length !== 0 + ? parseInt(rowsComment[0].comments) || 0 + : 0, + + countries: rowsCountries.map(r => ({ + isoCode: r.country, + viewers: r.viewers + })) + } + } + + static async getRetentionStats (video: MVideo): Promise { + const step = Math.max(Math.round(video.duration / 100), 1) + + const query = `WITH "total" AS (SELECT COUNT(*) AS viewers FROM "localVideoViewer" WHERE "videoId" = :videoId) ` + + `SELECT serie AS "second", ` + + `(COUNT("localVideoViewer".id)::float / (SELECT GREATEST("total"."viewers", 1) FROM "total")) AS "retention" ` + + `FROM generate_series(0, ${video.duration}, ${step}) serie ` + + `LEFT JOIN "localVideoViewer" ON "localVideoViewer"."videoId" = :videoId ` + + `AND EXISTS (` + + `SELECT 1 FROM "localVideoViewerWatchSection" ` + + `WHERE "localVideoViewer"."id" = "localVideoViewerWatchSection"."localVideoViewerId" ` + + `AND serie >= "localVideoViewerWatchSection"."watchStart" ` + + `AND serie <= "localVideoViewerWatchSection"."watchEnd"` + + `)` + + `GROUP BY serie ` + + `ORDER BY serie ASC` + + const queryOptions = { + type: QueryTypes.SELECT as QueryTypes.SELECT, + replacements: { videoId: video.id } + } + + const rows = await LocalVideoViewerModel.sequelize.query(query, queryOptions) + + return { + data: rows.map(r => ({ + second: r.second, + retentionPercent: parseFloat(r.retention) * 100 + })) + } + } + + static async getTimeserieStats (options: { + video: MVideo + metric: VideoStatsTimeserieMetric + }): Promise { + const { video, metric } = options + + const selectMetrics: { [ id in VideoStatsTimeserieMetric ]: string } = { + viewers: 'COUNT("localVideoViewer"."id")', + aggregateWatchTime: 'SUM("localVideoViewer"."watchTime")' + } + + const query = `WITH days AS ( ` + + `SELECT (current_date::timestamp - (serie || ' days')::interval)::timestamptz AS day + FROM generate_series(0, ${STATS_TIMESERIE.MAX_DAYS - 1}) serie` + + `) ` + + `SELECT days.day AS date, COALESCE(${selectMetrics[metric]}, 0) AS value ` + + `FROM days ` + + `LEFT JOIN "localVideoViewer" ON "localVideoViewer"."videoId" = :videoId ` + + `AND date_trunc('day', "localVideoViewer"."startDate") = date_trunc('day', days.day) ` + + `GROUP BY day ` + + `ORDER BY day ` + + const queryOptions = { + type: QueryTypes.SELECT as QueryTypes.SELECT, + replacements: { videoId: video.id } + } + + const rows = await LocalVideoViewerModel.sequelize.query(query, queryOptions) + + return { + data: rows.map(r => ({ + date: r.date, + value: parseInt(r.value) + })) + } + } + + toActivityPubObject (this: MLocalVideoViewerWithWatchSections): WatchActionObject { + const location = this.country + ? { + location: { + addressCountry: this.country + } + } + : {} + + return { + id: this.url, + type: 'WatchAction', + duration: getActivityStreamDuration(this.watchTime), + startTime: this.startDate.toISOString(), + endTime: this.endDate.toISOString(), + + object: this.Video.url, + uuid: this.uuid, + actionStatus: 'CompletedActionStatus', + + watchSections: this.WatchSections.map(w => ({ + startTimestamp: w.watchStart, + endTimestamp: w.watchEnd + })), + + ...location + } + } +} diff --git a/server/models/video/video-view.ts b/server/models/view/video-view.ts similarity index 96% rename from server/models/video/video-view.ts rename to server/models/view/video-view.ts index d72df100f..df462e631 100644 --- a/server/models/video/video-view.ts +++ b/server/models/view/video-view.ts @@ -1,7 +1,7 @@ import { literal, Op } from 'sequelize' import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table } from 'sequelize-typescript' import { AttributesOnly } from '@shared/typescript-utils' -import { VideoModel } from './video' +import { VideoModel } from '../video/video' @Table({ tableName: 'videoView', diff --git a/server/tests/api/activitypub/client.ts b/server/tests/api/activitypub/client.ts index e69ab3cb9..655fa30d0 100644 --- a/server/tests/api/activitypub/client.ts +++ b/server/tests/api/activitypub/client.ts @@ -2,6 +2,8 @@ import 'mocha' import * as chai from 'chai' +import { processViewersStats } from '@server/tests/shared' +import { HttpStatusCode, VideoPlaylistPrivacy, WatchActionObject } from '@shared/models' import { cleanupTests, createMultipleServers, @@ -11,7 +13,6 @@ import { setAccessTokensToServers, setDefaultVideoChannel } from '@shared/server-commands' -import { HttpStatusCode, VideoPlaylistPrivacy } from '@shared/models' const expect = chai.expect @@ -115,6 +116,23 @@ describe('Test activitypub', function () { expect(res.header.location).to.equal('http://localhost:' + servers[0].port + '/videos/watch/' + video.uuid) }) + it('Should return the watch action', async function () { + this.timeout(50000) + + await servers[0].views.simulateViewer({ id: video.uuid, currentTimes: [ 0, 2 ] }) + await processViewersStats(servers) + + const res = await makeActivityPubGetRequest(servers[0].url, '/videos/local-viewer/1', HttpStatusCode.OK_200) + + const object: WatchActionObject = res.body + expect(object.type).to.equal('WatchAction') + expect(object.duration).to.equal('PT2S') + expect(object.actionStatus).to.equal('CompletedActionStatus') + expect(object.watchSections).to.have.lengthOf(1) + expect(object.watchSections[0].startTimestamp).to.equal(0) + expect(object.watchSections[0].endTimestamp).to.equal(2) + }) + after(async function () { await cleanupTests(servers) }) diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts index c9adeef4a..259d7e783 100644 --- a/server/tests/api/check-params/index.ts +++ b/server/tests/api/check-params/index.ts @@ -33,3 +33,4 @@ import './videos-common-filters' import './video-files' import './videos-history' import './videos-overviews' +import './views' diff --git a/server/tests/api/check-params/videos-history.ts b/server/tests/api/check-params/videos-history.ts index 82f38b7b4..c1b2d8bf3 100644 --- a/server/tests/api/check-params/videos-history.ts +++ b/server/tests/api/check-params/videos-history.ts @@ -17,7 +17,7 @@ import { describe('Test videos history API validator', function () { const myHistoryPath = '/api/v1/users/me/history/videos' const myHistoryRemove = myHistoryPath + '/remove' - let watchingPath: string + let viewPath: string let server: PeerTubeServer let videoId: number @@ -31,51 +31,15 @@ describe('Test videos history API validator', function () { await setAccessTokensToServers([ server ]) const { id, uuid } = await server.videos.upload() - watchingPath = '/api/v1/videos/' + uuid + '/watching' + viewPath = '/api/v1/videos/' + uuid + '/views' videoId = id }) describe('When notifying a user is watching a video', function () { - it('Should fail with an unauthenticated user', async function () { + it('Should fail with a bad token', async function () { const fields = { currentTime: 5 } - await makePutBodyRequest({ url: server.url, path: watchingPath, fields, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with an incorrect video id', async function () { - const fields = { currentTime: 5 } - const path = '/api/v1/videos/blabla/watching' - await makePutBodyRequest({ - url: server.url, - path, - fields, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with an unknown video', async function () { - const fields = { currentTime: 5 } - const path = '/api/v1/videos/d91fff41-c24d-4508-8e13-3bd5902c3b02/watching' - - await makePutBodyRequest({ - url: server.url, - path, - fields, - token: server.accessToken, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with a bad current time', async function () { - const fields = { currentTime: 'hello' } - await makePutBodyRequest({ - url: server.url, - path: watchingPath, - fields, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) + await makePutBodyRequest({ url: server.url, path: viewPath, fields, token: 'bad', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) }) it('Should succeed with the correct parameters', async function () { @@ -83,7 +47,7 @@ describe('Test videos history API validator', function () { await makePutBodyRequest({ url: server.url, - path: watchingPath, + path: viewPath, fields, token: server.accessToken, expectedStatus: HttpStatusCode.NO_CONTENT_204 diff --git a/server/tests/api/check-params/views.ts b/server/tests/api/check-params/views.ts new file mode 100644 index 000000000..185b04af1 --- /dev/null +++ b/server/tests/api/check-params/views.ts @@ -0,0 +1,157 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import 'mocha' +import { HttpStatusCode, VideoPrivacy } from '@shared/models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel +} from '@shared/server-commands' + +describe('Test videos views', function () { + let servers: PeerTubeServer[] + let liveVideoId: string + let videoId: string + let remoteVideoId: string + let userAccessToken: string + + before(async function () { + this.timeout(30000) + + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await servers[0].config.enableLive({ allowReplay: false, transcoding: false }); + + ({ uuid: videoId } = await servers[0].videos.quickUpload({ name: 'video' })); + ({ uuid: remoteVideoId } = await servers[1].videos.quickUpload({ name: 'video' })); + ({ uuid: liveVideoId } = await servers[0].live.create({ + fields: { + name: 'live', + privacy: VideoPrivacy.PUBLIC, + channelId: servers[0].store.channel.id + } + })) + + userAccessToken = await servers[0].users.generateUserAndToken('user') + + await doubleFollow(servers[0], servers[1]) + }) + + describe('When viewing a video', async function () { + + // TODO: implement it when we'll remove backward compatibility in REST API + it('Should fail without current time') + + it('Should fail with an invalid current time', async function () { + await servers[0].views.view({ id: videoId, currentTime: -1, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await servers[0].views.view({ id: videoId, currentTime: 10, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should succeed with correct parameters', async function () { + await servers[0].views.view({ id: videoId, currentTime: 1 }) + }) + }) + + describe('When getting overall stats', function () { + + it('Should fail with a remote video', async function () { + await servers[0].videoStats.getOverallStats({ videoId: remoteVideoId, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail without token', async function () { + await servers[0].videoStats.getOverallStats({ videoId: videoId, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with another token', async function () { + await servers[0].videoStats.getOverallStats({ + videoId: videoId, + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await servers[0].videoStats.getOverallStats({ videoId }) + }) + }) + + describe('When getting timeserie stats', function () { + + it('Should fail with a remote video', async function () { + await servers[0].videoStats.getTimeserieStats({ + videoId: remoteVideoId, + metric: 'viewers', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail without token', async function () { + await servers[0].videoStats.getTimeserieStats({ + videoId: videoId, + token: null, + metric: 'viewers', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with another token', async function () { + await servers[0].videoStats.getTimeserieStats({ + videoId: videoId, + token: userAccessToken, + metric: 'viewers', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with an invalid metric', async function () { + await servers[0].videoStats.getTimeserieStats({ videoId, metric: 'hello' as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should succeed with the correct parameters', async function () { + await servers[0].videoStats.getTimeserieStats({ videoId, metric: 'viewers' }) + }) + }) + + describe('When getting retention stats', function () { + + it('Should fail with a remote video', async function () { + await servers[0].videoStats.getRetentionStats({ + videoId: remoteVideoId, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail without token', async function () { + await servers[0].videoStats.getRetentionStats({ + videoId: videoId, + token: null, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with another token', async function () { + await servers[0].videoStats.getRetentionStats({ + videoId: videoId, + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail on live video', async function () { + await servers[0].videoStats.getRetentionStats({ videoId: liveVideoId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should succeed with the correct parameters', async function () { + await servers[0].videoStats.getRetentionStats({ videoId }) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/server/tests/api/live/index.ts b/server/tests/api/live/index.ts index 105416b8d..71bc150d8 100644 --- a/server/tests/api/live/index.ts +++ b/server/tests/api/live/index.ts @@ -3,5 +3,4 @@ import './live-socket-messages' import './live-permanent' import './live-rtmps' import './live-save-replay' -import './live-views' import './live' diff --git a/server/tests/api/live/live-socket-messages.ts b/server/tests/api/live/live-socket-messages.ts index 50b16443e..7668ed5b9 100644 --- a/server/tests/api/live/live-socket-messages.ts +++ b/server/tests/api/live/live-socket-messages.ts @@ -140,8 +140,8 @@ describe('Test live', function () { expect(localLastVideoViews).to.equal(0) expect(remoteLastVideoViews).to.equal(0) - await servers[0].videos.view({ id: liveVideoUUID }) - await servers[1].videos.view({ id: liveVideoUUID }) + await servers[0].views.simulateView({ id: liveVideoUUID }) + await servers[1].views.simulateView({ id: liveVideoUUID }) await waitJobs(servers) diff --git a/server/tests/api/live/live-views.ts b/server/tests/api/live/live-views.ts deleted file mode 100644 index 446d0913c..000000000 --- a/server/tests/api/live/live-views.ts +++ /dev/null @@ -1,132 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import 'mocha' -import * as chai from 'chai' -import { FfmpegCommand } from 'fluent-ffmpeg' -import { wait } from '@shared/core-utils' -import { VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - stopFfmpeg, - waitJobs, - waitUntilLivePublishedOnAllServers -} from '@shared/server-commands' - -const expect = chai.expect - -describe('Live views', function () { - let servers: PeerTubeServer[] = [] - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2) - - // Get the access tokens - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - await servers[0].config.updateCustomSubConfig({ - newConfig: { - live: { - enabled: true, - allowReplay: true, - transcoding: { - enabled: false - } - } - } - }) - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - }) - - let liveVideoId: string - let command: FfmpegCommand - - async function countViewers (expectedViewers: number) { - for (const server of servers) { - const video = await server.videos.get({ id: liveVideoId }) - expect(video.viewers).to.equal(expectedViewers) - } - } - - async function countViews (expectedViews: number) { - for (const server of servers) { - const video = await server.videos.get({ id: liveVideoId }) - expect(video.views).to.equal(expectedViews) - } - } - - before(async function () { - this.timeout(30000) - - const liveAttributes = { - name: 'live video', - channelId: servers[0].store.channel.id, - privacy: VideoPrivacy.PUBLIC - } - - const live = await servers[0].live.create({ fields: liveAttributes }) - liveVideoId = live.uuid - - command = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoId }) - await waitUntilLivePublishedOnAllServers(servers, liveVideoId) - await waitJobs(servers) - }) - - it('Should display no views and viewers for a live', async function () { - await countViews(0) - await countViewers(0) - }) - - it('Should view a live twice and display 1 view/viewer', async function () { - this.timeout(30000) - - await servers[0].videos.view({ id: liveVideoId }) - await servers[0].videos.view({ id: liveVideoId }) - - await waitJobs(servers) - await countViewers(1) - - await wait(7000) - await countViews(1) - }) - - it('Should wait and display 0 viewers while still have 1 view', async function () { - this.timeout(30000) - - await wait(12000) - await waitJobs(servers) - - await countViews(1) - await countViewers(0) - }) - - it('Should view a live on a remote and on local and display 2 viewers and 3 views', async function () { - this.timeout(30000) - - await servers[0].videos.view({ id: liveVideoId }) - await servers[1].videos.view({ id: liveVideoId }) - await servers[1].videos.view({ id: liveVideoId }) - await waitJobs(servers) - - await countViewers(2) - - await wait(7000) - await waitJobs(servers) - - await countViews(3) - }) - - after(async function () { - await stopFfmpeg(command) - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index 3f2286278..0f7ffcb4c 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts @@ -87,7 +87,7 @@ async function createServers (strategy: VideoRedundancyStrategy | null, addition const { id } = await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } }) video1Server2 = await servers[1].videos.get({ id }) - await servers[1].videos.view({ id }) + await servers[1].views.simulateView({ id }) } await waitJobs(servers) @@ -447,8 +447,8 @@ describe('Test videos redundancy', function () { it('Should view 2 times the first video to have > min_views config', async function () { this.timeout(80000) - await servers[0].videos.view({ id: video1Server2.uuid }) - await servers[2].videos.view({ id: video1Server2.uuid }) + await servers[0].views.simulateView({ id: video1Server2.uuid }) + await servers[2].views.simulateView({ id: video1Server2.uuid }) await wait(10000) await waitJobs(servers) @@ -516,8 +516,8 @@ describe('Test videos redundancy', function () { it('Should have 1 redundancy on the first video', async function () { this.timeout(160000) - await servers[0].videos.view({ id: video1Server2.uuid }) - await servers[2].videos.view({ id: video1Server2.uuid }) + await servers[0].views.simulateView({ id: video1Server2.uuid }) + await servers[2].views.simulateView({ id: video1Server2.uuid }) await wait(10000) await waitJobs(servers) diff --git a/server/tests/api/server/reverse-proxy.ts b/server/tests/api/server/reverse-proxy.ts index 968d98e96..fa2063536 100644 --- a/server/tests/api/server/reverse-proxy.ts +++ b/server/tests/api/server/reverse-proxy.ts @@ -41,8 +41,8 @@ describe('Test application behind a reverse proxy', function () { it('Should view a video only once with the same IP by default', async function () { this.timeout(20000) - await server.videos.view({ id: videoId }) - await server.videos.view({ id: videoId }) + await server.views.simulateView({ id: videoId }) + await server.views.simulateView({ id: videoId }) // Wait the repeatable job await wait(8000) @@ -54,8 +54,8 @@ describe('Test application behind a reverse proxy', function () { it('Should view a video 2 times with the X-Forwarded-For header set', async function () { this.timeout(20000) - await server.videos.view({ id: videoId, xForwardedFor: '0.0.0.1,127.0.0.1' }) - await server.videos.view({ id: videoId, xForwardedFor: '0.0.0.2,127.0.0.1' }) + await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.1,127.0.0.1' }) + await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.2,127.0.0.1' }) // Wait the repeatable job await wait(8000) @@ -67,8 +67,8 @@ describe('Test application behind a reverse proxy', function () { it('Should view a video only once with the same client IP in the X-Forwarded-For header', async function () { this.timeout(20000) - await server.videos.view({ id: videoId, xForwardedFor: '0.0.0.4,0.0.0.3,::ffff:127.0.0.1' }) - await server.videos.view({ id: videoId, xForwardedFor: '0.0.0.5,0.0.0.3,127.0.0.1' }) + await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.4,0.0.0.3,::ffff:127.0.0.1' }) + await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.5,0.0.0.3,127.0.0.1' }) // Wait the repeatable job await wait(8000) @@ -80,8 +80,8 @@ describe('Test application behind a reverse proxy', function () { it('Should view a video two times with a different client IP in the X-Forwarded-For header', async function () { this.timeout(20000) - await server.videos.view({ id: videoId, xForwardedFor: '0.0.0.8,0.0.0.6,127.0.0.1' }) - await server.videos.view({ id: videoId, xForwardedFor: '0.0.0.8,0.0.0.7,127.0.0.1' }) + await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.8,0.0.0.6,127.0.0.1' }) + await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.8,0.0.0.7,127.0.0.1' }) // Wait the repeatable job await wait(8000) diff --git a/server/tests/api/server/stats.ts b/server/tests/api/server/stats.ts index 2296c0cb9..a9ae236fb 100644 --- a/server/tests/api/server/stats.ts +++ b/server/tests/api/server/stats.ts @@ -38,7 +38,7 @@ describe('Test stats (excluding redundancy)', function () { await servers[0].comments.createThread({ videoId: uuid, text: 'comment' }) - await servers[0].videos.view({ id: uuid }) + await servers[0].views.simulateView({ id: uuid }) // Wait the video views repeatable job await wait(8000) diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts index 7dc826353..27b119f30 100644 --- a/server/tests/api/videos/index.ts +++ b/server/tests/api/videos/index.ts @@ -16,4 +16,3 @@ import './video-schedule-update' import './videos-common-filters' import './videos-history' import './videos-overview' -import './videos-views-cleaner' diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts index a9df262dc..84c1515a3 100644 --- a/server/tests/api/videos/multiple-servers.ts +++ b/server/tests/api/videos/multiple-servers.ts @@ -504,21 +504,22 @@ describe('Test multiple servers', function () { it('Should view multiple videos on owned servers', async function () { this.timeout(30000) - await servers[2].videos.view({ id: localVideosServer3[0] }) + await servers[2].views.simulateView({ id: localVideosServer3[0] }) await wait(1000) - await servers[2].videos.view({ id: localVideosServer3[0] }) - await servers[2].videos.view({ id: localVideosServer3[1] }) + await servers[2].views.simulateView({ id: localVideosServer3[0] }) + await servers[2].views.simulateView({ id: localVideosServer3[1] }) await wait(1000) - await servers[2].videos.view({ id: localVideosServer3[0] }) - await servers[2].videos.view({ id: localVideosServer3[0] }) + await servers[2].views.simulateView({ id: localVideosServer3[0] }) + await servers[2].views.simulateView({ id: localVideosServer3[0] }) await waitJobs(servers) - // Wait the repeatable job - await wait(6000) + for (const server of servers) { + await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } }) + } await waitJobs(servers) @@ -537,23 +538,24 @@ describe('Test multiple servers', function () { this.timeout(45000) const tasks: Promise[] = [] - tasks.push(servers[0].videos.view({ id: remoteVideosServer1[0] })) - tasks.push(servers[1].videos.view({ id: remoteVideosServer2[0] })) - tasks.push(servers[1].videos.view({ id: remoteVideosServer2[0] })) - tasks.push(servers[2].videos.view({ id: remoteVideosServer3[0] })) - tasks.push(servers[2].videos.view({ id: remoteVideosServer3[1] })) - tasks.push(servers[2].videos.view({ id: remoteVideosServer3[1] })) - tasks.push(servers[2].videos.view({ id: remoteVideosServer3[1] })) - tasks.push(servers[2].videos.view({ id: localVideosServer3[1] })) - tasks.push(servers[2].videos.view({ id: localVideosServer3[1] })) - tasks.push(servers[2].videos.view({ id: localVideosServer3[1] })) + tasks.push(servers[0].views.simulateView({ id: remoteVideosServer1[0] })) + tasks.push(servers[1].views.simulateView({ id: remoteVideosServer2[0] })) + tasks.push(servers[1].views.simulateView({ id: remoteVideosServer2[0] })) + tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[0] })) + tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[1] })) + tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[1] })) + tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[1] })) + tasks.push(servers[2].views.simulateView({ id: localVideosServer3[1] })) + tasks.push(servers[2].views.simulateView({ id: localVideosServer3[1] })) + tasks.push(servers[2].views.simulateView({ id: localVideosServer3[1] })) await Promise.all(tasks) await waitJobs(servers) - // Wait the repeatable job - await wait(16000) + for (const server of servers) { + await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } }) + } await waitJobs(servers) diff --git a/server/tests/api/videos/single-server.ts b/server/tests/api/videos/single-server.ts index d37043aef..0e429fef7 100644 --- a/server/tests/api/videos/single-server.ts +++ b/server/tests/api/videos/single-server.ts @@ -179,22 +179,21 @@ describe('Test a single server', function () { it('Should have the views updated', async function () { this.timeout(20000) - await server.videos.view({ id: videoId }) - await server.videos.view({ id: videoId }) - await server.videos.view({ id: videoId }) + await server.views.simulateView({ id: videoId }) + await server.views.simulateView({ id: videoId }) + await server.views.simulateView({ id: videoId }) await wait(1500) - await server.videos.view({ id: videoId }) - await server.videos.view({ id: videoId }) + await server.views.simulateView({ id: videoId }) + await server.views.simulateView({ id: videoId }) await wait(1500) - await server.videos.view({ id: videoId }) - await server.videos.view({ id: videoId }) + await server.views.simulateView({ id: videoId }) + await server.views.simulateView({ id: videoId }) - // Wait the repeatable job - await wait(8000) + await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } }) const video = await server.videos.get({ id: videoId }) expect(video.views).to.equal(3) diff --git a/server/tests/api/videos/video-channels.ts b/server/tests/api/videos/video-channels.ts index 09a4bfa70..6f495c42d 100644 --- a/server/tests/api/videos/video-channels.ts +++ b/server/tests/api/videos/video-channels.ts @@ -466,8 +466,8 @@ describe('Test video channels', function () { { // video has been posted on channel servers[0].store.videoChannel.id since last update - await servers[0].videos.view({ id: videoUUID, xForwardedFor: '0.0.0.1,127.0.0.1' }) - await servers[0].videos.view({ id: videoUUID, xForwardedFor: '0.0.0.2,127.0.0.1' }) + await servers[0].views.simulateView({ id: videoUUID, xForwardedFor: '0.0.0.1,127.0.0.1' }) + await servers[0].views.simulateView({ id: videoUUID, xForwardedFor: '0.0.0.2,127.0.0.1' }) // Wait the repeatable job await wait(8000) diff --git a/server/tests/api/videos/videos-history.ts b/server/tests/api/videos/videos-history.ts index 8648c97f0..b1b3ff10a 100644 --- a/server/tests/api/videos/videos-history.ts +++ b/server/tests/api/videos/videos-history.ts @@ -3,15 +3,8 @@ import 'mocha' import * as chai from 'chai' import { wait } from '@shared/core-utils' -import { HttpStatusCode, Video } from '@shared/models' -import { - cleanupTests, - createSingleServer, - HistoryCommand, - killallServers, - PeerTubeServer, - setAccessTokensToServers -} from '@shared/server-commands' +import { Video } from '@shared/models' +import { cleanupTests, createSingleServer, killallServers, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' const expect = chai.expect @@ -23,7 +16,6 @@ describe('Test videos history', function () { let video3UUID: string let video3WatchedDate: Date let userAccessToken: string - let command: HistoryCommand before(async function () { this.timeout(30000) @@ -32,30 +24,26 @@ describe('Test videos history', function () { await setAccessTokensToServers([ server ]) - command = server.history + // 10 seconds long + const fixture = 'video_59fps.mp4' { - const { id, uuid } = await server.videos.upload({ attributes: { name: 'video 1' } }) + const { id, uuid } = await server.videos.upload({ attributes: { name: 'video 1', fixture } }) video1UUID = uuid video1Id = id } { - const { uuid } = await server.videos.upload({ attributes: { name: 'video 2' } }) + const { uuid } = await server.videos.upload({ attributes: { name: 'video 2', fixture } }) video2UUID = uuid } { - const { uuid } = await server.videos.upload({ attributes: { name: 'video 3' } }) + const { uuid } = await server.videos.upload({ attributes: { name: 'video 3', fixture } }) video3UUID = uuid } - const user = { - username: 'user_1', - password: 'super password' - } - await server.users.create({ username: user.username, password: user.password }) - userAccessToken = await server.login.getAccessToken(user) + userAccessToken = await server.users.generateUserAndToken('user_1') }) it('Should get videos, without watching history', async function () { @@ -70,8 +58,8 @@ describe('Test videos history', function () { }) it('Should watch the first and second video', async function () { - await command.watchVideo({ videoId: video2UUID, currentTime: 8 }) - await command.watchVideo({ videoId: video1UUID, currentTime: 3 }) + await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 }) + await server.views.view({ id: video1UUID, token: server.accessToken, currentTime: 3 }) }) it('Should return the correct history when listing, searching and getting videos', async function () { @@ -124,9 +112,9 @@ describe('Test videos history', function () { it('Should have these videos when listing my history', async function () { video3WatchedDate = new Date() - await command.watchVideo({ videoId: video3UUID, currentTime: 2 }) + await server.views.view({ id: video3UUID, token: server.accessToken, currentTime: 2 }) - const body = await command.list() + const body = await server.history.list() expect(body.total).to.equal(3) @@ -137,14 +125,14 @@ describe('Test videos history', function () { }) it('Should not have videos history on another user', async function () { - const body = await command.list({ token: userAccessToken }) + const body = await server.history.list({ token: userAccessToken }) expect(body.total).to.equal(0) expect(body.data).to.have.lengthOf(0) }) it('Should be able to search through videos in my history', async function () { - const body = await command.list({ search: '2' }) + const body = await server.history.list({ search: '2' }) expect(body.total).to.equal(1) const videos = body.data @@ -152,11 +140,11 @@ describe('Test videos history', function () { }) it('Should clear my history', async function () { - await command.removeAll({ beforeDate: video3WatchedDate.toISOString() }) + await server.history.removeAll({ beforeDate: video3WatchedDate.toISOString() }) }) it('Should have my history cleared', async function () { - const body = await command.list() + const body = await server.history.list() expect(body.total).to.equal(1) const videos = body.data @@ -168,7 +156,10 @@ describe('Test videos history', function () { videosHistoryEnabled: false }) - await command.watchVideo({ videoId: video2UUID, currentTime: 8, expectedStatus: HttpStatusCode.CONFLICT_409 }) + await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 }) + + const { data } = await server.history.list() + expect(data[0].name).to.not.equal('video 2') }) it('Should re-enable videos history', async function () { @@ -176,14 +167,10 @@ describe('Test videos history', function () { videosHistoryEnabled: true }) - await command.watchVideo({ videoId: video1UUID, currentTime: 8 }) + await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 }) - const body = await command.list() - expect(body.total).to.equal(2) - - const videos = body.data - expect(videos[0].name).to.equal('video 1') - expect(videos[1].name).to.equal('video 3') + const { data } = await server.history.list() + expect(data[0].name).to.equal('video 2') }) it('Should not clean old history', async function () { @@ -197,7 +184,7 @@ describe('Test videos history', function () { // Should still have history - const body = await command.list() + const body = await server.history.list() expect(body.total).to.equal(2) }) @@ -210,25 +197,25 @@ describe('Test videos history', function () { await wait(6000) - const body = await command.list() + const body = await server.history.list() expect(body.total).to.equal(0) }) it('Should delete a specific history element', async function () { { - await command.watchVideo({ videoId: video1UUID, currentTime: 4 }) - await command.watchVideo({ videoId: video2UUID, currentTime: 8 }) + await server.views.view({ id: video1UUID, token: server.accessToken, currentTime: 4 }) + await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 }) } { - const body = await command.list() + const body = await server.history.list() expect(body.total).to.equal(2) } { - await command.removeElement({ videoId: video1Id }) + await server.history.removeElement({ videoId: video1Id }) - const body = await command.list() + const body = await server.history.list() expect(body.total).to.equal(1) expect(body.data[0].uuid).to.equal(video2UUID) } diff --git a/server/tests/api/views/index.ts b/server/tests/api/views/index.ts new file mode 100644 index 000000000..5e06b31fb --- /dev/null +++ b/server/tests/api/views/index.ts @@ -0,0 +1,5 @@ +export * from './video-views-counter' +export * from './video-views-overall-stats' +export * from './video-views-retention-stats' +export * from './video-views-timeserie-stats' +export * from './videos-views-cleaner' diff --git a/server/tests/api/views/video-views-counter.ts b/server/tests/api/views/video-views-counter.ts new file mode 100644 index 000000000..b68aaa350 --- /dev/null +++ b/server/tests/api/views/video-views-counter.ts @@ -0,0 +1,155 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import 'mocha' +import * as chai from 'chai' +import { FfmpegCommand } from 'fluent-ffmpeg' +import { prepareViewsServers, prepareViewsVideos, processViewsBuffer } from '@server/tests/shared' +import { wait } from '@shared/core-utils' +import { cleanupTests, PeerTubeServer, stopFfmpeg, waitJobs } from '@shared/server-commands' + +const expect = chai.expect + +describe('Test video views/viewers counters', function () { + let servers: PeerTubeServer[] + + async function checkCounter (field: 'views' | 'viewers', id: string, expected: number) { + for (const server of servers) { + const video = await server.videos.get({ id }) + + const messageSuffix = video.isLive + ? 'live video' + : 'vod video' + + expect(video[field]).to.equal(expected, `${field} not valid on server ${server.serverNumber} for ${messageSuffix} ${video.uuid}`) + } + } + + before(async function () { + this.timeout(120000) + + servers = await prepareViewsServers() + }) + + describe('Test views counter on VOD', function () { + let videoUUID: string + + before(async function () { + this.timeout(30000) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) + videoUUID = uuid + + await waitJobs(servers) + }) + + it('Should not view a video if watch time is below the threshold', async function () { + await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 2 ] }) + await processViewsBuffer(servers) + + await checkCounter('views', videoUUID, 0) + }) + + it('Should view a video if watch time is above the threshold', async function () { + await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 4 ] }) + await processViewsBuffer(servers) + + await checkCounter('views', videoUUID, 1) + }) + + it('Should not view again this video with the same IP', async function () { + await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 4 ] }) + await processViewsBuffer(servers) + + await checkCounter('views', videoUUID, 1) + }) + + it('Should view the video from server 2 and send the event', async function () { + await servers[1].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 4 ] }) + await waitJobs(servers) + await processViewsBuffer(servers) + + await checkCounter('views', videoUUID, 2) + }) + }) + + describe('Test views and viewers counters on live and VOD', function () { + let liveVideoId: string + let vodVideoId: string + let command: FfmpegCommand + + before(async function () { + this.timeout(60000); + + ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) + }) + + it('Should display no views and viewers', async function () { + await checkCounter('views', liveVideoId, 0) + await checkCounter('viewers', liveVideoId, 0) + + await checkCounter('views', vodVideoId, 0) + await checkCounter('viewers', vodVideoId, 0) + }) + + it('Should view twice and display 1 view/viewer', async function () { + this.timeout(30000) + + await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) + await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) + await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) + await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) + + await waitJobs(servers) + await checkCounter('viewers', liveVideoId, 1) + await checkCounter('viewers', vodVideoId, 1) + + await processViewsBuffer(servers) + + await checkCounter('views', liveVideoId, 1) + await checkCounter('views', vodVideoId, 1) + }) + + it('Should wait and display 0 viewers but still have 1 view', async function () { + this.timeout(30000) + + await wait(12000) + await waitJobs(servers) + + await checkCounter('views', liveVideoId, 1) + await checkCounter('viewers', liveVideoId, 0) + + await checkCounter('views', vodVideoId, 1) + await checkCounter('viewers', vodVideoId, 0) + }) + + it('Should view on a remote and on local and display 2 viewers and 3 views', async function () { + this.timeout(30000) + + await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) + await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) + await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) + + await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) + await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) + await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) + + await waitJobs(servers) + + await checkCounter('viewers', liveVideoId, 2) + await checkCounter('viewers', vodVideoId, 2) + + await processViewsBuffer(servers) + + await checkCounter('views', liveVideoId, 3) + await checkCounter('views', vodVideoId, 3) + }) + + after(async function () { + await stopFfmpeg(command) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/server/tests/api/views/video-views-overall-stats.ts b/server/tests/api/views/video-views-overall-stats.ts new file mode 100644 index 000000000..22761d6ec --- /dev/null +++ b/server/tests/api/views/video-views-overall-stats.ts @@ -0,0 +1,291 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import 'mocha' +import * as chai from 'chai' +import { FfmpegCommand } from 'fluent-ffmpeg' +import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@server/tests/shared' +import { cleanupTests, PeerTubeServer, stopFfmpeg, waitJobs } from '@shared/server-commands' + +const expect = chai.expect + +describe('Test views overall stats', function () { + let servers: PeerTubeServer[] + + before(async function () { + this.timeout(120000) + + servers = await prepareViewsServers() + }) + + describe('Test rates and comments of local videos on VOD', function () { + let vodVideoId: string + + before(async function () { + this.timeout(60000); + + ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true })) + }) + + it('Should have the appropriate likes', async function () { + this.timeout(60000) + + await servers[0].videos.rate({ id: vodVideoId, rating: 'like' }) + await servers[1].videos.rate({ id: vodVideoId, rating: 'like' }) + + await waitJobs(servers) + + const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) + + expect(stats.likes).to.equal(2) + expect(stats.dislikes).to.equal(0) + }) + + it('Should have the appropriate dislikes', async function () { + this.timeout(60000) + + await servers[0].videos.rate({ id: vodVideoId, rating: 'dislike' }) + await servers[1].videos.rate({ id: vodVideoId, rating: 'dislike' }) + + await waitJobs(servers) + + const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) + + expect(stats.likes).to.equal(0) + expect(stats.dislikes).to.equal(2) + }) + + it('Should have the appropriate comments', async function () { + this.timeout(60000) + + await servers[0].comments.createThread({ videoId: vodVideoId, text: 'root' }) + await servers[0].comments.addReplyToLastThread({ text: 'reply' }) + await servers[1].comments.createThread({ videoId: vodVideoId, text: 'root' }) + + await waitJobs(servers) + + const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) + expect(stats.comments).to.equal(3) + }) + }) + + describe('Test watch time stats of local videos on live and VOD', function () { + let vodVideoId: string + let liveVideoId: string + let command: FfmpegCommand + + before(async function () { + this.timeout(60000); + + ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) + }) + + it('Should display overall stats of a video with no viewers', async function () { + for (const videoId of [ liveVideoId, vodVideoId ]) { + const stats = await servers[0].videoStats.getOverallStats({ videoId }) + + expect(stats.views).to.equal(0) + expect(stats.averageWatchTime).to.equal(0) + expect(stats.totalWatchTime).to.equal(0) + } + }) + + it('Should display overall stats with 1 viewer below the watch time limit', async function () { + this.timeout(60000) + + for (const videoId of [ liveVideoId, vodVideoId ]) { + await servers[0].views.simulateViewer({ id: videoId, currentTimes: [ 0, 1 ] }) + } + + await processViewersStats(servers) + + for (const videoId of [ liveVideoId, vodVideoId ]) { + const stats = await servers[0].videoStats.getOverallStats({ videoId }) + + expect(stats.views).to.equal(0) + expect(stats.averageWatchTime).to.equal(1) + expect(stats.totalWatchTime).to.equal(1) + } + }) + + it('Should display overall stats with 2 viewers', async function () { + this.timeout(60000) + + { + await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 3 ] }) + await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35, 40 ] }) + + await processViewersStats(servers) + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) + expect(stats.views).to.equal(1) + expect(stats.averageWatchTime).to.equal(2) + expect(stats.totalWatchTime).to.equal(4) + } + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId }) + expect(stats.views).to.equal(1) + expect(stats.averageWatchTime).to.equal(21) + expect(stats.totalWatchTime).to.equal(41) + } + } + }) + + it('Should display overall stats with a remote viewer below the watch time limit', async function () { + this.timeout(60000) + + for (const videoId of [ liveVideoId, vodVideoId ]) { + await servers[1].views.simulateViewer({ id: videoId, currentTimes: [ 0, 2 ] }) + } + + await processViewersStats(servers) + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) + + expect(stats.views).to.equal(1) + expect(stats.averageWatchTime).to.equal(2) + expect(stats.totalWatchTime).to.equal(6) + } + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId }) + + expect(stats.views).to.equal(1) + expect(stats.averageWatchTime).to.equal(14) + expect(stats.totalWatchTime).to.equal(43) + } + }) + + it('Should display overall stats with a remote viewer above the watch time limit', async function () { + this.timeout(60000) + + await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) + await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 45 ] }) + await processViewersStats(servers) + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) + expect(stats.views).to.equal(2) + expect(stats.averageWatchTime).to.equal(3) + expect(stats.totalWatchTime).to.equal(11) + } + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId }) + expect(stats.views).to.equal(2) + expect(stats.averageWatchTime).to.equal(22) + expect(stats.totalWatchTime).to.equal(88) + } + }) + + after(async function () { + await stopFfmpeg(command) + }) + }) + + describe('Test watchers peak stats of local videos on VOD', function () { + let videoUUID: string + + before(async function () { + this.timeout(60000); + + ({ vodVideoId: videoUUID } = await prepareViewsVideos({ servers, live: true, vod: true })) + }) + + it('Should not have watchers peak', async function () { + const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID }) + + expect(stats.viewersPeak).to.equal(0) + expect(stats.viewersPeakDate).to.be.null + }) + + it('Should have watcher peak with 1 watcher', async function () { + this.timeout(60000) + + const before = new Date() + await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 0, 2 ] }) + const after = new Date() + + await processViewersStats(servers) + + const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID }) + + expect(stats.viewersPeak).to.equal(1) + expect(new Date(stats.viewersPeakDate)).to.be.above(before).and.below(after) + }) + + it('Should have watcher peak with 2 watchers', async function () { + this.timeout(60000) + + const before = new Date() + await servers[0].views.view({ id: videoUUID, currentTime: 0 }) + await servers[1].views.view({ id: videoUUID, currentTime: 0 }) + await servers[0].views.view({ id: videoUUID, currentTime: 2 }) + await servers[1].views.view({ id: videoUUID, currentTime: 2 }) + const after = new Date() + + await processViewersStats(servers) + + const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID }) + + expect(stats.viewersPeak).to.equal(2) + expect(new Date(stats.viewersPeakDate)).to.be.above(before).and.below(after) + }) + }) + + describe('Test countries', function () { + + it('Should not report countries if geoip is disabled', async function () { + this.timeout(60000) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) + await waitJobs(servers) + + await servers[1].views.view({ id: uuid, xForwardedFor: '8.8.8.8,127.0.0.1', currentTime: 1 }) + + await processViewersStats(servers) + + const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid }) + expect(stats.countries).to.have.lengthOf(0) + }) + + it('Should report countries if geoip is enabled', async function () { + this.timeout(60000) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) + await waitJobs(servers) + + await Promise.all([ + servers[0].kill(), + servers[1].kill() + ]) + + const config = { geo_ip: { enabled: true } } + await Promise.all([ + servers[0].run(config), + servers[1].run(config) + ]) + + await servers[0].views.view({ id: uuid, xForwardedFor: '8.8.8.8,127.0.0.1', currentTime: 1 }) + await servers[1].views.view({ id: uuid, xForwardedFor: '8.8.8.4,127.0.0.1', currentTime: 3 }) + await servers[1].views.view({ id: uuid, xForwardedFor: '80.67.169.12,127.0.0.1', currentTime: 2 }) + + await processViewersStats(servers) + + const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid }) + expect(stats.countries).to.have.lengthOf(2) + + expect(stats.countries[0].isoCode).to.equal('US') + expect(stats.countries[0].viewers).to.equal(2) + + expect(stats.countries[1].isoCode).to.equal('FR') + expect(stats.countries[1].viewers).to.equal(1) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/server/tests/api/views/video-views-retention-stats.ts b/server/tests/api/views/video-views-retention-stats.ts new file mode 100644 index 000000000..98be7bfdb --- /dev/null +++ b/server/tests/api/views/video-views-retention-stats.ts @@ -0,0 +1,56 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import 'mocha' +import * as chai from 'chai' +import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@server/tests/shared' +import { cleanupTests, PeerTubeServer } from '@shared/server-commands' + +const expect = chai.expect + +describe('Test views retention stats', function () { + let servers: PeerTubeServer[] + + before(async function () { + this.timeout(120000) + + servers = await prepareViewsServers() + }) + + describe('Test retention stats on VOD', function () { + let vodVideoId: string + + before(async function () { + this.timeout(60000); + + ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true })) + }) + + it('Should display empty retention', async function () { + const { data } = await servers[0].videoStats.getRetentionStats({ videoId: vodVideoId }) + expect(data).to.have.lengthOf(6) + + for (let i = 0; i < 6; i++) { + expect(data[i].second).to.equal(i) + expect(data[i].retentionPercent).to.equal(0) + } + }) + + it('Should display appropriate retention metrics', async function () { + await servers[0].views.simulateViewer({ xForwardedFor: '127.0.0.2,127.0.0.1', id: vodVideoId, currentTimes: [ 0, 1 ] }) + await servers[0].views.simulateViewer({ xForwardedFor: '127.0.0.3,127.0.0.1', id: vodVideoId, currentTimes: [ 1, 3 ] }) + await servers[1].views.simulateViewer({ xForwardedFor: '127.0.0.2,127.0.0.1', id: vodVideoId, currentTimes: [ 4 ] }) + await servers[1].views.simulateViewer({ xForwardedFor: '127.0.0.3,127.0.0.1', id: vodVideoId, currentTimes: [ 0, 1 ] }) + + await processViewersStats(servers) + + const { data } = await servers[0].videoStats.getRetentionStats({ videoId: vodVideoId }) + expect(data).to.have.lengthOf(6) + + expect(data.map(d => d.retentionPercent)).to.deep.equal([ 50, 75, 25, 25, 25, 0 ]) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/server/tests/api/views/video-views-timeserie-stats.ts b/server/tests/api/views/video-views-timeserie-stats.ts new file mode 100644 index 000000000..98c041cdf --- /dev/null +++ b/server/tests/api/views/video-views-timeserie-stats.ts @@ -0,0 +1,109 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import 'mocha' +import * as chai from 'chai' +import { FfmpegCommand } from 'fluent-ffmpeg' +import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@server/tests/shared' +import { VideoStatsTimeserie, VideoStatsTimeserieMetric } from '@shared/models' +import { cleanupTests, PeerTubeServer, stopFfmpeg } from '@shared/server-commands' + +const expect = chai.expect + +describe('Test views timeserie stats', function () { + const availableMetrics: VideoStatsTimeserieMetric[] = [ 'viewers' ] + + let servers: PeerTubeServer[] + + before(async function () { + this.timeout(120000) + + servers = await prepareViewsServers() + }) + + describe('Common metric tests', function () { + let vodVideoId: string + + before(async function () { + this.timeout(60000); + + ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true })) + }) + + it('Should display empty metric stats', async function () { + for (const metric of availableMetrics) { + const { data } = await servers[0].videoStats.getTimeserieStats({ videoId: vodVideoId, metric }) + + expect(data).to.have.lengthOf(30) + + for (const d of data) { + expect(d.value).to.equal(0) + } + } + }) + }) + + describe('Test viewer and watch time metrics on live and VOD', function () { + let vodVideoId: string + let liveVideoId: string + let command: FfmpegCommand + + function expectTimeserieData (result: VideoStatsTimeserie, lastValue: number) { + const { data } = result + expect(data).to.have.lengthOf(30) + + const last = data[data.length - 1] + + const today = new Date().getDate() + expect(new Date(last.date).getDate()).to.equal(today) + expect(last.value).to.equal(lastValue) + + for (let i = 0; i < data.length - 2; i++) { + expect(data[i].value).to.equal(0) + } + } + + before(async function () { + this.timeout(60000); + + ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) + }) + + it('Should display appropriate viewers metrics', async function () { + for (const videoId of [ vodVideoId, liveVideoId ]) { + await servers[0].views.simulateViewer({ id: videoId, currentTimes: [ 0, 3 ] }) + await servers[1].views.simulateViewer({ id: videoId, currentTimes: [ 0, 5 ] }) + } + + await processViewersStats(servers) + + for (const videoId of [ vodVideoId, liveVideoId ]) { + const result = await servers[0].videoStats.getTimeserieStats({ videoId, metric: 'viewers' }) + expectTimeserieData(result, 2) + } + }) + + it('Should display appropriate watch time metrics', async function () { + for (const videoId of [ vodVideoId, liveVideoId ]) { + const result = await servers[0].videoStats.getTimeserieStats({ videoId, metric: 'aggregateWatchTime' }) + expectTimeserieData(result, 8) + + await servers[1].views.simulateViewer({ id: videoId, currentTimes: [ 0, 1 ] }) + } + + await processViewersStats(servers) + + for (const videoId of [ vodVideoId, liveVideoId ]) { + const result = await servers[0].videoStats.getTimeserieStats({ videoId, metric: 'aggregateWatchTime' }) + expectTimeserieData(result, 9) + } + }) + + after(async function () { + await stopFfmpeg(command) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/server/tests/api/videos/videos-views-cleaner.ts b/server/tests/api/views/videos-views-cleaner.ts similarity index 90% rename from server/tests/api/videos/videos-views-cleaner.ts rename to server/tests/api/views/videos-views-cleaner.ts index e6815a4a8..ef988837f 100644 --- a/server/tests/api/videos/videos-views-cleaner.ts +++ b/server/tests/api/views/videos-views-cleaner.ts @@ -34,10 +34,10 @@ describe('Test video views cleaner', function () { await waitJobs(servers) - await servers[0].videos.view({ id: videoIdServer1 }) - await servers[1].videos.view({ id: videoIdServer1 }) - await servers[0].videos.view({ id: videoIdServer2 }) - await servers[1].videos.view({ id: videoIdServer2 }) + await servers[0].views.simulateView({ id: videoIdServer1 }) + await servers[1].views.simulateView({ id: videoIdServer1 }) + await servers[0].views.simulateView({ id: videoIdServer2 }) + await servers[1].views.simulateView({ id: videoIdServer2 }) await waitJobs(servers) }) diff --git a/server/tests/plugins/action-hooks.ts b/server/tests/plugins/action-hooks.ts index 8788a9644..57ede2701 100644 --- a/server/tests/plugins/action-hooks.ts +++ b/server/tests/plugins/action-hooks.ts @@ -1,6 +1,7 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import 'mocha' +import { ServerHookName, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models' import { cleanupTests, createMultipleServers, @@ -10,7 +11,6 @@ import { setAccessTokensToServers, setDefaultVideoChannel } from '@shared/server-commands' -import { ServerHookName, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models' describe('Test plugin action hooks', function () { let servers: PeerTubeServer[] @@ -61,7 +61,7 @@ describe('Test plugin action hooks', function () { }) it('Should run action:api.video.viewed', async function () { - await servers[0].videos.view({ id: videoUUID }) + await servers[0].views.simulateView({ id: videoUUID }) await checkHook('action:api.video.viewed') }) diff --git a/server/tests/plugins/plugin-helpers.ts b/server/tests/plugins/plugin-helpers.ts index 167429ef4..5e8d08dff 100644 --- a/server/tests/plugins/plugin-helpers.ts +++ b/server/tests/plugins/plugin-helpers.ts @@ -301,7 +301,7 @@ describe('Test plugin helpers', function () { // Should not throw -> video exists const video = await servers[0].videos.get({ id: videoUUID }) // Should delete the video - await servers[0].videos.view({ id: videoUUID }) + await servers[0].views.simulateView({ id: videoUUID }) await servers[0].servers.waitUntilLog('Video deleted by plugin four.') diff --git a/server/tests/shared/index.ts b/server/tests/shared/index.ts index 47019d6a8..9f7ade53d 100644 --- a/server/tests/shared/index.ts +++ b/server/tests/shared/index.ts @@ -13,3 +13,4 @@ export * from './streaming-playlists' export * from './tests' export * from './tracker' export * from './videos' +export * from './views' diff --git a/server/tests/shared/views.ts b/server/tests/shared/views.ts new file mode 100644 index 000000000..e6b289715 --- /dev/null +++ b/server/tests/shared/views.ts @@ -0,0 +1,93 @@ +import { FfmpegCommand } from 'fluent-ffmpeg' +import { wait } from '@shared/core-utils' +import { VideoCreateResult, VideoPrivacy } from '@shared/models' +import { + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs, + waitUntilLivePublishedOnAllServers +} from '@shared/server-commands' + +async function processViewersStats (servers: PeerTubeServer[]) { + await wait(6000) + + for (const server of servers) { + await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } }) + await server.debug.sendCommand({ body: { command: 'process-video-viewers' } }) + } + + await waitJobs(servers) +} + +async function processViewsBuffer (servers: PeerTubeServer[]) { + for (const server of servers) { + await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } }) + } + + await waitJobs(servers) +} + +async function prepareViewsServers () { + const servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await servers[0].config.updateCustomSubConfig({ + newConfig: { + live: { + enabled: true, + allowReplay: true, + transcoding: { + enabled: false + } + } + } + }) + + await doubleFollow(servers[0], servers[1]) + + return servers +} + +async function prepareViewsVideos (options: { + servers: PeerTubeServer[] + live: boolean + vod: boolean +}) { + const { servers } = options + + const liveAttributes = { + name: 'live video', + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + + let ffmpegCommand: FfmpegCommand + let live: VideoCreateResult + let vod: VideoCreateResult + + if (options.live) { + live = await servers[0].live.create({ fields: liveAttributes }) + + ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: live.uuid }) + await waitUntilLivePublishedOnAllServers(servers, live.uuid) + } + + if (options.vod) { + vod = await servers[0].videos.quickUpload({ name: 'video' }) + } + + await waitJobs(servers) + + return { liveVideoId: live?.uuid, vodVideoId: vod?.uuid, ffmpegCommand } +} + +export { + processViewersStats, + prepareViewsServers, + processViewsBuffer, + prepareViewsVideos +} diff --git a/server/types/express.d.ts b/server/types/express.d.ts index 91a8cf3d8..4537c57c6 100644 --- a/server/types/express.d.ts +++ b/server/types/express.d.ts @@ -185,6 +185,8 @@ declare module 'express' { externalAuth?: RegisterServerAuthExternalOptions plugin?: MPlugin + + localViewerFull?: MLocalVideoViewerWithWatchSections } } } diff --git a/server/types/models/video/index.ts b/server/types/models/video/index.ts index e586a4e42..5ddffcab5 100644 --- a/server/types/models/video/index.ts +++ b/server/types/models/video/index.ts @@ -1,3 +1,5 @@ +export * from './local-video-viewer-watch-section' +export * from './local-video-viewer' export * from './schedule-video-update' export * from './tag' export * from './thumbnail' diff --git a/server/types/models/video/local-video-viewer-watch-section.ts b/server/types/models/video/local-video-viewer-watch-section.ts new file mode 100644 index 000000000..be7a3bba0 --- /dev/null +++ b/server/types/models/video/local-video-viewer-watch-section.ts @@ -0,0 +1,5 @@ +import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' + +// ############################################################################ + +export type MLocalVideoViewerWatchSection = Omit diff --git a/server/types/models/video/local-video-viewer.ts b/server/types/models/video/local-video-viewer.ts new file mode 100644 index 000000000..b78ef5507 --- /dev/null +++ b/server/types/models/video/local-video-viewer.ts @@ -0,0 +1,19 @@ +import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' +import { PickWith } from '@shared/typescript-utils' +import { MLocalVideoViewerWatchSection } from './local-video-viewer-watch-section' +import { MVideo } from './video' + +type Use = PickWith + +// ############################################################################ + +export type MLocalVideoViewer = Omit + +export type MLocalVideoViewerVideo = + MLocalVideoViewer & + Use<'Video', MVideo> + +export type MLocalVideoViewerWithWatchSections = + MLocalVideoViewer & + Use<'Video', MVideo> & + Use<'WatchSections', MLocalVideoViewerWatchSection[]> diff --git a/shared/models/activitypub/activity.ts b/shared/models/activitypub/activity.ts index d6284e283..fd5d38316 100644 --- a/shared/models/activitypub/activity.ts +++ b/shared/models/activitypub/activity.ts @@ -1,6 +1,6 @@ import { ActivityPubActor } from './activitypub-actor' import { ActivityPubSignature } from './activitypub-signature' -import { ActivityFlagReasonObject, CacheFileObject, VideoObject } from './objects' +import { ActivityFlagReasonObject, CacheFileObject, VideoObject, WatchActionObject } from './objects' import { AbuseObject } from './objects/abuse-object' import { DislikeObject } from './objects/dislike-object' import { APObject } from './objects/object.model' @@ -52,7 +52,7 @@ export interface BaseActivity { export interface ActivityCreate extends BaseActivity { type: 'Create' - object: VideoObject | AbuseObject | DislikeObject | VideoCommentObject | CacheFileObject | PlaylistObject + object: VideoObject | AbuseObject | DislikeObject | VideoCommentObject | CacheFileObject | PlaylistObject | WatchActionObject } export interface ActivityUpdate extends BaseActivity { @@ -99,7 +99,9 @@ export interface ActivityView extends BaseActivity { type: 'View' actor: string object: APObject - expires: string + + // If sending a "viewer" event + expires?: string } export interface ActivityDislike extends BaseActivity { diff --git a/shared/models/activitypub/context.ts b/shared/models/activitypub/context.ts index 4ada3b083..e9df38207 100644 --- a/shared/models/activitypub/context.ts +++ b/shared/models/activitypub/context.ts @@ -12,4 +12,5 @@ export type ContextType = 'Rate' | 'Flag' | 'Actor' | - 'Collection' + 'Collection' | + 'WatchAction' diff --git a/shared/models/activitypub/objects/index.ts b/shared/models/activitypub/objects/index.ts index 9e2c6b728..47a8e847a 100644 --- a/shared/models/activitypub/objects/index.ts +++ b/shared/models/activitypub/objects/index.ts @@ -8,3 +8,4 @@ export * from './playlist-object' export * from './video-comment-object' export * from './video-torrent-object' export * from './view-object' +export * from './watch-action-object' diff --git a/shared/models/activitypub/objects/watch-action-object.ts b/shared/models/activitypub/objects/watch-action-object.ts new file mode 100644 index 000000000..ed336602f --- /dev/null +++ b/shared/models/activitypub/objects/watch-action-object.ts @@ -0,0 +1,22 @@ +export interface WatchActionObject { + id: string + type: 'WatchAction' + + startTime: string + endTime: string + + location?: { + addressCountry: string + } + + uuid: string + object: string + actionStatus: 'CompletedActionStatus' + + duration: string + + watchSections: { + startTimestamp: number + endTimestamp: number + }[] +} diff --git a/shared/models/server/debug.model.ts b/shared/models/server/debug.model.ts index 2ecabdeca..223d23362 100644 --- a/shared/models/server/debug.model.ts +++ b/shared/models/server/debug.model.ts @@ -4,5 +4,5 @@ export interface Debug { } export interface SendDebugCommand { - command: 'remove-dandling-resumable-uploads' + command: 'remove-dandling-resumable-uploads' | 'process-video-views-buffer' | 'process-video-viewers' } diff --git a/shared/models/users/index.ts b/shared/models/users/index.ts index a24ffee96..b25978587 100644 --- a/shared/models/users/index.ts +++ b/shared/models/users/index.ts @@ -12,5 +12,4 @@ export * from './user-scoped-token' export * from './user-update-me.model' export * from './user-update.model' export * from './user-video-quota.model' -export * from './user-watching-video.model' export * from './user.model' diff --git a/shared/models/users/user-watching-video.model.ts b/shared/models/users/user-watching-video.model.ts deleted file mode 100644 index c22480595..000000000 --- a/shared/models/users/user-watching-video.model.ts +++ /dev/null @@ -1,3 +0,0 @@ -export interface UserWatchingVideo { - currentTime: number -} diff --git a/shared/models/videos/index.ts b/shared/models/videos/index.ts index 705e8d0ff..05497bda1 100644 --- a/shared/models/videos/index.ts +++ b/shared/models/videos/index.ts @@ -9,6 +9,7 @@ export * from './file' export * from './import' export * from './playlist' export * from './rate' +export * from './stats' export * from './transcoding' export * from './nsfw-policy.type' @@ -32,5 +33,6 @@ export * from './video-streaming-playlist.model' export * from './video-streaming-playlist.type' export * from './video-update.model' +export * from './video-view.model' export * from './video.model' export * from './video-create-result.model' diff --git a/shared/models/videos/stats/index.ts b/shared/models/videos/stats/index.ts new file mode 100644 index 000000000..d1e9c167c --- /dev/null +++ b/shared/models/videos/stats/index.ts @@ -0,0 +1,4 @@ +export * from './video-stats-overall.model' +export * from './video-stats-retention.model' +export * from './video-stats-timeserie.model' +export * from './video-stats-timeserie-metric.type' diff --git a/shared/models/videos/stats/video-stats-overall.model.ts b/shared/models/videos/stats/video-stats-overall.model.ts new file mode 100644 index 000000000..f2a0470ef --- /dev/null +++ b/shared/models/videos/stats/video-stats-overall.model.ts @@ -0,0 +1,17 @@ +export interface VideoStatsOverall { + averageWatchTime: number + totalWatchTime: number + + viewersPeak: number + viewersPeakDate: string + + views: number + likes: number + dislikes: number + comments: number + + countries: { + isoCode: string + viewers: number + }[] +} diff --git a/shared/models/videos/stats/video-stats-retention.model.ts b/shared/models/videos/stats/video-stats-retention.model.ts new file mode 100644 index 000000000..e494888ed --- /dev/null +++ b/shared/models/videos/stats/video-stats-retention.model.ts @@ -0,0 +1,6 @@ +export interface VideoStatsRetention { + data: { + second: number + retentionPercent: number + }[] +} diff --git a/shared/models/videos/stats/video-stats-timeserie-metric.type.ts b/shared/models/videos/stats/video-stats-timeserie-metric.type.ts new file mode 100644 index 000000000..fc268d083 --- /dev/null +++ b/shared/models/videos/stats/video-stats-timeserie-metric.type.ts @@ -0,0 +1 @@ +export type VideoStatsTimeserieMetric = 'viewers' | 'aggregateWatchTime' diff --git a/shared/models/videos/stats/video-stats-timeserie.model.ts b/shared/models/videos/stats/video-stats-timeserie.model.ts new file mode 100644 index 000000000..d95e34f1d --- /dev/null +++ b/shared/models/videos/stats/video-stats-timeserie.model.ts @@ -0,0 +1,6 @@ +export interface VideoStatsTimeserie { + data: { + date: string + value: number + }[] +} diff --git a/shared/models/videos/video-view.model.ts b/shared/models/videos/video-view.model.ts new file mode 100644 index 000000000..f61211104 --- /dev/null +++ b/shared/models/videos/video-view.model.ts @@ -0,0 +1,6 @@ +export type VideoViewEvent = 'seek' + +export interface VideoView { + currentTime: number + viewEvent?: VideoViewEvent +} diff --git a/shared/models/videos/video.model.ts b/shared/models/videos/video.model.ts index f98eed012..d9765dbd6 100644 --- a/shared/models/videos/video.model.ts +++ b/shared/models/videos/video.model.ts @@ -39,8 +39,7 @@ export interface Video { url: string views: number - // If live - viewers?: number + viewers: number likes: number dislikes: number diff --git a/shared/server-commands/server/server.ts b/shared/server-commands/server/server.ts index 2bf31b5a4..0ad818a11 100644 --- a/shared/server-commands/server/server.ts +++ b/shared/server-commands/server/server.ts @@ -25,10 +25,12 @@ import { PlaylistsCommand, ServicesCommand, StreamingPlaylistsCommand, + VideosCommand, VideoStudioCommand, - VideosCommand + ViewsCommand } from '../videos' import { CommentsCommand } from '../videos/comments-command' +import { VideoStatsCommand } from '../videos/video-stats-command' import { ConfigCommand } from './config-command' import { ContactFormCommand } from './contact-form-command' import { DebugCommand } from './debug-command' @@ -127,6 +129,8 @@ export class PeerTubeServer { objectStorage?: ObjectStorageCommand videoStudio?: VideoStudioCommand videos?: VideosCommand + videoStats?: VideoStatsCommand + views?: ViewsCommand constructor (options: { serverNumber: number } | { url: string }) { if ((options as any).url) { @@ -397,5 +401,7 @@ export class PeerTubeServer { this.videos = new VideosCommand(this) this.objectStorage = new ObjectStorageCommand(this) this.videoStudio = new VideoStudioCommand(this) + this.videoStats = new VideoStatsCommand(this) + this.views = new ViewsCommand(this) } } diff --git a/shared/server-commands/videos/history-command.ts b/shared/server-commands/videos/history-command.ts index e9dc63462..d27afcff2 100644 --- a/shared/server-commands/videos/history-command.ts +++ b/shared/server-commands/videos/history-command.ts @@ -3,25 +3,6 @@ import { AbstractCommand, OverrideCommandOptions } from '../shared' export class HistoryCommand extends AbstractCommand { - watchVideo (options: OverrideCommandOptions & { - videoId: number | string - currentTime: number - }) { - const { videoId, currentTime } = options - - const path = '/api/v1/videos/' + videoId + '/watching' - const fields = { currentTime } - - return this.putBodyRequest({ - ...options, - - path, - fields, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - list (options: OverrideCommandOptions & { search?: string } = {}) { diff --git a/shared/server-commands/videos/index.ts b/shared/server-commands/videos/index.ts index c9ef6134d..b861731fb 100644 --- a/shared/server-commands/videos/index.ts +++ b/shared/server-commands/videos/index.ts @@ -13,4 +13,5 @@ export * from './services-command' export * from './streaming-playlists-command' export * from './comments-command' export * from './video-studio-command' +export * from './views-command' export * from './videos-command' diff --git a/shared/server-commands/videos/video-stats-command.ts b/shared/server-commands/videos/video-stats-command.ts new file mode 100644 index 000000000..90f7ffeaf --- /dev/null +++ b/shared/server-commands/videos/video-stats-command.ts @@ -0,0 +1,48 @@ +import { HttpStatusCode, VideoStatsOverall, VideoStatsRetention, VideoStatsTimeserie, VideoStatsTimeserieMetric } from '@shared/models' +import { AbstractCommand, OverrideCommandOptions } from '../shared' + +export class VideoStatsCommand extends AbstractCommand { + + getOverallStats (options: OverrideCommandOptions & { + videoId: number | string + }) { + const path = '/api/v1/videos/' + options.videoId + '/stats/overall' + + return this.getRequestBody({ + ...options, + path, + + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getTimeserieStats (options: OverrideCommandOptions & { + videoId: number | string + metric: VideoStatsTimeserieMetric + }) { + const path = '/api/v1/videos/' + options.videoId + '/stats/timeseries/' + options.metric + + return this.getRequestBody({ + ...options, + path, + + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getRetentionStats (options: OverrideCommandOptions & { + videoId: number | string + }) { + const path = '/api/v1/videos/' + options.videoId + '/stats/retention' + + return this.getRequestBody({ + ...options, + path, + + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } +} diff --git a/shared/server-commands/videos/videos-command.ts b/shared/server-commands/videos/videos-command.ts index 21753ddc4..2ac426f76 100644 --- a/shared/server-commands/videos/videos-command.ts +++ b/shared/server-commands/videos/videos-command.ts @@ -107,23 +107,6 @@ export class VideosCommand extends AbstractCommand { // --------------------------------------------------------------------------- - view (options: OverrideCommandOptions & { - id: number | string - xForwardedFor?: string - }) { - const { id, xForwardedFor } = options - const path = '/api/v1/videos/' + id + '/views' - - return this.postBodyRequest({ - ...options, - - path, - xForwardedFor, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - rate (options: OverrideCommandOptions & { id: number | string rating: UserVideoRateType diff --git a/shared/server-commands/videos/views-command.ts b/shared/server-commands/videos/views-command.ts new file mode 100644 index 000000000..01113f798 --- /dev/null +++ b/shared/server-commands/videos/views-command.ts @@ -0,0 +1,51 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */ +import { HttpStatusCode, VideoViewEvent } from '@shared/models' +import { AbstractCommand, OverrideCommandOptions } from '../shared' + +export class ViewsCommand extends AbstractCommand { + + view (options: OverrideCommandOptions & { + id: number | string + currentTime?: number + viewEvent?: VideoViewEvent + xForwardedFor?: string + }) { + const { id, xForwardedFor, viewEvent, currentTime } = options + const path = '/api/v1/videos/' + id + '/views' + + return this.postBodyRequest({ + ...options, + + path, + xForwardedFor, + fields: { + currentTime: currentTime ?? 1, + viewEvent + }, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + async simulateView (options: OverrideCommandOptions & { + id: number | string + xForwardedFor?: string + }) { + await this.view({ ...options, currentTime: 0 }) + await this.view({ ...options, currentTime: 5 }) + } + + async simulateViewer (options: OverrideCommandOptions & { + id: number | string + currentTimes: number[] + xForwardedFor?: string + }) { + let viewEvent: VideoViewEvent = 'seek' + + for (const currentTime of options.currentTimes) { + await this.view({ ...options, currentTime, viewEvent }) + + viewEvent = undefined + } + } +} diff --git a/yarn.lock b/yarn.lock index f6fed8fc0..a72f777c1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5905,6 +5905,14 @@ math-interval-parser@^2.0.1: resolved "https://registry.yarnpkg.com/math-interval-parser/-/math-interval-parser-2.0.1.tgz#e22cd6d15a0a7f4c03aec560db76513da615bed4" integrity sha512-VmlAmb0UJwlvMyx8iPhXUDnVW1F9IrGEd9CIOmv+XL8AErCUUuozoDMrgImvnYt2A+53qVX/tPW6YJurMKYsvA== +maxmind@^4.3.6: + version "4.3.6" + resolved "https://registry.yarnpkg.com/maxmind/-/maxmind-4.3.6.tgz#5e4aa2491eef8bd401f34be307776fa1fb5bc3ca" + integrity sha512-CwnEZqJX0T6b2rWrc0/V3n9hL/hWAMEn7fY09077YJUHiHx7cn/esA2ZIz8BpYLSJUf7cGVel0oUJa9jMwyQpg== + dependencies: + mmdb-lib "2.0.2" + tiny-lru "8.0.2" + md5@^2.2.1: version "2.3.0" resolved "https://registry.yarnpkg.com/md5/-/md5-2.3.0.tgz#c3da9a6aae3a30b46b7b0c349b87b110dc3bda4f" @@ -6092,6 +6100,11 @@ mkdirp@^1.0.3: resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== +mmdb-lib@2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/mmdb-lib/-/mmdb-lib-2.0.2.tgz#fe60404142c0456c19607c72caa15821731ae957" + integrity sha512-shi1I+fCPQonhTi7qyb6hr7hi87R7YS69FlfJiMFuJ12+grx0JyL56gLNzGTYXPU7EhAPkMLliGeyHer0K+AVA== + mocha@^9.0.0: version "9.2.2" resolved "https://registry.yarnpkg.com/mocha/-/mocha-9.2.2.tgz#d70db46bdb93ca57402c809333e5a84977a88fb9" @@ -8292,6 +8305,11 @@ timm@^1.6.1: resolved "https://registry.yarnpkg.com/timm/-/timm-1.7.1.tgz#96bab60c7d45b5a10a8a4d0f0117c6b7e5aff76f" integrity sha512-IjZc9KIotudix8bMaBW6QvMuq64BrJWFs1+4V0lXwWGQZwH+LnX87doAYhem4caOEusRP9/g6jVDQmZ8XOk1nw== +tiny-lru@8.0.2: + version "8.0.2" + resolved "https://registry.yarnpkg.com/tiny-lru/-/tiny-lru-8.0.2.tgz#812fccbe6e622ded552e3ff8a4c3b5ff34a85e4c" + integrity sha512-ApGvZ6vVvTNdsmt676grvCkUCGwzG9IqXma5Z07xJgiC5L7akUMof5U8G2JTI9Rz/ovtVhJBlY6mNhEvtjzOIg== + tinycolor2@^1.4.1: version "1.4.2" resolved "https://registry.yarnpkg.com/tinycolor2/-/tinycolor2-1.4.2.tgz#3f6a4d1071ad07676d7fa472e1fac40a719d8803"