add channel and playlist stats to server stats endpoint (#3747)

* add channel and playlist stats to nodeinfo

* add tests for active video channels stats

* fix tests for active channel stats
pull/3955/head
Rigel Kent 2021-04-12 11:19:07 +02:00 committed by GitHub
parent a472cf0330
commit fe19f600da
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 193 additions and 24 deletions

View File

@ -3,8 +3,10 @@ import { UserModel } from '@server/models/account/user'
import { ActorFollowModel } from '@server/models/activitypub/actor-follow' import { ActorFollowModel } from '@server/models/activitypub/actor-follow'
import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy' import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy'
import { VideoModel } from '@server/models/video/video' import { VideoModel } from '@server/models/video/video'
import { VideoChannelModel } from '@server/models/video/video-channel'
import { VideoCommentModel } from '@server/models/video/video-comment' import { VideoCommentModel } from '@server/models/video/video-comment'
import { VideoFileModel } from '@server/models/video/video-file' import { VideoFileModel } from '@server/models/video/video-file'
import { VideoPlaylistModel } from '@server/models/video/video-playlist'
import { ActivityType, ServerStats, VideoRedundancyStrategyWithManual } from '@shared/models' import { ActivityType, ServerStats, VideoRedundancyStrategyWithManual } from '@shared/models'
class StatsManager { class StatsManager {
@ -46,22 +48,37 @@ class StatsManager {
const { totalUsers, totalDailyActiveUsers, totalWeeklyActiveUsers, totalMonthlyActiveUsers } = await UserModel.getStats() const { totalUsers, totalDailyActiveUsers, totalWeeklyActiveUsers, totalMonthlyActiveUsers } = await UserModel.getStats()
const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats()
const { totalLocalVideoFilesSize } = await VideoFileModel.getStats() const { totalLocalVideoFilesSize } = await VideoFileModel.getStats()
const {
totalLocalVideoChannels,
totalLocalDailyActiveVideoChannels,
totalLocalWeeklyActiveVideoChannels,
totalLocalMonthlyActiveVideoChannels
} = await VideoChannelModel.getStats()
const { totalLocalPlaylists } = await VideoPlaylistModel.getStats()
const videosRedundancyStats = await this.buildRedundancyStats() const videosRedundancyStats = await this.buildRedundancyStats()
const data: ServerStats = { const data: ServerStats = {
totalLocalVideos,
totalLocalVideoViews,
totalLocalVideoFilesSize,
totalLocalVideoComments,
totalVideos,
totalVideoComments,
totalUsers, totalUsers,
totalDailyActiveUsers, totalDailyActiveUsers,
totalWeeklyActiveUsers, totalWeeklyActiveUsers,
totalMonthlyActiveUsers, totalMonthlyActiveUsers,
totalLocalVideos,
totalLocalVideoViews,
totalLocalVideoComments,
totalLocalVideoFilesSize,
totalVideos,
totalVideoComments,
totalLocalVideoChannels,
totalLocalDailyActiveVideoChannels,
totalLocalWeeklyActiveVideoChannels,
totalLocalMonthlyActiveVideoChannels,
totalLocalPlaylists,
totalInstanceFollowers, totalInstanceFollowers,
totalInstanceFollowing, totalInstanceFollowing,

View File

@ -1,4 +1,4 @@
import { FindOptions, Includeable, literal, Op, ScopeOptions } from 'sequelize' import { FindOptions, Includeable, literal, Op, QueryTypes, ScopeOptions } from 'sequelize'
import { import {
AllowNull, AllowNull,
BeforeDestroy, BeforeDestroy,
@ -338,6 +338,47 @@ export class VideoChannelModel extends Model {
return VideoChannelModel.count(query) return VideoChannelModel.count(query)
} }
static async getStats () {
function getActiveVideoChannels (days: number) {
const options = {
type: QueryTypes.SELECT as QueryTypes.SELECT,
raw: true
}
const query = `
SELECT COUNT(DISTINCT("VideoChannelModel"."id")) AS "count"
FROM "videoChannel" AS "VideoChannelModel"
INNER JOIN "video" AS "Videos"
ON "VideoChannelModel"."id" = "Videos"."channelId"
AND ("Videos"."publishedAt" > Now() - interval '${days}d')
INNER JOIN "account" AS "Account"
ON "VideoChannelModel"."accountId" = "Account"."id"
INNER JOIN "actor" AS "Account->Actor"
ON "Account"."actorId" = "Account->Actor"."id"
AND "Account->Actor"."serverId" IS NULL
LEFT OUTER JOIN "server" AS "Account->Actor->Server"
ON "Account->Actor"."serverId" = "Account->Actor->Server"."id"`
return VideoChannelModel.sequelize.query<{ count: string }>(query, options)
.then(r => parseInt(r[0].count, 10))
}
const totalLocalVideoChannels = await VideoChannelModel.count()
const totalLocalDailyActiveVideoChannels = await getActiveVideoChannels(1)
const totalLocalWeeklyActiveVideoChannels = await getActiveVideoChannels(7)
const totalLocalMonthlyActiveVideoChannels = await getActiveVideoChannels(30)
const totalHalfYearActiveVideoChannels = await getActiveVideoChannels(180)
return {
totalLocalVideoChannels,
totalLocalDailyActiveVideoChannels,
totalLocalWeeklyActiveVideoChannels,
totalLocalMonthlyActiveVideoChannels,
totalHalfYearActiveVideoChannels
}
}
static listForApi (parameters: { static listForApi (parameters: {
actorId: number actorId: number
start: number start: number

View File

@ -54,6 +54,7 @@ import { buildServerIdsFollowedBy, buildWhereIdOrUUID, getPlaylistSort, isOutdat
import { ThumbnailModel } from './thumbnail' import { ThumbnailModel } from './thumbnail'
import { ScopeNames as VideoChannelScopeNames, VideoChannelModel } from './video-channel' import { ScopeNames as VideoChannelScopeNames, VideoChannelModel } from './video-channel'
import { VideoPlaylistElementModel } from './video-playlist-element' import { VideoPlaylistElementModel } from './video-playlist-element'
import { ActorModel } from '../activitypub/actor'
enum ScopeNames { enum ScopeNames {
AVAILABLE_FOR_LIST = 'AVAILABLE_FOR_LIST', AVAILABLE_FOR_LIST = 'AVAILABLE_FOR_LIST',
@ -65,7 +66,7 @@ enum ScopeNames {
} }
type AvailableForListOptions = { type AvailableForListOptions = {
followerActorId: number followerActorId?: number
type?: VideoPlaylistType type?: VideoPlaylistType
accountId?: number accountId?: number
videoChannelId?: number videoChannelId?: number
@ -134,20 +135,26 @@ type AvailableForListOptions = {
privacy: VideoPlaylistPrivacy.PUBLIC privacy: VideoPlaylistPrivacy.PUBLIC
}) })
// Only list local playlists OR playlists that are on an instance followed by actorId // Only list local playlists
const inQueryInstanceFollow = buildServerIdsFollowedBy(options.followerActorId) const whereActorOr: WhereOptions[] = [
whereActor = {
[Op.or]: [
{ {
serverId: null serverId: null
}, }
{ ]
// … OR playlists that are on an instance followed by actorId
if (options.followerActorId) {
const inQueryInstanceFollow = buildServerIdsFollowedBy(options.followerActorId)
whereActorOr.push({
serverId: { serverId: {
[Op.in]: literal(inQueryInstanceFollow) [Op.in]: literal(inQueryInstanceFollow)
} }
})
} }
]
whereActor = {
[Op.or]: whereActorOr
} }
} }
@ -495,6 +502,33 @@ export class VideoPlaylistModel extends Model {
return '/video-playlists/embed/' + this.uuid return '/video-playlists/embed/' + this.uuid
} }
static async getStats () {
const totalLocalPlaylists = await VideoPlaylistModel.count({
include: [
{
model: AccountModel,
required: true,
include: [
{
model: ActorModel,
required: true,
where: {
serverId: null
}
}
]
}
],
where: {
privacy: VideoPlaylistPrivacy.PUBLIC
}
})
return {
totalLocalPlaylists
}
}
setAsRefreshed () { setAsRefreshed () {
this.changed('updatedAt', true) this.changed('updatedAt', true)

View File

@ -3,8 +3,10 @@
import 'mocha' import 'mocha'
import * as chai from 'chai' import * as chai from 'chai'
import { import {
addVideoChannel,
cleanupTests, cleanupTests,
createUser, createUser,
createVideoPlaylist,
doubleFollow, doubleFollow,
flushAndRunMultipleServers, flushAndRunMultipleServers,
follow, follow,
@ -21,12 +23,14 @@ import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
import { getStats } from '../../../../shared/extra-utils/server/stats' import { getStats } from '../../../../shared/extra-utils/server/stats'
import { addVideoCommentThread } from '../../../../shared/extra-utils/videos/video-comments' import { addVideoCommentThread } from '../../../../shared/extra-utils/videos/video-comments'
import { ServerStats } from '../../../../shared/models/server/server-stats.model' import { ServerStats } from '../../../../shared/models/server/server-stats.model'
import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model'
import { ActivityType } from '@shared/models' import { ActivityType } from '@shared/models'
const expect = chai.expect const expect = chai.expect
describe('Test stats (excluding redundancy)', function () { describe('Test stats (excluding redundancy)', function () {
let servers: ServerInfo[] = [] let servers: ServerInfo[] = []
let channelId
const user = { const user = {
username: 'user1', username: 'user1',
password: 'super_password' password: 'super_password'
@ -70,6 +74,7 @@ describe('Test stats (excluding redundancy)', function () {
expect(data.totalVideos).to.equal(1) expect(data.totalVideos).to.equal(1)
expect(data.totalInstanceFollowers).to.equal(2) expect(data.totalInstanceFollowers).to.equal(2)
expect(data.totalInstanceFollowing).to.equal(1) expect(data.totalInstanceFollowing).to.equal(1)
expect(data.totalLocalPlaylists).to.equal(0)
}) })
it('Should have the correct stats on instance 2', async function () { it('Should have the correct stats on instance 2', async function () {
@ -85,6 +90,7 @@ describe('Test stats (excluding redundancy)', function () {
expect(data.totalVideos).to.equal(1) expect(data.totalVideos).to.equal(1)
expect(data.totalInstanceFollowers).to.equal(1) expect(data.totalInstanceFollowers).to.equal(1)
expect(data.totalInstanceFollowing).to.equal(1) expect(data.totalInstanceFollowing).to.equal(1)
expect(data.totalLocalPlaylists).to.equal(0)
}) })
it('Should have the correct stats on instance 3', async function () { it('Should have the correct stats on instance 3', async function () {
@ -99,6 +105,7 @@ describe('Test stats (excluding redundancy)', function () {
expect(data.totalVideos).to.equal(1) expect(data.totalVideos).to.equal(1)
expect(data.totalInstanceFollowing).to.equal(1) expect(data.totalInstanceFollowing).to.equal(1)
expect(data.totalInstanceFollowers).to.equal(0) expect(data.totalInstanceFollowers).to.equal(0)
expect(data.totalLocalPlaylists).to.equal(0)
}) })
it('Should have the correct total videos stats after an unfollow', async function () { it('Should have the correct total videos stats after an unfollow', async function () {
@ -113,7 +120,7 @@ describe('Test stats (excluding redundancy)', function () {
expect(data.totalVideos).to.equal(0) expect(data.totalVideos).to.equal(0)
}) })
it('Should have the correct active users stats', async function () { it('Should have the correct active user stats', async function () {
const server = servers[0] const server = servers[0]
{ {
@ -135,6 +142,69 @@ describe('Test stats (excluding redundancy)', function () {
} }
}) })
it('Should have the correct active channel stats', async function () {
const server = servers[0]
{
const res = await getStats(server.url)
const data: ServerStats = res.body
expect(data.totalLocalDailyActiveVideoChannels).to.equal(1)
expect(data.totalLocalWeeklyActiveVideoChannels).to.equal(1)
expect(data.totalLocalMonthlyActiveVideoChannels).to.equal(1)
}
{
const channelAttributes = {
name: 'stats_channel',
displayName: 'My stats channel'
}
const resChannel = await addVideoChannel(server.url, server.accessToken, channelAttributes)
channelId = resChannel.body.videoChannel.id
const res = await getStats(server.url)
const data: ServerStats = res.body
expect(data.totalLocalDailyActiveVideoChannels).to.equal(1)
expect(data.totalLocalWeeklyActiveVideoChannels).to.equal(1)
expect(data.totalLocalMonthlyActiveVideoChannels).to.equal(1)
}
{
await uploadVideo(server.url, server.accessToken, { fixture: 'video_short.webm', channelId })
const res = await getStats(server.url)
const data: ServerStats = res.body
expect(data.totalLocalDailyActiveVideoChannels).to.equal(2)
expect(data.totalLocalWeeklyActiveVideoChannels).to.equal(2)
expect(data.totalLocalMonthlyActiveVideoChannels).to.equal(2)
}
})
it('Should have the correct playlist stats', async function () {
const server = servers[0]
{
const resStats = await getStats(server.url)
const dataStats: ServerStats = resStats.body
expect(dataStats.totalLocalPlaylists).to.equal(0)
}
{
await createVideoPlaylist({
url: server.url,
token: server.accessToken,
playlistAttrs: {
displayName: 'playlist for count',
privacy: VideoPlaylistPrivacy.PUBLIC,
videoChannelId: channelId
}
})
const resStats = await getStats(server.url)
const dataStats: ServerStats = resStats.body
expect(dataStats.totalLocalPlaylists).to.equal(1)
}
})
it('Should correctly count video file sizes if transcoding is enabled', async function () { it('Should correctly count video file sizes if transcoding is enabled', async function () {
this.timeout(60000) this.timeout(60000)
@ -173,8 +243,8 @@ describe('Test stats (excluding redundancy)', function () {
{ {
const res = await getStats(servers[0].url) const res = await getStats(servers[0].url)
const data: ServerStats = res.body const data: ServerStats = res.body
expect(data.totalLocalVideoFilesSize).to.be.greaterThan(300000) expect(data.totalLocalVideoFilesSize).to.be.greaterThan(500000)
expect(data.totalLocalVideoFilesSize).to.be.lessThan(400000) expect(data.totalLocalVideoFilesSize).to.be.lessThan(600000)
} }
}) })

View File

@ -13,6 +13,13 @@ export interface ServerStats {
totalVideos: number totalVideos: number
totalVideoComments: number totalVideoComments: number
totalLocalVideoChannels: number
totalLocalDailyActiveVideoChannels: number
totalLocalWeeklyActiveVideoChannels: number
totalLocalMonthlyActiveVideoChannels: number
totalLocalPlaylists: number
totalInstanceFollowers: number totalInstanceFollowers: number
totalInstanceFollowing: number totalInstanceFollowing: number