From db69d9491ee9e7fdff5919c58158745e2f6859bf Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 21 Feb 2024 13:48:52 +0100 Subject: [PATCH] Add abuse and registration requests stats --- .../+signup/+register/register.component.html | 2 +- .../+signup/+register/register.component.ts | 11 +- .../steps/register-step-about.component.html | 8 +- .../steps/register-step-about.component.ts | 6 + .../video-go-live.component.html | 2 +- .../metadata/video-attributes.component.html | 2 +- .../instance-banner.component.html | 2 +- .../instance-features-table.component.ts | 20 +- .../angular/days-duration-formatter.pipe.ts | 15 + .../app/shared/shared-main/angular/index.ts | 3 +- ...ipe.ts => time-duration-formatter.pipe.ts} | 4 +- .../shared/shared-main/shared-main.module.ts | 9 +- config/default.yaml | 9 + config/production.yaml.example | 9 + .../models/src/server/server-stats.model.ts | 8 + packages/tests/src/api/server/stats.ts | 567 +++++++++++------- server/core/controllers/api/abuse.ts | 13 +- .../controllers/api/users/registrations.ts | 4 + server/core/initializers/config.ts | 8 + server/core/initializers/constants.ts | 2 +- .../0820-abuse-registration-stats.ts | 34 ++ server/core/lib/stat-manager.ts | 34 ++ server/core/models/abuse/abuse.ts | 37 +- server/core/models/user/user-registration.ts | 36 +- support/doc/api/openapi.yaml | 20 + 25 files changed, 638 insertions(+), 227 deletions(-) create mode 100644 client/src/app/shared/shared-main/angular/days-duration-formatter.pipe.ts rename client/src/app/shared/shared-main/angular/{duration-formatter.pipe.ts => time-duration-formatter.pipe.ts} (86%) create mode 100644 server/core/initializers/migrations/0820-abuse-registration-stats.ts diff --git a/client/src/app/+signup/+register/register.component.html b/client/src/app/+signup/+register/register.component.html index 23adee02d..e035ddd14 100644 --- a/client/src/app/+signup/+register/register.component.html +++ b/client/src/app/+signup/+register/register.component.html @@ -23,7 +23,7 @@
on {{ instanceName }}
- +
diff --git a/client/src/app/+signup/+register/register.component.ts b/client/src/app/+signup/+register/register.component.ts index 396971d98..3a65bff2f 100644 --- a/client/src/app/+signup/+register/register.component.ts +++ b/client/src/app/+signup/+register/register.component.ts @@ -2,10 +2,10 @@ import { CdkStep } from '@angular/cdk/stepper' import { Component, OnInit, ViewChild } from '@angular/core' import { FormGroup } from '@angular/forms' import { ActivatedRoute } from '@angular/router' -import { AuthService } from '@app/core' +import { AuthService, ServerService } from '@app/core' import { HooksService } from '@app/core/plugins/hooks.service' import { InstanceAboutAccordionComponent } from '@app/shared/shared-instance' -import { ServerConfig, UserRegister } from '@peertube/peertube-models' +import { ServerConfig, ServerStats, UserRegister } from '@peertube/peertube-models' import { SignupService } from '../shared/signup.service' @Component({ @@ -45,12 +45,15 @@ export class RegisterComponent implements OnInit { signupDisabled = false + serverStats: ServerStats + private serverConfig: ServerConfig constructor ( private route: ActivatedRoute, private authService: AuthService, private signupService: SignupService, + private server: ServerService, private hooks: HooksService ) { } @@ -85,8 +88,10 @@ export class RegisterComponent implements OnInit { ? $localize`:Button on the registration form to finalize the account and channel creation:Signup` : this.defaultNextStepButtonLabel - this.hooks.runAction('action:signup.register.init', 'signup') + this.server.getServerStats() + .subscribe(stats => this.serverStats = stats) + this.hooks.runAction('action:signup.register.init', 'signup') } hasSameChannelAndAccountNames () { diff --git a/client/src/app/+signup/+register/steps/register-step-about.component.html b/client/src/app/+signup/+register/steps/register-step-about.component.html index 97c27a85c..bafe9ed33 100644 --- a/client/src/app/+signup/+register/steps/register-step-about.component.html +++ b/client/src/app/+signup/+register/steps/register-step-about.component.html @@ -1,4 +1,4 @@ - +

Why creating an account?

@@ -18,15 +18,15 @@

Moderators of {{ instanceName }} will have to approve your registration request once you have finished to fill the form. + + They usually respond within {{ averageResponseTime | myDaysDurationFormatter }}.

Do you use Mastodon, ActivityPub or a RSS feed aggregator?

-

- You can already follow {{ instanceName }} using your favorite tool. -

+

You can already follow {{ instanceName }} using your favorite tool.

diff --git a/client/src/app/+signup/+register/steps/register-step-about.component.ts b/client/src/app/+signup/+register/steps/register-step-about.component.ts index b176ffa59..f1223889e 100644 --- a/client/src/app/+signup/+register/steps/register-step-about.component.ts +++ b/client/src/app/+signup/+register/steps/register-step-about.component.ts @@ -1,5 +1,6 @@ import { Component, Input } from '@angular/core' import { ServerService } from '@app/core' +import { ServerStats } from '@peertube/peertube-models' @Component({ selector: 'my-register-step-about', @@ -9,6 +10,7 @@ import { ServerService } from '@app/core' export class RegisterStepAboutComponent { @Input() requiresApproval: boolean @Input() videoUploadDisabled: boolean + @Input() serverStats: ServerStats constructor (private serverService: ServerService) { @@ -17,4 +19,8 @@ export class RegisterStepAboutComponent { get instanceName () { return this.serverService.getHTMLConfig().instance.name } + + get averageResponseTime () { + return this.serverStats?.averageRegistrationRequestResponseTimeMs + } } diff --git a/client/src/app/+videos/+video-edit/video-add-components/video-go-live.component.html b/client/src/app/+videos/+video-edit/video-add-components/video-go-live.component.html index e23fd77c7..48ba38c87 100644 --- a/client/src/app/+videos/+video-edit/video-add-components/video-go-live.component.html +++ b/client/src/app/+videos/+video-edit/video-add-components/video-go-live.component.html @@ -45,7 +45,7 @@
- Max live duration is {{ getMaxLiveDuration() | myDurationFormatter }}. + Max live duration is {{ getMaxLiveDuration() | myTimeDurationFormatter }}. If your live reaches this limit, it will be automatically terminated.
diff --git a/client/src/app/+videos/+video-watch/shared/metadata/video-attributes.component.html b/client/src/app/+videos/+video-watch/shared/metadata/video-attributes.component.html index 74d31001e..54672c31a 100644 --- a/client/src/app/+videos/+video-watch/shared/metadata/video-attributes.component.html +++ b/client/src/app/+videos/+video-watch/shared/metadata/video-attributes.component.html @@ -65,7 +65,7 @@
Duration - {{ video.duration | myDurationFormatter }} + {{ video.duration | myTimeDurationFormatter }}
diff --git a/client/src/app/shared/shared-instance/instance-banner.component.html b/client/src/app/shared/shared-instance/instance-banner.component.html index 14ec89f8a..b0121cf2e 100644 --- a/client/src/app/shared/shared-instance/instance-banner.component.html +++ b/client/src/app/shared/shared-instance/instance-banner.component.html @@ -1,3 +1,3 @@ diff --git a/client/src/app/shared/shared-instance/instance-features-table.component.ts b/client/src/app/shared/shared-instance/instance-features-table.component.ts index 11c6cc0ac..ae53661b0 100644 --- a/client/src/app/shared/shared-instance/instance-features-table.component.ts +++ b/client/src/app/shared/shared-instance/instance-features-table.component.ts @@ -1,7 +1,8 @@ import { Component, OnInit } from '@angular/core' import { ServerService } from '@app/core' import { formatICU } from '@app/helpers' -import { ServerConfig } from '@peertube/peertube-models' +import { ServerConfig, ServerStats } from '@peertube/peertube-models' +import { DaysDurationFormatterPipe } from '../shared-main' @Component({ selector: 'my-instance-features-table', @@ -11,6 +12,7 @@ import { ServerConfig } from '@peertube/peertube-models' export class InstanceFeaturesTableComponent implements OnInit { quotaHelpIndication = '' serverConfig: ServerConfig + serverStats: ServerStats constructor ( private serverService: ServerService @@ -42,8 +44,12 @@ export class InstanceFeaturesTableComponent implements OnInit { this.serverService.getConfig() .subscribe(config => { this.serverConfig = config + this.buildQuotaHelpIndication() }) + + this.serverService.getServerStats() + .subscribe(stats => this.serverStats = stats) } buildNSFWLabel () { @@ -58,7 +64,17 @@ export class InstanceFeaturesTableComponent implements OnInit { const config = this.serverConfig.signup if (config.allowed !== true) return $localize`Disabled` - if (config.requiresApproval === true) return $localize`Requires approval by moderators` + + if (config.requiresApproval === true) { + const responseTimeMS = this.serverStats?.averageRegistrationRequestResponseTimeMs + + if (!responseTimeMS) { + return $localize`Requires approval by moderators` + } + + const responseTime = new DaysDurationFormatterPipe().transform(responseTimeMS) + return $localize`Requires approval by moderators (~ ${responseTime})` + } return $localize`Enabled` } diff --git a/client/src/app/shared/shared-main/angular/days-duration-formatter.pipe.ts b/client/src/app/shared/shared-main/angular/days-duration-formatter.pipe.ts new file mode 100644 index 000000000..59069d11d --- /dev/null +++ b/client/src/app/shared/shared-main/angular/days-duration-formatter.pipe.ts @@ -0,0 +1,15 @@ +import { Pipe, PipeTransform } from '@angular/core' + +@Pipe({ + name: 'myDaysDurationFormatter' +}) +export class DaysDurationFormatterPipe implements PipeTransform { + + transform (value: number): string { + const days = Math.floor(value / (3600 * 24 * 1000)) + + if (days <= 1) return $localize`1 day` + + return $localize`${days} days` + } +} diff --git a/client/src/app/shared/shared-main/angular/index.ts b/client/src/app/shared/shared-main/angular/index.ts index 6ea494129..8e1517f8f 100644 --- a/client/src/app/shared/shared-main/angular/index.ts +++ b/client/src/app/shared/shared-main/angular/index.ts @@ -1,11 +1,12 @@ export * from './auto-colspan.directive' export * from './autofocus.directive' export * from './bytes.pipe' +export * from './days-duration-formatter.pipe' export * from './defer-loading.directive' -export * from './duration-formatter.pipe' export * from './from-now.pipe' export * from './infinite-scroller.directive' export * from './link.component' export * from './login-link.component' export * from './number-formatter.pipe' export * from './peertube-template.directive' +export * from './time-duration-formatter.pipe' diff --git a/client/src/app/shared/shared-main/angular/duration-formatter.pipe.ts b/client/src/app/shared/shared-main/angular/time-duration-formatter.pipe.ts similarity index 86% rename from client/src/app/shared/shared-main/angular/duration-formatter.pipe.ts rename to client/src/app/shared/shared-main/angular/time-duration-formatter.pipe.ts index 29ff864ec..05a4f7db2 100644 --- a/client/src/app/shared/shared-main/angular/duration-formatter.pipe.ts +++ b/client/src/app/shared/shared-main/angular/time-duration-formatter.pipe.ts @@ -1,9 +1,9 @@ import { Pipe, PipeTransform } from '@angular/core' @Pipe({ - name: 'myDurationFormatter' + name: 'myTimeDurationFormatter' }) -export class DurationFormatterPipe implements PipeTransform { +export class TimeDurationFormatterPipe implements PipeTransform { transform (value: number): string { const hours = Math.floor(value / 3600) diff --git a/client/src/app/shared/shared-main/shared-main.module.ts b/client/src/app/shared/shared-main/shared-main.module.ts index 30c6cabf5..7f6b99293 100644 --- a/client/src/app/shared/shared-main/shared-main.module.ts +++ b/client/src/app/shared/shared-main/shared-main.module.ts @@ -22,7 +22,8 @@ import { AutofocusDirective, BytesPipe, DeferLoadingDirective, - DurationFormatterPipe, + TimeDurationFormatterPipe, + DaysDurationFormatterPipe, FromNowPipe, InfiniteScrollerDirective, LinkComponent, @@ -89,7 +90,8 @@ import { VideoChannelService } from './video-channel' FromNowPipe, NumberFormatterPipe, BytesPipe, - DurationFormatterPipe, + TimeDurationFormatterPipe, + DaysDurationFormatterPipe, AutofocusDirective, DeferLoadingDirective, AutoColspanDirective, @@ -152,7 +154,8 @@ import { VideoChannelService } from './video-channel' FromNowPipe, BytesPipe, NumberFormatterPipe, - DurationFormatterPipe, + TimeDurationFormatterPipe, + DaysDurationFormatterPipe, AutofocusDirective, DeferLoadingDirective, AutoColspanDirective, diff --git a/config/default.yaml b/config/default.yaml index 4ec113158..5a337341c 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -447,6 +447,15 @@ thumbnails: # Minimum value is 2 frames_to_analyze: 50 +stats: + # Display registration requests stats (average response time, total requests...) + registration_requests: + enabled: true + + # Display abuses stats (average response time, total abuses...) + abuses: + enabled: true + cache: previews: size: 500 # Max number of previews you want to cache diff --git a/config/production.yaml.example b/config/production.yaml.example index 28f4cf7a3..2cfe4e215 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -445,6 +445,15 @@ thumbnails: # Minimum value is 2 frames_to_analyze: 50 +stats: + # Display registration requests stats (average response time, total requests...) + registration_requests: + enabled: true + + # Display abuses stats (average response time, total abuses...) + abuses: + enabled: true + ############################################################################### # # From this point, almost all following keys can be overridden by the web interface diff --git a/packages/models/src/server/server-stats.model.ts b/packages/models/src/server/server-stats.model.ts index 5870ee73d..42f04ea94 100644 --- a/packages/models/src/server/server-stats.model.ts +++ b/packages/models/src/server/server-stats.model.ts @@ -36,6 +36,14 @@ export interface ServerStats extends ActivityPubMessagesSuccess, ActivityPubMess activityPubMessagesProcessedPerSecond: number totalActivityPubMessagesWaiting: number + + averageRegistrationRequestResponseTimeMs: number + totalRegistrationRequestsProcessed: number + totalRegistrationRequests: number + + averageAbuseResponseTimeMs: number + totalAbusesProcessed: number + totalAbuses: number } export interface VideosRedundancyStats { diff --git a/packages/tests/src/api/server/stats.ts b/packages/tests/src/api/server/stats.ts index 6c2d4afe1..248100fef 100644 --- a/packages/tests/src/api/server/stats.ts +++ b/packages/tests/src/api/server/stats.ts @@ -2,7 +2,7 @@ import { expect } from 'chai' import { wait } from '@peertube/peertube-core-utils' -import { ActivityType, VideoPlaylistPrivacy } from '@peertube/peertube-models' +import { AbuseState, ActivityType, VideoPlaylistPrivacy } from '@peertube/peertube-models' import { cleanupTests, createMultipleServers, @@ -17,10 +17,8 @@ import { describe('Test stats (excluding redundancy)', function () { let servers: PeerTubeServer[] = [] let channelId - const user = { - username: 'user1', - password: 'super_password' - } + const user = { username: 'user1', password: 'super_password' } + let userAccountId: number before(async function () { this.timeout(120000) @@ -33,7 +31,8 @@ describe('Test stats (excluding redundancy)', function () { await doubleFollow(servers[0], servers[1]) - await servers[0].users.create({ username: user.username, password: user.password }) + const { account } = await servers[0].users.create({ username: user.username, password: user.password }) + userAccountId = account.id const { uuid } = await servers[0].videos.upload({ attributes: { fixture: 'video_short.webm' } }) @@ -48,229 +47,393 @@ describe('Test stats (excluding redundancy)', function () { await waitJobs(servers) }) - it('Should have the correct stats on instance 1', async function () { - const data = await servers[0].stats.get() + describe('Total stats', function () { - expect(data.totalLocalVideoComments).to.equal(1) - expect(data.totalLocalVideos).to.equal(1) - expect(data.totalLocalVideoViews).to.equal(1) - expect(data.totalLocalVideoFilesSize).to.equal(218910) - expect(data.totalUsers).to.equal(2) - expect(data.totalVideoComments).to.equal(1) - expect(data.totalVideos).to.equal(1) - expect(data.totalInstanceFollowers).to.equal(2) - expect(data.totalInstanceFollowing).to.equal(1) - expect(data.totalLocalPlaylists).to.equal(0) - }) + it('Should have the correct stats on instance 1', async function () { + const data = await servers[0].stats.get() - it('Should have the correct stats on instance 2', async function () { - const data = await servers[1].stats.get() - - expect(data.totalLocalVideoComments).to.equal(0) - expect(data.totalLocalVideos).to.equal(0) - expect(data.totalLocalVideoViews).to.equal(0) - expect(data.totalLocalVideoFilesSize).to.equal(0) - expect(data.totalUsers).to.equal(1) - expect(data.totalVideoComments).to.equal(1) - expect(data.totalVideos).to.equal(1) - expect(data.totalInstanceFollowers).to.equal(1) - expect(data.totalInstanceFollowing).to.equal(1) - expect(data.totalLocalPlaylists).to.equal(0) - }) - - it('Should have the correct stats on instance 3', async function () { - const data = await servers[2].stats.get() - - expect(data.totalLocalVideoComments).to.equal(0) - expect(data.totalLocalVideos).to.equal(0) - expect(data.totalLocalVideoViews).to.equal(0) - expect(data.totalUsers).to.equal(1) - expect(data.totalVideoComments).to.equal(1) - expect(data.totalVideos).to.equal(1) - expect(data.totalInstanceFollowing).to.equal(1) - expect(data.totalInstanceFollowers).to.equal(0) - expect(data.totalLocalPlaylists).to.equal(0) - }) - - it('Should have the correct total videos stats after an unfollow', async function () { - this.timeout(15000) - - await servers[2].follows.unfollow({ target: servers[0] }) - await waitJobs(servers) - - const data = await servers[2].stats.get() - - expect(data.totalVideos).to.equal(0) - }) - - it('Should have the correct active user stats', async function () { - const server = servers[0] - - { - const data = await server.stats.get() - - expect(data.totalDailyActiveUsers).to.equal(1) - expect(data.totalWeeklyActiveUsers).to.equal(1) - expect(data.totalMonthlyActiveUsers).to.equal(1) - } - - { - await server.login.getAccessToken(user) - - const data = await server.stats.get() - - expect(data.totalDailyActiveUsers).to.equal(2) - expect(data.totalWeeklyActiveUsers).to.equal(2) - expect(data.totalMonthlyActiveUsers).to.equal(2) - } - }) - - it('Should have the correct active channel stats', async function () { - const server = servers[0] - - { - const data = await server.stats.get() - - expect(data.totalLocalVideoChannels).to.equal(2) - expect(data.totalLocalDailyActiveVideoChannels).to.equal(1) - expect(data.totalLocalWeeklyActiveVideoChannels).to.equal(1) - expect(data.totalLocalMonthlyActiveVideoChannels).to.equal(1) - } - - { - const attributes = { - name: 'stats_channel', - displayName: 'My stats channel' - } - const created = await server.channels.create({ attributes }) - channelId = created.id - - const data = await server.stats.get() - - expect(data.totalLocalVideoChannels).to.equal(3) - expect(data.totalLocalDailyActiveVideoChannels).to.equal(1) - expect(data.totalLocalWeeklyActiveVideoChannels).to.equal(1) - expect(data.totalLocalMonthlyActiveVideoChannels).to.equal(1) - } - - { - await server.videos.upload({ attributes: { fixture: 'video_short.webm', channelId } }) - - const data = await server.stats.get() - - expect(data.totalLocalVideoChannels).to.equal(3) - expect(data.totalLocalDailyActiveVideoChannels).to.equal(2) - expect(data.totalLocalWeeklyActiveVideoChannels).to.equal(2) - expect(data.totalLocalMonthlyActiveVideoChannels).to.equal(2) - } - }) - - it('Should have the correct playlist stats', async function () { - const server = servers[0] - - { - const data = await server.stats.get() + expect(data.totalLocalVideoComments).to.equal(1) + expect(data.totalLocalVideos).to.equal(1) + expect(data.totalLocalVideoViews).to.equal(1) + expect(data.totalLocalVideoFilesSize).to.equal(218910) + expect(data.totalUsers).to.equal(2) + expect(data.totalVideoComments).to.equal(1) + expect(data.totalVideos).to.equal(1) + expect(data.totalInstanceFollowers).to.equal(2) + expect(data.totalInstanceFollowing).to.equal(1) expect(data.totalLocalPlaylists).to.equal(0) - } + }) - { - await server.playlists.create({ - attributes: { - displayName: 'playlist for count', - privacy: VideoPlaylistPrivacy.PUBLIC, - videoChannelId: channelId - } - }) + it('Should have the correct stats on instance 2', async function () { + const data = await servers[1].stats.get() - const data = await server.stats.get() - expect(data.totalLocalPlaylists).to.equal(1) - } - }) + expect(data.totalLocalVideoComments).to.equal(0) + expect(data.totalLocalVideos).to.equal(0) + expect(data.totalLocalVideoViews).to.equal(0) + expect(data.totalLocalVideoFilesSize).to.equal(0) + expect(data.totalUsers).to.equal(1) + expect(data.totalVideoComments).to.equal(1) + expect(data.totalVideos).to.equal(1) + expect(data.totalInstanceFollowers).to.equal(1) + expect(data.totalInstanceFollowing).to.equal(1) + expect(data.totalLocalPlaylists).to.equal(0) + }) - it('Should correctly count video file sizes if transcoding is enabled', async function () { - this.timeout(120000) + it('Should have the correct stats on instance 3', async function () { + const data = await servers[2].stats.get() - await servers[0].config.updateCustomSubConfig({ - newConfig: { - transcoding: { - enabled: true, - webVideos: { - enabled: true - }, - hls: { - enabled: true - }, - resolutions: { - '0p': false, - '144p': false, - '240p': false, - '360p': false, - '480p': false, - '720p': false, - '1080p': false, - '1440p': false, - '2160p': false - } - } + expect(data.totalLocalVideoComments).to.equal(0) + expect(data.totalLocalVideos).to.equal(0) + expect(data.totalLocalVideoViews).to.equal(0) + expect(data.totalUsers).to.equal(1) + expect(data.totalVideoComments).to.equal(1) + expect(data.totalVideos).to.equal(1) + expect(data.totalInstanceFollowing).to.equal(1) + expect(data.totalInstanceFollowers).to.equal(0) + expect(data.totalLocalPlaylists).to.equal(0) + }) + + it('Should have the correct total videos stats after an unfollow', async function () { + this.timeout(15000) + + await servers[2].follows.unfollow({ target: servers[0] }) + await waitJobs(servers) + + const data = await servers[2].stats.get() + + expect(data.totalVideos).to.equal(0) + }) + + it('Should have the correct active user stats', async function () { + const server = servers[0] + + { + const data = await server.stats.get() + + expect(data.totalDailyActiveUsers).to.equal(1) + expect(data.totalWeeklyActiveUsers).to.equal(1) + expect(data.totalMonthlyActiveUsers).to.equal(1) + } + + { + await server.login.getAccessToken(user) + + const data = await server.stats.get() + + expect(data.totalDailyActiveUsers).to.equal(2) + expect(data.totalWeeklyActiveUsers).to.equal(2) + expect(data.totalMonthlyActiveUsers).to.equal(2) } }) - await servers[0].videos.upload({ attributes: { name: 'video', fixture: 'video_short.webm' } }) + it('Should have the correct active channel stats', async function () { + const server = servers[0] - await waitJobs(servers) + { + const data = await server.stats.get() - { - const data = await servers[1].stats.get() - expect(data.totalLocalVideoFilesSize).to.equal(0) - } + expect(data.totalLocalVideoChannels).to.equal(2) + expect(data.totalLocalDailyActiveVideoChannels).to.equal(1) + expect(data.totalLocalWeeklyActiveVideoChannels).to.equal(1) + expect(data.totalLocalMonthlyActiveVideoChannels).to.equal(1) + } - { - const data = await servers[0].stats.get() - expect(data.totalLocalVideoFilesSize).to.be.greaterThan(500000) - expect(data.totalLocalVideoFilesSize).to.be.lessThan(600000) - } + { + const attributes = { + name: 'stats_channel', + displayName: 'My stats channel' + } + const created = await server.channels.create({ attributes }) + channelId = created.id + + const data = await server.stats.get() + + expect(data.totalLocalVideoChannels).to.equal(3) + expect(data.totalLocalDailyActiveVideoChannels).to.equal(1) + expect(data.totalLocalWeeklyActiveVideoChannels).to.equal(1) + expect(data.totalLocalMonthlyActiveVideoChannels).to.equal(1) + } + + { + await server.videos.upload({ attributes: { fixture: 'video_short.webm', channelId } }) + + const data = await server.stats.get() + + expect(data.totalLocalVideoChannels).to.equal(3) + expect(data.totalLocalDailyActiveVideoChannels).to.equal(2) + expect(data.totalLocalWeeklyActiveVideoChannels).to.equal(2) + expect(data.totalLocalMonthlyActiveVideoChannels).to.equal(2) + } + }) + + it('Should have the correct playlist stats', async function () { + const server = servers[0] + + { + const data = await server.stats.get() + expect(data.totalLocalPlaylists).to.equal(0) + } + + { + await server.playlists.create({ + attributes: { + displayName: 'playlist for count', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: channelId + } + }) + + const data = await server.stats.get() + expect(data.totalLocalPlaylists).to.equal(1) + } + }) }) - it('Should have the correct AP stats', async function () { - this.timeout(240000) + describe('File sizes', function () { - await servers[0].config.disableTranscoding() + it('Should correctly count video file sizes if transcoding is enabled', async function () { + this.timeout(120000) - const first = await servers[1].stats.get() + await servers[0].config.updateCustomSubConfig({ + newConfig: { + transcoding: { + enabled: true, + webVideos: { + enabled: true + }, + hls: { + enabled: true + }, + resolutions: { + '0p': false, + '144p': false, + '240p': false, + '360p': false, + '480p': false, + '720p': false, + '1080p': false, + '1440p': false, + '2160p': false + } + } + } + }) - for (let i = 0; i < 10; i++) { - await servers[0].videos.upload({ attributes: { name: 'video' } }) - } + await servers[0].videos.upload({ attributes: { name: 'video', fixture: 'video_short.webm' } }) - await waitJobs(servers) + await waitJobs(servers) - await wait(6000) + { + const data = await servers[1].stats.get() + expect(data.totalLocalVideoFilesSize).to.equal(0) + } - const second = await servers[1].stats.get() - expect(second.totalActivityPubMessagesProcessed).to.be.greaterThan(first.totalActivityPubMessagesProcessed) + { + const data = await servers[0].stats.get() + expect(data.totalLocalVideoFilesSize).to.be.greaterThan(500000) + expect(data.totalLocalVideoFilesSize).to.be.lessThan(600000) + } + }) + }) - const apTypes: ActivityType[] = [ - 'Create', 'Update', 'Delete', 'Follow', 'Accept', 'Announce', 'Undo', 'Like', 'Reject', 'View', 'Dislike', 'Flag' - ] + describe('ActivityPub', function () { - const processed = apTypes.reduce( - (previous, type) => previous + second['totalActivityPub' + type + 'MessagesSuccesses'], - 0 - ) - expect(second.totalActivityPubMessagesProcessed).to.equal(processed) - expect(second.totalActivityPubMessagesSuccesses).to.equal(processed) + it('Should have the correct AP stats', async function () { + this.timeout(240000) - expect(second.totalActivityPubMessagesErrors).to.equal(0) + await servers[0].config.disableTranscoding() - for (const apType of apTypes) { - expect(second['totalActivityPub' + apType + 'MessagesErrors']).to.equal(0) - } + const first = await servers[1].stats.get() - await wait(6000) + for (let i = 0; i < 10; i++) { + await servers[0].videos.upload({ attributes: { name: 'video' } }) + } - const third = await servers[1].stats.get() - expect(third.totalActivityPubMessagesWaiting).to.equal(0) - expect(third.activityPubMessagesProcessedPerSecond).to.be.lessThan(second.activityPubMessagesProcessedPerSecond) + await waitJobs(servers) + + await wait(6000) + + const second = await servers[1].stats.get() + expect(second.totalActivityPubMessagesProcessed).to.be.greaterThan(first.totalActivityPubMessagesProcessed) + + const apTypes: ActivityType[] = [ + 'Create', 'Update', 'Delete', 'Follow', 'Accept', 'Announce', 'Undo', 'Like', 'Reject', 'View', 'Dislike', 'Flag' + ] + + const processed = apTypes.reduce( + (previous, type) => previous + second['totalActivityPub' + type + 'MessagesSuccesses'], + 0 + ) + expect(second.totalActivityPubMessagesProcessed).to.equal(processed) + expect(second.totalActivityPubMessagesSuccesses).to.equal(processed) + + expect(second.totalActivityPubMessagesErrors).to.equal(0) + + for (const apType of apTypes) { + expect(second['totalActivityPub' + apType + 'MessagesErrors']).to.equal(0) + } + + await wait(6000) + + const third = await servers[1].stats.get() + expect(third.totalActivityPubMessagesWaiting).to.equal(0) + expect(third.activityPubMessagesProcessedPerSecond).to.be.lessThan(second.activityPubMessagesProcessedPerSecond) + }) + }) + + describe('User registration requests stats', function () { + let id2: number + let beforeTimestamp: number + let lastResponseTime: number + + before(async function () { + await servers[0].config.enableSignup(true) + }) + + it('Should not have registration requests stats available', async function () { + const data = await servers[0].stats.get() + + expect(data.totalRegistrationRequests).to.equal(0) + expect(data.totalRegistrationRequestsProcessed).to.equal(0) + expect(data.averageRegistrationRequestResponseTimeMs).to.be.null + }) + + it('Should create registration requests, accept one and have correct stats', async function () { + beforeTimestamp = new Date().getTime() + + const { id: id1 } = await servers[0].registrations.requestRegistration({ username: 'user2', registrationReason: 'reason 1' }); + ({ id: id2 } = await servers[0].registrations.requestRegistration({ username: 'user3', registrationReason: 'reason 2' })) + await servers[0].registrations.requestRegistration({ username: 'user4', registrationReason: 'reason 3' }) + + await wait(1500) + + await servers[0].registrations.accept({ id: id1, moderationResponse: 'thanks' }) + + const middleTimestamp = new Date().getTime() + + { + const data = await servers[0].stats.get() + + expect(data.totalRegistrationRequests).to.equal(3) + expect(data.totalRegistrationRequestsProcessed).to.equal(1) + + expect(data.averageRegistrationRequestResponseTimeMs).to.be.greaterThan(1000) + expect(data.averageRegistrationRequestResponseTimeMs).to.be.below(middleTimestamp - beforeTimestamp) + + lastResponseTime = data.averageRegistrationRequestResponseTimeMs + } + }) + + it('Should accept another request and update stats', async function () { + await wait(1500) + + await servers[0].registrations.accept({ id: id2, moderationResponse: 'thanks' }) + + const lastTimestamp = new Date().getTime() + + { + const data = await servers[0].stats.get() + + expect(data.totalRegistrationRequests).to.equal(3) + expect(data.totalRegistrationRequestsProcessed).to.equal(2) + + expect(data.averageRegistrationRequestResponseTimeMs).to.be.greaterThan(lastResponseTime) + expect(data.averageRegistrationRequestResponseTimeMs).to.be.below(lastTimestamp - beforeTimestamp) + } + }) + }) + + describe('Abuses stats', function () { + let abuse2: number + let videoId: number + let commentId: number + let beforeTimestamp: number + let lastResponseTime: number + let userToken: string + + before(async function () { + userToken = await servers[0].users.generateUserAndToken('reporter'); + + ({ id: videoId } = await servers[0].videos.quickUpload({ name: 'to_report' })); + ({ id: commentId } = await servers[0].comments.createThread({ videoId, text: 'text' })) + }) + + it('Should not abuses stats available', async function () { + const data = await servers[0].stats.get() + + expect(data.totalAbuses).to.equal(0) + expect(data.totalAbusesProcessed).to.equal(0) + expect(data.averageAbuseResponseTimeMs).to.be.null + }) + + it('Should create abuses, process one and have correct stats', async function () { + beforeTimestamp = new Date().getTime() + + const { abuse: abuse1 } = await servers[0].abuses.report({ videoId, token: userToken, reason: 'abuse reason' }); + ({ abuse: { id: abuse2 } } = await servers[0].abuses.report({ accountId: userAccountId, token: userToken, reason: 'abuse reason' })) + await servers[0].abuses.report({ commentId, token: userToken, reason: 'abuse reason' }) + + await wait(1500) + + await servers[0].abuses.update({ abuseId: abuse1.id, body: { state: AbuseState.REJECTED } }) + + const middleTimestamp = new Date().getTime() + + { + const data = await servers[0].stats.get() + + expect(data.totalAbuses).to.equal(3) + expect(data.totalAbusesProcessed).to.equal(1) + + expect(data.averageAbuseResponseTimeMs).to.be.greaterThan(1000) + expect(data.averageAbuseResponseTimeMs).to.be.below(middleTimestamp - beforeTimestamp) + + lastResponseTime = data.averageAbuseResponseTimeMs + } + }) + + it('Should accept another request and update stats', async function () { + await wait(1500) + + await servers[0].abuses.addMessage({ abuseId: abuse2, message: 'my message' }) + + const lastTimestamp = new Date().getTime() + + { + const data = await servers[0].stats.get() + + expect(data.totalAbuses).to.equal(3) + expect(data.totalAbusesProcessed).to.equal(2) + + expect(data.averageAbuseResponseTimeMs).to.be.greaterThan(lastResponseTime) + expect(data.averageAbuseResponseTimeMs).to.be.below(lastTimestamp - beforeTimestamp) + } + }) + }) + + describe('Disabling stats', async function () { + + it('Should disable registration requests and abuses stats', async function () { + this.timeout(60000) + + await servers[0].kill() + await servers[0].run({ + stats: { + registration_requests: { enabled: false }, + abuses: { enabled: false } + } + }) + + const data = await servers[0].stats.get() + + expect(data.totalRegistrationRequests).to.be.null + expect(data.totalRegistrationRequestsProcessed).to.be.null + expect(data.averageRegistrationRequestResponseTimeMs).to.be.null + + expect(data.totalAbuses).to.be.null + expect(data.totalAbusesProcessed).to.be.null + expect(data.averageAbuseResponseTimeMs).to.be.null + }) }) after(async function () { diff --git a/server/core/controllers/api/abuse.ts b/server/core/controllers/api/abuse.ts index 78fd514c6..3734246ea 100644 --- a/server/core/controllers/api/abuse.ts +++ b/server/core/controllers/api/abuse.ts @@ -131,6 +131,10 @@ async function updateAbuse (req: express.Request, res: express.Response) { if (req.body.state !== undefined) { abuse.state = req.body.state + + // We consider the abuse has been processed when its state change + if (!abuse.processedAt) abuse.processedAt = new Date() + stateUpdated = true } @@ -229,14 +233,21 @@ async function listAbuseMessages (req: express.Request, res: express.Response) { async function addAbuseMessage (req: express.Request, res: express.Response) { const abuse = res.locals.abuse const user = res.locals.oauth.token.user + const byModerator = abuse.reporterAccountId !== user.Account.id const abuseMessage = await AbuseMessageModel.create({ message: req.body.message, - byModerator: abuse.reporterAccountId !== user.Account.id, + byModerator, accountId: user.Account.id, abuseId: abuse.id }) + // If a moderator created an abuse message, we consider it as processed + if (byModerator && !abuse.processedAt) { + abuse.processedAt = new Date() + await abuse.save() + } + AbuseModel.loadFull(abuse.id) .then(abuseFull => Notifier.Instance.notifyOnAbuseMessage(abuseFull, abuseMessage)) .catch(err => logger.error('Cannot notify on new abuse message', { err })) diff --git a/server/core/controllers/api/users/registrations.ts b/server/core/controllers/api/users/registrations.ts index 6c2a309e8..e38b215a7 100644 --- a/server/core/controllers/api/users/registrations.ts +++ b/server/core/controllers/api/users/registrations.ts @@ -160,6 +160,8 @@ async function acceptRegistration (req: express.Request, res: express.Response) registration.state = UserRegistrationState.ACCEPTED registration.moderationResponse = body.moderationResponse + if (!registration.processedAt) registration.processedAt = new Date() + await registration.save() logger.info('Registration of %s accepted', registration.username) @@ -178,6 +180,8 @@ async function rejectRegistration (req: express.Request, res: express.Response) registration.state = UserRegistrationState.REJECTED registration.moderationResponse = body.moderationResponse + if (!registration.processedAt) registration.processedAt = new Date() + await registration.save() if (body.preventEmailDelivery !== true) { diff --git a/server/core/initializers/config.ts b/server/core/initializers/config.ts index 99e120ff6..b13de5e97 100644 --- a/server/core/initializers/config.ts +++ b/server/core/initializers/config.ts @@ -363,6 +363,14 @@ const CONFIG = { FRAMES_TO_ANALYZE: config.get('thumbnails.generation_from_video.frames_to_analyze') } }, + STATS: { + REGISTRATION_REQUESTS: { + ENABLED: config.get('stats.registration_requests.enabled') + }, + ABUSES: { + ENABLED: config.get('stats.abuses.enabled') + } + }, ADMIN: { get EMAIL () { return config.get('admin.email') } }, diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts index 143b586d0..26d9bc115 100644 --- a/server/core/initializers/constants.ts +++ b/server/core/initializers/constants.ts @@ -45,7 +45,7 @@ import { cpus } from 'os' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 815 +const LAST_MIGRATION_VERSION = 820 // --------------------------------------------------------------------------- diff --git a/server/core/initializers/migrations/0820-abuse-registration-stats.ts b/server/core/initializers/migrations/0820-abuse-registration-stats.ts new file mode 100644 index 000000000..07fecfec6 --- /dev/null +++ b/server/core/initializers/migrations/0820-abuse-registration-stats.ts @@ -0,0 +1,34 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize +}): Promise { + { + const data = { + type: Sequelize.DATE, + defaultValue: null, + allowNull: true + } + await utils.queryInterface.addColumn('userRegistration', 'processedAt', data) + } + + { + const data = { + type: Sequelize.DATE, + defaultValue: null, + allowNull: true + } + await utils.queryInterface.addColumn('abuse', 'processedAt', data) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/core/lib/stat-manager.ts b/server/core/lib/stat-manager.ts index f27266571..2d9176b24 100644 --- a/server/core/lib/stat-manager.ts +++ b/server/core/lib/stat-manager.ts @@ -9,6 +9,9 @@ import { VideoCommentModel } from '@server/models/video/video-comment.js' import { VideoFileModel } from '@server/models/video/video-file.js' import { VideoPlaylistModel } from '@server/models/video/video-playlist.js' import { ActivityType, ServerStats, VideoRedundancyStrategyWithManual } from '@peertube/peertube-models' +import { UserRegistrationModel } from '@server/models/user/user-registration.js' +import { AbuseModel } from '@server/models/abuse/abuse.js' +import { pick } from '@peertube/peertube-core-utils' class StatsManager { @@ -85,6 +88,9 @@ class StatsManager { videosRedundancy: videosRedundancyStats, + ...await this.buildAbuseStats(), + ...await this.buildRegistrationRequestsStats(), + ...this.buildAPStats() } @@ -170,6 +176,34 @@ class StatsManager { } } + private async buildRegistrationRequestsStats () { + if (!CONFIG.STATS.REGISTRATION_REQUESTS.ENABLED) { + return { + averageRegistrationRequestResponseTimeMs: null, + totalRegistrationRequests: null, + totalRegistrationRequestsProcessed: null + } + } + + const res = await UserRegistrationModel.getStats() + + return pick(res, [ 'averageRegistrationRequestResponseTimeMs', 'totalRegistrationRequests', 'totalRegistrationRequestsProcessed' ]) + } + + private async buildAbuseStats () { + if (!CONFIG.STATS.ABUSES.ENABLED) { + return { + averageAbuseResponseTimeMs: null, + totalAbuses: null, + totalAbusesProcessed: null + } + } + + const res = await AbuseModel.getStats() + + return pick(res, [ 'averageAbuseResponseTimeMs', 'totalAbuses', 'totalAbusesProcessed' ]) + } + static get Instance () { return this.instance || (this.instance = new this()) } diff --git a/server/core/models/abuse/abuse.ts b/server/core/models/abuse/abuse.ts index 8a8e292e4..fa33369b6 100644 --- a/server/core/models/abuse/abuse.ts +++ b/server/core/models/abuse/abuse.ts @@ -1,4 +1,4 @@ -import { abusePredefinedReasonsMap } from '@peertube/peertube-core-utils' +import { abusePredefinedReasonsMap, forceNumber } from '@peertube/peertube-core-utils' import { AbuseFilter, AbuseObject, @@ -41,7 +41,7 @@ import { MUserAccountId } from '../../types/models/index.js' import { AccountModel, ScopeNames as AccountScopeNames, SummaryOptions as AccountSummaryOptions } from '../account/account.js' -import { getSort, throwIfNotValid } from '../shared/index.js' +import { getSort, parseAggregateResult, throwIfNotValid } from '../shared/index.js' import { ThumbnailModel } from '../video/thumbnail.js' import { VideoBlacklistModel } from '../video/video-blacklist.js' import { SummaryOptions as ChannelSummaryOptions, VideoChannelModel, ScopeNames as VideoChannelScopeNames } from '../video/video-channel.js' @@ -220,6 +220,10 @@ export class AbuseModel extends Model>> { @Column(DataType.ARRAY(DataType.INTEGER)) predefinedReasons: AbusePredefinedReasonsType[] + @AllowNull(true) + @Column + processedAt: Date + @CreatedAt createdAt: Date @@ -441,6 +445,35 @@ export class AbuseModel extends Model>> { return { total, data } } + // --------------------------------------------------------------------------- + + static getStats () { + const query = `SELECT ` + + `AVG(EXTRACT(EPOCH FROM ("processedAt" - "createdAt") * 1000)) ` + + `FILTER (WHERE "processedAt" IS NOT NULL AND "createdAt" > CURRENT_DATE - INTERVAL '3 months')` + + `AS "avgResponseTime", ` + + `COUNT(*) FILTER (WHERE "processedAt" IS NOT NULL) AS "processedAbuses", ` + + `COUNT(*) AS "totalAbuses" ` + + `FROM "abuse"` + + return AbuseModel.sequelize.query(query, { + type: QueryTypes.SELECT, + raw: true + }).then(([ row ]) => { + return { + totalAbuses: parseAggregateResult(row.totalAbuses), + + totalAbusesProcessed: parseAggregateResult(row.processedAbuses), + + averageAbuseResponseTimeMs: row?.avgResponseTime + ? forceNumber(row.avgResponseTime) + : null + } + }) + } + + // --------------------------------------------------------------------------- + buildBaseVideoCommentAbuse (this: MAbuseUserFormattable) { // Associated video comment could have been destroyed if the video has been deleted if (!this.VideoCommentAbuse?.VideoComment) return null diff --git a/server/core/models/user/user-registration.ts b/server/core/models/user/user-registration.ts index c4bf50b1d..b9c19fa8e 100644 --- a/server/core/models/user/user-registration.ts +++ b/server/core/models/user/user-registration.ts @@ -9,7 +9,7 @@ import { isVideoChannelDisplayNameValid } from '@server/helpers/custom-validator import { cryptPassword } from '@server/helpers/peertube-crypto.js' import { USER_REGISTRATION_STATES } from '@server/initializers/constants.js' import { MRegistration, MRegistrationFormattable } from '@server/types/models/index.js' -import { FindOptions, Op, WhereOptions } from 'sequelize' +import { FindOptions, Op, QueryTypes, WhereOptions } from 'sequelize' import { AllowNull, BeforeCreate, @@ -25,8 +25,9 @@ import { UpdatedAt } from 'sequelize-typescript' import { isUserDisplayNameValid, isUserEmailVerifiedValid, isUserPasswordValid } from '../../helpers/custom-validators/users.js' -import { getSort, throwIfNotValid } from '../shared/index.js' +import { getSort, parseAggregateResult, throwIfNotValid } from '../shared/index.js' import { UserModel } from './user.js' +import { forceNumber } from '@peertube/peertube-core-utils' @Table({ tableName: 'userRegistration', @@ -100,6 +101,10 @@ export class UserRegistrationModel extends Model CURRENT_DATE - INTERVAL '3 months')` + + `AS "avgResponseTime", ` + + `COUNT(*) FILTER (WHERE "processedAt" IS NOT NULL) AS "processedRequests", ` + + `COUNT(*) AS "totalRequests" ` + + `FROM "userRegistration"` + + return UserRegistrationModel.sequelize.query(query, { + type: QueryTypes.SELECT, + raw: true + }).then(([ row ]) => { + return { + totalRegistrationRequests: parseAggregateResult(row.totalRequests), + + totalRegistrationRequestsProcessed: parseAggregateResult(row.processedRequests), + + averageRegistrationRequestResponseTimeMs: row?.avgResponseTime + ? forceNumber(row.avgResponseTime) + : null + } + }) + } + + // --------------------------------------------------------------------------- + toFormattedJSON (this: MRegistrationFormattable): UserRegistration { return { id: this.id, diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yaml index fed1ad869..8c9782839 100644 --- a/support/doc/api/openapi.yaml +++ b/support/doc/api/openapi.yaml @@ -8580,6 +8580,26 @@ components: totalActivityPubMessagesWaiting: type: number + averageRegistrationRequestResponseTimeMs: + type: number + description: "**PeerTube >= 6.1** Value is null if the admin disabled registration requests stats" + totalRegistrationRequestsProcessed: + type: number + description: "**PeerTube >= 6.1** Value is null if the admin disabled registration requests stats" + totalRegistrationRequests: + type: number + description: "**PeerTube >= 6.1** Value is null if the admin disabled registration requests stats" + + averageAbuseResponseTimeMs: + type: number + description: "**PeerTube >= 6.1** Value is null if the admin disabled abuses stats" + totalAbusesProcessed: + type: number + description: "**PeerTube >= 6.1** Value is null if the admin disabled abuses stats" + totalAbuses: + type: number + description: "**PeerTube >= 6.1** Value is null if the admin disabled abuses stats" + ServerConfigAbout: properties: instance: