From b764380ac23f4e9d4677d08acdc3474c2931a16d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 10 Jan 2020 10:11:28 +0100 Subject: [PATCH] Add ability to list redundancies --- client/package.json | 1 + client/src/app/+admin/admin.component.html | 2 +- client/src/app/+admin/admin.module.ts | 13 +- .../app/+admin/follows/follows.component.html | 6 +- .../src/app/+admin/follows/follows.routes.ts | 5 + client/src/app/+admin/follows/index.ts | 1 + .../shared/redundancy-checkbox.component.ts | 2 +- .../follows/shared/redundancy.service.ts | 28 -- .../follows/video-redundancies-list/index.ts | 1 + .../video-redundancies-list.component.html | 82 ++++ .../video-redundancies-list.component.scss | 37 ++ .../video-redundancies-list.component.ts | 178 +++++++++ ...ideo-redundancy-information.component.html | 24 ++ ...ideo-redundancy-information.component.scss | 8 + .../video-redundancy-information.component.ts | 11 + .../app/+admin/system/jobs/jobs.component.ts | 15 +- client/src/app/core/server/server.service.ts | 7 + .../shared/images/global-icon.component.ts | 1 - .../instance/instance-statistics.component.ts | 23 +- client/src/app/shared/shared.module.ts | 2 + .../app/shared/video/redundancy.service.ts | 73 ++++ .../video/video-actions-dropdown.component.ts | 28 +- .../shared/video/video-miniature.component.ts | 3 +- client/src/app/shared/video/video.model.ts | 4 + client/yarn.lock | 35 +- config/test.yaml | 8 +- package.json | 2 +- scripts/dev/index.sh | 4 +- server/controllers/api/server/follows.ts | 4 +- server/controllers/api/server/redundancy.ts | 84 +++- server/controllers/api/server/stats.ts | 10 +- .../activitypub/cache-file.ts | 2 +- .../custom-validators/video-redundancies.ts | 12 + server/helpers/webtorrent.ts | 4 +- server/initializers/config.ts | 4 +- server/initializers/constants.ts | 25 +- .../migrations/0475-redundancy-expires-on.ts | 27 ++ server/lib/activitypub/cache-file.ts | 4 +- .../job-queue/handlers/video-redundancy.ts | 20 + server/lib/job-queue/job-queue.ts | 13 +- server/lib/redundancy.ts | 8 +- .../lib/schedulers/update-videos-scheduler.ts | 1 - .../schedulers/videos-redundancy-scheduler.ts | 59 ++- server/middlewares/sort.ts | 23 +- server/middlewares/validators/redundancy.ts | 74 +++- server/middlewares/validators/sort.ts | 3 + server/models/redundancy/video-redundancy.ts | 186 ++++++++- server/tests/api/check-params/redundancy.ts | 141 ++++++- server/tests/api/redundancy/index.ts | 1 + .../tests/api/redundancy/manage-redundancy.ts | 373 ++++++++++++++++++ server/tests/api/redundancy/redundancy.ts | 138 +++++-- server/typings/models/video/video-file.ts | 5 +- .../models/video/video-streaming-playlist.ts | 6 +- server/typings/models/video/video.ts | 13 +- shared/extra-utils/server/redundancy.ts | 72 +++- shared/extra-utils/videos/videos.ts | 18 +- shared/models/redundancy/index.ts | 4 +- .../video-redundancies-filters.model.ts | 1 + .../redundancy/video-redundancy.model.ts | 33 ++ ...ts => videos-redundancy-strategy.model.ts} | 3 +- shared/models/server/job.model.ts | 3 +- shared/models/server/server-stats.model.ts | 18 +- shared/models/users/user-right.enum.ts | 4 +- shared/models/videos/video.model.ts | 2 +- 64 files changed, 1807 insertions(+), 195 deletions(-) delete mode 100644 client/src/app/+admin/follows/shared/redundancy.service.ts create mode 100644 client/src/app/+admin/follows/video-redundancies-list/index.ts create mode 100644 client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.html create mode 100644 client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.scss create mode 100644 client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts create mode 100644 client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.html create mode 100644 client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.scss create mode 100644 client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts create mode 100644 client/src/app/shared/video/redundancy.service.ts create mode 100644 server/helpers/custom-validators/video-redundancies.ts create mode 100644 server/initializers/migrations/0475-redundancy-expires-on.ts create mode 100644 server/lib/job-queue/handlers/video-redundancy.ts create mode 100644 server/tests/api/redundancy/manage-redundancy.ts create mode 100644 shared/models/redundancy/video-redundancies-filters.model.ts create mode 100644 shared/models/redundancy/video-redundancy.model.ts rename shared/models/redundancy/{videos-redundancy.model.ts => videos-redundancy-strategy.model.ts} (67%) diff --git a/client/package.json b/client/package.json index cd0a82aa4..7205dbe8f 100644 --- a/client/package.json +++ b/client/package.json @@ -77,6 +77,7 @@ "bootstrap": "^4.1.3", "buffer": "^5.1.0", "cache-chunk-store": "^3.0.0", + "chart.js": "^2.9.3", "codelyzer": "^5.0.1", "core-js": "^3.1.4", "css-loader": "^3.1.0", diff --git a/client/src/app/+admin/admin.component.html b/client/src/app/+admin/admin.component.html index 9a3d90c18..0d06aaedc 100644 --- a/client/src/app/+admin/admin.component.html +++ b/client/src/app/+admin/admin.component.html @@ -5,7 +5,7 @@ - Manage follows + Follows & redundancies diff --git a/client/src/app/+admin/admin.module.ts b/client/src/app/+admin/admin.module.ts index 9c56b5750..fdbe70314 100644 --- a/client/src/app/+admin/admin.module.ts +++ b/client/src/app/+admin/admin.module.ts @@ -5,7 +5,7 @@ import { TableModule } from 'primeng/table' import { SharedModule } from '../shared' import { AdminRoutingModule } from './admin-routing.module' import { AdminComponent } from './admin.component' -import { FollowersListComponent, FollowingAddComponent, FollowsComponent } from './follows' +import { FollowersListComponent, FollowingAddComponent, FollowsComponent, VideoRedundanciesListComponent } from './follows' import { FollowingListComponent } from './follows/following-list/following-list.component' import { UserCreateComponent, UserListComponent, UserPasswordComponent, UsersComponent, UserUpdateComponent } from './users' import { @@ -16,7 +16,6 @@ import { } from './moderation' import { ModerationComponent } from '@app/+admin/moderation/moderation.component' import { RedundancyCheckboxComponent } from '@app/+admin/follows/shared/redundancy-checkbox.component' -import { RedundancyService } from '@app/+admin/follows/shared/redundancy.service' import { InstanceAccountBlocklistComponent, InstanceServerBlocklistComponent } from '@app/+admin/moderation/instance-blocklist' import { JobsComponent } from '@app/+admin/system/jobs/jobs.component' import { JobService, LogsComponent, LogsService, SystemComponent } from '@app/+admin/system' @@ -27,13 +26,18 @@ import { PluginSearchComponent } from '@app/+admin/plugins/plugin-search/plugin- import { PluginShowInstalledComponent } from '@app/+admin/plugins/plugin-show-installed/plugin-show-installed.component' import { SelectButtonModule } from 'primeng/selectbutton' import { PluginApiService } from '@app/+admin/plugins/shared/plugin-api.service' +import { VideoRedundancyInformationComponent } from '@app/+admin/follows/video-redundancies-list/video-redundancy-information.component' +import { ChartModule } from 'primeng/chart' @NgModule({ imports: [ AdminRoutingModule, + + SharedModule, + TableModule, SelectButtonModule, - SharedModule + ChartModule ], declarations: [ @@ -44,6 +48,8 @@ import { PluginApiService } from '@app/+admin/plugins/shared/plugin-api.service' FollowersListComponent, FollowingListComponent, RedundancyCheckboxComponent, + VideoRedundanciesListComponent, + VideoRedundancyInformationComponent, UsersComponent, UserCreateComponent, @@ -78,7 +84,6 @@ import { PluginApiService } from '@app/+admin/plugins/shared/plugin-api.service' ], providers: [ - RedundancyService, JobService, LogsService, DebugService, diff --git a/client/src/app/+admin/follows/follows.component.html b/client/src/app/+admin/follows/follows.component.html index 21d477132..46581daf9 100644 --- a/client/src/app/+admin/follows/follows.component.html +++ b/client/src/app/+admin/follows/follows.component.html @@ -1,5 +1,5 @@
-
Manage follows
+
Follows & redundancies
Following @@ -7,7 +7,9 @@ Follow Followers + + Video redundancies
- \ No newline at end of file + diff --git a/client/src/app/+admin/follows/follows.routes.ts b/client/src/app/+admin/follows/follows.routes.ts index e84c79e82..298733eb0 100644 --- a/client/src/app/+admin/follows/follows.routes.ts +++ b/client/src/app/+admin/follows/follows.routes.ts @@ -6,6 +6,7 @@ import { FollowingAddComponent } from './following-add' import { FollowersListComponent } from './followers-list' import { UserRight } from '../../../../../shared' import { FollowingListComponent } from './following-list/following-list.component' +import { VideoRedundanciesListComponent } from '@app/+admin/follows/video-redundancies-list' export const FollowsRoutes: Routes = [ { @@ -47,6 +48,10 @@ export const FollowsRoutes: Routes = [ title: 'Add follow' } } + }, + { + path: 'video-redundancies-list', + component: VideoRedundanciesListComponent } ] } diff --git a/client/src/app/+admin/follows/index.ts b/client/src/app/+admin/follows/index.ts index e94f33710..4fcb35cb1 100644 --- a/client/src/app/+admin/follows/index.ts +++ b/client/src/app/+admin/follows/index.ts @@ -1,5 +1,6 @@ export * from './following-add' export * from './followers-list' export * from './following-list' +export * from './video-redundancies-list' export * from './follows.component' export * from './follows.routes' diff --git a/client/src/app/+admin/follows/shared/redundancy-checkbox.component.ts b/client/src/app/+admin/follows/shared/redundancy-checkbox.component.ts index fa1da26bf..9d7883d97 100644 --- a/client/src/app/+admin/follows/shared/redundancy-checkbox.component.ts +++ b/client/src/app/+admin/follows/shared/redundancy-checkbox.component.ts @@ -1,7 +1,7 @@ import { Component, Input } from '@angular/core' import { Notifier } from '@app/core' import { I18n } from '@ngx-translate/i18n-polyfill' -import { RedundancyService } from '@app/+admin/follows/shared/redundancy.service' +import { RedundancyService } from '@app/shared/video/redundancy.service' @Component({ selector: 'my-redundancy-checkbox', diff --git a/client/src/app/+admin/follows/shared/redundancy.service.ts b/client/src/app/+admin/follows/shared/redundancy.service.ts deleted file mode 100644 index 87ae01c04..000000000 --- a/client/src/app/+admin/follows/shared/redundancy.service.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { catchError, map } from 'rxjs/operators' -import { HttpClient } from '@angular/common/http' -import { Injectable } from '@angular/core' -import { RestExtractor } from '@app/shared' -import { environment } from '../../../../environments/environment' - -@Injectable() -export class RedundancyService { - static BASE_USER_SUBSCRIPTIONS_URL = environment.apiUrl + '/api/v1/server/redundancy' - - constructor ( - private authHttp: HttpClient, - private restExtractor: RestExtractor - ) { } - - updateRedundancy (host: string, redundancyAllowed: boolean) { - const url = RedundancyService.BASE_USER_SUBSCRIPTIONS_URL + '/' + host - - const body = { redundancyAllowed } - - return this.authHttp.put(url, body) - .pipe( - map(this.restExtractor.extractDataBool), - catchError(err => this.restExtractor.handleError(err)) - ) - } - -} diff --git a/client/src/app/+admin/follows/video-redundancies-list/index.ts b/client/src/app/+admin/follows/video-redundancies-list/index.ts new file mode 100644 index 000000000..6a7c7f483 --- /dev/null +++ b/client/src/app/+admin/follows/video-redundancies-list/index.ts @@ -0,0 +1 @@ +export * from './video-redundancies-list.component' diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.html b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.html new file mode 100644 index 000000000..80c66ec60 --- /dev/null +++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.html @@ -0,0 +1,82 @@ +
+
Video redundancies list
+ +
+ + +
+ +
+
+
+ + + + + Strategy + Video name + Video URL + Total size + + + + + + + {{ getRedundancyStrategy(redundancy) }} + + {{ redundancy.name }} + + + {{ redundancy.url }} + + + {{ getTotalSize(redundancy) | bytes: 1 }} + + + + + + + + + + +
+ +
+ + + + + +
+ +
+ + + + + + +
+
Enabled strategies stats
+ +
+ +
+ No redundancy strategy is enabled on your instance. +
+ +
+ +
+ +
+
diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.scss b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.scss new file mode 100644 index 000000000..05018c281 --- /dev/null +++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.scss @@ -0,0 +1,37 @@ +@import '_variables'; +@import '_mixins'; + +.expansion-block { + margin-bottom: 20px; +} + +.admin-sub-header { + align-items: flex-end; + + .select-filter-block { + &:not(:last-child) { + margin-right: 10px; + } + + label { + margin-bottom: 2px; + } + + .peertube-select-container { + @include peertube-select-container(auto); + } + } +} + +.redundancies-charts { + margin-top: 50px; + + .chart-blocks { + display: flex; + justify-content: center; + + .chart-block { + margin: 0 20px; + } + } +} diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts new file mode 100644 index 000000000..4b41d1d86 --- /dev/null +++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts @@ -0,0 +1,178 @@ +import { Component, OnInit } from '@angular/core' +import { Notifier, ServerService } from '@app/core' +import { SortMeta } from 'primeng/api' +import { ConfirmService } from '../../../core/confirm/confirm.service' +import { RestPagination, RestTable } from '../../../shared' +import { I18n } from '@ngx-translate/i18n-polyfill' +import { VideoRedundanciesTarget, VideoRedundancy } from '@shared/models' +import { peertubeLocalStorage } from '@app/shared/misc/peertube-web-storage' +import { VideosRedundancyStats } from '@shared/models/server' +import { BytesPipe } from 'ngx-pipes' +import { RedundancyService } from '@app/shared/video/redundancy.service' + +@Component({ + selector: 'my-video-redundancies-list', + templateUrl: './video-redundancies-list.component.html', + styleUrls: [ './video-redundancies-list.component.scss' ] +}) +export class VideoRedundanciesListComponent extends RestTable implements OnInit { + private static LOCAL_STORAGE_DISPLAY_TYPE = 'video-redundancies-list-display-type' + + videoRedundancies: VideoRedundancy[] = [] + totalRecords = 0 + rowsPerPage = 10 + + sort: SortMeta = { field: 'name', order: 1 } + pagination: RestPagination = { count: this.rowsPerPage, start: 0 } + displayType: VideoRedundanciesTarget = 'my-videos' + + redundanciesGraphsData: { stats: VideosRedundancyStats, graphData: object, options: object }[] = [] + + noRedundancies = false + + private bytesPipe: BytesPipe + + constructor ( + private notifier: Notifier, + private confirmService: ConfirmService, + private redundancyService: RedundancyService, + private serverService: ServerService, + private i18n: I18n + ) { + super() + + this.bytesPipe = new BytesPipe() + } + + ngOnInit () { + this.loadSelectLocalStorage() + + this.initialize() + + this.serverService.getServerStats() + .subscribe(res => { + const redundancies = res.videosRedundancy + + if (redundancies.length === 0) this.noRedundancies = true + + for (const r of redundancies) { + this.buildPieData(r) + } + }) + } + + isDisplayingRemoteVideos () { + return this.displayType === 'remote-videos' + } + + getTotalSize (redundancy: VideoRedundancy) { + return redundancy.redundancies.files.reduce((a, b) => a + b.size, 0) + + redundancy.redundancies.streamingPlaylists.reduce((a, b) => a + b.size, 0) + } + + onDisplayTypeChanged () { + this.pagination.start = 0 + this.saveSelectLocalStorage() + + this.loadData() + } + + getRedundancyStrategy (redundancy: VideoRedundancy) { + if (redundancy.redundancies.files.length !== 0) return redundancy.redundancies.files[0].strategy + if (redundancy.redundancies.streamingPlaylists.length !== 0) return redundancy.redundancies.streamingPlaylists[0].strategy + + return '' + } + + buildPieData (stats: VideosRedundancyStats) { + const totalSize = stats.totalSize + ? stats.totalSize - stats.totalUsed + : stats.totalUsed + + if (totalSize === 0) return + + this.redundanciesGraphsData.push({ + stats, + graphData: { + labels: [ this.i18n('Used'), this.i18n('Available') ], + datasets: [ + { + data: [ stats.totalUsed, totalSize ], + backgroundColor: [ + '#FF6384', + '#36A2EB' + ], + hoverBackgroundColor: [ + '#FF6384', + '#36A2EB' + ] + } + ] + }, + options: { + title: { + display: true, + text: stats.strategy + }, + + tooltips: { + callbacks: { + label: (tooltipItem: any, data: any) => { + const dataset = data.datasets[tooltipItem.datasetIndex] + let label = data.labels[tooltipItem.index] + if (label) label += ': ' + else label = '' + + label += this.bytesPipe.transform(dataset.data[tooltipItem.index], 1) + return label + } + } + } + } + }) + } + + async removeRedundancy (redundancy: VideoRedundancy) { + const message = this.i18n('Do you really want to remove this video redundancy?') + const res = await this.confirmService.confirm(message, this.i18n('Remove redundancy')) + if (res === false) return + + this.redundancyService.removeVideoRedundancies(redundancy) + .subscribe( + () => { + this.notifier.success(this.i18n('Video redundancies removed!')) + this.loadData() + }, + + err => this.notifier.error(err.message) + ) + + } + + protected loadData () { + const options = { + pagination: this.pagination, + sort: this.sort, + target: this.displayType + } + + this.redundancyService.listVideoRedundancies(options) + .subscribe( + resultList => { + this.videoRedundancies = resultList.data + this.totalRecords = resultList.total + }, + + err => this.notifier.error(err.message) + ) + } + + private loadSelectLocalStorage () { + const displayType = peertubeLocalStorage.getItem(VideoRedundanciesListComponent.LOCAL_STORAGE_DISPLAY_TYPE) + if (displayType) this.displayType = displayType as VideoRedundanciesTarget + } + + private saveSelectLocalStorage () { + peertubeLocalStorage.setItem(VideoRedundanciesListComponent.LOCAL_STORAGE_DISPLAY_TYPE, this.displayType) + } +} diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.html b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.html new file mode 100644 index 000000000..a379520e3 --- /dev/null +++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.html @@ -0,0 +1,24 @@ +
+ Url + {{ redundancyElement.fileUrl }} +
+ +
+ Created on + {{ redundancyElement.createdAt | date: 'medium' }} +
+ +
+ Expires on + {{ redundancyElement.expiresOn | date: 'medium' }} +
+ +
+ Size + {{ redundancyElement.size | bytes: 1 }} +
+ +
+ Strategy + {{ redundancyElement.strategy }} +
diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.scss b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.scss new file mode 100644 index 000000000..6b09fbb01 --- /dev/null +++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.scss @@ -0,0 +1,8 @@ +@import '_variables'; +@import '_mixins'; + +.label { + display: inline-block; + min-width: 100px; + font-weight: $font-semibold; +} diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts new file mode 100644 index 000000000..6f3090c08 --- /dev/null +++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts @@ -0,0 +1,11 @@ +import { Component, Input } from '@angular/core' +import { FileRedundancyInformation, StreamingPlaylistRedundancyInformation } from '@shared/models' + +@Component({ + selector: 'my-video-redundancy-information', + templateUrl: './video-redundancy-information.component.html', + styleUrls: [ './video-redundancy-information.component.scss' ] +}) +export class VideoRedundancyInformationComponent { + @Input() redundancyElement: FileRedundancyInformation | StreamingPlaylistRedundancyInformation +} diff --git a/client/src/app/+admin/system/jobs/jobs.component.ts b/client/src/app/+admin/system/jobs/jobs.component.ts index 20c8ea71a..bc40452cf 100644 --- a/client/src/app/+admin/system/jobs/jobs.component.ts +++ b/client/src/app/+admin/system/jobs/jobs.component.ts @@ -16,8 +16,8 @@ import { JobTypeClient } from '../../../../types/job-type-client.type' styleUrls: [ './jobs.component.scss' ] }) export class JobsComponent extends RestTable implements OnInit { - private static JOB_STATE_LOCAL_STORAGE_STATE = 'jobs-list-state' - private static JOB_STATE_LOCAL_STORAGE_TYPE = 'jobs-list-type' + private static LOCAL_STORAGE_STATE = 'jobs-list-state' + private static LOCAL_STORAGE_TYPE = 'jobs-list-type' jobState: JobStateClient = 'waiting' jobStates: JobStateClient[] = [ 'active', 'completed', 'failed', 'waiting', 'delayed' ] @@ -34,7 +34,8 @@ export class JobsComponent extends RestTable implements OnInit { 'video-file-import', 'video-import', 'videos-views', - 'activitypub-refresher' + 'activitypub-refresher', + 'video-redundancy' ] jobs: Job[] = [] @@ -77,15 +78,15 @@ export class JobsComponent extends RestTable implements OnInit { } private loadJobStateAndType () { - const state = peertubeLocalStorage.getItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_STATE) + const state = peertubeLocalStorage.getItem(JobsComponent.LOCAL_STORAGE_STATE) if (state) this.jobState = state as JobState - const type = peertubeLocalStorage.getItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_TYPE) + const type = peertubeLocalStorage.getItem(JobsComponent.LOCAL_STORAGE_TYPE) if (type) this.jobType = type as JobType } private saveJobStateAndType () { - peertubeLocalStorage.setItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_STATE, this.jobState) - peertubeLocalStorage.setItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_TYPE, this.jobType) + peertubeLocalStorage.setItem(JobsComponent.LOCAL_STORAGE_STATE, this.jobState) + peertubeLocalStorage.setItem(JobsComponent.LOCAL_STORAGE_TYPE, this.jobType) } } diff --git a/client/src/app/core/server/server.service.ts b/client/src/app/core/server/server.service.ts index cdcbcb528..1f6cfb596 100644 --- a/client/src/app/core/server/server.service.ts +++ b/client/src/app/core/server/server.service.ts @@ -9,6 +9,7 @@ import { VideoConstant } from '../../../../../shared/models/videos' import { isDefaultLocale, peertubeTranslate } from '../../../../../shared/models/i18n' import { getDevLocale, isOnDevLocale } from '@app/shared/i18n/i18n-utils' import { sortBy } from '@app/shared/misc/utils' +import { ServerStats } from '@shared/models/server' @Injectable() export class ServerService { @@ -16,6 +17,8 @@ export class ServerService { private static BASE_VIDEO_URL = environment.apiUrl + '/api/v1/videos/' private static BASE_VIDEO_PLAYLIST_URL = environment.apiUrl + '/api/v1/video-playlists/' private static BASE_LOCALE_URL = environment.apiUrl + '/client/locales/' + private static BASE_STATS_URL = environment.apiUrl + '/api/v1/server/stats' + private static CONFIG_LOCAL_STORAGE_KEY = 'server-config' configReloaded = new Subject() @@ -235,6 +238,10 @@ export class ServerService { return this.localeObservable.pipe(first()) } + getServerStats () { + return this.http.get(ServerService.BASE_STATS_URL) + } + private loadAttributeEnum ( baseUrl: string, attributeName: 'categories' | 'licences' | 'languages' | 'privacies', diff --git a/client/src/app/shared/images/global-icon.component.ts b/client/src/app/shared/images/global-icon.component.ts index 806aca347..b6e641228 100644 --- a/client/src/app/shared/images/global-icon.component.ts +++ b/client/src/app/shared/images/global-icon.component.ts @@ -1,6 +1,5 @@ import { ChangeDetectionStrategy, Component, ElementRef, Input, OnInit } from '@angular/core' import { HooksService } from '@app/core/plugins/hooks.service' -import { I18n } from '@ngx-translate/i18n-polyfill' const icons = { 'add': require('!!raw-loader?!../../../assets/images/global/add.svg'), diff --git a/client/src/app/shared/instance/instance-statistics.component.ts b/client/src/app/shared/instance/instance-statistics.component.ts index 8ec728f05..40aa8a4c0 100644 --- a/client/src/app/shared/instance/instance-statistics.component.ts +++ b/client/src/app/shared/instance/instance-statistics.component.ts @@ -1,9 +1,6 @@ -import { map } from 'rxjs/operators' -import { HttpClient } from '@angular/common/http' import { Component, OnInit } from '@angular/core' -import { I18n } from '@ngx-translate/i18n-polyfill' import { ServerStats } from '@shared/models/server' -import { environment } from '../../../environments/environment' +import { ServerService } from '@app/core' @Component({ selector: 'my-instance-statistics', @@ -11,27 +8,15 @@ import { environment } from '../../../environments/environment' styleUrls: [ './instance-statistics.component.scss' ] }) export class InstanceStatisticsComponent implements OnInit { - private static BASE_STATS_URL = environment.apiUrl + '/api/v1/server/stats' - serverStats: ServerStats = null constructor ( - private http: HttpClient, - private i18n: I18n + private serverService: ServerService ) { } ngOnInit () { - this.getStats() - .subscribe( - res => { - this.serverStats = res - } - ) - } - - getStats () { - return this.http - .get(InstanceStatisticsComponent.BASE_STATS_URL) + this.serverService.getServerStats() + .subscribe(res => this.serverStats = res) } } diff --git a/client/src/app/shared/shared.module.ts b/client/src/app/shared/shared.module.ts index b2eb13f73..d06d37d8c 100644 --- a/client/src/app/shared/shared.module.ts +++ b/client/src/app/shared/shared.module.ts @@ -98,6 +98,7 @@ import { FollowService } from '@app/shared/instance/follow.service' import { MultiSelectModule } from 'primeng/multiselect' import { FeatureBooleanComponent } from '@app/shared/instance/feature-boolean.component' import { InputReadonlyCopyComponent } from '@app/shared/forms/input-readonly-copy.component' +import { RedundancyService } from '@app/shared/video/redundancy.service' @NgModule({ imports: [ @@ -300,6 +301,7 @@ import { InputReadonlyCopyComponent } from '@app/shared/forms/input-readonly-cop UserNotificationService, FollowService, + RedundancyService, I18n ] diff --git a/client/src/app/shared/video/redundancy.service.ts b/client/src/app/shared/video/redundancy.service.ts new file mode 100644 index 000000000..fb918d73b --- /dev/null +++ b/client/src/app/shared/video/redundancy.service.ts @@ -0,0 +1,73 @@ +import { catchError, map, toArray } from 'rxjs/operators' +import { HttpClient, HttpParams } from '@angular/common/http' +import { Injectable } from '@angular/core' +import { RestExtractor, RestPagination, RestService } from '@app/shared/rest' +import { SortMeta } from 'primeng/api' +import { ResultList, Video, VideoRedundanciesTarget, VideoRedundancy } from '@shared/models' +import { concat, Observable } from 'rxjs' +import { environment } from '../../../environments/environment' + +@Injectable() +export class RedundancyService { + static BASE_REDUNDANCY_URL = environment.apiUrl + '/api/v1/server/redundancy' + + constructor ( + private authHttp: HttpClient, + private restService: RestService, + private restExtractor: RestExtractor + ) { } + + updateRedundancy (host: string, redundancyAllowed: boolean) { + const url = RedundancyService.BASE_REDUNDANCY_URL + '/' + host + + const body = { redundancyAllowed } + + return this.authHttp.put(url, body) + .pipe( + map(this.restExtractor.extractDataBool), + catchError(err => this.restExtractor.handleError(err)) + ) + } + + listVideoRedundancies (options: { + pagination: RestPagination, + sort: SortMeta, + target?: VideoRedundanciesTarget + }): Observable> { + const { pagination, sort, target } = options + + let params = new HttpParams() + params = this.restService.addRestGetParams(params, pagination, sort) + + if (target) params = params.append('target', target) + + return this.authHttp.get>(RedundancyService.BASE_REDUNDANCY_URL + '/videos', { params }) + .pipe( + catchError(res => this.restExtractor.handleError(res)) + ) + } + + addVideoRedundancy (video: Video) { + return this.authHttp.post(RedundancyService.BASE_REDUNDANCY_URL + '/videos', { videoId: video.id }) + .pipe( + catchError(res => this.restExtractor.handleError(res)) + ) + } + + removeVideoRedundancies (redundancy: VideoRedundancy) { + const observables = redundancy.redundancies.streamingPlaylists.map(r => r.id) + .concat(redundancy.redundancies.files.map(r => r.id)) + .map(id => this.removeRedundancy(id)) + + return concat(...observables) + .pipe(toArray()) + } + + private removeRedundancy (redundancyId: number) { + return this.authHttp.delete(RedundancyService.BASE_REDUNDANCY_URL + '/videos/' + redundancyId) + .pipe( + map(this.restExtractor.extractDataBool), + catchError(res => this.restExtractor.handleError(res)) + ) + } +} diff --git a/client/src/app/shared/video/video-actions-dropdown.component.ts b/client/src/app/shared/video/video-actions-dropdown.component.ts index afdeab18d..390d74c52 100644 --- a/client/src/app/shared/video/video-actions-dropdown.component.ts +++ b/client/src/app/shared/video/video-actions-dropdown.component.ts @@ -14,6 +14,7 @@ import { VideoBlacklistComponent } from '@app/shared/video/modals/video-blacklis import { VideoBlacklistService } from '@app/shared/video-blacklist' import { ScreenService } from '@app/shared/misc/screen.service' import { VideoCaption } from '@shared/models' +import { RedundancyService } from '@app/shared/video/redundancy.service' export type VideoActionsDisplayType = { playlist?: boolean @@ -22,6 +23,7 @@ export type VideoActionsDisplayType = { blacklist?: boolean delete?: boolean report?: boolean + duplicate?: boolean } @Component({ @@ -46,7 +48,8 @@ export class VideoActionsDropdownComponent implements OnChanges { update: true, blacklist: true, delete: true, - report: true + report: true, + duplicate: true } @Input() placement = 'left' @@ -74,6 +77,7 @@ export class VideoActionsDropdownComponent implements OnChanges { private screenService: ScreenService, private videoService: VideoService, private blocklistService: BlocklistService, + private redundancyService: RedundancyService, private i18n: I18n ) { } @@ -144,6 +148,10 @@ export class VideoActionsDropdownComponent implements OnChanges { return this.video && this.video instanceof VideoDetails && this.video.downloadEnabled } + canVideoBeDuplicated () { + return this.video.canBeDuplicatedBy(this.user) + } + /* Action handlers */ async unblacklistVideo () { @@ -186,6 +194,18 @@ export class VideoActionsDropdownComponent implements OnChanges { ) } + duplicateVideo () { + this.redundancyService.addVideoRedundancy(this.video) + .subscribe( + () => { + const message = this.i18n('This video will be duplicated by your instance.') + this.notifier.success(message) + }, + + err => this.notifier.error(err.message) + ) + } + onVideoBlacklisted () { this.videoBlacklisted.emit() } @@ -233,6 +253,12 @@ export class VideoActionsDropdownComponent implements OnChanges { iconName: 'undo', isDisplayed: () => this.authService.isLoggedIn() && this.displayOptions.blacklist && this.isVideoUnblacklistable() }, + { + label: this.i18n('Duplicate (redundancy)'), + handler: () => this.duplicateVideo(), + isDisplayed: () => this.authService.isLoggedIn() && this.displayOptions.duplicate && this.canVideoBeDuplicated(), + iconName: 'cloud-download' + }, { label: this.i18n('Delete'), handler: () => this.removeVideo(), diff --git a/client/src/app/shared/video/video-miniature.component.ts b/client/src/app/shared/video/video-miniature.component.ts index 598a7a983..1dfb3eec7 100644 --- a/client/src/app/shared/video/video-miniature.component.ts +++ b/client/src/app/shared/video/video-miniature.component.ts @@ -64,7 +64,8 @@ export class VideoMiniatureComponent implements OnInit { update: true, blacklist: true, delete: true, - report: true + report: true, + duplicate: false } showActions = false serverConfig: ServerConfig diff --git a/client/src/app/shared/video/video.model.ts b/client/src/app/shared/video/video.model.ts index fb98d5382..9eeaf41b0 100644 --- a/client/src/app/shared/video/video.model.ts +++ b/client/src/app/shared/video/video.model.ts @@ -152,4 +152,8 @@ export class Video implements VideoServerModel { isUpdatableBy (user: AuthUser) { return user && this.isLocal === true && (this.account.name === user.username || user.hasRight(UserRight.UPDATE_ANY_VIDEO)) } + + canBeDuplicatedBy (user: AuthUser) { + return user && this.isLocal === false && user.hasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES) + } } diff --git a/client/yarn.lock b/client/yarn.lock index 0855a2570..c900ae549 100644 --- a/client/yarn.lock +++ b/client/yarn.lock @@ -2586,6 +2586,29 @@ chardet@^0.7.0: resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e" integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA== +chart.js@^2.9.3: + version "2.9.3" + resolved "https://registry.yarnpkg.com/chart.js/-/chart.js-2.9.3.tgz#ae3884114dafd381bc600f5b35a189138aac1ef7" + integrity sha512-+2jlOobSk52c1VU6fzkh3UwqHMdSlgH1xFv9FKMqHiNCpXsGPQa/+81AFa+i3jZ253Mq9aAycPwDjnn1XbRNNw== + dependencies: + chartjs-color "^2.1.0" + moment "^2.10.2" + +chartjs-color-string@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/chartjs-color-string/-/chartjs-color-string-0.6.0.tgz#1df096621c0e70720a64f4135ea171d051402f71" + integrity sha512-TIB5OKn1hPJvO7JcteW4WY/63v6KwEdt6udfnDE9iCAZgy+V4SrbSxoIbTw/xkUIapjEI4ExGtD0+6D3KyFd7A== + dependencies: + color-name "^1.0.0" + +chartjs-color@^2.1.0: + version "2.4.1" + resolved "https://registry.yarnpkg.com/chartjs-color/-/chartjs-color-2.4.1.tgz#6118bba202fe1ea79dd7f7c0f9da93467296c3b0" + integrity sha512-haqOg1+Yebys/Ts/9bLo/BqUcONQOdr/hoEr2LLTRl6C5LXctUdHxsCYfvQVg5JIxITrfCNUDr4ntqmQk9+/0w== + dependencies: + chartjs-color-string "^0.6.0" + color-convert "^1.9.3" + check-types@^8.0.3: version "8.0.3" resolved "https://registry.yarnpkg.com/check-types/-/check-types-8.0.3.tgz#3356cca19c889544f2d7a95ed49ce508a0ecf552" @@ -2800,7 +2823,7 @@ collection-visit@^1.0.0: map-visit "^1.0.0" object-visit "^1.0.0" -color-convert@^1.9.0: +color-convert@^1.9.0, color-convert@^1.9.3: version "1.9.3" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== @@ -2812,6 +2835,11 @@ color-name@1.1.3: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= +color-name@^1.0.0: + version "1.1.4" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" + integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== + colors@1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/colors/-/colors-1.1.2.tgz#168a4701756b6a7f51a12ce0c97bfa28c084ed63" @@ -6941,6 +6969,11 @@ mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1: dependencies: minimist "0.0.8" +moment@^2.10.2: + version "2.24.0" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.24.0.tgz#0d055d53f5052aa653c9f6eb68bb5d12bf5c2b5b" + integrity sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg== + mousetrap@^1.6.0: version "1.6.3" resolved "https://registry.yarnpkg.com/mousetrap/-/mousetrap-1.6.3.tgz#80fee49665fd478bccf072c9d46bdf1bfed3558a" diff --git a/config/test.yaml b/config/test.yaml index 5758c1887..74979f3a7 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -40,18 +40,18 @@ contact_form: redundancy: videos: - check_interval: '10 minutes' + check_interval: '1 minute' strategies: - - size: '10MB' + size: '1000MB' min_lifetime: '10 minutes' strategy: 'most-views' - - size: '10MB' + size: '1000MB' min_lifetime: '10 minutes' strategy: 'trending' - - size: '10MB' + size: '1000MB' min_lifetime: '10 minutes' strategy: 'recently-added' min_views: 1 diff --git a/package.json b/package.json index d4858725a..fed390c82 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "i18n:create-custom-files": "node ./dist/scripts/i18n/create-custom-files.js", "reset-password": "node ./dist/scripts/reset-password.js", "play": "scripty", - "dev": "scripty", + "dev": "sh ./scripts/dev/index.sh", "dev:server": "sh ./scripts/dev/server.sh", "dev:embed": "scripty", "dev:client": "sh ./scripts/dev/client.sh", diff --git a/scripts/dev/index.sh b/scripts/dev/index.sh index d221d2fc8..9e89516b8 100755 --- a/scripts/dev/index.sh +++ b/scripts/dev/index.sh @@ -3,5 +3,5 @@ set -eu NODE_ENV=test npm run concurrently -- -k \ - "npm run dev:client -- --skip-server" \ - "npm run dev:server" + "sh scripts/dev/client.sh --skip-server" \ + "sh scripts/dev/server.sh" diff --git a/server/controllers/api/server/follows.ts b/server/controllers/api/server/follows.ts index 29a403a43..c69de21fb 100644 --- a/server/controllers/api/server/follows.ts +++ b/server/controllers/api/server/follows.ts @@ -24,7 +24,7 @@ import { } from '../../../middlewares/validators' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { JobQueue } from '../../../lib/job-queue' -import { removeRedundancyOf } from '../../../lib/redundancy' +import { removeRedundanciesOfServer } from '../../../lib/redundancy' import { sequelizeTypescript } from '../../../initializers/database' import { autoFollowBackIfNeeded } from '../../../lib/activitypub/follow' @@ -153,7 +153,7 @@ async function removeFollowing (req: express.Request, res: express.Response) { await server.save({ transaction: t }) // Async, could be long - removeRedundancyOf(server.id) + removeRedundanciesOfServer(server.id) .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, err)) await follow.destroy({ transaction: t }) diff --git a/server/controllers/api/server/redundancy.ts b/server/controllers/api/server/redundancy.ts index 4ea6164a3..a11c1a74f 100644 --- a/server/controllers/api/server/redundancy.ts +++ b/server/controllers/api/server/redundancy.ts @@ -1,9 +1,24 @@ import * as express from 'express' import { UserRight } from '../../../../shared/models/users' -import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares' -import { updateServerRedundancyValidator } from '../../../middlewares/validators/redundancy' -import { removeRedundancyOf } from '../../../lib/redundancy' +import { + asyncMiddleware, + authenticate, + ensureUserHasRight, + paginationValidator, + setDefaultPagination, + setDefaultVideoRedundanciesSort, + videoRedundanciesSortValidator +} from '../../../middlewares' +import { + listVideoRedundanciesValidator, + updateServerRedundancyValidator, + addVideoRedundancyValidator, + removeVideoRedundancyValidator +} from '../../../middlewares/validators/redundancy' +import { removeRedundanciesOfServer, removeVideoRedundancy } from '../../../lib/redundancy' import { logger } from '../../../helpers/logger' +import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy' +import { JobQueue } from '@server/lib/job-queue' const serverRedundancyRouter = express.Router() @@ -14,6 +29,31 @@ serverRedundancyRouter.put('/redundancy/:host', asyncMiddleware(updateRedundancy) ) +serverRedundancyRouter.get('/redundancy/videos', + authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES), + listVideoRedundanciesValidator, + paginationValidator, + videoRedundanciesSortValidator, + setDefaultVideoRedundanciesSort, + setDefaultPagination, + asyncMiddleware(listVideoRedundancies) +) + +serverRedundancyRouter.post('/redundancy/videos', + authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES), + addVideoRedundancyValidator, + asyncMiddleware(addVideoRedundancy) +) + +serverRedundancyRouter.delete('/redundancy/videos/:redundancyId', + authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES), + removeVideoRedundancyValidator, + asyncMiddleware(removeVideoRedundancyController) +) + // --------------------------------------------------------------------------- export { @@ -22,6 +62,42 @@ export { // --------------------------------------------------------------------------- +async function listVideoRedundancies (req: express.Request, res: express.Response) { + const resultList = await VideoRedundancyModel.listForApi({ + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + target: req.query.target, + strategy: req.query.strategy + }) + + const result = { + total: resultList.total, + data: resultList.data.map(r => VideoRedundancyModel.toFormattedJSONStatic(r)) + } + + return res.json(result) +} + +async function addVideoRedundancy (req: express.Request, res: express.Response) { + const payload = { + videoId: res.locals.onlyVideo.id + } + + await JobQueue.Instance.createJob({ + type: 'video-redundancy', + payload + }) + + return res.sendStatus(204) +} + +async function removeVideoRedundancyController (req: express.Request, res: express.Response) { + await removeVideoRedundancy(res.locals.videoRedundancy) + + return res.sendStatus(204) +} + async function updateRedundancy (req: express.Request, res: express.Response) { const server = res.locals.server @@ -30,7 +106,7 @@ async function updateRedundancy (req: express.Request, res: express.Response) { await server.save() // Async, could be long - removeRedundancyOf(server.id) + removeRedundanciesOfServer(server.id) .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, { err })) return res.sendStatus(204) diff --git a/server/controllers/api/server/stats.ts b/server/controllers/api/server/stats.ts index 3616c074d..6d508a481 100644 --- a/server/controllers/api/server/stats.ts +++ b/server/controllers/api/server/stats.ts @@ -10,6 +10,7 @@ import { ROUTE_CACHE_LIFETIME } from '../../../initializers/constants' import { cacheRoute } from '../../../middlewares/cache' import { VideoFileModel } from '../../../models/video/video-file' import { CONFIG } from '../../../initializers/config' +import { VideoRedundancyStrategyWithManual } from '@shared/models' const statsRouter = express.Router() @@ -25,8 +26,15 @@ async function getStats (req: express.Request, res: express.Response) { const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() const { totalLocalVideoFilesSize } = await VideoFileModel.getStats() + const strategies: { strategy: VideoRedundancyStrategyWithManual, size: number }[] = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES + .map(r => ({ + strategy: r.strategy, + size: r.size + })) + strategies.push({ strategy: 'manual', size: null }) + const videosRedundancyStats = await Promise.all( - CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.map(r => { + strategies.map(r => { return VideoRedundancyModel.getStats(r.strategy) .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size })) }) diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts index 21d5c53ca..c5b3b4d9f 100644 --- a/server/helpers/custom-validators/activitypub/cache-file.ts +++ b/server/helpers/custom-validators/activitypub/cache-file.ts @@ -6,7 +6,7 @@ import { CacheFileObject } from '../../../../shared/models/activitypub/objects' function isCacheFileObjectValid (object: CacheFileObject) { return exists(object) && object.type === 'CacheFile' && - isDateValid(object.expires) && + (object.expires === null || isDateValid(object.expires)) && isActivityPubUrlValid(object.object) && (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url)) } diff --git a/server/helpers/custom-validators/video-redundancies.ts b/server/helpers/custom-validators/video-redundancies.ts new file mode 100644 index 000000000..50a559c4f --- /dev/null +++ b/server/helpers/custom-validators/video-redundancies.ts @@ -0,0 +1,12 @@ +import { exists } from './misc' + +function isVideoRedundancyTarget (value: any) { + return exists(value) && + (value === 'my-videos' || value === 'remote-videos') +} + +// --------------------------------------------------------------------------- + +export { + isVideoRedundancyTarget +} diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index 3a99518c6..8a5d030df 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -9,12 +9,12 @@ import { promisify2 } from './core-utils' import { MVideo } from '@server/typings/models/video/video' import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/typings/models/video/video-file' import { isStreamingPlaylist, MStreamingPlaylistVideo } from '@server/typings/models/video/video-streaming-playlist' -import { STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' +import { WEBSERVER } from '@server/initializers/constants' import * as parseTorrent from 'parse-torrent' import * as magnetUtil from 'magnet-uri' import { isArray } from '@server/helpers/custom-validators/misc' import { extractVideo } from '@server/lib/videos' -import { getTorrentFileName, getVideoFilename, getVideoFilePath } from '@server/lib/video-paths' +import { getTorrentFileName, getVideoFilePath } from '@server/lib/video-paths' const createTorrentPromise = promisify2(createTorrent) diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 7fd77f3e8..fd8bf09fc 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -1,6 +1,6 @@ import { IConfig } from 'config' import { dirname, join } from 'path' -import { VideosRedundancy } from '../../shared/models' +import { VideosRedundancyStrategy } from '../../shared/models' // Do not use barrels, remain constants as independent as possible import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils' import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' @@ -304,7 +304,7 @@ function getLocalConfigFilePath () { return join(dirname(configSources[ 0 ].name), filename + '.json') } -function buildVideosRedundancy (objs: any[]): VideosRedundancy[] { +function buildVideosRedundancy (objs: any[]): VideosRedundancyStrategy[] { if (!objs) return [] if (!Array.isArray(objs)) return objs diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 032f63c8f..e01ab8943 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -14,7 +14,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 470 +const LAST_MIGRATION_VERSION = 475 // --------------------------------------------------------------------------- @@ -73,7 +73,9 @@ const SORTABLE_COLUMNS = { PLUGINS: [ 'name', 'createdAt', 'updatedAt' ], - AVAILABLE_PLUGINS: [ 'npmName', 'popularity' ] + AVAILABLE_PLUGINS: [ 'npmName', 'popularity' ], + + VIDEO_REDUNDANCIES: [ 'name' ] } const OAUTH_LIFETIME = { @@ -117,45 +119,44 @@ const REMOTE_SCHEME = { WS: 'wss' } -// TODO: remove 'video-file' -const JOB_ATTEMPTS: { [id in (JobType | 'video-file')]: number } = { +const JOB_ATTEMPTS: { [id in JobType]: number } = { 'activitypub-http-broadcast': 5, 'activitypub-http-unicast': 5, 'activitypub-http-fetcher': 5, 'activitypub-follow': 5, 'video-file-import': 1, 'video-transcoding': 1, - 'video-file': 1, 'video-import': 1, 'email': 5, 'videos-views': 1, - 'activitypub-refresher': 1 + 'activitypub-refresher': 1, + 'video-redundancy': 1 } -const JOB_CONCURRENCY: { [id in (JobType | 'video-file')]: number } = { +const JOB_CONCURRENCY: { [id in JobType]: number } = { 'activitypub-http-broadcast': 1, 'activitypub-http-unicast': 5, 'activitypub-http-fetcher': 1, 'activitypub-follow': 1, 'video-file-import': 1, 'video-transcoding': 1, - 'video-file': 1, 'video-import': 1, 'email': 5, 'videos-views': 1, - 'activitypub-refresher': 1 + 'activitypub-refresher': 1, + 'video-redundancy': 1 } -const JOB_TTL: { [id in (JobType | 'video-file')]: number } = { +const JOB_TTL: { [id in JobType]: number } = { 'activitypub-http-broadcast': 60000 * 10, // 10 minutes 'activitypub-http-unicast': 60000 * 10, // 10 minutes 'activitypub-http-fetcher': 60000 * 10, // 10 minutes 'activitypub-follow': 60000 * 10, // 10 minutes 'video-file-import': 1000 * 3600, // 1 hour 'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long - 'video-file': 1000 * 3600 * 48, // 2 days, transcoding could be long 'video-import': 1000 * 3600 * 2, // hours 'email': 60000 * 10, // 10 minutes 'videos-views': undefined, // Unlimited - 'activitypub-refresher': 60000 * 10 // 10 minutes + 'activitypub-refresher': 60000 * 10, // 10 minutes + 'video-redundancy': 1000 * 3600 * 3 // 3 hours } const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = { 'videos-views': { diff --git a/server/initializers/migrations/0475-redundancy-expires-on.ts b/server/initializers/migrations/0475-redundancy-expires-on.ts new file mode 100644 index 000000000..7e392c8c0 --- /dev/null +++ b/server/initializers/migrations/0475-redundancy-expires-on.ts @@ -0,0 +1,27 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction, + queryInterface: Sequelize.QueryInterface, + sequelize: Sequelize.Sequelize, + db: any +}): Promise { + { + const data = { + type: Sequelize.DATE, + allowNull: true, + defaultValue: null + } + + await utils.queryInterface.changeColumn('videoRedundancy', 'expiresOn', data) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts index 65b2dcb49..8252e95e9 100644 --- a/server/lib/activitypub/cache-file.ts +++ b/server/lib/activitypub/cache-file.ts @@ -13,7 +13,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) return { - expiresOn: new Date(cacheFileObject.expires), + expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, url: cacheFileObject.id, fileUrl: url.href, strategy: null, @@ -30,7 +30,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`) return { - expiresOn: new Date(cacheFileObject.expires), + expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, url: cacheFileObject.id, fileUrl: url.href, strategy: null, diff --git a/server/lib/job-queue/handlers/video-redundancy.ts b/server/lib/job-queue/handlers/video-redundancy.ts new file mode 100644 index 000000000..319d7090e --- /dev/null +++ b/server/lib/job-queue/handlers/video-redundancy.ts @@ -0,0 +1,20 @@ +import * as Bull from 'bull' +import { logger } from '../../../helpers/logger' +import { VideosRedundancyScheduler } from '@server/lib/schedulers/videos-redundancy-scheduler' + +export type VideoRedundancyPayload = { + videoId: number +} + +async function processVideoRedundancy (job: Bull.Job) { + const payload = job.data as VideoRedundancyPayload + logger.info('Processing video redundancy in job %d.', job.id) + + return VideosRedundancyScheduler.Instance.createManualRedundancy(payload.videoId) +} + +// --------------------------------------------------------------------------- + +export { + processVideoRedundancy +} diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index ec601e9ea..8bbf58f2b 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts @@ -13,6 +13,7 @@ import { processVideoImport, VideoImportPayload } from './handlers/video-import' import { processVideosViews } from './handlers/video-views' import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher' import { processVideoFileImport, VideoFileImportPayload } from './handlers/video-file-import' +import { processVideoRedundancy, VideoRedundancyPayload } from '@server/lib/job-queue/handlers/video-redundancy' type CreateJobArgument = { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | @@ -24,20 +25,21 @@ type CreateJobArgument = { type: 'email', payload: EmailPayload } | { type: 'video-import', payload: VideoImportPayload } | { type: 'activitypub-refresher', payload: RefreshPayload } | - { type: 'videos-views', payload: {} } + { type: 'videos-views', payload: {} } | + { type: 'video-redundancy', payload: VideoRedundancyPayload } -const handlers: { [ id in (JobType | 'video-file') ]: (job: Bull.Job) => Promise} = { +const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise} = { 'activitypub-http-broadcast': processActivityPubHttpBroadcast, 'activitypub-http-unicast': processActivityPubHttpUnicast, 'activitypub-http-fetcher': processActivityPubHttpFetcher, 'activitypub-follow': processActivityPubFollow, 'video-file-import': processVideoFileImport, 'video-transcoding': processVideoTranscoding, - 'video-file': processVideoTranscoding, // TODO: remove it (changed in 1.3) 'email': processEmail, 'video-import': processVideoImport, 'videos-views': processVideosViews, - 'activitypub-refresher': refreshAPObject + 'activitypub-refresher': refreshAPObject, + 'video-redundancy': processVideoRedundancy } const jobTypes: JobType[] = [ @@ -50,7 +52,8 @@ const jobTypes: JobType[] = [ 'video-file-import', 'video-import', 'videos-views', - 'activitypub-refresher' + 'activitypub-refresher', + 'video-redundancy' ] class JobQueue { diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts index 1b4ecd7c0..78d84e02e 100644 --- a/server/lib/redundancy.ts +++ b/server/lib/redundancy.ts @@ -13,10 +13,10 @@ async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t? await videoRedundancy.destroy({ transaction: t }) } -async function removeRedundancyOf (serverId: number) { - const videosRedundancy = await VideoRedundancyModel.listLocalOfServer(serverId) +async function removeRedundanciesOfServer (serverId: number) { + const redundancies = await VideoRedundancyModel.listLocalOfServer(serverId) - for (const redundancy of videosRedundancy) { + for (const redundancy of redundancies) { await removeVideoRedundancy(redundancy) } } @@ -24,6 +24,6 @@ async function removeRedundancyOf (serverId: number) { // --------------------------------------------------------------------------- export { - removeRedundancyOf, + removeRedundanciesOfServer, removeVideoRedundancy } diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts index 350a335d3..956780a77 100644 --- a/server/lib/schedulers/update-videos-scheduler.ts +++ b/server/lib/schedulers/update-videos-scheduler.ts @@ -4,7 +4,6 @@ import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-upda import { retryTransactionWrapper } from '../../helpers/database-utils' import { federateVideoIfNeeded } from '../activitypub' import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { VideoPrivacy } from '../../../shared/models/videos' import { Notifier } from '../notifier' import { sequelizeTypescript } from '../../initializers/database' import { MVideoFullLight } from '@server/typings/models' diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index c1c91b656..6e61cbe7d 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -1,7 +1,7 @@ import { AbstractScheduler } from './abstract-scheduler' import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER } from '../../initializers/constants' import { logger } from '../../helpers/logger' -import { VideosRedundancy } from '../../../shared/models/redundancy' +import { VideosRedundancyStrategy } from '../../../shared/models/redundancy' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent' import { join } from 'path' @@ -25,9 +25,10 @@ import { MVideoWithAllFiles } from '@server/typings/models' import { getVideoFilename } from '../video-paths' +import { VideoModel } from '@server/models/video/video' type CandidateToDuplicate = { - redundancy: VideosRedundancy, + redundancy: VideosRedundancyStrategy, video: MVideoWithAllFiles, files: MVideoFile[], streamingPlaylists: MStreamingPlaylistFiles[] @@ -41,7 +42,7 @@ function isMVideoRedundancyFileVideo ( export class VideosRedundancyScheduler extends AbstractScheduler { - private static instance: AbstractScheduler + private static instance: VideosRedundancyScheduler protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL @@ -49,6 +50,22 @@ export class VideosRedundancyScheduler extends AbstractScheduler { super() } + async createManualRedundancy (videoId: number) { + const videoToDuplicate = await VideoModel.loadWithFiles(videoId) + + if (!videoToDuplicate) { + logger.warn('Video to manually duplicate %d does not exist anymore.', videoId) + return + } + + return this.createVideoRedundancies({ + video: videoToDuplicate, + redundancy: null, + files: videoToDuplicate.VideoFiles, + streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists + }) + } + protected async internalExecute () { for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy) @@ -94,7 +111,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { for (const redundancyModel of expired) { try { const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) - const candidate = { + const candidate: CandidateToDuplicate = { redundancy: redundancyConfig, video: null, files: [], @@ -140,7 +157,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } - private findVideoToDuplicate (cache: VideosRedundancy) { + private findVideoToDuplicate (cache: VideosRedundancyStrategy) { if (cache.strategy === 'most-views') { return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) } @@ -187,13 +204,21 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } - private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: MVideoAccountLight, fileArg: MVideoFile) { + private async createVideoFileRedundancy (redundancy: VideosRedundancyStrategy | null, video: MVideoAccountLight, fileArg: MVideoFile) { + let strategy = 'manual' + let expiresOn: Date = null + + if (redundancy) { + strategy = redundancy.strategy + expiresOn = this.buildNewExpiration(redundancy.minLifetime) + } + const file = fileArg as MVideoFileVideo file.Video = video const serverActor = await getServerActor() - logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) + logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy) const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() const magnetUri = generateMagnetUri(video, file, baseUrlHttp, baseUrlWs) @@ -204,10 +229,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler { await move(tmpPath, destPath, { overwrite: true }) const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ - expiresOn: this.buildNewExpiration(redundancy.minLifetime), + expiresOn, url: getVideoCacheFileActivityPubUrl(file), fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL), - strategy: redundancy.strategy, + strategy, videoFileId: file.id, actorId: serverActor.id }) @@ -220,25 +245,33 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } private async createStreamingPlaylistRedundancy ( - redundancy: VideosRedundancy, + redundancy: VideosRedundancyStrategy, video: MVideoAccountLight, playlistArg: MStreamingPlaylist ) { + let strategy = 'manual' + let expiresOn: Date = null + + if (redundancy) { + strategy = redundancy.strategy + expiresOn = this.buildNewExpiration(redundancy.minLifetime) + } + const playlist = playlistArg as MStreamingPlaylistVideo playlist.Video = video const serverActor = await getServerActor() - logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy) + logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy) const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ - expiresOn: this.buildNewExpiration(redundancy.minLifetime), + expiresOn, url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL), - strategy: redundancy.strategy, + strategy, videoStreamingPlaylistId: playlist.id, actorId: serverActor.id }) diff --git a/server/middlewares/sort.ts b/server/middlewares/sort.ts index 8c27e8237..75238228f 100644 --- a/server/middlewares/sort.ts +++ b/server/middlewares/sort.ts @@ -1,17 +1,11 @@ import * as express from 'express' import { SortType } from '../models/utils' -function setDefaultSort (req: express.Request, res: express.Response, next: express.NextFunction) { - if (!req.query.sort) req.query.sort = '-createdAt' +const setDefaultSort = setDefaultSortFactory('-createdAt') - return next() -} +const setDefaultVideoRedundanciesSort = setDefaultSortFactory('name') -function setDefaultSearchSort (req: express.Request, res: express.Response, next: express.NextFunction) { - if (!req.query.sort) req.query.sort = '-match' - - return next() -} +const setDefaultSearchSort = setDefaultSortFactory('-match') function setBlacklistSort (req: express.Request, res: express.Response, next: express.NextFunction) { let newSort: SortType = { sortModel: undefined, sortValue: '' } @@ -39,5 +33,16 @@ function setBlacklistSort (req: express.Request, res: express.Response, next: ex export { setDefaultSort, setDefaultSearchSort, + setDefaultVideoRedundanciesSort, setBlacklistSort } + +// --------------------------------------------------------------------------- + +function setDefaultSortFactory (sort: string) { + return (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (!req.query.sort) req.query.sort = sort + + return next() + } +} diff --git a/server/middlewares/validators/redundancy.ts b/server/middlewares/validators/redundancy.ts index 8098e3a44..16b42fc0d 100644 --- a/server/middlewares/validators/redundancy.ts +++ b/server/middlewares/validators/redundancy.ts @@ -1,12 +1,13 @@ import * as express from 'express' -import { body, param } from 'express-validator' -import { exists, isBooleanValid, isIdOrUUIDValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc' +import { body, param, query } from 'express-validator' +import { exists, isBooleanValid, isIdOrUUIDValid, isIdValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc' import { logger } from '../../helpers/logger' import { areValidationErrors } from './utils' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' import { isHostValid } from '../../helpers/custom-validators/servers' import { ServerModel } from '../../models/server/server' import { doesVideoExist } from '../../helpers/middlewares' +import { isVideoRedundancyTarget } from '@server/helpers/custom-validators/video-redundancies' const videoFileRedundancyGetValidator = [ param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), @@ -101,10 +102,77 @@ const updateServerRedundancyValidator = [ } ] +const listVideoRedundanciesValidator = [ + query('target') + .custom(isVideoRedundancyTarget).withMessage('Should have a valid video redundancies target'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking listVideoRedundanciesValidator parameters', { parameters: req.query }) + + if (areValidationErrors(req, res)) return + + return next() + } +] + +const addVideoRedundancyValidator = [ + body('videoId') + .custom(isIdValid) + .withMessage('Should have a valid video id'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking addVideoRedundancyValidator parameters', { parameters: req.query }) + + if (areValidationErrors(req, res)) return + + if (!await doesVideoExist(req.body.videoId, res, 'only-video')) return + + if (res.locals.onlyVideo.remote === false) { + return res.status(400) + .json({ error: 'Cannot create a redundancy on a local video' }) + .end() + } + + const alreadyExists = await VideoRedundancyModel.isLocalByVideoUUIDExists(res.locals.onlyVideo.uuid) + if (alreadyExists) { + return res.status(409) + .json({ error: 'This video is already duplicated by your instance.' }) + } + + return next() + } +] + +const removeVideoRedundancyValidator = [ + param('redundancyId') + .custom(isIdValid) + .withMessage('Should have a valid redundancy id'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking removeVideoRedundancyValidator parameters', { parameters: req.query }) + + if (areValidationErrors(req, res)) return + + const redundancy = await VideoRedundancyModel.loadByIdWithVideo(parseInt(req.params.redundancyId, 10)) + if (!redundancy) { + return res.status(404) + .json({ error: 'Video redundancy not found' }) + .end() + } + + res.locals.videoRedundancy = redundancy + + return next() + } +] + // --------------------------------------------------------------------------- export { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator, - updateServerRedundancyValidator + updateServerRedundancyValidator, + listVideoRedundanciesValidator, + addVideoRedundancyValidator, + removeVideoRedundancyValidator } diff --git a/server/middlewares/validators/sort.ts b/server/middlewares/validators/sort.ts index c75e701d6..b76dab722 100644 --- a/server/middlewares/validators/sort.ts +++ b/server/middlewares/validators/sort.ts @@ -23,6 +23,7 @@ const SORTABLE_USER_NOTIFICATIONS_COLUMNS = createSortableColumns(SORTABLE_COLUM const SORTABLE_VIDEO_PLAYLISTS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_PLAYLISTS) const SORTABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.PLUGINS) const SORTABLE_AVAILABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.AVAILABLE_PLUGINS) +const SORTABLE_VIDEO_REDUNDANCIES_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES) const usersSortValidator = checkSort(SORTABLE_USERS_COLUMNS) const accountsSortValidator = checkSort(SORTABLE_ACCOUNTS_COLUMNS) @@ -45,6 +46,7 @@ const userNotificationsSortValidator = checkSort(SORTABLE_USER_NOTIFICATIONS_COL const videoPlaylistsSortValidator = checkSort(SORTABLE_VIDEO_PLAYLISTS_COLUMNS) const pluginsSortValidator = checkSort(SORTABLE_PLUGINS_COLUMNS) const availablePluginsSortValidator = checkSort(SORTABLE_AVAILABLE_PLUGINS_COLUMNS) +const videoRedundanciesSortValidator = checkSort(SORTABLE_VIDEO_REDUNDANCIES_COLUMNS) // --------------------------------------------------------------------------- @@ -69,5 +71,6 @@ export { serversBlocklistSortValidator, userNotificationsSortValidator, videoPlaylistsSortValidator, + videoRedundanciesSortValidator, pluginsSortValidator } diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index 8c9a7eabf..4e66d72e3 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -13,13 +13,13 @@ import { UpdatedAt } from 'sequelize-typescript' import { ActorModel } from '../activitypub/actor' -import { getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils' +import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils' import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc' import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants' import { VideoFileModel } from '../video/video-file' import { getServerActor } from '../../helpers/utils' import { VideoModel } from '../video/video' -import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' +import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy' import { logger } from '../../helpers/logger' import { CacheFileObject, VideoPrivacy } from '../../../shared' import { VideoChannelModel } from '../video/video-channel' @@ -27,10 +27,16 @@ import { ServerModel } from '../server/server' import { sample } from 'lodash' import { isTestInstance } from '../../helpers/core-utils' import * as Bluebird from 'bluebird' -import { col, FindOptions, fn, literal, Op, Transaction } from 'sequelize' +import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize' import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist' import { CONFIG } from '../../initializers/config' -import { MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/typings/models' +import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/typings/models' +import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model' +import { + FileRedundancyInformation, + StreamingPlaylistRedundancyInformation, + VideoRedundancy +} from '@shared/models/redundancy/video-redundancy.model' export enum ScopeNames { WITH_VIDEO = 'WITH_VIDEO' @@ -86,7 +92,7 @@ export class VideoRedundancyModel extends Model { @UpdatedAt updatedAt: Date - @AllowNull(false) + @AllowNull(true) @Column expiresOn: Date @@ -193,6 +199,15 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) } + static loadByIdWithVideo (id: number, transaction?: Transaction): Bluebird { + const query = { + where: { id }, + transaction + } + + return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) + } + static loadByUrl (url: string, transaction?: Transaction): Bluebird { const query = { where: { @@ -394,7 +409,8 @@ export class VideoRedundancyModel extends Model { [Op.ne]: actor.id }, expiresOn: { - [ Op.lt ]: new Date() + [ Op.lt ]: new Date(), + [ Op.ne ]: null } } } @@ -447,7 +463,112 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.findAll(query) } - static async getStats (strategy: VideoRedundancyStrategy) { + static listForApi (options: { + start: number, + count: number, + sort: string, + target: VideoRedundanciesTarget, + strategy?: string + }) { + const { start, count, sort, target, strategy } = options + let redundancyWhere: WhereOptions = {} + let videosWhere: WhereOptions = {} + let redundancySqlSuffix = '' + + if (target === 'my-videos') { + Object.assign(videosWhere, { remote: false }) + } else if (target === 'remote-videos') { + Object.assign(videosWhere, { remote: true }) + Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } }) + redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL' + } + + if (strategy) { + Object.assign(redundancyWhere, { strategy: strategy }) + } + + const videoFilterWhere = { + [Op.and]: [ + { + [ Op.or ]: [ + { + id: { + [ Op.in ]: literal( + '(' + + 'SELECT "videoId" FROM "videoFile" ' + + 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' + + redundancySqlSuffix + + ')' + ) + } + }, + { + id: { + [ Op.in ]: literal( + '(' + + 'select "videoId" FROM "videoStreamingPlaylist" ' + + 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' + + redundancySqlSuffix + + ')' + ) + } + } + ] + }, + + videosWhere + ] + } + + // /!\ On video model /!\ + const findOptions = { + offset: start, + limit: count, + order: getSort(sort), + include: [ + { + required: false, + model: VideoFileModel.unscoped(), + include: [ + { + model: VideoRedundancyModel.unscoped(), + required: false, + where: redundancyWhere + } + ] + }, + { + required: false, + model: VideoStreamingPlaylistModel.unscoped(), + include: [ + { + model: VideoRedundancyModel.unscoped(), + required: false, + where: redundancyWhere + }, + { + model: VideoFileModel.unscoped(), + required: false + } + ] + } + ], + where: videoFilterWhere + } + + // /!\ On video model /!\ + const countOptions = { + where: videoFilterWhere + } + + return Promise.all([ + VideoModel.findAll(findOptions), + + VideoModel.count(countOptions) + ]).then(([ data, total ]) => ({ total, data })) + } + + static async getStats (strategy: VideoRedundancyStrategyWithManual) { const actor = await getServerActor() const query: FindOptions = { @@ -478,6 +599,53 @@ export class VideoRedundancyModel extends Model { })) } + static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy { + let filesRedundancies: FileRedundancyInformation[] = [] + let streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = [] + + for (const file of video.VideoFiles) { + for (const redundancy of file.RedundancyVideos) { + filesRedundancies.push({ + id: redundancy.id, + fileUrl: redundancy.fileUrl, + strategy: redundancy.strategy, + createdAt: redundancy.createdAt, + updatedAt: redundancy.updatedAt, + expiresOn: redundancy.expiresOn, + size: file.size + }) + } + } + + for (const playlist of video.VideoStreamingPlaylists) { + const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0) + + for (const redundancy of playlist.RedundancyVideos) { + streamingPlaylistsRedundancies.push({ + id: redundancy.id, + fileUrl: redundancy.fileUrl, + strategy: redundancy.strategy, + createdAt: redundancy.createdAt, + updatedAt: redundancy.updatedAt, + expiresOn: redundancy.expiresOn, + size + }) + } + } + + return { + id: video.id, + name: video.name, + url: video.url, + uuid: video.uuid, + + redundancies: { + files: filesRedundancies, + streamingPlaylists: streamingPlaylistsRedundancies + } + } + } + getVideo () { if (this.VideoFile) return this.VideoFile.Video @@ -494,7 +662,7 @@ export class VideoRedundancyModel extends Model { id: this.url, type: 'CacheFile' as 'CacheFile', object: this.VideoStreamingPlaylist.Video.url, - expires: this.expiresOn.toISOString(), + expires: this.expiresOn ? this.expiresOn.toISOString() : null, url: { type: 'Link', mediaType: 'application/x-mpegURL', @@ -507,7 +675,7 @@ export class VideoRedundancyModel extends Model { id: this.url, type: 'CacheFile' as 'CacheFile', object: this.VideoFile.Video.url, - expires: this.expiresOn.toISOString(), + expires: this.expiresOn ? this.expiresOn.toISOString() : null, url: { type: 'Link', mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any, diff --git a/server/tests/api/check-params/redundancy.ts b/server/tests/api/check-params/redundancy.ts index 6471da840..7012a39ee 100644 --- a/server/tests/api/check-params/redundancy.ts +++ b/server/tests/api/check-params/redundancy.ts @@ -3,21 +3,25 @@ import 'mocha' import { + checkBadCountPagination, + checkBadSortPagination, + checkBadStartPagination, cleanupTests, createUser, doubleFollow, - flushAndRunMultipleServers, - flushTests, - killallServers, + flushAndRunMultipleServers, makeDeleteRequest, + makeGetRequest, makePostBodyRequest, makePutBodyRequest, ServerInfo, - setAccessTokensToServers, - userLogin + setAccessTokensToServers, uploadVideoAndGetId, + userLogin, waitJobs } from '../../../../shared/extra-utils' describe('Test server redundancy API validators', function () { let servers: ServerInfo[] let userAccessToken = null + let videoIdLocal: number + let videoIdRemote: number // --------------------------------------------------------------- @@ -36,9 +40,134 @@ describe('Test server redundancy API validators', function () { await createUser({ url: servers[ 0 ].url, accessToken: servers[ 0 ].accessToken, username: user.username, password: user.password }) userAccessToken = await userLogin(servers[0], user) + + videoIdLocal = (await uploadVideoAndGetId({ server: servers[0], videoName: 'video' })).id + videoIdRemote = (await uploadVideoAndGetId({ server: servers[1], videoName: 'video' })).id + + await waitJobs(servers) }) - describe('When updating redundancy', function () { + describe('When listing redundancies', function () { + const path = '/api/v1/server/redundancy/videos' + + let url: string + let token: string + + before(function () { + url = servers[0].url + token = servers[0].accessToken + }) + + it('Should fail with an invalid token', async function () { + await makeGetRequest({ url, path, token: 'fake_token', statusCodeExpected: 401 }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makeGetRequest({ url, path, token: userAccessToken, statusCodeExpected: 403 }) + }) + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(url, path, servers[0].accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(url, path, servers[0].accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(url, path, servers[0].accessToken) + }) + + it('Should fail with a bad target', async function () { + await makeGetRequest({ url, path, token, query: { target: 'bad target' } }) + }) + + it('Should fail without target', async function () { + await makeGetRequest({ url, path, token }) + }) + + it('Should succeed with the correct params', async function () { + await makeGetRequest({ url, path, token, query: { target: 'my-videos' }, statusCodeExpected: 200 }) + }) + }) + + describe('When manually adding a redundancy', function () { + const path = '/api/v1/server/redundancy/videos' + + let url: string + let token: string + + before(function () { + url = servers[0].url + token = servers[0].accessToken + }) + + it('Should fail with an invalid token', async function () { + await makePostBodyRequest({ url, path, token: 'fake_token', statusCodeExpected: 401 }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makePostBodyRequest({ url, path, token: userAccessToken, statusCodeExpected: 403 }) + }) + + it('Should fail without a video id', async function () { + await makePostBodyRequest({ url, path, token }) + }) + + it('Should fail with an incorrect video id', async function () { + await makePostBodyRequest({ url, path, token, fields: { videoId: 'peertube' } }) + }) + + it('Should fail with a not found video id', async function () { + await makePostBodyRequest({ url, path, token, fields: { videoId: 6565 }, statusCodeExpected: 404 }) + }) + + it('Should fail with a local a video id', async function () { + await makePostBodyRequest({ url, path, token, fields: { videoId: videoIdLocal } }) + }) + + it('Should succeed with the correct params', async function () { + await makePostBodyRequest({ url, path, token, fields: { videoId: videoIdRemote }, statusCodeExpected: 204 }) + }) + + it('Should fail if the video is already duplicated', async function () { + this.timeout(30000) + + await waitJobs(servers) + + await makePostBodyRequest({ url, path, token, fields: { videoId: videoIdRemote }, statusCodeExpected: 409 }) + }) + }) + + describe('When manually removing a redundancy', function () { + const path = '/api/v1/server/redundancy/videos/' + + let url: string + let token: string + + before(function () { + url = servers[0].url + token = servers[0].accessToken + }) + + it('Should fail with an invalid token', async function () { + await makeDeleteRequest({ url, path: path + '1', token: 'fake_token', statusCodeExpected: 401 }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makeDeleteRequest({ url, path: path + '1', token: userAccessToken, statusCodeExpected: 403 }) + }) + + it('Should fail with an incorrect video id', async function () { + await makeDeleteRequest({ url, path: path + 'toto', token }) + }) + + it('Should fail with a not found video redundancy', async function () { + await makeDeleteRequest({ url, path: path + '454545', token, statusCodeExpected: 404 }) + }) + }) + + describe('When updating server redundancy', function () { const path = '/api/v1/server/redundancy' it('Should fail with an invalid token', async function () { diff --git a/server/tests/api/redundancy/index.ts b/server/tests/api/redundancy/index.ts index 8e69b95a6..5359055b0 100644 --- a/server/tests/api/redundancy/index.ts +++ b/server/tests/api/redundancy/index.ts @@ -1 +1,2 @@ import './redundancy' +import './manage-redundancy' diff --git a/server/tests/api/redundancy/manage-redundancy.ts b/server/tests/api/redundancy/manage-redundancy.ts new file mode 100644 index 000000000..6a8937f24 --- /dev/null +++ b/server/tests/api/redundancy/manage-redundancy.ts @@ -0,0 +1,373 @@ +/* tslint:disable:no-unused-expression */ + +import * as chai from 'chai' +import 'mocha' +import { + cleanupTests, + doubleFollow, + flushAndRunMultipleServers, + getLocalIdByUUID, + ServerInfo, + setAccessTokensToServers, + uploadVideo, + uploadVideoAndGetId, + waitUntilLog +} from '../../../../shared/extra-utils' +import { waitJobs } from '../../../../shared/extra-utils/server/jobs' +import { addVideoRedundancy, listVideoRedundancies, removeVideoRedundancy, updateRedundancy } from '@shared/extra-utils/server/redundancy' +import { VideoPrivacy, VideoRedundanciesTarget, VideoRedundancy } from '@shared/models' + +const expect = chai.expect + +describe('Test manage videos redundancy', function () { + const targets: VideoRedundanciesTarget[] = [ 'my-videos', 'remote-videos' ] + + let servers: ServerInfo[] + let video1Server2UUID: string + let video2Server2UUID: string + let redundanciesToRemove: number[] = [] + + before(async function () { + this.timeout(120000) + + const config = { + transcoding: { + hls: { + enabled: true + } + }, + redundancy: { + videos: { + check_interval: '1 second', + strategies: [ + { + strategy: 'recently-added', + min_lifetime: '1 hour', + size: '10MB', + min_views: 0 + } + ] + } + } + } + servers = await flushAndRunMultipleServers(3, config) + + // Get the access tokens + await setAccessTokensToServers(servers) + + { + const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) + video1Server2UUID = res.body.video.uuid + } + + { + const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) + video2Server2UUID = res.body.video.uuid + } + + await waitJobs(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[ 0 ], servers[ 1 ]) + await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) + + await waitJobs(servers) + }) + + it('Should not have redundancies on server 3', async function () { + for (const target of targets) { + const res = await listVideoRedundancies({ + url: servers[2].url, + accessToken: servers[2].accessToken, + target + }) + + expect(res.body.total).to.equal(0) + expect(res.body.data).to.have.lengthOf(0) + } + }) + + it('Should not have "remote-videos" redundancies on server 2', async function () { + this.timeout(120000) + + await waitJobs(servers) + await waitUntilLog(servers[0], 'Duplicated ', 10) + await waitJobs(servers) + + const res = await listVideoRedundancies({ + url: servers[1].url, + accessToken: servers[1].accessToken, + target: 'remote-videos' + }) + + expect(res.body.total).to.equal(0) + expect(res.body.data).to.have.lengthOf(0) + }) + + it('Should have "my-videos" redundancies on server 2', async function () { + this.timeout(120000) + + const res = await listVideoRedundancies({ + url: servers[1].url, + accessToken: servers[1].accessToken, + target: 'my-videos' + }) + + expect(res.body.total).to.equal(2) + + const videos = res.body.data as VideoRedundancy[] + expect(videos).to.have.lengthOf(2) + + const videos1 = videos.find(v => v.uuid === video1Server2UUID) + const videos2 = videos.find(v => v.uuid === video2Server2UUID) + + expect(videos1.name).to.equal('video 1 server 2') + expect(videos2.name).to.equal('video 2 server 2') + + expect(videos1.redundancies.files).to.have.lengthOf(4) + expect(videos1.redundancies.streamingPlaylists).to.have.lengthOf(1) + + const redundancies = videos1.redundancies.files.concat(videos1.redundancies.streamingPlaylists) + + for (const r of redundancies) { + expect(r.strategy).to.be.null + expect(r.fileUrl).to.exist + expect(r.createdAt).to.exist + expect(r.updatedAt).to.exist + expect(r.expiresOn).to.exist + } + }) + + it('Should not have "my-videos" redundancies on server 1', async function () { + const res = await listVideoRedundancies({ + url: servers[0].url, + accessToken: servers[0].accessToken, + target: 'my-videos' + }) + + expect(res.body.total).to.equal(0) + expect(res.body.data).to.have.lengthOf(0) + }) + + it('Should have "remote-videos" redundancies on server 1', async function () { + this.timeout(120000) + + const res = await listVideoRedundancies({ + url: servers[0].url, + accessToken: servers[0].accessToken, + target: 'remote-videos' + }) + + expect(res.body.total).to.equal(2) + + const videos = res.body.data as VideoRedundancy[] + expect(videos).to.have.lengthOf(2) + + const videos1 = videos.find(v => v.uuid === video1Server2UUID) + const videos2 = videos.find(v => v.uuid === video2Server2UUID) + + expect(videos1.name).to.equal('video 1 server 2') + expect(videos2.name).to.equal('video 2 server 2') + + expect(videos1.redundancies.files).to.have.lengthOf(4) + expect(videos1.redundancies.streamingPlaylists).to.have.lengthOf(1) + + const redundancies = videos1.redundancies.files.concat(videos1.redundancies.streamingPlaylists) + + for (const r of redundancies) { + expect(r.strategy).to.equal('recently-added') + expect(r.fileUrl).to.exist + expect(r.createdAt).to.exist + expect(r.updatedAt).to.exist + expect(r.expiresOn).to.exist + } + }) + + it('Should correctly paginate and sort results', async function () { + { + const res = await listVideoRedundancies({ + url: servers[0].url, + accessToken: servers[0].accessToken, + target: 'remote-videos', + sort: 'name', + start: 0, + count: 2 + }) + + const videos = res.body.data + expect(videos[ 0 ].name).to.equal('video 1 server 2') + expect(videos[ 1 ].name).to.equal('video 2 server 2') + } + + { + const res = await listVideoRedundancies({ + url: servers[0].url, + accessToken: servers[0].accessToken, + target: 'remote-videos', + sort: '-name', + start: 0, + count: 2 + }) + + const videos = res.body.data + expect(videos[ 0 ].name).to.equal('video 2 server 2') + expect(videos[ 1 ].name).to.equal('video 1 server 2') + } + + { + const res = await listVideoRedundancies({ + url: servers[0].url, + accessToken: servers[0].accessToken, + target: 'remote-videos', + sort: '-name', + start: 1, + count: 1 + }) + + const videos = res.body.data + expect(videos[ 0 ].name).to.equal('video 1 server 2') + } + }) + + it('Should manually add a redundancy and list it', async function () { + this.timeout(120000) + + const uuid = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video 3 server 2', privacy: VideoPrivacy.UNLISTED })).uuid + await waitJobs(servers) + const videoId = await getLocalIdByUUID(servers[0].url, uuid) + + await addVideoRedundancy({ + url: servers[0].url, + accessToken: servers[0].accessToken, + videoId + }) + + await waitJobs(servers) + await waitUntilLog(servers[0], 'Duplicated ', 15) + await waitJobs(servers) + + { + const res = await listVideoRedundancies({ + url: servers[0].url, + accessToken: servers[0].accessToken, + target: 'remote-videos', + sort: '-name', + start: 0, + count: 5 + }) + + const videos = res.body.data + expect(videos[ 0 ].name).to.equal('video 3 server 2') + + const video = videos[ 0 ] + expect(video.redundancies.files).to.have.lengthOf(4) + expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) + + const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists) + + for (const r of redundancies) { + redundanciesToRemove.push(r.id) + + expect(r.strategy).to.equal('manual') + expect(r.fileUrl).to.exist + expect(r.createdAt).to.exist + expect(r.updatedAt).to.exist + expect(r.expiresOn).to.be.null + } + } + + const res = await listVideoRedundancies({ + url: servers[1].url, + accessToken: servers[1].accessToken, + target: 'my-videos', + sort: '-name', + start: 0, + count: 5 + }) + + const videos = res.body.data + expect(videos[ 0 ].name).to.equal('video 3 server 2') + + const video = videos[ 0 ] + expect(video.redundancies.files).to.have.lengthOf(4) + expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) + + const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists) + + for (const r of redundancies) { + expect(r.strategy).to.be.null + expect(r.fileUrl).to.exist + expect(r.createdAt).to.exist + expect(r.updatedAt).to.exist + expect(r.expiresOn).to.be.null + } + }) + + it('Should manually remove a redundancy and remove it from the list', async function () { + this.timeout(120000) + + for (const redundancyId of redundanciesToRemove) { + await removeVideoRedundancy({ + url: servers[ 0 ].url, + accessToken: servers[ 0 ].accessToken, + redundancyId + }) + } + + { + const res = await listVideoRedundancies({ + url: servers[0].url, + accessToken: servers[0].accessToken, + target: 'remote-videos', + sort: '-name', + start: 0, + count: 5 + }) + + const videos = res.body.data + expect(videos).to.have.lengthOf(2) + + expect(videos[ 0 ].name).to.equal('video 2 server 2') + + redundanciesToRemove = [] + const video = videos[ 0 ] + expect(video.redundancies.files).to.have.lengthOf(4) + expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) + + const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists) + + for (const r of redundancies) { + redundanciesToRemove.push(r.id) + } + } + }) + + it('Should remove another (auto) redundancy', async function () { + { + for (const redundancyId of redundanciesToRemove) { + await removeVideoRedundancy({ + url: servers[ 0 ].url, + accessToken: servers[ 0 ].accessToken, + redundancyId + }) + } + + const res = await listVideoRedundancies({ + url: servers[0].url, + accessToken: servers[0].accessToken, + target: 'remote-videos', + sort: '-name', + start: 0, + count: 5 + }) + + const videos = res.body.data + expect(videos[ 0 ].name).to.equal('video 1 server 2') + expect(videos).to.have.lengthOf(1) + } + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index 1cdf93aa1..f5bf130d5 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts @@ -5,7 +5,8 @@ import 'mocha' import { VideoDetails } from '../../../../shared/models/videos' import { checkSegmentHash, - checkVideoFilesWereRemoved, cleanupTests, + checkVideoFilesWereRemoved, + cleanupTests, doubleFollow, flushAndRunMultipleServers, getFollowingListPaginationAndSort, @@ -28,11 +29,16 @@ import { import { waitJobs } from '../../../../shared/extra-utils/server/jobs' import * as magnetUtil from 'magnet-uri' -import { updateRedundancy } from '../../../../shared/extra-utils/server/redundancy' +import { + addVideoRedundancy, + listVideoRedundancies, + removeVideoRedundancy, + updateRedundancy +} from '../../../../shared/extra-utils/server/redundancy' import { ActorFollow } from '../../../../shared/models/actors' import { readdir } from 'fs-extra' import { join } from 'path' -import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy' +import { VideoRedundancy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../../shared/models/redundancy' import { getStats } from '../../../../shared/extra-utils/server/stats' import { ServerStats } from '../../../../shared/models/server/server-stats.model' @@ -40,6 +46,7 @@ const expect = chai.expect let servers: ServerInfo[] = [] let video1Server2UUID: string +let video1Server2Id: number function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) { const parsed = magnetUtil.decode(file.magnetUri) @@ -52,7 +59,19 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) } -async function flushAndRunServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { +async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}) { + const strategies: any[] = [] + + if (strategy !== null) { + strategies.push( + immutableAssign({ + min_lifetime: '1 hour', + strategy: strategy, + size: '400KB' + }, additionalParams) + ) + } + const config = { transcoding: { hls: { @@ -62,16 +81,11 @@ async function flushAndRunServers (strategy: VideoRedundancyStrategy, additional redundancy: { videos: { check_interval: '5 seconds', - strategies: [ - immutableAssign({ - min_lifetime: '1 hour', - strategy: strategy, - size: '400KB' - }, additionalParams) - ] + strategies } } } + servers = await flushAndRunMultipleServers(3, config) // Get the access tokens @@ -80,6 +94,7 @@ async function flushAndRunServers (strategy: VideoRedundancyStrategy, additional { const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) video1Server2UUID = res.body.video.uuid + video1Server2Id = res.body.video.id await viewVideo(servers[ 1 ].url, video1Server2UUID) } @@ -216,29 +231,38 @@ async function check1PlaylistRedundancies (videoUUID?: string) { } } -async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { +async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) { + let totalSize: number = null + let statsLength = 1 + + if (strategy !== 'manual') { + totalSize = 409600 + statsLength = 2 + } + const res = await getStats(servers[0].url) const data: ServerStats = res.body - expect(data.videosRedundancy).to.have.lengthOf(1) - const stat = data.videosRedundancy[0] + expect(data.videosRedundancy).to.have.lengthOf(statsLength) + const stat = data.videosRedundancy[0] expect(stat.strategy).to.equal(strategy) - expect(stat.totalSize).to.equal(409600) + expect(stat.totalSize).to.equal(totalSize) + + return stat +} + +async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategyWithManual) { + const stat = await checkStatsGlobal(strategy) + expect(stat.totalUsed).to.be.at.least(1).and.below(409601) expect(stat.totalVideoFiles).to.equal(4) expect(stat.totalVideos).to.equal(1) } -async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { - const res = await getStats(servers[0].url) - const data: ServerStats = res.body +async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategyWithManual) { + const stat = await checkStatsGlobal(strategy) - expect(data.videosRedundancy).to.have.lengthOf(1) - - const stat = data.videosRedundancy[0] - expect(stat.strategy).to.equal(strategy) - expect(stat.totalSize).to.equal(409600) expect(stat.totalUsed).to.equal(0) expect(stat.totalVideoFiles).to.equal(0) expect(stat.totalVideos).to.equal(0) @@ -446,6 +470,74 @@ describe('Test videos redundancy', function () { }) }) + describe('With manual strategy', function () { + before(function () { + this.timeout(120000) + + return flushAndRunServers(null) + }) + + it('Should have 1 webseed on the first video', async function () { + await check1WebSeed() + await check0PlaylistRedundancies() + await checkStatsWith1Webseed('manual') + }) + + it('Should create a redundancy on first video', async function () { + await addVideoRedundancy({ + url: servers[0].url, + accessToken: servers[0].accessToken, + videoId: video1Server2Id + }) + }) + + it('Should have 2 webseeds on the first video', async function () { + this.timeout(80000) + + await waitJobs(servers) + await waitUntilLog(servers[0], 'Duplicated ', 5) + await waitJobs(servers) + + await check2Webseeds() + await check1PlaylistRedundancies() + await checkStatsWith2Webseed('manual') + }) + + it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () { + this.timeout(80000) + + const res = await listVideoRedundancies({ + url: servers[0].url, + accessToken: servers[0].accessToken, + target: 'remote-videos' + }) + + const videos = res.body.data as VideoRedundancy[] + expect(videos).to.have.lengthOf(1) + + const video = videos[0] + for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) { + await removeVideoRedundancy({ + url: servers[0].url, + accessToken: servers[0].accessToken, + redundancyId: r.id + }) + } + + await waitJobs(servers) + await wait(5000) + + await check1WebSeed() + await check0PlaylistRedundancies() + + await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) + }) + + after(async function () { + await cleanupTests(servers) + }) + }) + describe('Test expiration', function () { const strategy = 'recently-added' diff --git a/server/typings/models/video/video-file.ts b/server/typings/models/video/video-file.ts index 352fe3d32..139b22b2c 100644 --- a/server/typings/models/video/video-file.ts +++ b/server/typings/models/video/video-file.ts @@ -1,7 +1,7 @@ import { VideoFileModel } from '../../../models/video/video-file' import { PickWith, PickWithOpt } from '../../utils' import { MVideo, MVideoUUID } from './video' -import { MVideoRedundancyFileUrl } from './video-redundancy' +import { MVideoRedundancy, MVideoRedundancyFileUrl } from './video-redundancy' import { MStreamingPlaylistVideo, MStreamingPlaylist } from './video-streaming-playlist' type Use = PickWith @@ -22,6 +22,9 @@ export type MVideoFileStreamingPlaylistVideo = MVideoFile & export type MVideoFileVideoUUID = MVideoFile & Use<'Video', MVideoUUID> +export type MVideoFileRedundanciesAll = MVideoFile & + PickWithOpt + export type MVideoFileRedundanciesOpt = MVideoFile & PickWithOpt diff --git a/server/typings/models/video/video-streaming-playlist.ts b/server/typings/models/video/video-streaming-playlist.ts index 436c0c072..6fd489945 100644 --- a/server/typings/models/video/video-streaming-playlist.ts +++ b/server/typings/models/video/video-streaming-playlist.ts @@ -1,6 +1,6 @@ import { VideoStreamingPlaylistModel } from '../../../models/video/video-streaming-playlist' import { PickWith, PickWithOpt } from '../../utils' -import { MVideoRedundancyFileUrl } from './video-redundancy' +import { MVideoRedundancyFileUrl, MVideoRedundancy } from './video-redundancy' import { MVideo } from './video' import { MVideoFile } from './video-file' @@ -20,6 +20,10 @@ export type MStreamingPlaylistFilesVideo = MStreamingPlaylist & Use<'VideoFiles', MVideoFile[]> & Use<'Video', MVideo> +export type MStreamingPlaylistRedundanciesAll = MStreamingPlaylist & + Use<'VideoFiles', MVideoFile[]> & + Use<'RedundancyVideos', MVideoRedundancy[]> + export type MStreamingPlaylistRedundancies = MStreamingPlaylist & Use<'VideoFiles', MVideoFile[]> & Use<'RedundancyVideos', MVideoRedundancyFileUrl[]> diff --git a/server/typings/models/video/video.ts b/server/typings/models/video/video.ts index 7f69a91de..bcc5e5028 100644 --- a/server/typings/models/video/video.ts +++ b/server/typings/models/video/video.ts @@ -10,8 +10,13 @@ import { } from './video-channels' import { MTag } from './tag' import { MVideoCaptionLanguage } from './video-caption' -import { MStreamingPlaylistFiles, MStreamingPlaylistRedundancies, MStreamingPlaylistRedundanciesOpt } from './video-streaming-playlist' -import { MVideoFile, MVideoFileRedundanciesOpt } from './video-file' +import { + MStreamingPlaylistFiles, + MStreamingPlaylistRedundancies, + MStreamingPlaylistRedundanciesAll, + MStreamingPlaylistRedundanciesOpt +} from './video-streaming-playlist' +import { MVideoFile, MVideoFileRedundanciesAll, MVideoFileRedundanciesOpt } from './video-file' import { MThumbnail } from './thumbnail' import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist' import { MScheduleVideoUpdate } from './schedule-video-update' @@ -158,6 +163,10 @@ export type MVideoForUser = MVideo & Use<'VideoBlacklist', MVideoBlacklistLight> & Use<'Thumbnails', MThumbnail[]> +export type MVideoForRedundancyAPI = MVideo & + Use<'VideoFiles', MVideoFileRedundanciesAll[]> & + Use<'VideoStreamingPlaylists', MStreamingPlaylistRedundanciesAll[]> + // ############################################################################ // Format for API or AP object diff --git a/shared/extra-utils/server/redundancy.ts b/shared/extra-utils/server/redundancy.ts index c39ff2c8b..7b488e23e 100644 --- a/shared/extra-utils/server/redundancy.ts +++ b/shared/extra-utils/server/redundancy.ts @@ -1,6 +1,7 @@ -import { makePutBodyRequest } from '../requests/requests' +import { makeDeleteRequest, makeGetRequest, makePostBodyRequest, makePutBodyRequest } from '../requests/requests' +import { VideoRedundanciesTarget } from '@shared/models' -async function updateRedundancy (url: string, accessToken: string, host: string, redundancyAllowed: boolean, expectedStatus = 204) { +function updateRedundancy (url: string, accessToken: string, host: string, redundancyAllowed: boolean, expectedStatus = 204) { const path = '/api/v1/server/redundancy/' + host return makePutBodyRequest({ @@ -12,6 +13,69 @@ async function updateRedundancy (url: string, accessToken: string, host: string, }) } -export { - updateRedundancy +function listVideoRedundancies (options: { + url: string + accessToken: string, + target: VideoRedundanciesTarget, + start?: number, + count?: number, + sort?: string, + statusCodeExpected?: number +}) { + const path = '/api/v1/server/redundancy/videos' + + const { url, accessToken, target, statusCodeExpected, start, count, sort } = options + + return makeGetRequest({ + url, + token: accessToken, + path, + query: { + start: start ?? 0, + count: count ?? 5, + sort: sort ?? 'name', + target + }, + statusCodeExpected: statusCodeExpected || 200 + }) +} + +function addVideoRedundancy (options: { + url: string, + accessToken: string, + videoId: number +}) { + const path = '/api/v1/server/redundancy/videos' + const { url, accessToken, videoId } = options + + return makePostBodyRequest({ + url, + token: accessToken, + path, + fields: { videoId }, + statusCodeExpected: 204 + }) +} + +function removeVideoRedundancy (options: { + url: string, + accessToken: string, + redundancyId: number +}) { + const { url, accessToken, redundancyId } = options + const path = '/api/v1/server/redundancy/videos/' + redundancyId + + return makeDeleteRequest({ + url, + token: accessToken, + path, + statusCodeExpected: 204 + }) +} + +export { + updateRedundancy, + listVideoRedundancies, + addVideoRedundancy, + removeVideoRedundancy } diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts index 7a77a03ad..aa13273ae 100644 --- a/shared/extra-utils/videos/videos.ts +++ b/shared/extra-utils/videos/videos.ts @@ -607,15 +607,28 @@ async function videoUUIDToId (url: string, id: number | string) { return res.body.id } -async function uploadVideoAndGetId (options: { server: ServerInfo, videoName: string, nsfw?: boolean, token?: string }) { +async function uploadVideoAndGetId (options: { + server: ServerInfo, + videoName: string, + nsfw?: boolean, + privacy?: VideoPrivacy, + token?: string +}) { const videoAttrs: any = { name: options.videoName } if (options.nsfw) videoAttrs.nsfw = options.nsfw + if (options.privacy) videoAttrs.privacy = options.privacy const res = await uploadVideo(options.server.url, options.token || options.server.accessToken, videoAttrs) return { id: res.body.video.id, uuid: res.body.video.uuid } } +async function getLocalIdByUUID (url: string, uuid: string) { + const res = await getVideo(url, uuid) + + return res.body.id +} + // --------------------------------------------------------------------------- export { @@ -645,5 +658,6 @@ export { completeVideoCheck, checkVideoFilesWereRemoved, getPlaylistVideos, - uploadVideoAndGetId + uploadVideoAndGetId, + getLocalIdByUUID } diff --git a/shared/models/redundancy/index.ts b/shared/models/redundancy/index.ts index 61bf0fca7..649cc489f 100644 --- a/shared/models/redundancy/index.ts +++ b/shared/models/redundancy/index.ts @@ -1 +1,3 @@ -export * from './videos-redundancy.model' +export * from './videos-redundancy-strategy.model' +export * from './video-redundancies-filters.model' +export * from './video-redundancy.model' diff --git a/shared/models/redundancy/video-redundancies-filters.model.ts b/shared/models/redundancy/video-redundancies-filters.model.ts new file mode 100644 index 000000000..05ba7dfd3 --- /dev/null +++ b/shared/models/redundancy/video-redundancies-filters.model.ts @@ -0,0 +1 @@ +export type VideoRedundanciesTarget = 'my-videos' | 'remote-videos' diff --git a/shared/models/redundancy/video-redundancy.model.ts b/shared/models/redundancy/video-redundancy.model.ts new file mode 100644 index 000000000..014f69634 --- /dev/null +++ b/shared/models/redundancy/video-redundancy.model.ts @@ -0,0 +1,33 @@ +export interface VideoRedundancy { + id: number + name: string + url: string + uuid: string + + redundancies: { + files: FileRedundancyInformation[] + + streamingPlaylists: StreamingPlaylistRedundancyInformation[] + } +} + +interface RedundancyInformation { + id: number + fileUrl: string + strategy: string + + createdAt: Date | string + updatedAt: Date | string + + expiresOn: Date | string + + size: number +} + +export interface FileRedundancyInformation extends RedundancyInformation { + +} + +export interface StreamingPlaylistRedundancyInformation extends RedundancyInformation { + +} diff --git a/shared/models/redundancy/videos-redundancy.model.ts b/shared/models/redundancy/videos-redundancy-strategy.model.ts similarity index 67% rename from shared/models/redundancy/videos-redundancy.model.ts rename to shared/models/redundancy/videos-redundancy-strategy.model.ts index a8c2743c1..15409abf0 100644 --- a/shared/models/redundancy/videos-redundancy.model.ts +++ b/shared/models/redundancy/videos-redundancy-strategy.model.ts @@ -1,4 +1,5 @@ export type VideoRedundancyStrategy = 'most-views' | 'trending' | 'recently-added' +export type VideoRedundancyStrategyWithManual = VideoRedundancyStrategy | 'manual' export type MostViewsRedundancyStrategy = { strategy: 'most-views' @@ -19,4 +20,4 @@ export type RecentlyAddedStrategy = { minLifetime: number } -export type VideosRedundancy = MostViewsRedundancyStrategy | TrendingRedundancyStrategy | RecentlyAddedStrategy +export type VideosRedundancyStrategy = MostViewsRedundancyStrategy | TrendingRedundancyStrategy | RecentlyAddedStrategy diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index b82a633b2..19fd4c659 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts @@ -9,7 +9,8 @@ export type JobType = 'activitypub-http-unicast' | 'email' | 'video-import' | 'videos-views' | - 'activitypub-refresher' + 'activitypub-refresher' | + 'video-redundancy' export interface Job { id: number diff --git a/shared/models/server/server-stats.model.ts b/shared/models/server/server-stats.model.ts index 74f3de5d3..11778e6ed 100644 --- a/shared/models/server/server-stats.model.ts +++ b/shared/models/server/server-stats.model.ts @@ -1,4 +1,4 @@ -import { VideoRedundancyStrategy } from '../redundancy' +import { VideoRedundancyStrategyWithManual } from '../redundancy' export interface ServerStats { totalUsers: number @@ -13,11 +13,13 @@ export interface ServerStats { totalInstanceFollowers: number totalInstanceFollowing: number - videosRedundancy: { - strategy: VideoRedundancyStrategy - totalSize: number - totalUsed: number - totalVideoFiles: number - totalVideos: number - }[] + videosRedundancy: VideosRedundancyStats[] +} + +export interface VideosRedundancyStats { + strategy: VideoRedundancyStrategyWithManual + totalSize: number + totalUsed: number + totalVideoFiles: number + totalVideos: number } diff --git a/shared/models/users/user-right.enum.ts b/shared/models/users/user-right.enum.ts index 4a28a229d..2f88a65de 100644 --- a/shared/models/users/user-right.enum.ts +++ b/shared/models/users/user-right.enum.ts @@ -33,5 +33,7 @@ export enum UserRight { SEE_ALL_VIDEOS, CHANGE_VIDEO_OWNERSHIP, - MANAGE_PLUGINS + MANAGE_PLUGINS, + + MANAGE_VIDEOS_REDUNDANCIES } diff --git a/shared/models/videos/video.model.ts b/shared/models/videos/video.model.ts index 7576439fe..a69152759 100644 --- a/shared/models/videos/video.model.ts +++ b/shared/models/videos/video.model.ts @@ -1,4 +1,4 @@ -import { AccountSummary, VideoChannelSummary, VideoResolution, VideoState } from '../../index' +import { AccountSummary, VideoChannelSummary, VideoState } from '../../index' import { Account } from '../actors' import { VideoChannel } from './channel/video-channel.model' import { VideoPrivacy } from './video-privacy.enum'