diff --git a/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.html b/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.html index bbf946df0..9701e7f85 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.html +++ b/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.html @@ -52,6 +52,20 @@
{{ formErrors.cache.torrents.size }}
+ +
+ + +
+ + {getCacheSize('storyboards'), plural, =1 {cached storyboard} other {cached storyboards}} +
+ +
{{ formErrors.cache.storyboards.size }}
+
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.ts index 79a98f288..06c5e6221 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.ts @@ -10,7 +10,7 @@ export class EditAdvancedConfigurationComponent { @Input() form: FormGroup @Input() formErrors: any - getCacheSize (type: 'captions' | 'previews' | 'torrents') { + getCacheSize (type: 'captions' | 'previews' | 'torrents' | 'storyboards') { return this.form.value['cache'][type]['size'] } } diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts index 2c3b7560d..9219d608b 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts @@ -9,8 +9,7 @@ import { Notifier } from '@app/core' import { ServerService } from '@app/core/server/server.service' import { ADMIN_EMAIL_VALIDATOR, - CACHE_CAPTIONS_SIZE_VALIDATOR, - CACHE_PREVIEWS_SIZE_VALIDATOR, + CACHE_SIZE_VALIDATOR, CONCURRENCY_VALIDATOR, INDEX_URL_VALIDATOR, INSTANCE_NAME_VALIDATOR, @@ -120,13 +119,16 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit { }, cache: { previews: { - size: CACHE_PREVIEWS_SIZE_VALIDATOR + size: CACHE_SIZE_VALIDATOR }, captions: { - size: CACHE_CAPTIONS_SIZE_VALIDATOR + size: CACHE_SIZE_VALIDATOR }, torrents: { - size: CACHE_CAPTIONS_SIZE_VALIDATOR + size: CACHE_SIZE_VALIDATOR + }, + storyboards: { + size: CACHE_SIZE_VALIDATOR } }, signup: { diff --git a/client/src/app/+videos/+video-watch/video-watch.component.ts b/client/src/app/+videos/+video-watch/video-watch.component.ts index aba3ee086..43744789d 100644 --- a/client/src/app/+videos/+video-watch/video-watch.component.ts +++ b/client/src/app/+videos/+video-watch/video-watch.component.ts @@ -33,6 +33,7 @@ import { LiveVideo, PeerTubeProblemDocument, ServerErrorCode, + Storyboard, VideoCaption, VideoPrivacy, VideoState @@ -69,6 +70,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy { videoCaptions: VideoCaption[] = [] liveVideo: LiveVideo videoPassword: string + storyboards: Storyboard[] = [] playlistPosition: number playlist: VideoPlaylist = null @@ -285,9 +287,10 @@ export class VideoWatchComponent implements OnInit, OnDestroy { forkJoin([ videoAndLiveObs, this.videoCaptionService.listCaptions(videoId, videoPassword), + this.videoService.getStoryboards(videoId), this.userService.getAnonymousOrLoggedUser() ]).subscribe({ - next: ([ { video, live, videoFileToken }, captionsResult, loggedInOrAnonymousUser ]) => { + next: ([ { video, live, videoFileToken }, captionsResult, storyboards, loggedInOrAnonymousUser ]) => { const queryParams = this.route.snapshot.queryParams const urlOptions = { @@ -309,6 +312,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy { video, live, videoCaptions: captionsResult.data, + storyboards, videoFileToken, videoPassword, loggedInOrAnonymousUser, @@ -414,6 +418,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy { video: VideoDetails live: LiveVideo videoCaptions: VideoCaption[] + storyboards: Storyboard[] videoFileToken: string videoPassword: string @@ -421,7 +426,17 @@ export class VideoWatchComponent implements OnInit, OnDestroy { loggedInOrAnonymousUser: User forceAutoplay: boolean }) { - const { video, live, videoCaptions, urlOptions, videoFileToken, videoPassword, loggedInOrAnonymousUser, forceAutoplay } = options + const { + video, + live, + videoCaptions, + storyboards, + urlOptions, + videoFileToken, + videoPassword, + loggedInOrAnonymousUser, + forceAutoplay + } = options this.subscribeToLiveEventsIfNeeded(this.video, video) @@ -430,6 +445,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy { this.liveVideo = live this.videoFileToken = videoFileToken this.videoPassword = videoPassword + this.storyboards = storyboards // Re init attributes this.playerPlaceholderImgSrc = undefined @@ -485,6 +501,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy { const params = { video: this.video, videoCaptions: this.videoCaptions, + storyboards: this.storyboards, liveVideo: this.liveVideo, videoFileToken: this.videoFileToken, videoPassword: this.videoPassword, @@ -636,6 +653,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy { video: VideoDetails liveVideo: LiveVideo videoCaptions: VideoCaption[] + storyboards: Storyboard[] videoFileToken: string videoPassword: string @@ -646,7 +664,17 @@ export class VideoWatchComponent implements OnInit, OnDestroy { forceAutoplay: boolean user?: AuthUser // Keep for plugins }) { - const { video, liveVideo, videoCaptions, videoFileToken, videoPassword, urlOptions, loggedInOrAnonymousUser, forceAutoplay } = params + const { + video, + liveVideo, + videoCaptions, + storyboards, + videoFileToken, + videoPassword, + urlOptions, + loggedInOrAnonymousUser, + forceAutoplay + } = params const getStartTime = () => { const byUrl = urlOptions.startTime !== undefined @@ -673,6 +701,15 @@ export class VideoWatchComponent implements OnInit, OnDestroy { src: environment.apiUrl + c.captionPath })) + const storyboard = storyboards.length !== 0 + ? { + url: environment.apiUrl + storyboards[0].storyboardPath, + height: storyboards[0].spriteHeight, + width: storyboards[0].spriteWidth, + interval: storyboards[0].spriteDuration + } + : undefined + const liveOptions = video.isLive ? { latencyMode: liveVideo.latencyMode } : undefined @@ -734,6 +771,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy { videoPassword: () => videoPassword, videoCaptions: playerCaptions, + storyboard, videoShortUUID: video.shortUUID, videoUUID: video.uuid, @@ -767,6 +805,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy { else mode = 'webtorrent' } + // FIXME: remove, we don't support these old web browsers anymore // p2p-media-loader needs TextEncoder, fallback on WebTorrent if not available if (typeof TextEncoder === 'undefined') { mode = 'webtorrent' diff --git a/client/src/app/shared/form-validators/custom-config-validators.ts b/client/src/app/shared/form-validators/custom-config-validators.ts index ff0813f7d..3672e5610 100644 --- a/client/src/app/shared/form-validators/custom-config-validators.ts +++ b/client/src/app/shared/form-validators/custom-config-validators.ts @@ -22,21 +22,12 @@ export const SERVICES_TWITTER_USERNAME_VALIDATOR: BuildFormValidator = { } } -export const CACHE_PREVIEWS_SIZE_VALIDATOR: BuildFormValidator = { +export const CACHE_SIZE_VALIDATOR: BuildFormValidator = { VALIDATORS: [ Validators.required, Validators.min(1), Validators.pattern('[0-9]+') ], MESSAGES: { - required: $localize`Previews cache size is required.`, - min: $localize`Previews cache size must be greater than 1.`, - pattern: $localize`Previews cache size must be a number.` - } -} - -export const CACHE_CAPTIONS_SIZE_VALIDATOR: BuildFormValidator = { - VALIDATORS: [ Validators.required, Validators.min(1), Validators.pattern('[0-9]+') ], - MESSAGES: { - required: $localize`Captions cache size is required.`, - min: $localize`Captions cache size must be greater than 1.`, - pattern: $localize`Captions cache size must be a number.` + required: $localize`Cache size is required.`, + min: $localize`Cache size must be greater than 1.`, + pattern: $localize`Cache size must be a number.` } } diff --git a/client/src/app/shared/shared-main/video/video.service.ts b/client/src/app/shared/shared-main/video/video.service.ts index d67a2e192..c2e3d7511 100644 --- a/client/src/app/shared/shared-main/video/video.service.ts +++ b/client/src/app/shared/shared-main/video/video.service.ts @@ -11,6 +11,7 @@ import { FeedFormat, NSFWPolicyType, ResultList, + Storyboard, UserVideoRate, UserVideoRateType, UserVideoRateUpdate, @@ -344,6 +345,25 @@ export class VideoService { ) } + // --------------------------------------------------------------------------- + + getStoryboards (videoId: string | number) { + return this.authHttp + .get<{ storyboards: Storyboard[] }>(VideoService.BASE_VIDEO_URL + '/' + videoId + '/storyboards') + .pipe( + map(({ storyboards }) => storyboards), + catchError(err => { + if (err.status === 404) { + return of([]) + } + + this.restExtractor.handleError(err) + }) + ) + } + + // --------------------------------------------------------------------------- + getSource (videoId: number) { return this.authHttp .get<{ source: VideoSource }>(VideoService.BASE_VIDEO_URL + '/' + videoId + '/source') @@ -358,6 +378,8 @@ export class VideoService { ) } + // --------------------------------------------------------------------------- + setVideoLike (id: string, videoPassword: string) { return this.setVideoRate(id, 'like', videoPassword) } @@ -370,6 +392,8 @@ export class VideoService { return this.setVideoRate(id, 'none', videoPassword) } + // --------------------------------------------------------------------------- + getUserVideoRating (id: string) { const url = UserService.BASE_USERS_URL + 'me/videos/' + id + '/rating' diff --git a/client/src/assets/player/peertube-player-manager.ts b/client/src/assets/player/peertube-player-manager.ts index 2781850b9..66d9c7298 100644 --- a/client/src/assets/player/peertube-player-manager.ts +++ b/client/src/assets/player/peertube-player-manager.ts @@ -6,6 +6,7 @@ import './shared/stats/stats-plugin' import './shared/bezels/bezels-plugin' import './shared/peertube/peertube-plugin' import './shared/resolutions/peertube-resolutions-plugin' +import './shared/control-bar/storyboard-plugin' import './shared/control-bar/next-previous-video-button' import './shared/control-bar/p2p-info-button' import './shared/control-bar/peertube-link-button' @@ -42,6 +43,12 @@ CaptionsButton.prototype.controlText_ = 'Subtitles/CC' // We just want to display 'Off' instead of 'captions off', keep a space so the variable == true (hacky I know) CaptionsButton.prototype.label_ = ' ' +// TODO: remove when https://github.com/videojs/video.js/pull/7598 is merged +const PlayProgressBar = videojs.getComponent('PlayProgressBar') as any +if (PlayProgressBar.prototype.options_.children.includes('timeTooltip') !== true) { + PlayProgressBar.prototype.options_.children.push('timeTooltip') +} + export class PeertubePlayerManager { private static playerElementClassName: string private static playerElementAttributes: { name: string, value: string }[] = [] @@ -135,6 +142,10 @@ export class PeertubePlayerManager { p2pEnabled: options.common.p2pEnabled }) + if (options.common.storyboard) { + player.storyboard(options.common.storyboard) + } + player.on('p2pInfo', (_, data: PlayerNetworkInfo) => { if (data.source !== 'p2p-media-loader' || isNaN(data.bandwidthEstimate)) return diff --git a/client/src/assets/player/shared/control-bar/index.ts b/client/src/assets/player/shared/control-bar/index.ts index e71e90713..24877c267 100644 --- a/client/src/assets/player/shared/control-bar/index.ts +++ b/client/src/assets/player/shared/control-bar/index.ts @@ -3,4 +3,5 @@ export * from './p2p-info-button' export * from './peertube-link-button' export * from './peertube-live-display' export * from './peertube-load-progress-bar' +export * from './storyboard-plugin' export * from './theater-button' diff --git a/client/src/assets/player/shared/control-bar/storyboard-plugin.ts b/client/src/assets/player/shared/control-bar/storyboard-plugin.ts new file mode 100644 index 000000000..c1843f595 --- /dev/null +++ b/client/src/assets/player/shared/control-bar/storyboard-plugin.ts @@ -0,0 +1,184 @@ +import videojs from 'video.js' +import { StoryboardOptions } from '../../types' + +// Big thanks to this beautiful plugin: https://github.com/phloxic/videojs-sprite-thumbnails +// Adapted to respect peertube player style + +const Plugin = videojs.getPlugin('plugin') + +class StoryboardPlugin extends Plugin { + private url: string + private height: number + private width: number + private interval: number + + private cached: boolean + + private mouseTimeTooltip: videojs.MouseTimeDisplay + private seekBar: { el(): HTMLElement, mouseTimeDisplay: any, playProgressBar: any } + private progress: any + + private spritePlaceholder: HTMLElement + + private readonly sprites: { [id: string]: HTMLImageElement } = {} + + private readonly boundedHijackMouseTooltip: typeof StoryboardPlugin.prototype.hijackMouseTooltip + + constructor (player: videojs.Player, options: videojs.ComponentOptions & StoryboardOptions) { + super(player, options) + + this.url = options.url + this.height = options.height + this.width = options.width + this.interval = options.interval + + this.boundedHijackMouseTooltip = this.hijackMouseTooltip.bind(this) + + this.player.ready(() => { + player.addClass('vjs-storyboard') + + this.init() + }) + } + + init () { + const controls = this.player.controlBar as any + + // default control bar component tree is expected + // https://docs.videojs.com/tutorial-components.html#default-component-tree + this.progress = controls?.progressControl + this.seekBar = this.progress?.seekBar + + this.mouseTimeTooltip = this.seekBar?.mouseTimeDisplay?.timeTooltip + + this.spritePlaceholder = videojs.dom.createEl('div', { className: 'vjs-storyboard-sprite-placeholder' }) as HTMLElement + this.seekBar?.el()?.appendChild(this.spritePlaceholder) + + this.player.on([ 'ready', 'loadstart' ], evt => { + if (evt !== 'ready') { + const spriteSource = this.player.currentSources().find(source => { + return Object.prototype.hasOwnProperty.call(source, 'storyboard') + }) as any + const spriteOpts = spriteSource?.['storyboard'] as StoryboardOptions + + if (spriteOpts) { + this.url = spriteOpts.url + this.height = spriteOpts.height + this.width = spriteOpts.width + this.interval = spriteOpts.interval + } + } + + this.cached = !!this.sprites[this.url] + + this.load() + }) + } + + private load () { + const spriteEvents = [ 'mousemove', 'touchmove' ] + + if (this.isReady()) { + if (!this.cached) { + this.sprites[this.url] = videojs.dom.createEl('img', { + src: this.url + }) + } + this.progress.on(spriteEvents, this.boundedHijackMouseTooltip) + } else { + this.progress.off(spriteEvents, this.boundedHijackMouseTooltip) + + this.resetMouseTooltip() + } + } + + private hijackMouseTooltip (evt: Event) { + const sprite = this.sprites[this.url] + const imgWidth = sprite.naturalWidth + const imgHeight = sprite.naturalHeight + const seekBarEl = this.seekBar.el() + + if (!sprite.complete || !imgWidth || !imgHeight) { + this.resetMouseTooltip() + return + } + + this.player.requestNamedAnimationFrame('StoryBoardPlugin#hijackMouseTooltip', () => { + const seekBarRect = videojs.dom.getBoundingClientRect(seekBarEl) + const playerRect = videojs.dom.getBoundingClientRect(this.player.el()) + + if (!seekBarRect || !playerRect) return + + const seekBarX = videojs.dom.getPointerPosition(seekBarEl, evt).x + let position = seekBarX * this.player.duration() + + const maxPosition = Math.round((imgHeight / this.height) * (imgWidth / this.width)) - 1 + position = Math.min(position / this.interval, maxPosition) + + const responsive = 600 + const playerWidth = this.player.currentWidth() + const scaleFactor = responsive && playerWidth < responsive + ? playerWidth / responsive + : 1 + const columns = imgWidth / this.width + + const scaledWidth = this.width * scaleFactor + const scaledHeight = this.height * scaleFactor + const cleft = Math.floor(position % columns) * -scaledWidth + const ctop = Math.floor(position / columns) * -scaledHeight + + const bgSize = `${imgWidth * scaleFactor}px ${imgHeight * scaleFactor}px` + const topOffset = -scaledHeight - 60 + + const previewHalfSize = Math.round(scaledWidth / 2) + let left = seekBarRect.width * seekBarX - previewHalfSize + + // Seek bar doesn't take all the player width, so we can add/minus a few more pixels + const minLeft = playerRect.left - seekBarRect.left + const maxLeft = seekBarRect.width - scaledWidth + (playerRect.right - seekBarRect.right) + + if (left < minLeft) left = minLeft + if (left > maxLeft) left = maxLeft + + const tooltipStyle: { [id: string]: string } = { + 'background-image': `url("${this.url}")`, + 'background-repeat': 'no-repeat', + 'background-position': `${cleft}px ${ctop}px`, + 'background-size': bgSize, + + 'color': '#fff', + 'text-shadow': '1px 1px #000', + + 'position': 'relative', + + 'top': `${topOffset}px`, + + 'border': '1px solid #000', + + // border should not overlay thumbnail area + 'width': `${scaledWidth + 2}px`, + 'height': `${scaledHeight + 2}px` + } + + tooltipStyle.left = `${left}px` + + for (const [ key, value ] of Object.entries(tooltipStyle)) { + this.spritePlaceholder.style.setProperty(key, value) + } + }) + } + + private resetMouseTooltip () { + if (this.spritePlaceholder) { + this.spritePlaceholder.style.cssText = '' + } + } + + private isReady () { + return this.mouseTimeTooltip && this.width && this.height && this.url + } +} + +videojs.registerPlugin('storyboard', StoryboardPlugin) + +export { StoryboardPlugin } diff --git a/client/src/assets/player/types/manager-options.ts b/client/src/assets/player/types/manager-options.ts index 1f3a0aa2e..a73341b4c 100644 --- a/client/src/assets/player/types/manager-options.ts +++ b/client/src/assets/player/types/manager-options.ts @@ -1,6 +1,6 @@ import { PluginsManager } from '@root-helpers/plugins-manager' import { LiveVideoLatencyMode, VideoFile } from '@shared/models' -import { PlaylistPluginOptions, VideoJSCaption } from './peertube-videojs-typings' +import { PlaylistPluginOptions, VideoJSCaption, VideoJSStoryboard } from './peertube-videojs-typings' export type PlayerMode = 'webtorrent' | 'p2p-media-loader' @@ -78,6 +78,7 @@ export interface CommonOptions extends CustomizationOptions { language?: string videoCaptions: VideoJSCaption[] + storyboard: VideoJSStoryboard videoUUID: string videoShortUUID: string diff --git a/client/src/assets/player/types/peertube-videojs-typings.ts b/client/src/assets/player/types/peertube-videojs-typings.ts index 723c42c5d..30d2b287f 100644 --- a/client/src/assets/player/types/peertube-videojs-typings.ts +++ b/client/src/assets/player/types/peertube-videojs-typings.ts @@ -49,6 +49,8 @@ declare module 'video.js' { stats (options?: StatsCardOptions): StatsForNerdsPlugin + storyboard (options: StoryboardOptions): void + textTracks (): TextTrackList & { tracks_: (TextTrack & { id: string, label: string, src: string })[] } @@ -89,6 +91,13 @@ type VideoJSCaption = { src: string } +type VideoJSStoryboard = { + url: string + width: number + height: number + interval: number +} + type PeerTubePluginOptions = { mode: PlayerMode @@ -118,6 +127,13 @@ type MetricsPluginOptions = { videoUUID: string } +type StoryboardOptions = { + url: string + width: number + height: number + interval: number +} + type PlaylistPluginOptions = { elements: VideoPlaylistElement[] @@ -238,6 +254,7 @@ type PlaylistItemOptions = { export { PlayerNetworkInfo, + VideoJSStoryboard, PlaylistItemOptions, NextPreviousVideoButtonOptions, ResolutionUpdateData, @@ -251,6 +268,7 @@ export { PeerTubeResolution, VideoJSPluginOptions, LoadedQualityData, + StoryboardOptions, PeerTubeLinkButtonOptions, PeerTubeP2PInfoButtonOptions } diff --git a/client/src/sass/player/control-bar.scss b/client/src/sass/player/control-bar.scss index 96b3adf66..02d5fa169 100644 --- a/client/src/sass/player/control-bar.scss +++ b/client/src/sass/player/control-bar.scss @@ -3,6 +3,20 @@ @use '_mixins' as *; @use './_player-variables' as *; +// Like the time tooltip +.video-js .vjs-progress-holder .vjs-storyboard-sprite-placeholder { + display: none; +} + +.video-js .vjs-progress-control:hover .vjs-storyboard-sprite-placeholder, +.video-js .vjs-progress-control:hover .vjs-progress-holder:focus .vjs-storyboard-sprite-placeholder { + display: block; + + // Ensure that we maintain a font-size of ~10px. + font-size: 0.6em; + visibility: visible; +} + .video-js.vjs-peertube-skin .vjs-control-bar { z-index: 100; @@ -79,6 +93,7 @@ top: -0.3em; } + // Only used on mobile .vjs-time-tooltip { display: none; } diff --git a/client/src/sass/player/mobile.scss b/client/src/sass/player/mobile.scss index 84d7a00f1..d150c54ee 100644 --- a/client/src/sass/player/mobile.scss +++ b/client/src/sass/player/mobile.scss @@ -6,6 +6,31 @@ /* Special mobile style */ .video-js.vjs-peertube-skin.vjs-is-mobile { + // No hover means we can't display the storyboard/time tooltip on mouse hover + // Use the time tooltip in progress control instead + .vjs-mouse-display { + display: none !important; + } + + .vjs-storyboard-sprite-placeholder { + display: none; + } + + .vjs-progress-control .vjs-sliding { + + .vjs-time-tooltip, + .vjs-storyboard-sprite-placeholder { + display: block !important; + + visibility: visible !important; + } + + .vjs-time-tooltip { + color: #fff; + background-color: rgba(0, 0, 0, 0.8); + } + } + .vjs-control-bar { .vjs-progress-control .vjs-slider .vjs-play-progress { // Always display the circle on mobile diff --git a/config/default.yaml b/config/default.yaml index 5d0eab4f5..e54c93ac5 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -136,6 +136,7 @@ storage: logs: 'storage/logs/' previews: 'storage/previews/' thumbnails: 'storage/thumbnails/' + storyboards: 'storage/storyboards/' torrents: 'storage/torrents/' captions: 'storage/captions/' cache: 'storage/cache/' @@ -396,6 +397,8 @@ cache: size: 500 # Max number of video captions/subtitles you want to cache torrents: size: 500 # Max number of video torrents you want to cache + storyboards: + size: 500 # Max number of video storyboards you want to cache admin: # Used to generate the root user at first startup diff --git a/config/production.yaml.example b/config/production.yaml.example index 5514f1af6..83ee48dae 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -134,6 +134,7 @@ storage: logs: '/var/www/peertube/storage/logs/' previews: '/var/www/peertube/storage/previews/' thumbnails: '/var/www/peertube/storage/thumbnails/' + storyboards: '/var/www/peertube/storage/storyboards/' torrents: '/var/www/peertube/storage/torrents/' captions: '/var/www/peertube/storage/captions/' cache: '/var/www/peertube/storage/cache/' @@ -406,6 +407,8 @@ cache: size: 500 # Max number of video captions/subtitles you want to cache torrents: size: 500 # Max number of video torrents you want to cache + storyboards: + size: 500 # Max number of video storyboards you want to cache admin: # Used to generate the root user at first startup diff --git a/config/test-1.yaml b/config/test-1.yaml index 7b62e3d0c..45ec27e63 100644 --- a/config/test-1.yaml +++ b/config/test-1.yaml @@ -19,6 +19,7 @@ storage: logs: 'test1/logs/' previews: 'test1/previews/' thumbnails: 'test1/thumbnails/' + storyboards: 'test1/storyboards/' torrents: 'test1/torrents/' captions: 'test1/captions/' cache: 'test1/cache/' diff --git a/config/test-2.yaml b/config/test-2.yaml index ba36369a6..7a06e5650 100644 --- a/config/test-2.yaml +++ b/config/test-2.yaml @@ -19,6 +19,7 @@ storage: logs: 'test2/logs/' previews: 'test2/previews/' thumbnails: 'test2/thumbnails/' + storyboards: 'test2/storyboards/' torrents: 'test2/torrents/' captions: 'test2/captions/' cache: 'test2/cache/' diff --git a/config/test-3.yaml b/config/test-3.yaml index 6adec7953..4b1563369 100644 --- a/config/test-3.yaml +++ b/config/test-3.yaml @@ -19,6 +19,7 @@ storage: logs: 'test3/logs/' previews: 'test3/previews/' thumbnails: 'test3/thumbnails/' + storyboards: 'test3/storyboards/' torrents: 'test3/torrents/' captions: 'test3/captions/' cache: 'test3/cache/' diff --git a/config/test-4.yaml b/config/test-4.yaml index f042aee46..248db4db9 100644 --- a/config/test-4.yaml +++ b/config/test-4.yaml @@ -19,6 +19,7 @@ storage: logs: 'test4/logs/' previews: 'test4/previews/' thumbnails: 'test4/thumbnails/' + storyboards: 'test4/storyboards/' torrents: 'test4/torrents/' captions: 'test4/captions/' cache: 'test4/cache/' diff --git a/config/test-5.yaml b/config/test-5.yaml index ad90fec04..04e2cd78d 100644 --- a/config/test-5.yaml +++ b/config/test-5.yaml @@ -19,6 +19,7 @@ storage: logs: 'test5/logs/' previews: 'test5/previews/' thumbnails: 'test5/thumbnails/' + storyboards: 'test5/storyboards/' torrents: 'test5/torrents/' captions: 'test5/captions/' cache: 'test5/cache/' diff --git a/config/test-6.yaml b/config/test-6.yaml index a579f1f01..25efe0054 100644 --- a/config/test-6.yaml +++ b/config/test-6.yaml @@ -19,6 +19,7 @@ storage: logs: 'test6/logs/' previews: 'test6/previews/' thumbnails: 'test6/thumbnails/' + storyboards: 'test6/storyboards/' torrents: 'test6/torrents/' captions: 'test6/captions/' cache: 'test6/cache/' diff --git a/config/test.yaml b/config/test.yaml index 361064af1..41ec0917e 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -73,6 +73,8 @@ cache: size: 1 torrents: size: 1 + storyboards: + size: 1 signup: enabled: true diff --git a/server.ts b/server.ts index a7a723b24..5d3acb2cd 100644 --- a/server.ts +++ b/server.ts @@ -101,7 +101,7 @@ loadLanguages() import { installApplication } from './server/initializers/installer' import { Emailer } from './server/lib/emailer' import { JobQueue } from './server/lib/job-queue' -import { VideosPreviewCache, VideosCaptionCache } from './server/lib/files-cache' +import { VideosPreviewCache, VideosCaptionCache, VideosStoryboardCache } from './server/lib/files-cache' import { activityPubRouter, apiRouter, @@ -316,6 +316,7 @@ async function startApplication () { VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE) VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE) VideosTorrentCache.Instance.init(CONFIG.CACHE.TORRENTS.SIZE, FILES_CACHE.TORRENTS.MAX_AGE) + VideosStoryboardCache.Instance.init(CONFIG.CACHE.STORYBOARDS.SIZE, FILES_CACHE.STORYBOARDS.MAX_AGE) // Enable Schedulers ActorFollowScheduler.Instance.enable() diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts index 750e3091c..166fc2a22 100644 --- a/server/controllers/activitypub/client.ts +++ b/server/controllers/activitypub/client.ts @@ -33,7 +33,6 @@ import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from ' import { AccountModel } from '../../models/account/account' import { AccountVideoRateModel } from '../../models/account/account-video-rate' import { ActorFollowModel } from '../../models/actor/actor-follow' -import { VideoCaptionModel } from '../../models/video/video-caption' import { VideoCommentModel } from '../../models/video/video-comment' import { VideoPlaylistModel } from '../../models/video/video-playlist' import { VideoShareModel } from '../../models/video/video-share' @@ -242,14 +241,13 @@ async function videoController (req: express.Request, res: express.Response) { if (redirectIfNotOwned(video.url, res)) return // We need captions to render AP object - const captions = await VideoCaptionModel.listVideoCaptions(video.id) - const videoWithCaptions = Object.assign(video, { VideoCaptions: captions }) + const videoAP = await video.lightAPToFullAP(undefined) - const audience = getAudience(videoWithCaptions.VideoChannel.Account.Actor, videoWithCaptions.privacy === VideoPrivacy.PUBLIC) - const videoObject = audiencify(await videoWithCaptions.toActivityPubObject(), audience) + const audience = getAudience(videoAP.VideoChannel.Account.Actor, videoAP.privacy === VideoPrivacy.PUBLIC) + const videoObject = audiencify(await videoAP.toActivityPubObject(), audience) if (req.path.endsWith('/activity')) { - const data = buildCreateActivity(videoWithCaptions.url, video.VideoChannel.Account.Actor, videoObject, audience) + const data = buildCreateActivity(videoAP.url, video.VideoChannel.Account.Actor, videoObject, audience) return activityPubResponse(activityPubContextify(data, 'Video'), res) } diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index 228eae109..c1f6756de 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -190,6 +190,9 @@ function customConfig (): CustomConfig { }, torrents: { size: CONFIG.CACHE.TORRENTS.SIZE + }, + storyboards: { + size: CONFIG.CACHE.STORYBOARDS.SIZE } }, signup: { diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index d0eecf812..bbdda5b29 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -41,6 +41,7 @@ import { liveRouter } from './live' import { ownershipVideoRouter } from './ownership' import { rateVideoRouter } from './rate' import { statsRouter } from './stats' +import { storyboardRouter } from './storyboard' import { studioRouter } from './studio' import { tokenRouter } from './token' import { transcodingRouter } from './transcoding' @@ -70,6 +71,7 @@ videosRouter.use('/', filesRouter) videosRouter.use('/', transcodingRouter) videosRouter.use('/', tokenRouter) videosRouter.use('/', videoPasswordRouter) +videosRouter.use('/', storyboardRouter) videosRouter.get('/categories', openapiOperationDoc({ operationId: 'getCategories' }), diff --git a/server/controllers/api/videos/storyboard.ts b/server/controllers/api/videos/storyboard.ts new file mode 100644 index 000000000..47a22011d --- /dev/null +++ b/server/controllers/api/videos/storyboard.ts @@ -0,0 +1,29 @@ +import express from 'express' +import { getVideoWithAttributes } from '@server/helpers/video' +import { StoryboardModel } from '@server/models/video/storyboard' +import { asyncMiddleware, videosGetValidator } from '../../../middlewares' + +const storyboardRouter = express.Router() + +storyboardRouter.get('/:id/storyboards', + asyncMiddleware(videosGetValidator), + asyncMiddleware(listStoryboards) +) + +// --------------------------------------------------------------------------- + +export { + storyboardRouter +} + +// --------------------------------------------------------------------------- + +async function listStoryboards (req: express.Request, res: express.Response) { + const video = getVideoWithAttributes(res) + + const storyboards = await StoryboardModel.listStoryboardsOf(video) + + return res.json({ + storyboards: storyboards.map(s => s.toFormattedJSON()) + }) +} diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts index 073eb480f..86ab4591e 100644 --- a/server/controllers/api/videos/upload.ts +++ b/server/controllers/api/videos/upload.ts @@ -234,6 +234,15 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide } }, + { + type: 'generate-video-storyboard' as 'generate-video-storyboard', + payload: { + videoUUID: video.uuid, + // No need to federate, we process these jobs sequentially + federate: false + } + }, + { type: 'notify', payload: { diff --git a/server/controllers/lazy-static.ts b/server/controllers/lazy-static.ts index b082e41f6..6ffd39730 100644 --- a/server/controllers/lazy-static.ts +++ b/server/controllers/lazy-static.ts @@ -5,7 +5,7 @@ import { MActorImage } from '@server/types/models' import { HttpStatusCode } from '../../shared/models/http/http-error-codes' import { logger } from '../helpers/logger' import { ACTOR_IMAGES_SIZE, LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/constants' -import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache' +import { VideosCaptionCache, VideosPreviewCache, VideosStoryboardCache } from '../lib/files-cache' import { actorImagePathUnsafeCache, downloadActorImageFromWorker } from '../lib/local-actor' import { asyncMiddleware, handleStaticError } from '../middlewares' import { ActorImageModel } from '../models/actor/actor-image' @@ -32,6 +32,12 @@ lazyStaticRouter.use( handleStaticError ) +lazyStaticRouter.use( + LAZY_STATIC_PATHS.STORYBOARDS + ':filename', + asyncMiddleware(getStoryboard), + handleStaticError +) + lazyStaticRouter.use( LAZY_STATIC_PATHS.VIDEO_CAPTIONS + ':filename', asyncMiddleware(getVideoCaption), @@ -126,6 +132,13 @@ async function getPreview (req: express.Request, res: express.Response) { return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }) } +async function getStoryboard (req: express.Request, res: express.Response) { + const result = await VideosStoryboardCache.Instance.getFilePath(req.params.filename) + if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end() + + return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }) +} + async function getVideoCaption (req: express.Request, res: express.Response) { const result = await VideosCaptionCache.Instance.getFilePath(req.params.filename) if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end() diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index 97b3577af..573a29754 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -1,6 +1,6 @@ import validator from 'validator' import { logger } from '@server/helpers/logger' -import { ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject } from '@shared/models' +import { ActivityPubStoryboard, ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject, VideoObject } from '@shared/models' import { LiveVideoLatencyMode, VideoState } from '../../../../shared/models/videos' import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers/constants' import { peertubeTruncate } from '../../core-utils' @@ -48,6 +48,10 @@ function sanitizeAndCheckVideoTorrentObject (video: any) { logger.debug('Video has invalid icons', { video }) return false } + if (!setValidStoryboard(video)) { + logger.debug('Video has invalid preview (storyboard)', { video }) + return false + } // Default attributes if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED @@ -201,3 +205,36 @@ function setRemoteVideoContent (video: any) { return true } + +function setValidStoryboard (video: VideoObject) { + if (!video.preview) return true + if (!Array.isArray(video.preview)) return false + + video.preview = video.preview.filter(p => isStorybordValid(p)) + + return true +} + +function isStorybordValid (preview: ActivityPubStoryboard) { + if (!preview) return false + + if ( + preview.type !== 'Image' || + !isArray(preview.rel) || + !preview.rel.includes('storyboard') + ) { + return false + } + + preview.url = preview.url.filter(u => { + return u.mediaType === 'image/jpeg' && + isActivityPubUrlValid(u.href) && + validator.isInt(u.width + '', { min: 0 }) && + validator.isInt(u.height + '', { min: 0 }) && + validator.isInt(u.tileWidth + '', { min: 0 }) && + validator.isInt(u.tileHeight + '', { min: 0 }) && + isActivityPubVideoDurationValid(u.tileDuration) + }) + + return preview.url.length !== 0 +} diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index 0a315ea70..939b73344 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts @@ -29,7 +29,8 @@ function checkMissedConfig () { 'video_channels.max_per_user', 'csp.enabled', 'csp.report_only', 'csp.report_uri', 'security.frameguard.enabled', 'security.powered_by_header.enabled', - 'cache.previews.size', 'cache.captions.size', 'cache.torrents.size', 'admin.email', 'contact_form.enabled', + 'cache.previews.size', 'cache.captions.size', 'cache.torrents.size', 'cache.storyboards.size', + 'admin.email', 'contact_form.enabled', 'signup.enabled', 'signup.limit', 'signup.requires_approval', 'signup.requires_email_verification', 'signup.minimum_age', 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', 'redundancy.videos.strategies', 'redundancy.videos.check_interval', diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 51ac5d0ce..60ab6e204 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -112,6 +112,7 @@ const CONFIG = { STREAMING_PLAYLISTS_DIR: buildPath(config.get('storage.streaming_playlists')), REDUNDANCY_DIR: buildPath(config.get('storage.redundancy')), THUMBNAILS_DIR: buildPath(config.get('storage.thumbnails')), + STORYBOARDS_DIR: buildPath(config.get('storage.storyboards')), PREVIEWS_DIR: buildPath(config.get('storage.previews')), CAPTIONS_DIR: buildPath(config.get('storage.captions')), TORRENTS_DIR: buildPath(config.get('storage.torrents')), @@ -482,6 +483,9 @@ const CONFIG = { }, TORRENTS: { get SIZE () { return config.get('cache.torrents.size') } + }, + STORYBOARDS: { + get SIZE () { return config.get('cache.storyboards.size') } } }, INSTANCE: { diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index e2f34fe16..3a643a60b 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -174,6 +174,7 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = { 'after-video-channel-import': 1, 'move-to-object-storage': 3, 'transcoding-job-builder': 1, + 'generate-video-storyboard': 1, 'notify': 1, 'federate-video': 1 } @@ -198,6 +199,7 @@ const JOB_CONCURRENCY: { [id in Exclude i.width > THUMBNAILS_SIZE.minWidth) @@ -166,6 +169,26 @@ function getCaptionAttributesFromObject (video: MVideoId, videoObject: VideoObje })) } +function getStoryboardAttributeFromObject (video: MVideoId, videoObject: VideoObject) { + if (!isArray(videoObject.preview)) return undefined + + const storyboard = videoObject.preview.find(p => p.rel.includes('storyboard')) + if (!storyboard) return undefined + + const url = arrayify(storyboard.url).find(u => u.mediaType === 'image/jpeg') + + return { + filename: generateImageFilename(extname(url.href)), + totalHeight: url.height, + totalWidth: url.width, + spriteHeight: url.tileHeight, + spriteWidth: url.tileWidth, + spriteDuration: getDurationFromActivityStream(url.tileDuration), + fileUrl: url.href, + videoId: video.id + } +} + function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) { const privacy = to.includes(ACTIVITY_PUB.PUBLIC) ? VideoPrivacy.PUBLIC @@ -228,6 +251,7 @@ export { getLiveAttributesFromObject, getCaptionAttributesFromObject, + getStoryboardAttributeFromObject, getVideoAttributesFromObject } diff --git a/server/lib/activitypub/videos/updater.ts b/server/lib/activitypub/videos/updater.ts index 6ddd2301b..3a0886523 100644 --- a/server/lib/activitypub/videos/updater.ts +++ b/server/lib/activitypub/videos/updater.ts @@ -57,6 +57,7 @@ export class APVideoUpdater extends APVideoAbstractBuilder { await Promise.all([ runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)), runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)), + runInReadCommittedTransaction(t => this.setStoryboard(videoUpdated, t)), this.setOrDeleteLive(videoUpdated), this.setPreview(videoUpdated) ]) @@ -138,6 +139,10 @@ export class APVideoUpdater extends APVideoAbstractBuilder { await this.insertOrReplaceCaptions(videoUpdated, t) } + private async setStoryboard (videoUpdated: MVideoFullLight, t: Transaction) { + await this.insertOrReplaceStoryboard(videoUpdated, t) + } + private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction?: Transaction) { if (!this.video.isLive) return diff --git a/server/lib/files-cache/index.ts b/server/lib/files-cache/index.ts index e5853f7d6..59cec7215 100644 --- a/server/lib/files-cache/index.ts +++ b/server/lib/files-cache/index.ts @@ -1,3 +1,4 @@ -export * from './videos-preview-cache' export * from './videos-caption-cache' +export * from './videos-preview-cache' +export * from './videos-storyboard-cache' export * from './videos-torrent-cache' diff --git a/server/lib/files-cache/videos-storyboard-cache.ts b/server/lib/files-cache/videos-storyboard-cache.ts new file mode 100644 index 000000000..b0a55104f --- /dev/null +++ b/server/lib/files-cache/videos-storyboard-cache.ts @@ -0,0 +1,53 @@ +import { join } from 'path' +import { logger } from '@server/helpers/logger' +import { doRequestAndSaveToFile } from '@server/helpers/requests' +import { StoryboardModel } from '@server/models/video/storyboard' +import { FILES_CACHE } from '../../initializers/constants' +import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' + +class VideosStoryboardCache extends AbstractVideoStaticFileCache { + + private static instance: VideosStoryboardCache + + private constructor () { + super() + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + async getFilePathImpl (filename: string) { + const storyboard = await StoryboardModel.loadWithVideoByFilename(filename) + if (!storyboard) return undefined + + if (storyboard.Video.isOwned()) return { isOwned: true, path: storyboard.getPath() } + + return this.loadRemoteFile(storyboard.filename) + } + + // Key is the storyboard filename + protected async loadRemoteFile (key: string) { + const storyboard = await StoryboardModel.loadWithVideoByFilename(key) + if (!storyboard) return undefined + + const destPath = join(FILES_CACHE.STORYBOARDS.DIRECTORY, storyboard.filename) + const remoteUrl = storyboard.getOriginFileUrl(storyboard.Video) + + try { + await doRequestAndSaveToFile(remoteUrl, destPath) + + logger.debug('Fetched remote storyboard %s to %s.', remoteUrl, destPath) + + return { isOwned: false, path: destPath } + } catch (err) { + logger.info('Cannot fetch remote storyboard file %s.', remoteUrl, { err }) + + return undefined + } + } +} + +export { + VideosStoryboardCache +} diff --git a/server/lib/job-queue/handlers/generate-storyboard.ts b/server/lib/job-queue/handlers/generate-storyboard.ts new file mode 100644 index 000000000..652cac272 --- /dev/null +++ b/server/lib/job-queue/handlers/generate-storyboard.ts @@ -0,0 +1,138 @@ +import { Job } from 'bullmq' +import { join } from 'path' +import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' +import { generateImageFilename, getImageSize } from '@server/helpers/image-utils' +import { logger, loggerTagsFactory } from '@server/helpers/logger' +import { CONFIG } from '@server/initializers/config' +import { STORYBOARD } from '@server/initializers/constants' +import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' +import { VideoPathManager } from '@server/lib/video-path-manager' +import { StoryboardModel } from '@server/models/video/storyboard' +import { VideoModel } from '@server/models/video/video' +import { MVideo } from '@server/types/models' +import { FFmpegImage, isAudioFile } from '@shared/ffmpeg' +import { GenerateStoryboardPayload } from '@shared/models' + +const lTagsBase = loggerTagsFactory('storyboard') + +async function processGenerateStoryboard (job: Job): Promise { + const payload = job.data as GenerateStoryboardPayload + const lTags = lTagsBase(payload.videoUUID) + + logger.info('Processing generate storyboard of %s in job %s.', payload.videoUUID, job.id, lTags) + + const video = await VideoModel.loadFull(payload.videoUUID) + if (!video) { + logger.info('Video %s does not exist anymore, skipping storyboard generation.', payload.videoUUID, lTags) + return + } + + const inputFile = video.getMaxQualityFile() + + await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => { + const isAudio = await isAudioFile(videoPath) + + if (isAudio) { + logger.info('Do not generate a storyboard of %s since the video does not have a video stream', payload.videoUUID, lTags) + return + } + + const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')) + + const filename = generateImageFilename() + const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename) + + const totalSprites = buildTotalSprites(video) + const spriteDuration = Math.round(video.duration / totalSprites) + + const spritesCount = findGridSize({ + toFind: totalSprites, + maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT + }) + + logger.debug( + 'Generating storyboard from video of %s to %s', video.uuid, destination, + { ...lTags, spritesCount, spriteDuration, videoDuration: video.duration } + ) + + await ffmpeg.generateStoryboardFromVideo({ + destination, + path: videoPath, + sprites: { + size: STORYBOARD.SPRITE_SIZE, + count: spritesCount, + duration: spriteDuration + } + }) + + const imageSize = await getImageSize(destination) + + const existing = await StoryboardModel.loadByVideo(video.id) + if (existing) await existing.destroy() + + await StoryboardModel.create({ + filename, + totalHeight: imageSize.height, + totalWidth: imageSize.width, + spriteHeight: STORYBOARD.SPRITE_SIZE.height, + spriteWidth: STORYBOARD.SPRITE_SIZE.width, + spriteDuration, + videoId: video.id + }) + + logger.info('Storyboard generation %s ended for video %s.', destination, video.uuid, lTags) + }) + + if (payload.federate) { + await federateVideoIfNeeded(video, false) + } +} + +// --------------------------------------------------------------------------- + +export { + processGenerateStoryboard +} + +function buildTotalSprites (video: MVideo) { + const maxSprites = STORYBOARD.SPRITE_SIZE.height * STORYBOARD.SPRITE_SIZE.width + const totalSprites = Math.min(Math.ceil(video.duration), maxSprites) + + // We can generate a single line + if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return totalSprites + + return findGridFit(totalSprites, STORYBOARD.SPRITES_MAX_EDGE_COUNT) +} + +function findGridSize (options: { + toFind: number + maxEdgeCount: number +}) { + const { toFind, maxEdgeCount } = options + + for (let i = 1; i <= maxEdgeCount; i++) { + for (let j = i; j <= maxEdgeCount; j++) { + if (toFind === i * j) return { width: j, height: i } + } + } + + throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`) +} + +function findGridFit (value: number, maxMultiplier: number) { + for (let i = value; i--; i > 0) { + if (!isPrimeWithin(i, maxMultiplier)) return i + } + + throw new Error('Could not find prime number below ' + value) +} + +function isPrimeWithin (value: number, maxMultiplier: number) { + if (value < 2) return false + + for (let i = 2, end = Math.min(Math.sqrt(value), maxMultiplier); i <= end; i++) { + if (value % i === 0 && value / i <= maxMultiplier) return false + } + + return true +} diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index cdd362f6e..c1355dcef 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -306,6 +306,15 @@ async function afterImportSuccess (options: { Notifier.Instance.notifyOnNewVideoIfNeeded(video) } + // Generate the storyboard in the job queue, and don't forget to federate an update after + await JobQueue.Instance.createJob({ + type: 'generate-video-storyboard' as 'generate-video-storyboard', + payload: { + videoUUID: video.uuid, + federate: true + } + }) + if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { await JobQueue.Instance.createJob( await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT }) diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts index 49feb53f2..95d4f5e64 100644 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ b/server/lib/job-queue/handlers/video-live-ending.ts @@ -1,6 +1,8 @@ import { Job } from 'bullmq' import { readdir, remove } from 'fs-extra' import { join } from 'path' +import { peertubeTruncate } from '@server/helpers/core-utils' +import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' @@ -20,8 +22,7 @@ import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@serv import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg' import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' import { logger, loggerTagsFactory } from '../../../helpers/logger' -import { peertubeTruncate } from '@server/helpers/core-utils' -import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' +import { JobQueue } from '../job-queue' const lTags = loggerTagsFactory('live', 'job') @@ -147,6 +148,8 @@ async function saveReplayToExternalVideo (options: { } await moveToNextState({ video: replayVideo, isNewVideo: true }) + + await createStoryboardJob(replayVideo) } async function replaceLiveByReplay (options: { @@ -186,6 +189,7 @@ async function replaceLiveByReplay (options: { await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory }) + // FIXME: should not happen in this function if (permanentLive) { // Remove session replay await remove(replayDirectory) } else { // We won't stream again in this live, we can delete the base replay directory @@ -213,6 +217,8 @@ async function replaceLiveByReplay (options: { // We consider this is a new video await moveToNextState({ video: videoWithFiles, isNewVideo: true }) + + await createStoryboardJob(videoWithFiles) } async function assignReplayFilesToVideo (options: { @@ -277,3 +283,13 @@ async function cleanupLiveAndFederate (options: { logger.warn('Cannot federate live after cleanup', { videoId: video.id, err }) } } + +function createStoryboardJob (video: MVideo) { + return JobQueue.Instance.createJob({ + type: 'generate-video-storyboard' as 'generate-video-storyboard', + payload: { + videoUUID: video.uuid, + federate: true + } + }) +} diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 03f6fbea7..177bca285 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts @@ -25,6 +25,7 @@ import { DeleteResumableUploadMetaFilePayload, EmailPayload, FederateVideoPayload, + GenerateStoryboardPayload, JobState, JobType, ManageVideoTorrentPayload, @@ -65,6 +66,7 @@ import { processVideoLiveEnding } from './handlers/video-live-ending' import { processVideoStudioEdition } from './handlers/video-studio-edition' import { processVideoTranscoding } from './handlers/video-transcoding' import { processVideosViewsStats } from './handlers/video-views-stats' +import { processGenerateStoryboard } from './handlers/generate-storyboard' export type CreateJobArgument = { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | @@ -91,7 +93,8 @@ export type CreateJobArgument = { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } | { type: 'notify', payload: NotifyPayload } | { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | - { type: 'federate-video', payload: FederateVideoPayload } + { type: 'federate-video', payload: FederateVideoPayload } | + { type: 'generate-video-storyboard', payload: GenerateStoryboardPayload } export type CreateJobOptions = { delay?: number @@ -122,7 +125,8 @@ const handlers: { [id in JobType]: (job: Job) => Promise } = { 'video-redundancy': processVideoRedundancy, 'video-studio-edition': processVideoStudioEdition, 'video-transcoding': processVideoTranscoding, - 'videos-views-stats': processVideosViewsStats + 'videos-views-stats': processVideosViewsStats, + 'generate-video-storyboard': processGenerateStoryboard } const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise } = { @@ -141,10 +145,11 @@ const jobTypes: JobType[] = [ 'after-video-channel-import', 'email', 'federate-video', - 'transcoding-job-builder', + 'generate-video-storyboard', 'manage-video-torrent', 'move-to-object-storage', 'notify', + 'transcoding-job-builder', 'video-channel-import', 'video-file-import', 'video-import', diff --git a/server/lib/redis.ts b/server/lib/redis.ts index 8430b2227..48d9986b5 100644 --- a/server/lib/redis.ts +++ b/server/lib/redis.ts @@ -325,8 +325,8 @@ class Redis { const value = await this.getValue('resumable-upload-' + uploadId) return value - ? JSON.parse(value) - : '' + ? JSON.parse(value) as { video: { id: number, shortUUID: string, uuid: string } } + : undefined } deleteUploadSession (uploadId: string) { diff --git a/server/lib/transcoding/web-transcoding.ts b/server/lib/transcoding/web-transcoding.ts index 7cc8f20bc..a499db422 100644 --- a/server/lib/transcoding/web-transcoding.ts +++ b/server/lib/transcoding/web-transcoding.ts @@ -9,6 +9,7 @@ import { ffprobePromise, getVideoStreamDuration, getVideoStreamFPS, TranscodeVOD import { VideoResolution, VideoStorage } from '@shared/models' import { CONFIG } from '../../initializers/config' import { VideoFileModel } from '../../models/video/video-file' +import { JobQueue } from '../job-queue' import { generateWebTorrentVideoFilename } from '../paths' import { buildFileMetadata } from '../video-file' import { VideoPathManager } from '../video-path-manager' @@ -198,7 +199,8 @@ export async function mergeAudioVideofile (options: { return onWebTorrentVideoFileTranscoding({ video, videoFile: inputVideoFile, - videoOutputPath + videoOutputPath, + wasAudioFile: true }) }) @@ -212,8 +214,9 @@ export async function onWebTorrentVideoFileTranscoding (options: { video: MVideoFullLight videoFile: MVideoFile videoOutputPath: string + wasAudioFile?: boolean // default false }) { - const { video, videoFile, videoOutputPath } = options + const { video, videoFile, videoOutputPath, wasAudioFile } = options const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) @@ -242,6 +245,17 @@ export async function onWebTorrentVideoFileTranscoding (options: { await VideoFileModel.customUpsert(videoFile, 'video', undefined) video.VideoFiles = await video.$get('VideoFiles') + if (wasAudioFile) { + await JobQueue.Instance.createJob({ + type: 'generate-video-storyboard' as 'generate-video-storyboard', + payload: { + videoUUID: video.uuid, + // No need to federate, we process these jobs sequentially + federate: false + } + }) + } + return { video, videoFile } } finally { mutexReleaser() diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts index a0074cb24..7029a857f 100644 --- a/server/middlewares/validators/config.ts +++ b/server/middlewares/validators/config.ts @@ -25,6 +25,7 @@ const customConfigUpdateValidator = [ body('cache.previews.size').isInt(), body('cache.captions.size').isInt(), body('cache.torrents.size').isInt(), + body('cache.storyboards.size').isInt(), body('signup.enabled').isBoolean(), body('signup.limit').isInt(), diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts index f2001e432..4179545b8 100644 --- a/server/models/video/formatter/video-format-utils.ts +++ b/server/models/video/formatter/video-format-utils.ts @@ -5,6 +5,7 @@ import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls' import { VideoViewsManager } from '@server/lib/views/video-views-manager' import { uuidToShort } from '@shared/extra-utils' import { + ActivityPubStoryboard, ActivityTagObject, ActivityUrlObject, Video, @@ -347,29 +348,17 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject { name: t.name })) - let language - if (video.language) { - language = { - identifier: video.language, - name: getLanguageLabel(video.language) - } - } + const language = video.language + ? { identifier: video.language, name: getLanguageLabel(video.language) } + : undefined - let category - if (video.category) { - category = { - identifier: video.category + '', - name: getCategoryLabel(video.category) - } - } + const category = video.category + ? { identifier: video.category + '', name: getCategoryLabel(video.category) } + : undefined - let licence - if (video.licence) { - licence = { - identifier: video.licence + '', - name: getLicenceLabel(video.licence) - } - } + const licence = video.licence + ? { identifier: video.licence + '', name: getLicenceLabel(video.licence) } + : undefined const url: ActivityUrlObject[] = [ // HTML url should be the first element in the array so Mastodon correctly displays the embed @@ -465,6 +454,8 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject { height: i.height })), + preview: buildPreviewAPAttribute(video), + url, likes: getLocalVideoLikesActivityPubUrl(video), @@ -541,3 +532,30 @@ function buildLiveAPAttributes (video: MVideoAP) { latencyMode: video.VideoLive.latencyMode } } + +function buildPreviewAPAttribute (video: MVideoAP): ActivityPubStoryboard[] { + if (!video.Storyboard) return undefined + + const storyboard = video.Storyboard + + return [ + { + type: 'Image', + rel: [ 'storyboard' ], + url: [ + { + mediaType: 'image/jpeg', + + href: storyboard.getOriginFileUrl(video), + + width: storyboard.totalWidth, + height: storyboard.totalHeight, + + tileWidth: storyboard.spriteWidth, + tileHeight: storyboard.spriteHeight, + tileDuration: getActivityStreamDuration(storyboard.spriteDuration) + } + ] + } + ] +} diff --git a/server/models/video/storyboard.ts b/server/models/video/storyboard.ts new file mode 100644 index 000000000..65a044c98 --- /dev/null +++ b/server/models/video/storyboard.ts @@ -0,0 +1,169 @@ +import { remove } from 'fs-extra' +import { join } from 'path' +import { AfterDestroy, AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { CONFIG } from '@server/initializers/config' +import { MStoryboard, MStoryboardVideo, MVideo } from '@server/types/models' +import { Storyboard } from '@shared/models' +import { AttributesOnly } from '@shared/typescript-utils' +import { logger } from '../../helpers/logger' +import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, WEBSERVER } from '../../initializers/constants' +import { VideoModel } from './video' +import { Transaction } from 'sequelize' + +@Table({ + tableName: 'storyboard', + indexes: [ + { + fields: [ 'videoId' ], + unique: true + }, + { + fields: [ 'filename' ], + unique: true + } + ] +}) +export class StoryboardModel extends Model>> { + + @AllowNull(false) + @Column + filename: string + + @AllowNull(false) + @Column + totalHeight: number + + @AllowNull(false) + @Column + totalWidth: number + + @AllowNull(false) + @Column + spriteHeight: number + + @AllowNull(false) + @Column + spriteWidth: number + + @AllowNull(false) + @Column + spriteDuration: number + + @AllowNull(true) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.COMMONS.URL.max)) + fileUrl: string + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'CASCADE' + }) + Video: VideoModel + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AfterDestroy + static removeInstanceFile (instance: StoryboardModel) { + logger.info('Removing storyboard file %s.', instance.filename) + + // Don't block the transaction + instance.removeFile() + .catch(err => logger.error('Cannot remove storyboard file %s.', instance.filename, { err })) + } + + static loadByVideo (videoId: number, transaction?: Transaction): Promise { + const query = { + where: { + videoId + }, + transaction + } + + return StoryboardModel.findOne(query) + } + + static loadByFilename (filename: string): Promise { + const query = { + where: { + filename + } + } + + return StoryboardModel.findOne(query) + } + + static loadWithVideoByFilename (filename: string): Promise { + const query = { + where: { + filename + }, + include: [ + { + model: VideoModel.unscoped(), + required: true + } + ] + } + + return StoryboardModel.findOne(query) + } + + // --------------------------------------------------------------------------- + + static async listStoryboardsOf (video: MVideo): Promise { + const query = { + where: { + videoId: video.id + } + } + + const storyboards = await StoryboardModel.findAll(query) + + return storyboards.map(s => Object.assign(s, { Video: video })) + } + + // --------------------------------------------------------------------------- + + getOriginFileUrl (video: MVideo) { + if (video.isOwned()) { + return WEBSERVER.URL + this.getLocalStaticPath() + } + + return this.fileUrl + } + + getLocalStaticPath () { + return LAZY_STATIC_PATHS.STORYBOARDS + this.filename + } + + getPath () { + return join(CONFIG.STORAGE.STORYBOARDS_DIR, this.filename) + } + + removeFile () { + return remove(this.getPath()) + } + + toFormattedJSON (this: MStoryboardVideo): Storyboard { + return { + storyboardPath: this.getLocalStaticPath(), + + totalHeight: this.totalHeight, + totalWidth: this.totalWidth, + + spriteWidth: this.spriteWidth, + spriteHeight: this.spriteHeight, + + spriteDuration: this.spriteDuration + } + } +} diff --git a/server/models/video/video-caption.ts b/server/models/video/video-caption.ts index 1fb1cae82..dd4cefd65 100644 --- a/server/models/video/video-caption.ts +++ b/server/models/video/video-caption.ts @@ -15,7 +15,7 @@ import { Table, UpdatedAt } from 'sequelize-typescript' -import { MVideo, MVideoCaption, MVideoCaptionFormattable, MVideoCaptionVideo } from '@server/types/models' +import { MVideo, MVideoCaption, MVideoCaptionFormattable, MVideoCaptionLanguageUrl, MVideoCaptionVideo } from '@server/types/models' import { buildUUID } from '@shared/extra-utils' import { AttributesOnly } from '@shared/typescript-utils' import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model' @@ -225,7 +225,7 @@ export class VideoCaptionModel extends Model>> { }) VideoJobInfo: VideoJobInfoModel + @HasOne(() => StoryboardModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) + Storyboard: StoryboardModel + @AfterCreate static notifyCreate (video: MVideo) { InternalEventEmitter.Instance.emit('video-created', { video }) @@ -903,6 +916,10 @@ export class VideoModel extends Model>> { model: VideoCaptionModel.unscoped(), required: false }, + { + model: StoryboardModel.unscoped(), + required: false + }, { attributes: [ 'id', 'url' ], model: VideoShareModel.unscoped(), @@ -1768,6 +1785,32 @@ export class VideoModel extends Model>> { ) } + async lightAPToFullAP (this: MVideoAPLight, transaction: Transaction): Promise { + const videoAP = this as MVideoAP + + const getCaptions = () => { + if (isArray(videoAP.VideoCaptions)) return videoAP.VideoCaptions + + return this.$get('VideoCaptions', { + attributes: [ 'filename', 'language', 'fileUrl' ], + transaction + }) as Promise + } + + const getStoryboard = () => { + if (videoAP.Storyboard) return videoAP.Storyboard + + return this.$get('Storyboard', { transaction }) as Promise + } + + const [ captions, storyboard ] = await Promise.all([ getCaptions(), getStoryboard() ]) + + return Object.assign(this, { + VideoCaptions: captions, + Storyboard: storyboard + }) + } + getTruncatedDescription () { if (!this.description) return null diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index 472cad182..3c752cc3e 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts @@ -74,6 +74,9 @@ describe('Test config API validators', function () { }, torrents: { size: 4 + }, + storyboards: { + size: 5 } }, signup: { diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts index 400d312d3..c2a7ccd78 100644 --- a/server/tests/api/check-params/index.ts +++ b/server/tests/api/check-params/index.ts @@ -34,6 +34,7 @@ import './video-comments' import './video-files' import './video-imports' import './video-playlists' +import './video-storyboards' import './video-source' import './video-studio' import './video-token' diff --git a/server/tests/api/check-params/video-storyboards.ts b/server/tests/api/check-params/video-storyboards.ts new file mode 100644 index 000000000..a43d8fc48 --- /dev/null +++ b/server/tests/api/check-params/video-storyboards.ts @@ -0,0 +1,45 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { HttpStatusCode, VideoPrivacy } from '@shared/models' +import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' + +describe('Test video storyboards API validator', function () { + let server: PeerTubeServer + + let publicVideo: { uuid: string } + let privateVideo: { uuid: string } + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + + publicVideo = await server.videos.quickUpload({ name: 'public' }) + privateVideo = await server.videos.quickUpload({ name: 'private', privacy: VideoPrivacy.PRIVATE }) + }) + + it('Should fail without a valid uuid', async function () { + await server.storyboard.list({ id: '4da6fde3-88f7-4d16-b119-108df563d0b0', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should receive 404 when passing a non existing video id', async function () { + await server.storyboard.list({ id: '4da6fde3-88f7-4d16-b119-108df5630b06', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should not get the private storyboard without the appropriate token', async function () { + await server.storyboard.list({ id: privateVideo.uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401, token: null }) + await server.storyboard.list({ id: publicVideo.uuid, expectedStatus: HttpStatusCode.OK_200, token: null }) + }) + + it('Should succeed with the correct parameters', async function () { + await server.storyboard.list({ id: privateVideo.uuid }) + await server.storyboard.list({ id: publicVideo.uuid }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/server/tests/api/check-params/videos-overviews.ts b/server/tests/api/check-params/videos-overviews.ts index f9cdb7ab3..ae7de24dd 100644 --- a/server/tests/api/check-params/videos-overviews.ts +++ b/server/tests/api/check-params/videos-overviews.ts @@ -2,7 +2,7 @@ import { cleanupTests, createSingleServer, PeerTubeServer } from '@shared/server-commands' -describe('Test videos overview', function () { +describe('Test videos overview API validator', function () { let server: PeerTubeServer // --------------------------------------------------------------- diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index 011ba268c..efa7b50e3 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts @@ -46,6 +46,7 @@ function checkInitialConfig (server: PeerTubeServer, data: CustomConfig) { expect(data.cache.previews.size).to.equal(1) expect(data.cache.captions.size).to.equal(1) expect(data.cache.torrents.size).to.equal(1) + expect(data.cache.storyboards.size).to.equal(1) expect(data.signup.enabled).to.be.true expect(data.signup.limit).to.equal(4) @@ -154,6 +155,7 @@ function checkUpdatedConfig (data: CustomConfig) { expect(data.cache.previews.size).to.equal(2) expect(data.cache.captions.size).to.equal(3) expect(data.cache.torrents.size).to.equal(4) + expect(data.cache.storyboards.size).to.equal(5) expect(data.signup.enabled).to.be.false expect(data.signup.limit).to.equal(5) @@ -290,6 +292,9 @@ const newCustomConfig: CustomConfig = { }, torrents: { size: 4 + }, + storyboards: { + size: 5 } }, signup: { diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts index 357c08199..9c79b3aa6 100644 --- a/server/tests/api/videos/index.ts +++ b/server/tests/api/videos/index.ts @@ -20,3 +20,4 @@ import './videos-history' import './videos-overview' import './video-source' import './video-static-file-privacy' +import './video-storyboard' diff --git a/server/tests/api/videos/video-storyboard.ts b/server/tests/api/videos/video-storyboard.ts new file mode 100644 index 000000000..7ccdca8f7 --- /dev/null +++ b/server/tests/api/videos/video-storyboard.ts @@ -0,0 +1,184 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { FIXTURE_URLS } from '@server/tests/shared' +import { areHttpImportTestsDisabled } from '@shared/core-utils' +import { HttpStatusCode, VideoPrivacy } from '@shared/models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + makeGetRequest, + PeerTubeServer, + sendRTMPStream, + setAccessTokensToServers, + setDefaultVideoChannel, + stopFfmpeg, + waitJobs +} from '@shared/server-commands' + +async function checkStoryboard (options: { + server: PeerTubeServer + uuid: string + tilesCount?: number + minSize?: number +}) { + const { server, uuid, tilesCount, minSize = 1000 } = options + + const { storyboards } = await server.storyboard.list({ id: uuid }) + + expect(storyboards).to.have.lengthOf(1) + + const storyboard = storyboards[0] + + expect(storyboard.spriteDuration).to.equal(1) + expect(storyboard.spriteHeight).to.equal(108) + expect(storyboard.spriteWidth).to.equal(192) + expect(storyboard.storyboardPath).to.exist + + if (tilesCount) { + expect(storyboard.totalWidth).to.equal(192 * Math.min(tilesCount, 10)) + expect(storyboard.totalHeight).to.equal(108 * Math.max((tilesCount / 10), 1)) + } + + const { body } = await makeGetRequest({ url: server.url, path: storyboard.storyboardPath, expectedStatus: HttpStatusCode.OK_200 }) + expect(body.length).to.be.above(minSize) +} + +describe('Test video storyboard', function () { + let servers: PeerTubeServer[] + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await doubleFollow(servers[0], servers[1]) + }) + + it('Should generate a storyboard after upload without transcoding', async function () { + this.timeout(60000) + + // 5s video + const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_short.webm' }) + await waitJobs(servers) + + for (const server of servers) { + await checkStoryboard({ server, uuid, tilesCount: 5 }) + } + }) + + it('Should generate a storyboard after upload without transcoding with a long video', async function () { + this.timeout(60000) + + // 124s video + const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_very_long_10p.mp4' }) + await waitJobs(servers) + + for (const server of servers) { + await checkStoryboard({ server, uuid, tilesCount: 100 }) + } + }) + + it('Should generate a storyboard after upload with transcoding', async function () { + this.timeout(60000) + + await servers[0].config.enableMinimumTranscoding() + + // 5s video + const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_short.webm' }) + await waitJobs(servers) + + for (const server of servers) { + await checkStoryboard({ server, uuid, tilesCount: 5 }) + } + }) + + it('Should generate a storyboard after an audio upload', async function () { + this.timeout(60000) + + // 6s audio + const attributes = { name: 'audio', fixture: 'sample.ogg' } + const { uuid } = await servers[0].videos.upload({ attributes, mode: 'legacy' }) + await waitJobs(servers) + + for (const server of servers) { + await checkStoryboard({ server, uuid, tilesCount: 6, minSize: 250 }) + } + }) + + it('Should generate a storyboard after HTTP import', async function () { + this.timeout(60000) + + if (areHttpImportTestsDisabled()) return + + // 3s video + const { video } = await servers[0].imports.importVideo({ + attributes: { + targetUrl: FIXTURE_URLS.goodVideo, + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + }) + await waitJobs(servers) + + for (const server of servers) { + await checkStoryboard({ server, uuid: video.uuid, tilesCount: 3 }) + } + }) + + it('Should generate a storyboard after torrent import', async function () { + this.timeout(60000) + + if (areHttpImportTestsDisabled()) return + + // 10s video + const { video } = await servers[0].imports.importVideo({ + attributes: { + magnetUri: FIXTURE_URLS.magnet, + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + }) + await waitJobs(servers) + + for (const server of servers) { + await checkStoryboard({ server, uuid: video.uuid, tilesCount: 10 }) + } + }) + + it('Should generate a storyboard after a live', async function () { + this.timeout(240000) + + await servers[0].config.enableLive({ allowReplay: true, transcoding: true, resolutions: 'min' }) + + const { live, video } = await servers[0].live.quickCreate({ + saveReplay: true, + permanentLive: false, + privacy: VideoPrivacy.PUBLIC + }) + + const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) + await servers[0].live.waitUntilPublished({ videoId: video.id }) + + await stopFfmpeg(ffmpegCommand) + + await servers[0].live.waitUntilReplacedByReplay({ videoId: video.id }) + await waitJobs(servers) + + for (const server of servers) { + await checkStoryboard({ server, uuid: video.uuid }) + } + }) + + it('Should generate a storyboard with different video durations', async function () { + this.timeout(60000) + + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/server/tests/fixtures/video_very_long_10p.mp4 b/server/tests/fixtures/video_very_long_10p.mp4 new file mode 100644 index 000000000..852297933 Binary files /dev/null and b/server/tests/fixtures/video_very_long_10p.mp4 differ diff --git a/server/types/models/video/index.ts b/server/types/models/video/index.ts index 0ac032290..7f05db666 100644 --- a/server/types/models/video/index.ts +++ b/server/types/models/video/index.ts @@ -1,6 +1,7 @@ export * from './local-video-viewer-watch-section' export * from './local-video-viewer-watch-section' export * from './local-video-viewer' +export * from './storyboard' export * from './schedule-video-update' export * from './tag' export * from './thumbnail' diff --git a/server/types/models/video/storyboard.ts b/server/types/models/video/storyboard.ts new file mode 100644 index 000000000..a0403d4f0 --- /dev/null +++ b/server/types/models/video/storyboard.ts @@ -0,0 +1,15 @@ +import { StoryboardModel } from '@server/models/video/storyboard' +import { PickWith } from '@shared/typescript-utils' +import { MVideo } from './video' + +type Use = PickWith + +// ############################################################################ + +export type MStoryboard = Omit + +// ############################################################################ + +export type MStoryboardVideo = + MStoryboard & + Use<'Video', MVideo> diff --git a/server/types/models/video/video-caption.ts b/server/types/models/video/video-caption.ts index 8cd801064..d3adec362 100644 --- a/server/types/models/video/video-caption.ts +++ b/server/types/models/video/video-caption.ts @@ -11,7 +11,7 @@ export type MVideoCaption = Omit // ############################################################################ export type MVideoCaptionLanguage = Pick -export type MVideoCaptionLanguageUrl = Pick +export type MVideoCaptionLanguageUrl = Pick export type MVideoCaptionVideo = MVideoCaption & diff --git a/server/types/models/video/video.ts b/server/types/models/video/video.ts index 8021e56bb..53ee94269 100644 --- a/server/types/models/video/video.ts +++ b/server/types/models/video/video.ts @@ -3,6 +3,7 @@ import { VideoModel } from '../../../models/video/video' import { MTrackerUrl } from '../server/tracker' import { MUserVideoHistoryTime } from '../user/user-video-history' import { MScheduleVideoUpdate } from './schedule-video-update' +import { MStoryboard } from './storyboard' import { MTag } from './tag' import { MThumbnail } from './thumbnail' import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist' @@ -32,7 +33,7 @@ type Use = PickWith export type MVideo = Omit + 'ScheduleVideoUpdate' | 'VideoBlacklist' | 'VideoImport' | 'VideoCaptions' | 'VideoLive' | 'Trackers' | 'VideoPasswords' | 'Storyboard'> // ############################################################################ @@ -173,9 +174,10 @@ export type MVideoAP = Use<'VideoBlacklist', MVideoBlacklistUnfederated> & Use<'VideoFiles', MVideoFileRedundanciesOpt[]> & Use<'Thumbnails', MThumbnail[]> & - Use<'VideoLive', MVideoLive> + Use<'VideoLive', MVideoLive> & + Use<'Storyboard', MStoryboard> -export type MVideoAPWithoutCaption = Omit +export type MVideoAPLight = Omit export type MVideoDetails = MVideo & diff --git a/shared/ffmpeg/ffmpeg-images.ts b/shared/ffmpeg/ffmpeg-images.ts index 2db63bd8b..27305382c 100644 --- a/shared/ffmpeg/ffmpeg-images.ts +++ b/shared/ffmpeg/ffmpeg-images.ts @@ -56,4 +56,41 @@ export class FFmpegImage { .thumbnail(thumbnailOptions) }) } + + async generateStoryboardFromVideo (options: { + path: string + destination: string + + sprites: { + size: { + width: number + height: number + } + + count: { + width: number + height: number + } + + duration: number + } + }) { + const { path, destination, sprites } = options + + const command = this.commandWrapper.buildCommand(path) + + const filter = [ + `setpts=N/round(FRAME_RATE)/TB`, + `select='not(mod(t,${options.sprites.duration}))'`, + `scale=${sprites.size.width}:${sprites.size.height}`, + `tile=layout=${sprites.count.width}x${sprites.count.height}` + ].join(',') + + command.outputOption('-filter_complex', filter) + command.outputOption('-frames:v', '1') + command.outputOption('-q:v', '2') + command.output(destination) + + return this.commandWrapper.runCommand() + } } diff --git a/shared/models/activitypub/objects/index.ts b/shared/models/activitypub/objects/index.ts index 9aa3c462c..a2e040b32 100644 --- a/shared/models/activitypub/objects/index.ts +++ b/shared/models/activitypub/objects/index.ts @@ -6,5 +6,5 @@ export * from './object.model' export * from './playlist-element-object' export * from './playlist-object' export * from './video-comment-object' -export * from './video-torrent-object' +export * from './video-object' export * from './watch-action-object' diff --git a/shared/models/activitypub/objects/video-torrent-object.ts b/shared/models/activitypub/objects/video-object.ts similarity index 79% rename from shared/models/activitypub/objects/video-torrent-object.ts rename to shared/models/activitypub/objects/video-object.ts index 23d54bdbd..a252a2df0 100644 --- a/shared/models/activitypub/objects/video-torrent-object.ts +++ b/shared/models/activitypub/objects/video-object.ts @@ -51,6 +51,22 @@ export interface VideoObject { attributedTo: ActivityPubAttributedTo[] + preview?: ActivityPubStoryboard[] + to?: string[] cc?: string[] } + +export interface ActivityPubStoryboard { + type: 'Image' + rel: [ 'storyboard' ] + url: { + href: string + mediaType: string + width: number + height: number + tileWidth: number + tileHeight: number + tileDuration: string + }[] +} diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts index 4202589f3..1012312f3 100644 --- a/shared/models/server/custom-config.model.ts +++ b/shared/models/server/custom-config.model.ts @@ -78,6 +78,10 @@ export interface CustomConfig { torrents: { size: number } + + storyboards: { + size: number + } } signup: { diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index 22ecee324..9c40079fb 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts @@ -30,6 +30,7 @@ export type JobType = | 'video-studio-edition' | 'video-transcoding' | 'videos-views-stats' + | 'generate-video-storyboard' export interface Job { id: number | string @@ -294,3 +295,10 @@ export interface TranscodingJobBuilderPayload { priority?: number }[][] } + +// --------------------------------------------------------------------------- + +export interface GenerateStoryboardPayload { + videoUUID: string + federate: boolean +} diff --git a/shared/models/videos/index.ts b/shared/models/videos/index.ts index 80be1854b..b3ce6ad3f 100644 --- a/shared/models/videos/index.ts +++ b/shared/models/videos/index.ts @@ -15,6 +15,7 @@ export * from './channel-sync' export * from './nsfw-policy.type' +export * from './storyboard.model' export * from './thumbnail.type' export * from './video-constant.model' diff --git a/shared/models/videos/storyboard.model.ts b/shared/models/videos/storyboard.model.ts new file mode 100644 index 000000000..c92c81f09 --- /dev/null +++ b/shared/models/videos/storyboard.model.ts @@ -0,0 +1,11 @@ +export interface Storyboard { + storyboardPath: string + + totalHeight: number + totalWidth: number + + spriteHeight: number + spriteWidth: number + + spriteDuration: number +} diff --git a/shared/server-commands/server/config-command.ts b/shared/server-commands/server/config-command.ts index b94bd2625..114db8091 100644 --- a/shared/server-commands/server/config-command.ts +++ b/shared/server-commands/server/config-command.ts @@ -159,6 +159,10 @@ export class ConfigCommand extends AbstractCommand { newConfig: { transcoding: { enabled: true, + + allowAudioFiles: true, + allowAdditionalExtensions: true, + resolutions: { ...ConfigCommand.getCustomConfigResolutions(false), @@ -368,6 +372,9 @@ export class ConfigCommand extends AbstractCommand { }, torrents: { size: 4 + }, + storyboards: { + size: 5 } }, signup: { diff --git a/shared/server-commands/server/jobs.ts b/shared/server-commands/server/jobs.ts index ff3098063..8f131fba4 100644 --- a/shared/server-commands/server/jobs.ts +++ b/shared/server-commands/server/jobs.ts @@ -33,6 +33,8 @@ async function waitJobs ( // Check if each server has pending request for (const server of servers) { + if (process.env.DEBUG) console.log('Checking ' + server.url) + for (const state of states) { const jobPromise = server.jobs.list({ @@ -45,6 +47,10 @@ async function waitJobs ( .then(jobs => { if (jobs.length !== 0) { pendingRequests = true + + if (process.env.DEBUG) { + console.log(jobs) + } } }) @@ -55,6 +61,10 @@ async function waitJobs ( .then(obj => { if (obj.activityPubMessagesWaiting !== 0) { pendingRequests = true + + if (process.env.DEBUG) { + console.log('AP messages waiting: ' + obj.activityPubMessagesWaiting) + } } }) tasks.push(debugPromise) @@ -65,12 +75,15 @@ async function waitJobs ( for (const job of data) { if (job.state.id !== RunnerJobState.COMPLETED) { pendingRequests = true + + if (process.env.DEBUG) { + console.log(job) + } } } }) tasks.push(runnerJobsPromise) } - } return tasks diff --git a/shared/server-commands/server/server.ts b/shared/server-commands/server/server.ts index 0911e22b0..6aa4296b0 100644 --- a/shared/server-commands/server/server.ts +++ b/shared/server-commands/server/server.ts @@ -35,6 +35,7 @@ import { VideoPasswordsCommand, PlaylistsCommand, ServicesCommand, + StoryboardCommand, StreamingPlaylistsCommand, VideosCommand, VideoStudioCommand, @@ -149,6 +150,8 @@ export class PeerTubeServer { registrations?: RegistrationsCommand videoPasswords?: VideoPasswordsCommand + storyboard?: StoryboardCommand + runners?: RunnersCommand runnerRegistrationTokens?: RunnerRegistrationTokensCommand runnerJobs?: RunnerJobsCommand @@ -436,6 +439,8 @@ export class PeerTubeServer { this.videoToken = new VideoTokenCommand(this) this.registrations = new RegistrationsCommand(this) + this.storyboard = new StoryboardCommand(this) + this.runners = new RunnersCommand(this) this.runnerRegistrationTokens = new RunnerRegistrationTokensCommand(this) this.runnerJobs = new RunnerJobsCommand(this) diff --git a/shared/server-commands/videos/index.ts b/shared/server-commands/videos/index.ts index da36b5b6b..106d80af0 100644 --- a/shared/server-commands/videos/index.ts +++ b/shared/server-commands/videos/index.ts @@ -11,6 +11,7 @@ export * from './live-command' export * from './live' export * from './playlists-command' export * from './services-command' +export * from './storyboard-command' export * from './streaming-playlists-command' export * from './comments-command' export * from './video-studio-command' diff --git a/shared/server-commands/videos/storyboard-command.ts b/shared/server-commands/videos/storyboard-command.ts new file mode 100644 index 000000000..06d90fc12 --- /dev/null +++ b/shared/server-commands/videos/storyboard-command.ts @@ -0,0 +1,19 @@ +import { HttpStatusCode, Storyboard } from '@shared/models' +import { AbstractCommand, OverrideCommandOptions } from '../shared' + +export class StoryboardCommand extends AbstractCommand { + + list (options: OverrideCommandOptions & { + id: number | string + }) { + const path = '/api/v1/videos/' + options.id + '/storyboards' + + return this.getRequestBody<{ storyboards: Storyboard[] }>({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } +} diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yaml index ff94f802b..cd0e6ffd8 100644 --- a/support/doc/api/openapi.yaml +++ b/support/doc/api/openapi.yaml @@ -3668,6 +3668,27 @@ paths: items: $ref: '#/components/schemas/VideoBlacklist' + /api/v1/videos/{id}/storyboards: + get: + summary: List storyboards of a video + operationId: listVideoStoryboards + tags: + - Video + parameters: + - $ref: '#/components/parameters/idOrUUID' + responses: + '200': + description: successful operation + content: + application/json: + schema: + type: object + properties: + storyboards: + type: array + items: + $ref: '#/components/schemas/Storyboard' + /api/v1/videos/{id}/captions: get: summary: List captions of a video @@ -7509,6 +7530,20 @@ components: type: array items: $ref: '#/components/schemas/VideoCommentThreadTree' + Storyboard: + properties: + storyboardPath: + type: string + totalHeight: + type: integer + totalWidth: + type: integer + spriteHeight: + type: integer + spriteWidth: + type: integer + spriteDuration: + type: integer VideoCaption: properties: language: