diff --git a/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.html b/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.html
index bbf946df0..9701e7f85 100644
--- a/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.html
+++ b/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.html
@@ -52,6 +52,20 @@
{{ formErrors.cache.torrents.size }}
+
+
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.ts
index 79a98f288..06c5e6221 100644
--- a/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.ts
+++ b/client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.ts
@@ -10,7 +10,7 @@ export class EditAdvancedConfigurationComponent {
@Input() form: FormGroup
@Input() formErrors: any
- getCacheSize (type: 'captions' | 'previews' | 'torrents') {
+ getCacheSize (type: 'captions' | 'previews' | 'torrents' | 'storyboards') {
return this.form.value['cache'][type]['size']
}
}
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts
index 2c3b7560d..9219d608b 100644
--- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts
+++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts
@@ -9,8 +9,7 @@ import { Notifier } from '@app/core'
import { ServerService } from '@app/core/server/server.service'
import {
ADMIN_EMAIL_VALIDATOR,
- CACHE_CAPTIONS_SIZE_VALIDATOR,
- CACHE_PREVIEWS_SIZE_VALIDATOR,
+ CACHE_SIZE_VALIDATOR,
CONCURRENCY_VALIDATOR,
INDEX_URL_VALIDATOR,
INSTANCE_NAME_VALIDATOR,
@@ -120,13 +119,16 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit {
},
cache: {
previews: {
- size: CACHE_PREVIEWS_SIZE_VALIDATOR
+ size: CACHE_SIZE_VALIDATOR
},
captions: {
- size: CACHE_CAPTIONS_SIZE_VALIDATOR
+ size: CACHE_SIZE_VALIDATOR
},
torrents: {
- size: CACHE_CAPTIONS_SIZE_VALIDATOR
+ size: CACHE_SIZE_VALIDATOR
+ },
+ storyboards: {
+ size: CACHE_SIZE_VALIDATOR
}
},
signup: {
diff --git a/client/src/app/+videos/+video-watch/video-watch.component.ts b/client/src/app/+videos/+video-watch/video-watch.component.ts
index aba3ee086..43744789d 100644
--- a/client/src/app/+videos/+video-watch/video-watch.component.ts
+++ b/client/src/app/+videos/+video-watch/video-watch.component.ts
@@ -33,6 +33,7 @@ import {
LiveVideo,
PeerTubeProblemDocument,
ServerErrorCode,
+ Storyboard,
VideoCaption,
VideoPrivacy,
VideoState
@@ -69,6 +70,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
videoCaptions: VideoCaption[] = []
liveVideo: LiveVideo
videoPassword: string
+ storyboards: Storyboard[] = []
playlistPosition: number
playlist: VideoPlaylist = null
@@ -285,9 +287,10 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
forkJoin([
videoAndLiveObs,
this.videoCaptionService.listCaptions(videoId, videoPassword),
+ this.videoService.getStoryboards(videoId),
this.userService.getAnonymousOrLoggedUser()
]).subscribe({
- next: ([ { video, live, videoFileToken }, captionsResult, loggedInOrAnonymousUser ]) => {
+ next: ([ { video, live, videoFileToken }, captionsResult, storyboards, loggedInOrAnonymousUser ]) => {
const queryParams = this.route.snapshot.queryParams
const urlOptions = {
@@ -309,6 +312,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
video,
live,
videoCaptions: captionsResult.data,
+ storyboards,
videoFileToken,
videoPassword,
loggedInOrAnonymousUser,
@@ -414,6 +418,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
video: VideoDetails
live: LiveVideo
videoCaptions: VideoCaption[]
+ storyboards: Storyboard[]
videoFileToken: string
videoPassword: string
@@ -421,7 +426,17 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
loggedInOrAnonymousUser: User
forceAutoplay: boolean
}) {
- const { video, live, videoCaptions, urlOptions, videoFileToken, videoPassword, loggedInOrAnonymousUser, forceAutoplay } = options
+ const {
+ video,
+ live,
+ videoCaptions,
+ storyboards,
+ urlOptions,
+ videoFileToken,
+ videoPassword,
+ loggedInOrAnonymousUser,
+ forceAutoplay
+ } = options
this.subscribeToLiveEventsIfNeeded(this.video, video)
@@ -430,6 +445,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
this.liveVideo = live
this.videoFileToken = videoFileToken
this.videoPassword = videoPassword
+ this.storyboards = storyboards
// Re init attributes
this.playerPlaceholderImgSrc = undefined
@@ -485,6 +501,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
const params = {
video: this.video,
videoCaptions: this.videoCaptions,
+ storyboards: this.storyboards,
liveVideo: this.liveVideo,
videoFileToken: this.videoFileToken,
videoPassword: this.videoPassword,
@@ -636,6 +653,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
video: VideoDetails
liveVideo: LiveVideo
videoCaptions: VideoCaption[]
+ storyboards: Storyboard[]
videoFileToken: string
videoPassword: string
@@ -646,7 +664,17 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
forceAutoplay: boolean
user?: AuthUser // Keep for plugins
}) {
- const { video, liveVideo, videoCaptions, videoFileToken, videoPassword, urlOptions, loggedInOrAnonymousUser, forceAutoplay } = params
+ const {
+ video,
+ liveVideo,
+ videoCaptions,
+ storyboards,
+ videoFileToken,
+ videoPassword,
+ urlOptions,
+ loggedInOrAnonymousUser,
+ forceAutoplay
+ } = params
const getStartTime = () => {
const byUrl = urlOptions.startTime !== undefined
@@ -673,6 +701,15 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
src: environment.apiUrl + c.captionPath
}))
+ const storyboard = storyboards.length !== 0
+ ? {
+ url: environment.apiUrl + storyboards[0].storyboardPath,
+ height: storyboards[0].spriteHeight,
+ width: storyboards[0].spriteWidth,
+ interval: storyboards[0].spriteDuration
+ }
+ : undefined
+
const liveOptions = video.isLive
? { latencyMode: liveVideo.latencyMode }
: undefined
@@ -734,6 +771,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
videoPassword: () => videoPassword,
videoCaptions: playerCaptions,
+ storyboard,
videoShortUUID: video.shortUUID,
videoUUID: video.uuid,
@@ -767,6 +805,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
else mode = 'webtorrent'
}
+ // FIXME: remove, we don't support these old web browsers anymore
// p2p-media-loader needs TextEncoder, fallback on WebTorrent if not available
if (typeof TextEncoder === 'undefined') {
mode = 'webtorrent'
diff --git a/client/src/app/shared/form-validators/custom-config-validators.ts b/client/src/app/shared/form-validators/custom-config-validators.ts
index ff0813f7d..3672e5610 100644
--- a/client/src/app/shared/form-validators/custom-config-validators.ts
+++ b/client/src/app/shared/form-validators/custom-config-validators.ts
@@ -22,21 +22,12 @@ export const SERVICES_TWITTER_USERNAME_VALIDATOR: BuildFormValidator = {
}
}
-export const CACHE_PREVIEWS_SIZE_VALIDATOR: BuildFormValidator = {
+export const CACHE_SIZE_VALIDATOR: BuildFormValidator = {
VALIDATORS: [ Validators.required, Validators.min(1), Validators.pattern('[0-9]+') ],
MESSAGES: {
- required: $localize`Previews cache size is required.`,
- min: $localize`Previews cache size must be greater than 1.`,
- pattern: $localize`Previews cache size must be a number.`
- }
-}
-
-export const CACHE_CAPTIONS_SIZE_VALIDATOR: BuildFormValidator = {
- VALIDATORS: [ Validators.required, Validators.min(1), Validators.pattern('[0-9]+') ],
- MESSAGES: {
- required: $localize`Captions cache size is required.`,
- min: $localize`Captions cache size must be greater than 1.`,
- pattern: $localize`Captions cache size must be a number.`
+ required: $localize`Cache size is required.`,
+ min: $localize`Cache size must be greater than 1.`,
+ pattern: $localize`Cache size must be a number.`
}
}
diff --git a/client/src/app/shared/shared-main/video/video.service.ts b/client/src/app/shared/shared-main/video/video.service.ts
index d67a2e192..c2e3d7511 100644
--- a/client/src/app/shared/shared-main/video/video.service.ts
+++ b/client/src/app/shared/shared-main/video/video.service.ts
@@ -11,6 +11,7 @@ import {
FeedFormat,
NSFWPolicyType,
ResultList,
+ Storyboard,
UserVideoRate,
UserVideoRateType,
UserVideoRateUpdate,
@@ -344,6 +345,25 @@ export class VideoService {
)
}
+ // ---------------------------------------------------------------------------
+
+ getStoryboards (videoId: string | number) {
+ return this.authHttp
+ .get<{ storyboards: Storyboard[] }>(VideoService.BASE_VIDEO_URL + '/' + videoId + '/storyboards')
+ .pipe(
+ map(({ storyboards }) => storyboards),
+ catchError(err => {
+ if (err.status === 404) {
+ return of([])
+ }
+
+ this.restExtractor.handleError(err)
+ })
+ )
+ }
+
+ // ---------------------------------------------------------------------------
+
getSource (videoId: number) {
return this.authHttp
.get<{ source: VideoSource }>(VideoService.BASE_VIDEO_URL + '/' + videoId + '/source')
@@ -358,6 +378,8 @@ export class VideoService {
)
}
+ // ---------------------------------------------------------------------------
+
setVideoLike (id: string, videoPassword: string) {
return this.setVideoRate(id, 'like', videoPassword)
}
@@ -370,6 +392,8 @@ export class VideoService {
return this.setVideoRate(id, 'none', videoPassword)
}
+ // ---------------------------------------------------------------------------
+
getUserVideoRating (id: string) {
const url = UserService.BASE_USERS_URL + 'me/videos/' + id + '/rating'
diff --git a/client/src/assets/player/peertube-player-manager.ts b/client/src/assets/player/peertube-player-manager.ts
index 2781850b9..66d9c7298 100644
--- a/client/src/assets/player/peertube-player-manager.ts
+++ b/client/src/assets/player/peertube-player-manager.ts
@@ -6,6 +6,7 @@ import './shared/stats/stats-plugin'
import './shared/bezels/bezels-plugin'
import './shared/peertube/peertube-plugin'
import './shared/resolutions/peertube-resolutions-plugin'
+import './shared/control-bar/storyboard-plugin'
import './shared/control-bar/next-previous-video-button'
import './shared/control-bar/p2p-info-button'
import './shared/control-bar/peertube-link-button'
@@ -42,6 +43,12 @@ CaptionsButton.prototype.controlText_ = 'Subtitles/CC'
// We just want to display 'Off' instead of 'captions off', keep a space so the variable == true (hacky I know)
CaptionsButton.prototype.label_ = ' '
+// TODO: remove when https://github.com/videojs/video.js/pull/7598 is merged
+const PlayProgressBar = videojs.getComponent('PlayProgressBar') as any
+if (PlayProgressBar.prototype.options_.children.includes('timeTooltip') !== true) {
+ PlayProgressBar.prototype.options_.children.push('timeTooltip')
+}
+
export class PeertubePlayerManager {
private static playerElementClassName: string
private static playerElementAttributes: { name: string, value: string }[] = []
@@ -135,6 +142,10 @@ export class PeertubePlayerManager {
p2pEnabled: options.common.p2pEnabled
})
+ if (options.common.storyboard) {
+ player.storyboard(options.common.storyboard)
+ }
+
player.on('p2pInfo', (_, data: PlayerNetworkInfo) => {
if (data.source !== 'p2p-media-loader' || isNaN(data.bandwidthEstimate)) return
diff --git a/client/src/assets/player/shared/control-bar/index.ts b/client/src/assets/player/shared/control-bar/index.ts
index e71e90713..24877c267 100644
--- a/client/src/assets/player/shared/control-bar/index.ts
+++ b/client/src/assets/player/shared/control-bar/index.ts
@@ -3,4 +3,5 @@ export * from './p2p-info-button'
export * from './peertube-link-button'
export * from './peertube-live-display'
export * from './peertube-load-progress-bar'
+export * from './storyboard-plugin'
export * from './theater-button'
diff --git a/client/src/assets/player/shared/control-bar/storyboard-plugin.ts b/client/src/assets/player/shared/control-bar/storyboard-plugin.ts
new file mode 100644
index 000000000..c1843f595
--- /dev/null
+++ b/client/src/assets/player/shared/control-bar/storyboard-plugin.ts
@@ -0,0 +1,184 @@
+import videojs from 'video.js'
+import { StoryboardOptions } from '../../types'
+
+// Big thanks to this beautiful plugin: https://github.com/phloxic/videojs-sprite-thumbnails
+// Adapted to respect peertube player style
+
+const Plugin = videojs.getPlugin('plugin')
+
+class StoryboardPlugin extends Plugin {
+ private url: string
+ private height: number
+ private width: number
+ private interval: number
+
+ private cached: boolean
+
+ private mouseTimeTooltip: videojs.MouseTimeDisplay
+ private seekBar: { el(): HTMLElement, mouseTimeDisplay: any, playProgressBar: any }
+ private progress: any
+
+ private spritePlaceholder: HTMLElement
+
+ private readonly sprites: { [id: string]: HTMLImageElement } = {}
+
+ private readonly boundedHijackMouseTooltip: typeof StoryboardPlugin.prototype.hijackMouseTooltip
+
+ constructor (player: videojs.Player, options: videojs.ComponentOptions & StoryboardOptions) {
+ super(player, options)
+
+ this.url = options.url
+ this.height = options.height
+ this.width = options.width
+ this.interval = options.interval
+
+ this.boundedHijackMouseTooltip = this.hijackMouseTooltip.bind(this)
+
+ this.player.ready(() => {
+ player.addClass('vjs-storyboard')
+
+ this.init()
+ })
+ }
+
+ init () {
+ const controls = this.player.controlBar as any
+
+ // default control bar component tree is expected
+ // https://docs.videojs.com/tutorial-components.html#default-component-tree
+ this.progress = controls?.progressControl
+ this.seekBar = this.progress?.seekBar
+
+ this.mouseTimeTooltip = this.seekBar?.mouseTimeDisplay?.timeTooltip
+
+ this.spritePlaceholder = videojs.dom.createEl('div', { className: 'vjs-storyboard-sprite-placeholder' }) as HTMLElement
+ this.seekBar?.el()?.appendChild(this.spritePlaceholder)
+
+ this.player.on([ 'ready', 'loadstart' ], evt => {
+ if (evt !== 'ready') {
+ const spriteSource = this.player.currentSources().find(source => {
+ return Object.prototype.hasOwnProperty.call(source, 'storyboard')
+ }) as any
+ const spriteOpts = spriteSource?.['storyboard'] as StoryboardOptions
+
+ if (spriteOpts) {
+ this.url = spriteOpts.url
+ this.height = spriteOpts.height
+ this.width = spriteOpts.width
+ this.interval = spriteOpts.interval
+ }
+ }
+
+ this.cached = !!this.sprites[this.url]
+
+ this.load()
+ })
+ }
+
+ private load () {
+ const spriteEvents = [ 'mousemove', 'touchmove' ]
+
+ if (this.isReady()) {
+ if (!this.cached) {
+ this.sprites[this.url] = videojs.dom.createEl('img', {
+ src: this.url
+ })
+ }
+ this.progress.on(spriteEvents, this.boundedHijackMouseTooltip)
+ } else {
+ this.progress.off(spriteEvents, this.boundedHijackMouseTooltip)
+
+ this.resetMouseTooltip()
+ }
+ }
+
+ private hijackMouseTooltip (evt: Event) {
+ const sprite = this.sprites[this.url]
+ const imgWidth = sprite.naturalWidth
+ const imgHeight = sprite.naturalHeight
+ const seekBarEl = this.seekBar.el()
+
+ if (!sprite.complete || !imgWidth || !imgHeight) {
+ this.resetMouseTooltip()
+ return
+ }
+
+ this.player.requestNamedAnimationFrame('StoryBoardPlugin#hijackMouseTooltip', () => {
+ const seekBarRect = videojs.dom.getBoundingClientRect(seekBarEl)
+ const playerRect = videojs.dom.getBoundingClientRect(this.player.el())
+
+ if (!seekBarRect || !playerRect) return
+
+ const seekBarX = videojs.dom.getPointerPosition(seekBarEl, evt).x
+ let position = seekBarX * this.player.duration()
+
+ const maxPosition = Math.round((imgHeight / this.height) * (imgWidth / this.width)) - 1
+ position = Math.min(position / this.interval, maxPosition)
+
+ const responsive = 600
+ const playerWidth = this.player.currentWidth()
+ const scaleFactor = responsive && playerWidth < responsive
+ ? playerWidth / responsive
+ : 1
+ const columns = imgWidth / this.width
+
+ const scaledWidth = this.width * scaleFactor
+ const scaledHeight = this.height * scaleFactor
+ const cleft = Math.floor(position % columns) * -scaledWidth
+ const ctop = Math.floor(position / columns) * -scaledHeight
+
+ const bgSize = `${imgWidth * scaleFactor}px ${imgHeight * scaleFactor}px`
+ const topOffset = -scaledHeight - 60
+
+ const previewHalfSize = Math.round(scaledWidth / 2)
+ let left = seekBarRect.width * seekBarX - previewHalfSize
+
+ // Seek bar doesn't take all the player width, so we can add/minus a few more pixels
+ const minLeft = playerRect.left - seekBarRect.left
+ const maxLeft = seekBarRect.width - scaledWidth + (playerRect.right - seekBarRect.right)
+
+ if (left < minLeft) left = minLeft
+ if (left > maxLeft) left = maxLeft
+
+ const tooltipStyle: { [id: string]: string } = {
+ 'background-image': `url("${this.url}")`,
+ 'background-repeat': 'no-repeat',
+ 'background-position': `${cleft}px ${ctop}px`,
+ 'background-size': bgSize,
+
+ 'color': '#fff',
+ 'text-shadow': '1px 1px #000',
+
+ 'position': 'relative',
+
+ 'top': `${topOffset}px`,
+
+ 'border': '1px solid #000',
+
+ // border should not overlay thumbnail area
+ 'width': `${scaledWidth + 2}px`,
+ 'height': `${scaledHeight + 2}px`
+ }
+
+ tooltipStyle.left = `${left}px`
+
+ for (const [ key, value ] of Object.entries(tooltipStyle)) {
+ this.spritePlaceholder.style.setProperty(key, value)
+ }
+ })
+ }
+
+ private resetMouseTooltip () {
+ if (this.spritePlaceholder) {
+ this.spritePlaceholder.style.cssText = ''
+ }
+ }
+
+ private isReady () {
+ return this.mouseTimeTooltip && this.width && this.height && this.url
+ }
+}
+
+videojs.registerPlugin('storyboard', StoryboardPlugin)
+
+export { StoryboardPlugin }
diff --git a/client/src/assets/player/types/manager-options.ts b/client/src/assets/player/types/manager-options.ts
index 1f3a0aa2e..a73341b4c 100644
--- a/client/src/assets/player/types/manager-options.ts
+++ b/client/src/assets/player/types/manager-options.ts
@@ -1,6 +1,6 @@
import { PluginsManager } from '@root-helpers/plugins-manager'
import { LiveVideoLatencyMode, VideoFile } from '@shared/models'
-import { PlaylistPluginOptions, VideoJSCaption } from './peertube-videojs-typings'
+import { PlaylistPluginOptions, VideoJSCaption, VideoJSStoryboard } from './peertube-videojs-typings'
export type PlayerMode = 'webtorrent' | 'p2p-media-loader'
@@ -78,6 +78,7 @@ export interface CommonOptions extends CustomizationOptions {
language?: string
videoCaptions: VideoJSCaption[]
+ storyboard: VideoJSStoryboard
videoUUID: string
videoShortUUID: string
diff --git a/client/src/assets/player/types/peertube-videojs-typings.ts b/client/src/assets/player/types/peertube-videojs-typings.ts
index 723c42c5d..30d2b287f 100644
--- a/client/src/assets/player/types/peertube-videojs-typings.ts
+++ b/client/src/assets/player/types/peertube-videojs-typings.ts
@@ -49,6 +49,8 @@ declare module 'video.js' {
stats (options?: StatsCardOptions): StatsForNerdsPlugin
+ storyboard (options: StoryboardOptions): void
+
textTracks (): TextTrackList & {
tracks_: (TextTrack & { id: string, label: string, src: string })[]
}
@@ -89,6 +91,13 @@ type VideoJSCaption = {
src: string
}
+type VideoJSStoryboard = {
+ url: string
+ width: number
+ height: number
+ interval: number
+}
+
type PeerTubePluginOptions = {
mode: PlayerMode
@@ -118,6 +127,13 @@ type MetricsPluginOptions = {
videoUUID: string
}
+type StoryboardOptions = {
+ url: string
+ width: number
+ height: number
+ interval: number
+}
+
type PlaylistPluginOptions = {
elements: VideoPlaylistElement[]
@@ -238,6 +254,7 @@ type PlaylistItemOptions = {
export {
PlayerNetworkInfo,
+ VideoJSStoryboard,
PlaylistItemOptions,
NextPreviousVideoButtonOptions,
ResolutionUpdateData,
@@ -251,6 +268,7 @@ export {
PeerTubeResolution,
VideoJSPluginOptions,
LoadedQualityData,
+ StoryboardOptions,
PeerTubeLinkButtonOptions,
PeerTubeP2PInfoButtonOptions
}
diff --git a/client/src/sass/player/control-bar.scss b/client/src/sass/player/control-bar.scss
index 96b3adf66..02d5fa169 100644
--- a/client/src/sass/player/control-bar.scss
+++ b/client/src/sass/player/control-bar.scss
@@ -3,6 +3,20 @@
@use '_mixins' as *;
@use './_player-variables' as *;
+// Like the time tooltip
+.video-js .vjs-progress-holder .vjs-storyboard-sprite-placeholder {
+ display: none;
+}
+
+.video-js .vjs-progress-control:hover .vjs-storyboard-sprite-placeholder,
+.video-js .vjs-progress-control:hover .vjs-progress-holder:focus .vjs-storyboard-sprite-placeholder {
+ display: block;
+
+ // Ensure that we maintain a font-size of ~10px.
+ font-size: 0.6em;
+ visibility: visible;
+}
+
.video-js.vjs-peertube-skin .vjs-control-bar {
z-index: 100;
@@ -79,6 +93,7 @@
top: -0.3em;
}
+ // Only used on mobile
.vjs-time-tooltip {
display: none;
}
diff --git a/client/src/sass/player/mobile.scss b/client/src/sass/player/mobile.scss
index 84d7a00f1..d150c54ee 100644
--- a/client/src/sass/player/mobile.scss
+++ b/client/src/sass/player/mobile.scss
@@ -6,6 +6,31 @@
/* Special mobile style */
.video-js.vjs-peertube-skin.vjs-is-mobile {
+ // No hover means we can't display the storyboard/time tooltip on mouse hover
+ // Use the time tooltip in progress control instead
+ .vjs-mouse-display {
+ display: none !important;
+ }
+
+ .vjs-storyboard-sprite-placeholder {
+ display: none;
+ }
+
+ .vjs-progress-control .vjs-sliding {
+
+ .vjs-time-tooltip,
+ .vjs-storyboard-sprite-placeholder {
+ display: block !important;
+
+ visibility: visible !important;
+ }
+
+ .vjs-time-tooltip {
+ color: #fff;
+ background-color: rgba(0, 0, 0, 0.8);
+ }
+ }
+
.vjs-control-bar {
.vjs-progress-control .vjs-slider .vjs-play-progress {
// Always display the circle on mobile
diff --git a/config/default.yaml b/config/default.yaml
index 5d0eab4f5..e54c93ac5 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -136,6 +136,7 @@ storage:
logs: 'storage/logs/'
previews: 'storage/previews/'
thumbnails: 'storage/thumbnails/'
+ storyboards: 'storage/storyboards/'
torrents: 'storage/torrents/'
captions: 'storage/captions/'
cache: 'storage/cache/'
@@ -396,6 +397,8 @@ cache:
size: 500 # Max number of video captions/subtitles you want to cache
torrents:
size: 500 # Max number of video torrents you want to cache
+ storyboards:
+ size: 500 # Max number of video storyboards you want to cache
admin:
# Used to generate the root user at first startup
diff --git a/config/production.yaml.example b/config/production.yaml.example
index 5514f1af6..83ee48dae 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -134,6 +134,7 @@ storage:
logs: '/var/www/peertube/storage/logs/'
previews: '/var/www/peertube/storage/previews/'
thumbnails: '/var/www/peertube/storage/thumbnails/'
+ storyboards: '/var/www/peertube/storage/storyboards/'
torrents: '/var/www/peertube/storage/torrents/'
captions: '/var/www/peertube/storage/captions/'
cache: '/var/www/peertube/storage/cache/'
@@ -406,6 +407,8 @@ cache:
size: 500 # Max number of video captions/subtitles you want to cache
torrents:
size: 500 # Max number of video torrents you want to cache
+ storyboards:
+ size: 500 # Max number of video storyboards you want to cache
admin:
# Used to generate the root user at first startup
diff --git a/config/test-1.yaml b/config/test-1.yaml
index 7b62e3d0c..45ec27e63 100644
--- a/config/test-1.yaml
+++ b/config/test-1.yaml
@@ -19,6 +19,7 @@ storage:
logs: 'test1/logs/'
previews: 'test1/previews/'
thumbnails: 'test1/thumbnails/'
+ storyboards: 'test1/storyboards/'
torrents: 'test1/torrents/'
captions: 'test1/captions/'
cache: 'test1/cache/'
diff --git a/config/test-2.yaml b/config/test-2.yaml
index ba36369a6..7a06e5650 100644
--- a/config/test-2.yaml
+++ b/config/test-2.yaml
@@ -19,6 +19,7 @@ storage:
logs: 'test2/logs/'
previews: 'test2/previews/'
thumbnails: 'test2/thumbnails/'
+ storyboards: 'test2/storyboards/'
torrents: 'test2/torrents/'
captions: 'test2/captions/'
cache: 'test2/cache/'
diff --git a/config/test-3.yaml b/config/test-3.yaml
index 6adec7953..4b1563369 100644
--- a/config/test-3.yaml
+++ b/config/test-3.yaml
@@ -19,6 +19,7 @@ storage:
logs: 'test3/logs/'
previews: 'test3/previews/'
thumbnails: 'test3/thumbnails/'
+ storyboards: 'test3/storyboards/'
torrents: 'test3/torrents/'
captions: 'test3/captions/'
cache: 'test3/cache/'
diff --git a/config/test-4.yaml b/config/test-4.yaml
index f042aee46..248db4db9 100644
--- a/config/test-4.yaml
+++ b/config/test-4.yaml
@@ -19,6 +19,7 @@ storage:
logs: 'test4/logs/'
previews: 'test4/previews/'
thumbnails: 'test4/thumbnails/'
+ storyboards: 'test4/storyboards/'
torrents: 'test4/torrents/'
captions: 'test4/captions/'
cache: 'test4/cache/'
diff --git a/config/test-5.yaml b/config/test-5.yaml
index ad90fec04..04e2cd78d 100644
--- a/config/test-5.yaml
+++ b/config/test-5.yaml
@@ -19,6 +19,7 @@ storage:
logs: 'test5/logs/'
previews: 'test5/previews/'
thumbnails: 'test5/thumbnails/'
+ storyboards: 'test5/storyboards/'
torrents: 'test5/torrents/'
captions: 'test5/captions/'
cache: 'test5/cache/'
diff --git a/config/test-6.yaml b/config/test-6.yaml
index a579f1f01..25efe0054 100644
--- a/config/test-6.yaml
+++ b/config/test-6.yaml
@@ -19,6 +19,7 @@ storage:
logs: 'test6/logs/'
previews: 'test6/previews/'
thumbnails: 'test6/thumbnails/'
+ storyboards: 'test6/storyboards/'
torrents: 'test6/torrents/'
captions: 'test6/captions/'
cache: 'test6/cache/'
diff --git a/config/test.yaml b/config/test.yaml
index 361064af1..41ec0917e 100644
--- a/config/test.yaml
+++ b/config/test.yaml
@@ -73,6 +73,8 @@ cache:
size: 1
torrents:
size: 1
+ storyboards:
+ size: 1
signup:
enabled: true
diff --git a/server.ts b/server.ts
index a7a723b24..5d3acb2cd 100644
--- a/server.ts
+++ b/server.ts
@@ -101,7 +101,7 @@ loadLanguages()
import { installApplication } from './server/initializers/installer'
import { Emailer } from './server/lib/emailer'
import { JobQueue } from './server/lib/job-queue'
-import { VideosPreviewCache, VideosCaptionCache } from './server/lib/files-cache'
+import { VideosPreviewCache, VideosCaptionCache, VideosStoryboardCache } from './server/lib/files-cache'
import {
activityPubRouter,
apiRouter,
@@ -316,6 +316,7 @@ async function startApplication () {
VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE)
VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE)
VideosTorrentCache.Instance.init(CONFIG.CACHE.TORRENTS.SIZE, FILES_CACHE.TORRENTS.MAX_AGE)
+ VideosStoryboardCache.Instance.init(CONFIG.CACHE.STORYBOARDS.SIZE, FILES_CACHE.STORYBOARDS.MAX_AGE)
// Enable Schedulers
ActorFollowScheduler.Instance.enable()
diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts
index 750e3091c..166fc2a22 100644
--- a/server/controllers/activitypub/client.ts
+++ b/server/controllers/activitypub/client.ts
@@ -33,7 +33,6 @@ import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '
import { AccountModel } from '../../models/account/account'
import { AccountVideoRateModel } from '../../models/account/account-video-rate'
import { ActorFollowModel } from '../../models/actor/actor-follow'
-import { VideoCaptionModel } from '../../models/video/video-caption'
import { VideoCommentModel } from '../../models/video/video-comment'
import { VideoPlaylistModel } from '../../models/video/video-playlist'
import { VideoShareModel } from '../../models/video/video-share'
@@ -242,14 +241,13 @@ async function videoController (req: express.Request, res: express.Response) {
if (redirectIfNotOwned(video.url, res)) return
// We need captions to render AP object
- const captions = await VideoCaptionModel.listVideoCaptions(video.id)
- const videoWithCaptions = Object.assign(video, { VideoCaptions: captions })
+ const videoAP = await video.lightAPToFullAP(undefined)
- const audience = getAudience(videoWithCaptions.VideoChannel.Account.Actor, videoWithCaptions.privacy === VideoPrivacy.PUBLIC)
- const videoObject = audiencify(await videoWithCaptions.toActivityPubObject(), audience)
+ const audience = getAudience(videoAP.VideoChannel.Account.Actor, videoAP.privacy === VideoPrivacy.PUBLIC)
+ const videoObject = audiencify(await videoAP.toActivityPubObject(), audience)
if (req.path.endsWith('/activity')) {
- const data = buildCreateActivity(videoWithCaptions.url, video.VideoChannel.Account.Actor, videoObject, audience)
+ const data = buildCreateActivity(videoAP.url, video.VideoChannel.Account.Actor, videoObject, audience)
return activityPubResponse(activityPubContextify(data, 'Video'), res)
}
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index 228eae109..c1f6756de 100644
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -190,6 +190,9 @@ function customConfig (): CustomConfig {
},
torrents: {
size: CONFIG.CACHE.TORRENTS.SIZE
+ },
+ storyboards: {
+ size: CONFIG.CACHE.STORYBOARDS.SIZE
}
},
signup: {
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index d0eecf812..bbdda5b29 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -41,6 +41,7 @@ import { liveRouter } from './live'
import { ownershipVideoRouter } from './ownership'
import { rateVideoRouter } from './rate'
import { statsRouter } from './stats'
+import { storyboardRouter } from './storyboard'
import { studioRouter } from './studio'
import { tokenRouter } from './token'
import { transcodingRouter } from './transcoding'
@@ -70,6 +71,7 @@ videosRouter.use('/', filesRouter)
videosRouter.use('/', transcodingRouter)
videosRouter.use('/', tokenRouter)
videosRouter.use('/', videoPasswordRouter)
+videosRouter.use('/', storyboardRouter)
videosRouter.get('/categories',
openapiOperationDoc({ operationId: 'getCategories' }),
diff --git a/server/controllers/api/videos/storyboard.ts b/server/controllers/api/videos/storyboard.ts
new file mode 100644
index 000000000..47a22011d
--- /dev/null
+++ b/server/controllers/api/videos/storyboard.ts
@@ -0,0 +1,29 @@
+import express from 'express'
+import { getVideoWithAttributes } from '@server/helpers/video'
+import { StoryboardModel } from '@server/models/video/storyboard'
+import { asyncMiddleware, videosGetValidator } from '../../../middlewares'
+
+const storyboardRouter = express.Router()
+
+storyboardRouter.get('/:id/storyboards',
+ asyncMiddleware(videosGetValidator),
+ asyncMiddleware(listStoryboards)
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+ storyboardRouter
+}
+
+// ---------------------------------------------------------------------------
+
+async function listStoryboards (req: express.Request, res: express.Response) {
+ const video = getVideoWithAttributes(res)
+
+ const storyboards = await StoryboardModel.listStoryboardsOf(video)
+
+ return res.json({
+ storyboards: storyboards.map(s => s.toFormattedJSON())
+ })
+}
diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts
index 073eb480f..86ab4591e 100644
--- a/server/controllers/api/videos/upload.ts
+++ b/server/controllers/api/videos/upload.ts
@@ -234,6 +234,15 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide
}
},
+ {
+ type: 'generate-video-storyboard' as 'generate-video-storyboard',
+ payload: {
+ videoUUID: video.uuid,
+ // No need to federate, we process these jobs sequentially
+ federate: false
+ }
+ },
+
{
type: 'notify',
payload: {
diff --git a/server/controllers/lazy-static.ts b/server/controllers/lazy-static.ts
index b082e41f6..6ffd39730 100644
--- a/server/controllers/lazy-static.ts
+++ b/server/controllers/lazy-static.ts
@@ -5,7 +5,7 @@ import { MActorImage } from '@server/types/models'
import { HttpStatusCode } from '../../shared/models/http/http-error-codes'
import { logger } from '../helpers/logger'
import { ACTOR_IMAGES_SIZE, LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/constants'
-import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache'
+import { VideosCaptionCache, VideosPreviewCache, VideosStoryboardCache } from '../lib/files-cache'
import { actorImagePathUnsafeCache, downloadActorImageFromWorker } from '../lib/local-actor'
import { asyncMiddleware, handleStaticError } from '../middlewares'
import { ActorImageModel } from '../models/actor/actor-image'
@@ -32,6 +32,12 @@ lazyStaticRouter.use(
handleStaticError
)
+lazyStaticRouter.use(
+ LAZY_STATIC_PATHS.STORYBOARDS + ':filename',
+ asyncMiddleware(getStoryboard),
+ handleStaticError
+)
+
lazyStaticRouter.use(
LAZY_STATIC_PATHS.VIDEO_CAPTIONS + ':filename',
asyncMiddleware(getVideoCaption),
@@ -126,6 +132,13 @@ async function getPreview (req: express.Request, res: express.Response) {
return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER })
}
+async function getStoryboard (req: express.Request, res: express.Response) {
+ const result = await VideosStoryboardCache.Instance.getFilePath(req.params.filename)
+ if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end()
+
+ return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER })
+}
+
async function getVideoCaption (req: express.Request, res: express.Response) {
const result = await VideosCaptionCache.Instance.getFilePath(req.params.filename)
if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end()
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index 97b3577af..573a29754 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -1,6 +1,6 @@
import validator from 'validator'
import { logger } from '@server/helpers/logger'
-import { ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject } from '@shared/models'
+import { ActivityPubStoryboard, ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject, VideoObject } from '@shared/models'
import { LiveVideoLatencyMode, VideoState } from '../../../../shared/models/videos'
import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers/constants'
import { peertubeTruncate } from '../../core-utils'
@@ -48,6 +48,10 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
logger.debug('Video has invalid icons', { video })
return false
}
+ if (!setValidStoryboard(video)) {
+ logger.debug('Video has invalid preview (storyboard)', { video })
+ return false
+ }
// Default attributes
if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED
@@ -201,3 +205,36 @@ function setRemoteVideoContent (video: any) {
return true
}
+
+function setValidStoryboard (video: VideoObject) {
+ if (!video.preview) return true
+ if (!Array.isArray(video.preview)) return false
+
+ video.preview = video.preview.filter(p => isStorybordValid(p))
+
+ return true
+}
+
+function isStorybordValid (preview: ActivityPubStoryboard) {
+ if (!preview) return false
+
+ if (
+ preview.type !== 'Image' ||
+ !isArray(preview.rel) ||
+ !preview.rel.includes('storyboard')
+ ) {
+ return false
+ }
+
+ preview.url = preview.url.filter(u => {
+ return u.mediaType === 'image/jpeg' &&
+ isActivityPubUrlValid(u.href) &&
+ validator.isInt(u.width + '', { min: 0 }) &&
+ validator.isInt(u.height + '', { min: 0 }) &&
+ validator.isInt(u.tileWidth + '', { min: 0 }) &&
+ validator.isInt(u.tileHeight + '', { min: 0 }) &&
+ isActivityPubVideoDurationValid(u.tileDuration)
+ })
+
+ return preview.url.length !== 0
+}
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index 0a315ea70..939b73344 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -29,7 +29,8 @@ function checkMissedConfig () {
'video_channels.max_per_user',
'csp.enabled', 'csp.report_only', 'csp.report_uri',
'security.frameguard.enabled', 'security.powered_by_header.enabled',
- 'cache.previews.size', 'cache.captions.size', 'cache.torrents.size', 'admin.email', 'contact_form.enabled',
+ 'cache.previews.size', 'cache.captions.size', 'cache.torrents.size', 'cache.storyboards.size',
+ 'admin.email', 'contact_form.enabled',
'signup.enabled', 'signup.limit', 'signup.requires_approval', 'signup.requires_email_verification', 'signup.minimum_age',
'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist',
'redundancy.videos.strategies', 'redundancy.videos.check_interval',
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 51ac5d0ce..60ab6e204 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -112,6 +112,7 @@ const CONFIG = {
STREAMING_PLAYLISTS_DIR: buildPath(config.get('storage.streaming_playlists')),
REDUNDANCY_DIR: buildPath(config.get('storage.redundancy')),
THUMBNAILS_DIR: buildPath(config.get('storage.thumbnails')),
+ STORYBOARDS_DIR: buildPath(config.get('storage.storyboards')),
PREVIEWS_DIR: buildPath(config.get('storage.previews')),
CAPTIONS_DIR: buildPath(config.get('storage.captions')),
TORRENTS_DIR: buildPath(config.get('storage.torrents')),
@@ -482,6 +483,9 @@ const CONFIG = {
},
TORRENTS: {
get SIZE () { return config.get('cache.torrents.size') }
+ },
+ STORYBOARDS: {
+ get SIZE () { return config.get('cache.storyboards.size') }
}
},
INSTANCE: {
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index e2f34fe16..3a643a60b 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -174,6 +174,7 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
'after-video-channel-import': 1,
'move-to-object-storage': 3,
'transcoding-job-builder': 1,
+ 'generate-video-storyboard': 1,
'notify': 1,
'federate-video': 1
}
@@ -198,6 +199,7 @@ const JOB_CONCURRENCY: { [id in Exclude i.width > THUMBNAILS_SIZE.minWidth)
@@ -166,6 +169,26 @@ function getCaptionAttributesFromObject (video: MVideoId, videoObject: VideoObje
}))
}
+function getStoryboardAttributeFromObject (video: MVideoId, videoObject: VideoObject) {
+ if (!isArray(videoObject.preview)) return undefined
+
+ const storyboard = videoObject.preview.find(p => p.rel.includes('storyboard'))
+ if (!storyboard) return undefined
+
+ const url = arrayify(storyboard.url).find(u => u.mediaType === 'image/jpeg')
+
+ return {
+ filename: generateImageFilename(extname(url.href)),
+ totalHeight: url.height,
+ totalWidth: url.width,
+ spriteHeight: url.tileHeight,
+ spriteWidth: url.tileWidth,
+ spriteDuration: getDurationFromActivityStream(url.tileDuration),
+ fileUrl: url.href,
+ videoId: video.id
+ }
+}
+
function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) {
const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
? VideoPrivacy.PUBLIC
@@ -228,6 +251,7 @@ export {
getLiveAttributesFromObject,
getCaptionAttributesFromObject,
+ getStoryboardAttributeFromObject,
getVideoAttributesFromObject
}
diff --git a/server/lib/activitypub/videos/updater.ts b/server/lib/activitypub/videos/updater.ts
index 6ddd2301b..3a0886523 100644
--- a/server/lib/activitypub/videos/updater.ts
+++ b/server/lib/activitypub/videos/updater.ts
@@ -57,6 +57,7 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
await Promise.all([
runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)),
runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)),
+ runInReadCommittedTransaction(t => this.setStoryboard(videoUpdated, t)),
this.setOrDeleteLive(videoUpdated),
this.setPreview(videoUpdated)
])
@@ -138,6 +139,10 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
await this.insertOrReplaceCaptions(videoUpdated, t)
}
+ private async setStoryboard (videoUpdated: MVideoFullLight, t: Transaction) {
+ await this.insertOrReplaceStoryboard(videoUpdated, t)
+ }
+
private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction?: Transaction) {
if (!this.video.isLive) return
diff --git a/server/lib/files-cache/index.ts b/server/lib/files-cache/index.ts
index e5853f7d6..59cec7215 100644
--- a/server/lib/files-cache/index.ts
+++ b/server/lib/files-cache/index.ts
@@ -1,3 +1,4 @@
-export * from './videos-preview-cache'
export * from './videos-caption-cache'
+export * from './videos-preview-cache'
+export * from './videos-storyboard-cache'
export * from './videos-torrent-cache'
diff --git a/server/lib/files-cache/videos-storyboard-cache.ts b/server/lib/files-cache/videos-storyboard-cache.ts
new file mode 100644
index 000000000..b0a55104f
--- /dev/null
+++ b/server/lib/files-cache/videos-storyboard-cache.ts
@@ -0,0 +1,53 @@
+import { join } from 'path'
+import { logger } from '@server/helpers/logger'
+import { doRequestAndSaveToFile } from '@server/helpers/requests'
+import { StoryboardModel } from '@server/models/video/storyboard'
+import { FILES_CACHE } from '../../initializers/constants'
+import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
+
+class VideosStoryboardCache extends AbstractVideoStaticFileCache {
+
+ private static instance: VideosStoryboardCache
+
+ private constructor () {
+ super()
+ }
+
+ static get Instance () {
+ return this.instance || (this.instance = new this())
+ }
+
+ async getFilePathImpl (filename: string) {
+ const storyboard = await StoryboardModel.loadWithVideoByFilename(filename)
+ if (!storyboard) return undefined
+
+ if (storyboard.Video.isOwned()) return { isOwned: true, path: storyboard.getPath() }
+
+ return this.loadRemoteFile(storyboard.filename)
+ }
+
+ // Key is the storyboard filename
+ protected async loadRemoteFile (key: string) {
+ const storyboard = await StoryboardModel.loadWithVideoByFilename(key)
+ if (!storyboard) return undefined
+
+ const destPath = join(FILES_CACHE.STORYBOARDS.DIRECTORY, storyboard.filename)
+ const remoteUrl = storyboard.getOriginFileUrl(storyboard.Video)
+
+ try {
+ await doRequestAndSaveToFile(remoteUrl, destPath)
+
+ logger.debug('Fetched remote storyboard %s to %s.', remoteUrl, destPath)
+
+ return { isOwned: false, path: destPath }
+ } catch (err) {
+ logger.info('Cannot fetch remote storyboard file %s.', remoteUrl, { err })
+
+ return undefined
+ }
+ }
+}
+
+export {
+ VideosStoryboardCache
+}
diff --git a/server/lib/job-queue/handlers/generate-storyboard.ts b/server/lib/job-queue/handlers/generate-storyboard.ts
new file mode 100644
index 000000000..652cac272
--- /dev/null
+++ b/server/lib/job-queue/handlers/generate-storyboard.ts
@@ -0,0 +1,138 @@
+import { Job } from 'bullmq'
+import { join } from 'path'
+import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg'
+import { generateImageFilename, getImageSize } from '@server/helpers/image-utils'
+import { logger, loggerTagsFactory } from '@server/helpers/logger'
+import { CONFIG } from '@server/initializers/config'
+import { STORYBOARD } from '@server/initializers/constants'
+import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
+import { VideoPathManager } from '@server/lib/video-path-manager'
+import { StoryboardModel } from '@server/models/video/storyboard'
+import { VideoModel } from '@server/models/video/video'
+import { MVideo } from '@server/types/models'
+import { FFmpegImage, isAudioFile } from '@shared/ffmpeg'
+import { GenerateStoryboardPayload } from '@shared/models'
+
+const lTagsBase = loggerTagsFactory('storyboard')
+
+async function processGenerateStoryboard (job: Job): Promise {
+ const payload = job.data as GenerateStoryboardPayload
+ const lTags = lTagsBase(payload.videoUUID)
+
+ logger.info('Processing generate storyboard of %s in job %s.', payload.videoUUID, job.id, lTags)
+
+ const video = await VideoModel.loadFull(payload.videoUUID)
+ if (!video) {
+ logger.info('Video %s does not exist anymore, skipping storyboard generation.', payload.videoUUID, lTags)
+ return
+ }
+
+ const inputFile = video.getMaxQualityFile()
+
+ await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {
+ const isAudio = await isAudioFile(videoPath)
+
+ if (isAudio) {
+ logger.info('Do not generate a storyboard of %s since the video does not have a video stream', payload.videoUUID, lTags)
+ return
+ }
+
+ const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail'))
+
+ const filename = generateImageFilename()
+ const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)
+
+ const totalSprites = buildTotalSprites(video)
+ const spriteDuration = Math.round(video.duration / totalSprites)
+
+ const spritesCount = findGridSize({
+ toFind: totalSprites,
+ maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT
+ })
+
+ logger.debug(
+ 'Generating storyboard from video of %s to %s', video.uuid, destination,
+ { ...lTags, spritesCount, spriteDuration, videoDuration: video.duration }
+ )
+
+ await ffmpeg.generateStoryboardFromVideo({
+ destination,
+ path: videoPath,
+ sprites: {
+ size: STORYBOARD.SPRITE_SIZE,
+ count: spritesCount,
+ duration: spriteDuration
+ }
+ })
+
+ const imageSize = await getImageSize(destination)
+
+ const existing = await StoryboardModel.loadByVideo(video.id)
+ if (existing) await existing.destroy()
+
+ await StoryboardModel.create({
+ filename,
+ totalHeight: imageSize.height,
+ totalWidth: imageSize.width,
+ spriteHeight: STORYBOARD.SPRITE_SIZE.height,
+ spriteWidth: STORYBOARD.SPRITE_SIZE.width,
+ spriteDuration,
+ videoId: video.id
+ })
+
+ logger.info('Storyboard generation %s ended for video %s.', destination, video.uuid, lTags)
+ })
+
+ if (payload.federate) {
+ await federateVideoIfNeeded(video, false)
+ }
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+ processGenerateStoryboard
+}
+
+function buildTotalSprites (video: MVideo) {
+ const maxSprites = STORYBOARD.SPRITE_SIZE.height * STORYBOARD.SPRITE_SIZE.width
+ const totalSprites = Math.min(Math.ceil(video.duration), maxSprites)
+
+ // We can generate a single line
+ if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return totalSprites
+
+ return findGridFit(totalSprites, STORYBOARD.SPRITES_MAX_EDGE_COUNT)
+}
+
+function findGridSize (options: {
+ toFind: number
+ maxEdgeCount: number
+}) {
+ const { toFind, maxEdgeCount } = options
+
+ for (let i = 1; i <= maxEdgeCount; i++) {
+ for (let j = i; j <= maxEdgeCount; j++) {
+ if (toFind === i * j) return { width: j, height: i }
+ }
+ }
+
+ throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`)
+}
+
+function findGridFit (value: number, maxMultiplier: number) {
+ for (let i = value; i--; i > 0) {
+ if (!isPrimeWithin(i, maxMultiplier)) return i
+ }
+
+ throw new Error('Could not find prime number below ' + value)
+}
+
+function isPrimeWithin (value: number, maxMultiplier: number) {
+ if (value < 2) return false
+
+ for (let i = 2, end = Math.min(Math.sqrt(value), maxMultiplier); i <= end; i++) {
+ if (value % i === 0 && value / i <= maxMultiplier) return false
+ }
+
+ return true
+}
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index cdd362f6e..c1355dcef 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -306,6 +306,15 @@ async function afterImportSuccess (options: {
Notifier.Instance.notifyOnNewVideoIfNeeded(video)
}
+ // Generate the storyboard in the job queue, and don't forget to federate an update after
+ await JobQueue.Instance.createJob({
+ type: 'generate-video-storyboard' as 'generate-video-storyboard',
+ payload: {
+ videoUUID: video.uuid,
+ federate: true
+ }
+ })
+
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
await JobQueue.Instance.createJob(
await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT })
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts
index 49feb53f2..95d4f5e64 100644
--- a/server/lib/job-queue/handlers/video-live-ending.ts
+++ b/server/lib/job-queue/handlers/video-live-ending.ts
@@ -1,6 +1,8 @@
import { Job } from 'bullmq'
import { readdir, remove } from 'fs-extra'
import { join } from 'path'
+import { peertubeTruncate } from '@server/helpers/core-utils'
+import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live'
@@ -20,8 +22,7 @@ import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@serv
import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg'
import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
import { logger, loggerTagsFactory } from '../../../helpers/logger'
-import { peertubeTruncate } from '@server/helpers/core-utils'
-import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
+import { JobQueue } from '../job-queue'
const lTags = loggerTagsFactory('live', 'job')
@@ -147,6 +148,8 @@ async function saveReplayToExternalVideo (options: {
}
await moveToNextState({ video: replayVideo, isNewVideo: true })
+
+ await createStoryboardJob(replayVideo)
}
async function replaceLiveByReplay (options: {
@@ -186,6 +189,7 @@ async function replaceLiveByReplay (options: {
await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory })
+ // FIXME: should not happen in this function
if (permanentLive) { // Remove session replay
await remove(replayDirectory)
} else { // We won't stream again in this live, we can delete the base replay directory
@@ -213,6 +217,8 @@ async function replaceLiveByReplay (options: {
// We consider this is a new video
await moveToNextState({ video: videoWithFiles, isNewVideo: true })
+
+ await createStoryboardJob(videoWithFiles)
}
async function assignReplayFilesToVideo (options: {
@@ -277,3 +283,13 @@ async function cleanupLiveAndFederate (options: {
logger.warn('Cannot federate live after cleanup', { videoId: video.id, err })
}
}
+
+function createStoryboardJob (video: MVideo) {
+ return JobQueue.Instance.createJob({
+ type: 'generate-video-storyboard' as 'generate-video-storyboard',
+ payload: {
+ videoUUID: video.uuid,
+ federate: true
+ }
+ })
+}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 03f6fbea7..177bca285 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -25,6 +25,7 @@ import {
DeleteResumableUploadMetaFilePayload,
EmailPayload,
FederateVideoPayload,
+ GenerateStoryboardPayload,
JobState,
JobType,
ManageVideoTorrentPayload,
@@ -65,6 +66,7 @@ import { processVideoLiveEnding } from './handlers/video-live-ending'
import { processVideoStudioEdition } from './handlers/video-studio-edition'
import { processVideoTranscoding } from './handlers/video-transcoding'
import { processVideosViewsStats } from './handlers/video-views-stats'
+import { processGenerateStoryboard } from './handlers/generate-storyboard'
export type CreateJobArgument =
{ type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -91,7 +93,8 @@ export type CreateJobArgument =
{ type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } |
{ type: 'notify', payload: NotifyPayload } |
{ type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
- { type: 'federate-video', payload: FederateVideoPayload }
+ { type: 'federate-video', payload: FederateVideoPayload } |
+ { type: 'generate-video-storyboard', payload: GenerateStoryboardPayload }
export type CreateJobOptions = {
delay?: number
@@ -122,7 +125,8 @@ const handlers: { [id in JobType]: (job: Job) => Promise } = {
'video-redundancy': processVideoRedundancy,
'video-studio-edition': processVideoStudioEdition,
'video-transcoding': processVideoTranscoding,
- 'videos-views-stats': processVideosViewsStats
+ 'videos-views-stats': processVideosViewsStats,
+ 'generate-video-storyboard': processGenerateStoryboard
}
const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise } = {
@@ -141,10 +145,11 @@ const jobTypes: JobType[] = [
'after-video-channel-import',
'email',
'federate-video',
- 'transcoding-job-builder',
+ 'generate-video-storyboard',
'manage-video-torrent',
'move-to-object-storage',
'notify',
+ 'transcoding-job-builder',
'video-channel-import',
'video-file-import',
'video-import',
diff --git a/server/lib/redis.ts b/server/lib/redis.ts
index 8430b2227..48d9986b5 100644
--- a/server/lib/redis.ts
+++ b/server/lib/redis.ts
@@ -325,8 +325,8 @@ class Redis {
const value = await this.getValue('resumable-upload-' + uploadId)
return value
- ? JSON.parse(value)
- : ''
+ ? JSON.parse(value) as { video: { id: number, shortUUID: string, uuid: string } }
+ : undefined
}
deleteUploadSession (uploadId: string) {
diff --git a/server/lib/transcoding/web-transcoding.ts b/server/lib/transcoding/web-transcoding.ts
index 7cc8f20bc..a499db422 100644
--- a/server/lib/transcoding/web-transcoding.ts
+++ b/server/lib/transcoding/web-transcoding.ts
@@ -9,6 +9,7 @@ import { ffprobePromise, getVideoStreamDuration, getVideoStreamFPS, TranscodeVOD
import { VideoResolution, VideoStorage } from '@shared/models'
import { CONFIG } from '../../initializers/config'
import { VideoFileModel } from '../../models/video/video-file'
+import { JobQueue } from '../job-queue'
import { generateWebTorrentVideoFilename } from '../paths'
import { buildFileMetadata } from '../video-file'
import { VideoPathManager } from '../video-path-manager'
@@ -198,7 +199,8 @@ export async function mergeAudioVideofile (options: {
return onWebTorrentVideoFileTranscoding({
video,
videoFile: inputVideoFile,
- videoOutputPath
+ videoOutputPath,
+ wasAudioFile: true
})
})
@@ -212,8 +214,9 @@ export async function onWebTorrentVideoFileTranscoding (options: {
video: MVideoFullLight
videoFile: MVideoFile
videoOutputPath: string
+ wasAudioFile?: boolean // default false
}) {
- const { video, videoFile, videoOutputPath } = options
+ const { video, videoFile, videoOutputPath, wasAudioFile } = options
const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
@@ -242,6 +245,17 @@ export async function onWebTorrentVideoFileTranscoding (options: {
await VideoFileModel.customUpsert(videoFile, 'video', undefined)
video.VideoFiles = await video.$get('VideoFiles')
+ if (wasAudioFile) {
+ await JobQueue.Instance.createJob({
+ type: 'generate-video-storyboard' as 'generate-video-storyboard',
+ payload: {
+ videoUUID: video.uuid,
+ // No need to federate, we process these jobs sequentially
+ federate: false
+ }
+ })
+ }
+
return { video, videoFile }
} finally {
mutexReleaser()
diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts
index a0074cb24..7029a857f 100644
--- a/server/middlewares/validators/config.ts
+++ b/server/middlewares/validators/config.ts
@@ -25,6 +25,7 @@ const customConfigUpdateValidator = [
body('cache.previews.size').isInt(),
body('cache.captions.size').isInt(),
body('cache.torrents.size').isInt(),
+ body('cache.storyboards.size').isInt(),
body('signup.enabled').isBoolean(),
body('signup.limit').isInt(),
diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts
index f2001e432..4179545b8 100644
--- a/server/models/video/formatter/video-format-utils.ts
+++ b/server/models/video/formatter/video-format-utils.ts
@@ -5,6 +5,7 @@ import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls'
import { VideoViewsManager } from '@server/lib/views/video-views-manager'
import { uuidToShort } from '@shared/extra-utils'
import {
+ ActivityPubStoryboard,
ActivityTagObject,
ActivityUrlObject,
Video,
@@ -347,29 +348,17 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
name: t.name
}))
- let language
- if (video.language) {
- language = {
- identifier: video.language,
- name: getLanguageLabel(video.language)
- }
- }
+ const language = video.language
+ ? { identifier: video.language, name: getLanguageLabel(video.language) }
+ : undefined
- let category
- if (video.category) {
- category = {
- identifier: video.category + '',
- name: getCategoryLabel(video.category)
- }
- }
+ const category = video.category
+ ? { identifier: video.category + '', name: getCategoryLabel(video.category) }
+ : undefined
- let licence
- if (video.licence) {
- licence = {
- identifier: video.licence + '',
- name: getLicenceLabel(video.licence)
- }
- }
+ const licence = video.licence
+ ? { identifier: video.licence + '', name: getLicenceLabel(video.licence) }
+ : undefined
const url: ActivityUrlObject[] = [
// HTML url should be the first element in the array so Mastodon correctly displays the embed
@@ -465,6 +454,8 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
height: i.height
})),
+ preview: buildPreviewAPAttribute(video),
+
url,
likes: getLocalVideoLikesActivityPubUrl(video),
@@ -541,3 +532,30 @@ function buildLiveAPAttributes (video: MVideoAP) {
latencyMode: video.VideoLive.latencyMode
}
}
+
+function buildPreviewAPAttribute (video: MVideoAP): ActivityPubStoryboard[] {
+ if (!video.Storyboard) return undefined
+
+ const storyboard = video.Storyboard
+
+ return [
+ {
+ type: 'Image',
+ rel: [ 'storyboard' ],
+ url: [
+ {
+ mediaType: 'image/jpeg',
+
+ href: storyboard.getOriginFileUrl(video),
+
+ width: storyboard.totalWidth,
+ height: storyboard.totalHeight,
+
+ tileWidth: storyboard.spriteWidth,
+ tileHeight: storyboard.spriteHeight,
+ tileDuration: getActivityStreamDuration(storyboard.spriteDuration)
+ }
+ ]
+ }
+ ]
+}
diff --git a/server/models/video/storyboard.ts b/server/models/video/storyboard.ts
new file mode 100644
index 000000000..65a044c98
--- /dev/null
+++ b/server/models/video/storyboard.ts
@@ -0,0 +1,169 @@
+import { remove } from 'fs-extra'
+import { join } from 'path'
+import { AfterDestroy, AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import { CONFIG } from '@server/initializers/config'
+import { MStoryboard, MStoryboardVideo, MVideo } from '@server/types/models'
+import { Storyboard } from '@shared/models'
+import { AttributesOnly } from '@shared/typescript-utils'
+import { logger } from '../../helpers/logger'
+import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, WEBSERVER } from '../../initializers/constants'
+import { VideoModel } from './video'
+import { Transaction } from 'sequelize'
+
+@Table({
+ tableName: 'storyboard',
+ indexes: [
+ {
+ fields: [ 'videoId' ],
+ unique: true
+ },
+ {
+ fields: [ 'filename' ],
+ unique: true
+ }
+ ]
+})
+export class StoryboardModel extends Model>> {
+
+ @AllowNull(false)
+ @Column
+ filename: string
+
+ @AllowNull(false)
+ @Column
+ totalHeight: number
+
+ @AllowNull(false)
+ @Column
+ totalWidth: number
+
+ @AllowNull(false)
+ @Column
+ spriteHeight: number
+
+ @AllowNull(false)
+ @Column
+ spriteWidth: number
+
+ @AllowNull(false)
+ @Column
+ spriteDuration: number
+
+ @AllowNull(true)
+ @Column(DataType.STRING(CONSTRAINTS_FIELDS.COMMONS.URL.max))
+ fileUrl: string
+
+ @ForeignKey(() => VideoModel)
+ @Column
+ videoId: number
+
+ @BelongsTo(() => VideoModel, {
+ foreignKey: {
+ allowNull: true
+ },
+ onDelete: 'CASCADE'
+ })
+ Video: VideoModel
+
+ @CreatedAt
+ createdAt: Date
+
+ @UpdatedAt
+ updatedAt: Date
+
+ @AfterDestroy
+ static removeInstanceFile (instance: StoryboardModel) {
+ logger.info('Removing storyboard file %s.', instance.filename)
+
+ // Don't block the transaction
+ instance.removeFile()
+ .catch(err => logger.error('Cannot remove storyboard file %s.', instance.filename, { err }))
+ }
+
+ static loadByVideo (videoId: number, transaction?: Transaction): Promise {
+ const query = {
+ where: {
+ videoId
+ },
+ transaction
+ }
+
+ return StoryboardModel.findOne(query)
+ }
+
+ static loadByFilename (filename: string): Promise {
+ const query = {
+ where: {
+ filename
+ }
+ }
+
+ return StoryboardModel.findOne(query)
+ }
+
+ static loadWithVideoByFilename (filename: string): Promise {
+ const query = {
+ where: {
+ filename
+ },
+ include: [
+ {
+ model: VideoModel.unscoped(),
+ required: true
+ }
+ ]
+ }
+
+ return StoryboardModel.findOne(query)
+ }
+
+ // ---------------------------------------------------------------------------
+
+ static async listStoryboardsOf (video: MVideo): Promise {
+ const query = {
+ where: {
+ videoId: video.id
+ }
+ }
+
+ const storyboards = await StoryboardModel.findAll(query)
+
+ return storyboards.map(s => Object.assign(s, { Video: video }))
+ }
+
+ // ---------------------------------------------------------------------------
+
+ getOriginFileUrl (video: MVideo) {
+ if (video.isOwned()) {
+ return WEBSERVER.URL + this.getLocalStaticPath()
+ }
+
+ return this.fileUrl
+ }
+
+ getLocalStaticPath () {
+ return LAZY_STATIC_PATHS.STORYBOARDS + this.filename
+ }
+
+ getPath () {
+ return join(CONFIG.STORAGE.STORYBOARDS_DIR, this.filename)
+ }
+
+ removeFile () {
+ return remove(this.getPath())
+ }
+
+ toFormattedJSON (this: MStoryboardVideo): Storyboard {
+ return {
+ storyboardPath: this.getLocalStaticPath(),
+
+ totalHeight: this.totalHeight,
+ totalWidth: this.totalWidth,
+
+ spriteWidth: this.spriteWidth,
+ spriteHeight: this.spriteHeight,
+
+ spriteDuration: this.spriteDuration
+ }
+ }
+}
diff --git a/server/models/video/video-caption.ts b/server/models/video/video-caption.ts
index 1fb1cae82..dd4cefd65 100644
--- a/server/models/video/video-caption.ts
+++ b/server/models/video/video-caption.ts
@@ -15,7 +15,7 @@ import {
Table,
UpdatedAt
} from 'sequelize-typescript'
-import { MVideo, MVideoCaption, MVideoCaptionFormattable, MVideoCaptionVideo } from '@server/types/models'
+import { MVideo, MVideoCaption, MVideoCaptionFormattable, MVideoCaptionLanguageUrl, MVideoCaptionVideo } from '@server/types/models'
import { buildUUID } from '@shared/extra-utils'
import { AttributesOnly } from '@shared/typescript-utils'
import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model'
@@ -225,7 +225,7 @@ export class VideoCaptionModel extends Model>> {
})
VideoJobInfo: VideoJobInfoModel
+ @HasOne(() => StoryboardModel, {
+ foreignKey: {
+ name: 'videoId',
+ allowNull: false
+ },
+ onDelete: 'cascade'
+ })
+ Storyboard: StoryboardModel
+
@AfterCreate
static notifyCreate (video: MVideo) {
InternalEventEmitter.Instance.emit('video-created', { video })
@@ -903,6 +916,10 @@ export class VideoModel extends Model>> {
model: VideoCaptionModel.unscoped(),
required: false
},
+ {
+ model: StoryboardModel.unscoped(),
+ required: false
+ },
{
attributes: [ 'id', 'url' ],
model: VideoShareModel.unscoped(),
@@ -1768,6 +1785,32 @@ export class VideoModel extends Model>> {
)
}
+ async lightAPToFullAP (this: MVideoAPLight, transaction: Transaction): Promise {
+ const videoAP = this as MVideoAP
+
+ const getCaptions = () => {
+ if (isArray(videoAP.VideoCaptions)) return videoAP.VideoCaptions
+
+ return this.$get('VideoCaptions', {
+ attributes: [ 'filename', 'language', 'fileUrl' ],
+ transaction
+ }) as Promise
+ }
+
+ const getStoryboard = () => {
+ if (videoAP.Storyboard) return videoAP.Storyboard
+
+ return this.$get('Storyboard', { transaction }) as Promise
+ }
+
+ const [ captions, storyboard ] = await Promise.all([ getCaptions(), getStoryboard() ])
+
+ return Object.assign(this, {
+ VideoCaptions: captions,
+ Storyboard: storyboard
+ })
+ }
+
getTruncatedDescription () {
if (!this.description) return null
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts
index 472cad182..3c752cc3e 100644
--- a/server/tests/api/check-params/config.ts
+++ b/server/tests/api/check-params/config.ts
@@ -74,6 +74,9 @@ describe('Test config API validators', function () {
},
torrents: {
size: 4
+ },
+ storyboards: {
+ size: 5
}
},
signup: {
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts
index 400d312d3..c2a7ccd78 100644
--- a/server/tests/api/check-params/index.ts
+++ b/server/tests/api/check-params/index.ts
@@ -34,6 +34,7 @@ import './video-comments'
import './video-files'
import './video-imports'
import './video-playlists'
+import './video-storyboards'
import './video-source'
import './video-studio'
import './video-token'
diff --git a/server/tests/api/check-params/video-storyboards.ts b/server/tests/api/check-params/video-storyboards.ts
new file mode 100644
index 000000000..a43d8fc48
--- /dev/null
+++ b/server/tests/api/check-params/video-storyboards.ts
@@ -0,0 +1,45 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import { HttpStatusCode, VideoPrivacy } from '@shared/models'
+import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
+
+describe('Test video storyboards API validator', function () {
+ let server: PeerTubeServer
+
+ let publicVideo: { uuid: string }
+ let privateVideo: { uuid: string }
+
+ // ---------------------------------------------------------------
+
+ before(async function () {
+ this.timeout(30000)
+
+ server = await createSingleServer(1)
+ await setAccessTokensToServers([ server ])
+
+ publicVideo = await server.videos.quickUpload({ name: 'public' })
+ privateVideo = await server.videos.quickUpload({ name: 'private', privacy: VideoPrivacy.PRIVATE })
+ })
+
+ it('Should fail without a valid uuid', async function () {
+ await server.storyboard.list({ id: '4da6fde3-88f7-4d16-b119-108df563d0b0', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
+ })
+
+ it('Should receive 404 when passing a non existing video id', async function () {
+ await server.storyboard.list({ id: '4da6fde3-88f7-4d16-b119-108df5630b06', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
+ })
+
+ it('Should not get the private storyboard without the appropriate token', async function () {
+ await server.storyboard.list({ id: privateVideo.uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401, token: null })
+ await server.storyboard.list({ id: publicVideo.uuid, expectedStatus: HttpStatusCode.OK_200, token: null })
+ })
+
+ it('Should succeed with the correct parameters', async function () {
+ await server.storyboard.list({ id: privateVideo.uuid })
+ await server.storyboard.list({ id: publicVideo.uuid })
+ })
+
+ after(async function () {
+ await cleanupTests([ server ])
+ })
+})
diff --git a/server/tests/api/check-params/videos-overviews.ts b/server/tests/api/check-params/videos-overviews.ts
index f9cdb7ab3..ae7de24dd 100644
--- a/server/tests/api/check-params/videos-overviews.ts
+++ b/server/tests/api/check-params/videos-overviews.ts
@@ -2,7 +2,7 @@
import { cleanupTests, createSingleServer, PeerTubeServer } from '@shared/server-commands'
-describe('Test videos overview', function () {
+describe('Test videos overview API validator', function () {
let server: PeerTubeServer
// ---------------------------------------------------------------
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts
index 011ba268c..efa7b50e3 100644
--- a/server/tests/api/server/config.ts
+++ b/server/tests/api/server/config.ts
@@ -46,6 +46,7 @@ function checkInitialConfig (server: PeerTubeServer, data: CustomConfig) {
expect(data.cache.previews.size).to.equal(1)
expect(data.cache.captions.size).to.equal(1)
expect(data.cache.torrents.size).to.equal(1)
+ expect(data.cache.storyboards.size).to.equal(1)
expect(data.signup.enabled).to.be.true
expect(data.signup.limit).to.equal(4)
@@ -154,6 +155,7 @@ function checkUpdatedConfig (data: CustomConfig) {
expect(data.cache.previews.size).to.equal(2)
expect(data.cache.captions.size).to.equal(3)
expect(data.cache.torrents.size).to.equal(4)
+ expect(data.cache.storyboards.size).to.equal(5)
expect(data.signup.enabled).to.be.false
expect(data.signup.limit).to.equal(5)
@@ -290,6 +292,9 @@ const newCustomConfig: CustomConfig = {
},
torrents: {
size: 4
+ },
+ storyboards: {
+ size: 5
}
},
signup: {
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts
index 357c08199..9c79b3aa6 100644
--- a/server/tests/api/videos/index.ts
+++ b/server/tests/api/videos/index.ts
@@ -20,3 +20,4 @@ import './videos-history'
import './videos-overview'
import './video-source'
import './video-static-file-privacy'
+import './video-storyboard'
diff --git a/server/tests/api/videos/video-storyboard.ts b/server/tests/api/videos/video-storyboard.ts
new file mode 100644
index 000000000..7ccdca8f7
--- /dev/null
+++ b/server/tests/api/videos/video-storyboard.ts
@@ -0,0 +1,184 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import { expect } from 'chai'
+import { FIXTURE_URLS } from '@server/tests/shared'
+import { areHttpImportTestsDisabled } from '@shared/core-utils'
+import { HttpStatusCode, VideoPrivacy } from '@shared/models'
+import {
+ cleanupTests,
+ createMultipleServers,
+ doubleFollow,
+ makeGetRequest,
+ PeerTubeServer,
+ sendRTMPStream,
+ setAccessTokensToServers,
+ setDefaultVideoChannel,
+ stopFfmpeg,
+ waitJobs
+} from '@shared/server-commands'
+
+async function checkStoryboard (options: {
+ server: PeerTubeServer
+ uuid: string
+ tilesCount?: number
+ minSize?: number
+}) {
+ const { server, uuid, tilesCount, minSize = 1000 } = options
+
+ const { storyboards } = await server.storyboard.list({ id: uuid })
+
+ expect(storyboards).to.have.lengthOf(1)
+
+ const storyboard = storyboards[0]
+
+ expect(storyboard.spriteDuration).to.equal(1)
+ expect(storyboard.spriteHeight).to.equal(108)
+ expect(storyboard.spriteWidth).to.equal(192)
+ expect(storyboard.storyboardPath).to.exist
+
+ if (tilesCount) {
+ expect(storyboard.totalWidth).to.equal(192 * Math.min(tilesCount, 10))
+ expect(storyboard.totalHeight).to.equal(108 * Math.max((tilesCount / 10), 1))
+ }
+
+ const { body } = await makeGetRequest({ url: server.url, path: storyboard.storyboardPath, expectedStatus: HttpStatusCode.OK_200 })
+ expect(body.length).to.be.above(minSize)
+}
+
+describe('Test video storyboard', function () {
+ let servers: PeerTubeServer[]
+
+ before(async function () {
+ this.timeout(120000)
+
+ servers = await createMultipleServers(2)
+ await setAccessTokensToServers(servers)
+ await setDefaultVideoChannel(servers)
+
+ await doubleFollow(servers[0], servers[1])
+ })
+
+ it('Should generate a storyboard after upload without transcoding', async function () {
+ this.timeout(60000)
+
+ // 5s video
+ const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_short.webm' })
+ await waitJobs(servers)
+
+ for (const server of servers) {
+ await checkStoryboard({ server, uuid, tilesCount: 5 })
+ }
+ })
+
+ it('Should generate a storyboard after upload without transcoding with a long video', async function () {
+ this.timeout(60000)
+
+ // 124s video
+ const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_very_long_10p.mp4' })
+ await waitJobs(servers)
+
+ for (const server of servers) {
+ await checkStoryboard({ server, uuid, tilesCount: 100 })
+ }
+ })
+
+ it('Should generate a storyboard after upload with transcoding', async function () {
+ this.timeout(60000)
+
+ await servers[0].config.enableMinimumTranscoding()
+
+ // 5s video
+ const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_short.webm' })
+ await waitJobs(servers)
+
+ for (const server of servers) {
+ await checkStoryboard({ server, uuid, tilesCount: 5 })
+ }
+ })
+
+ it('Should generate a storyboard after an audio upload', async function () {
+ this.timeout(60000)
+
+ // 6s audio
+ const attributes = { name: 'audio', fixture: 'sample.ogg' }
+ const { uuid } = await servers[0].videos.upload({ attributes, mode: 'legacy' })
+ await waitJobs(servers)
+
+ for (const server of servers) {
+ await checkStoryboard({ server, uuid, tilesCount: 6, minSize: 250 })
+ }
+ })
+
+ it('Should generate a storyboard after HTTP import', async function () {
+ this.timeout(60000)
+
+ if (areHttpImportTestsDisabled()) return
+
+ // 3s video
+ const { video } = await servers[0].imports.importVideo({
+ attributes: {
+ targetUrl: FIXTURE_URLS.goodVideo,
+ channelId: servers[0].store.channel.id,
+ privacy: VideoPrivacy.PUBLIC
+ }
+ })
+ await waitJobs(servers)
+
+ for (const server of servers) {
+ await checkStoryboard({ server, uuid: video.uuid, tilesCount: 3 })
+ }
+ })
+
+ it('Should generate a storyboard after torrent import', async function () {
+ this.timeout(60000)
+
+ if (areHttpImportTestsDisabled()) return
+
+ // 10s video
+ const { video } = await servers[0].imports.importVideo({
+ attributes: {
+ magnetUri: FIXTURE_URLS.magnet,
+ channelId: servers[0].store.channel.id,
+ privacy: VideoPrivacy.PUBLIC
+ }
+ })
+ await waitJobs(servers)
+
+ for (const server of servers) {
+ await checkStoryboard({ server, uuid: video.uuid, tilesCount: 10 })
+ }
+ })
+
+ it('Should generate a storyboard after a live', async function () {
+ this.timeout(240000)
+
+ await servers[0].config.enableLive({ allowReplay: true, transcoding: true, resolutions: 'min' })
+
+ const { live, video } = await servers[0].live.quickCreate({
+ saveReplay: true,
+ permanentLive: false,
+ privacy: VideoPrivacy.PUBLIC
+ })
+
+ const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
+ await servers[0].live.waitUntilPublished({ videoId: video.id })
+
+ await stopFfmpeg(ffmpegCommand)
+
+ await servers[0].live.waitUntilReplacedByReplay({ videoId: video.id })
+ await waitJobs(servers)
+
+ for (const server of servers) {
+ await checkStoryboard({ server, uuid: video.uuid })
+ }
+ })
+
+ it('Should generate a storyboard with different video durations', async function () {
+ this.timeout(60000)
+
+ })
+
+ after(async function () {
+ await cleanupTests(servers)
+ })
+})
diff --git a/server/tests/fixtures/video_very_long_10p.mp4 b/server/tests/fixtures/video_very_long_10p.mp4
new file mode 100644
index 000000000..852297933
Binary files /dev/null and b/server/tests/fixtures/video_very_long_10p.mp4 differ
diff --git a/server/types/models/video/index.ts b/server/types/models/video/index.ts
index 0ac032290..7f05db666 100644
--- a/server/types/models/video/index.ts
+++ b/server/types/models/video/index.ts
@@ -1,6 +1,7 @@
export * from './local-video-viewer-watch-section'
export * from './local-video-viewer-watch-section'
export * from './local-video-viewer'
+export * from './storyboard'
export * from './schedule-video-update'
export * from './tag'
export * from './thumbnail'
diff --git a/server/types/models/video/storyboard.ts b/server/types/models/video/storyboard.ts
new file mode 100644
index 000000000..a0403d4f0
--- /dev/null
+++ b/server/types/models/video/storyboard.ts
@@ -0,0 +1,15 @@
+import { StoryboardModel } from '@server/models/video/storyboard'
+import { PickWith } from '@shared/typescript-utils'
+import { MVideo } from './video'
+
+type Use = PickWith
+
+// ############################################################################
+
+export type MStoryboard = Omit
+
+// ############################################################################
+
+export type MStoryboardVideo =
+ MStoryboard &
+ Use<'Video', MVideo>
diff --git a/server/types/models/video/video-caption.ts b/server/types/models/video/video-caption.ts
index 8cd801064..d3adec362 100644
--- a/server/types/models/video/video-caption.ts
+++ b/server/types/models/video/video-caption.ts
@@ -11,7 +11,7 @@ export type MVideoCaption = Omit
// ############################################################################
export type MVideoCaptionLanguage = Pick
-export type MVideoCaptionLanguageUrl = Pick
+export type MVideoCaptionLanguageUrl = Pick
export type MVideoCaptionVideo =
MVideoCaption &
diff --git a/server/types/models/video/video.ts b/server/types/models/video/video.ts
index 8021e56bb..53ee94269 100644
--- a/server/types/models/video/video.ts
+++ b/server/types/models/video/video.ts
@@ -3,6 +3,7 @@ import { VideoModel } from '../../../models/video/video'
import { MTrackerUrl } from '../server/tracker'
import { MUserVideoHistoryTime } from '../user/user-video-history'
import { MScheduleVideoUpdate } from './schedule-video-update'
+import { MStoryboard } from './storyboard'
import { MTag } from './tag'
import { MThumbnail } from './thumbnail'
import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist'
@@ -32,7 +33,7 @@ type Use = PickWith
export type MVideo =
Omit
+ 'ScheduleVideoUpdate' | 'VideoBlacklist' | 'VideoImport' | 'VideoCaptions' | 'VideoLive' | 'Trackers' | 'VideoPasswords' | 'Storyboard'>
// ############################################################################
@@ -173,9 +174,10 @@ export type MVideoAP =
Use<'VideoBlacklist', MVideoBlacklistUnfederated> &
Use<'VideoFiles', MVideoFileRedundanciesOpt[]> &
Use<'Thumbnails', MThumbnail[]> &
- Use<'VideoLive', MVideoLive>
+ Use<'VideoLive', MVideoLive> &
+ Use<'Storyboard', MStoryboard>
-export type MVideoAPWithoutCaption = Omit
+export type MVideoAPLight = Omit
export type MVideoDetails =
MVideo &
diff --git a/shared/ffmpeg/ffmpeg-images.ts b/shared/ffmpeg/ffmpeg-images.ts
index 2db63bd8b..27305382c 100644
--- a/shared/ffmpeg/ffmpeg-images.ts
+++ b/shared/ffmpeg/ffmpeg-images.ts
@@ -56,4 +56,41 @@ export class FFmpegImage {
.thumbnail(thumbnailOptions)
})
}
+
+ async generateStoryboardFromVideo (options: {
+ path: string
+ destination: string
+
+ sprites: {
+ size: {
+ width: number
+ height: number
+ }
+
+ count: {
+ width: number
+ height: number
+ }
+
+ duration: number
+ }
+ }) {
+ const { path, destination, sprites } = options
+
+ const command = this.commandWrapper.buildCommand(path)
+
+ const filter = [
+ `setpts=N/round(FRAME_RATE)/TB`,
+ `select='not(mod(t,${options.sprites.duration}))'`,
+ `scale=${sprites.size.width}:${sprites.size.height}`,
+ `tile=layout=${sprites.count.width}x${sprites.count.height}`
+ ].join(',')
+
+ command.outputOption('-filter_complex', filter)
+ command.outputOption('-frames:v', '1')
+ command.outputOption('-q:v', '2')
+ command.output(destination)
+
+ return this.commandWrapper.runCommand()
+ }
}
diff --git a/shared/models/activitypub/objects/index.ts b/shared/models/activitypub/objects/index.ts
index 9aa3c462c..a2e040b32 100644
--- a/shared/models/activitypub/objects/index.ts
+++ b/shared/models/activitypub/objects/index.ts
@@ -6,5 +6,5 @@ export * from './object.model'
export * from './playlist-element-object'
export * from './playlist-object'
export * from './video-comment-object'
-export * from './video-torrent-object'
+export * from './video-object'
export * from './watch-action-object'
diff --git a/shared/models/activitypub/objects/video-torrent-object.ts b/shared/models/activitypub/objects/video-object.ts
similarity index 79%
rename from shared/models/activitypub/objects/video-torrent-object.ts
rename to shared/models/activitypub/objects/video-object.ts
index 23d54bdbd..a252a2df0 100644
--- a/shared/models/activitypub/objects/video-torrent-object.ts
+++ b/shared/models/activitypub/objects/video-object.ts
@@ -51,6 +51,22 @@ export interface VideoObject {
attributedTo: ActivityPubAttributedTo[]
+ preview?: ActivityPubStoryboard[]
+
to?: string[]
cc?: string[]
}
+
+export interface ActivityPubStoryboard {
+ type: 'Image'
+ rel: [ 'storyboard' ]
+ url: {
+ href: string
+ mediaType: string
+ width: number
+ height: number
+ tileWidth: number
+ tileHeight: number
+ tileDuration: string
+ }[]
+}
diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts
index 4202589f3..1012312f3 100644
--- a/shared/models/server/custom-config.model.ts
+++ b/shared/models/server/custom-config.model.ts
@@ -78,6 +78,10 @@ export interface CustomConfig {
torrents: {
size: number
}
+
+ storyboards: {
+ size: number
+ }
}
signup: {
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts
index 22ecee324..9c40079fb 100644
--- a/shared/models/server/job.model.ts
+++ b/shared/models/server/job.model.ts
@@ -30,6 +30,7 @@ export type JobType =
| 'video-studio-edition'
| 'video-transcoding'
| 'videos-views-stats'
+ | 'generate-video-storyboard'
export interface Job {
id: number | string
@@ -294,3 +295,10 @@ export interface TranscodingJobBuilderPayload {
priority?: number
}[][]
}
+
+// ---------------------------------------------------------------------------
+
+export interface GenerateStoryboardPayload {
+ videoUUID: string
+ federate: boolean
+}
diff --git a/shared/models/videos/index.ts b/shared/models/videos/index.ts
index 80be1854b..b3ce6ad3f 100644
--- a/shared/models/videos/index.ts
+++ b/shared/models/videos/index.ts
@@ -15,6 +15,7 @@ export * from './channel-sync'
export * from './nsfw-policy.type'
+export * from './storyboard.model'
export * from './thumbnail.type'
export * from './video-constant.model'
diff --git a/shared/models/videos/storyboard.model.ts b/shared/models/videos/storyboard.model.ts
new file mode 100644
index 000000000..c92c81f09
--- /dev/null
+++ b/shared/models/videos/storyboard.model.ts
@@ -0,0 +1,11 @@
+export interface Storyboard {
+ storyboardPath: string
+
+ totalHeight: number
+ totalWidth: number
+
+ spriteHeight: number
+ spriteWidth: number
+
+ spriteDuration: number
+}
diff --git a/shared/server-commands/server/config-command.ts b/shared/server-commands/server/config-command.ts
index b94bd2625..114db8091 100644
--- a/shared/server-commands/server/config-command.ts
+++ b/shared/server-commands/server/config-command.ts
@@ -159,6 +159,10 @@ export class ConfigCommand extends AbstractCommand {
newConfig: {
transcoding: {
enabled: true,
+
+ allowAudioFiles: true,
+ allowAdditionalExtensions: true,
+
resolutions: {
...ConfigCommand.getCustomConfigResolutions(false),
@@ -368,6 +372,9 @@ export class ConfigCommand extends AbstractCommand {
},
torrents: {
size: 4
+ },
+ storyboards: {
+ size: 5
}
},
signup: {
diff --git a/shared/server-commands/server/jobs.ts b/shared/server-commands/server/jobs.ts
index ff3098063..8f131fba4 100644
--- a/shared/server-commands/server/jobs.ts
+++ b/shared/server-commands/server/jobs.ts
@@ -33,6 +33,8 @@ async function waitJobs (
// Check if each server has pending request
for (const server of servers) {
+ if (process.env.DEBUG) console.log('Checking ' + server.url)
+
for (const state of states) {
const jobPromise = server.jobs.list({
@@ -45,6 +47,10 @@ async function waitJobs (
.then(jobs => {
if (jobs.length !== 0) {
pendingRequests = true
+
+ if (process.env.DEBUG) {
+ console.log(jobs)
+ }
}
})
@@ -55,6 +61,10 @@ async function waitJobs (
.then(obj => {
if (obj.activityPubMessagesWaiting !== 0) {
pendingRequests = true
+
+ if (process.env.DEBUG) {
+ console.log('AP messages waiting: ' + obj.activityPubMessagesWaiting)
+ }
}
})
tasks.push(debugPromise)
@@ -65,12 +75,15 @@ async function waitJobs (
for (const job of data) {
if (job.state.id !== RunnerJobState.COMPLETED) {
pendingRequests = true
+
+ if (process.env.DEBUG) {
+ console.log(job)
+ }
}
}
})
tasks.push(runnerJobsPromise)
}
-
}
return tasks
diff --git a/shared/server-commands/server/server.ts b/shared/server-commands/server/server.ts
index 0911e22b0..6aa4296b0 100644
--- a/shared/server-commands/server/server.ts
+++ b/shared/server-commands/server/server.ts
@@ -35,6 +35,7 @@ import {
VideoPasswordsCommand,
PlaylistsCommand,
ServicesCommand,
+ StoryboardCommand,
StreamingPlaylistsCommand,
VideosCommand,
VideoStudioCommand,
@@ -149,6 +150,8 @@ export class PeerTubeServer {
registrations?: RegistrationsCommand
videoPasswords?: VideoPasswordsCommand
+ storyboard?: StoryboardCommand
+
runners?: RunnersCommand
runnerRegistrationTokens?: RunnerRegistrationTokensCommand
runnerJobs?: RunnerJobsCommand
@@ -436,6 +439,8 @@ export class PeerTubeServer {
this.videoToken = new VideoTokenCommand(this)
this.registrations = new RegistrationsCommand(this)
+ this.storyboard = new StoryboardCommand(this)
+
this.runners = new RunnersCommand(this)
this.runnerRegistrationTokens = new RunnerRegistrationTokensCommand(this)
this.runnerJobs = new RunnerJobsCommand(this)
diff --git a/shared/server-commands/videos/index.ts b/shared/server-commands/videos/index.ts
index da36b5b6b..106d80af0 100644
--- a/shared/server-commands/videos/index.ts
+++ b/shared/server-commands/videos/index.ts
@@ -11,6 +11,7 @@ export * from './live-command'
export * from './live'
export * from './playlists-command'
export * from './services-command'
+export * from './storyboard-command'
export * from './streaming-playlists-command'
export * from './comments-command'
export * from './video-studio-command'
diff --git a/shared/server-commands/videos/storyboard-command.ts b/shared/server-commands/videos/storyboard-command.ts
new file mode 100644
index 000000000..06d90fc12
--- /dev/null
+++ b/shared/server-commands/videos/storyboard-command.ts
@@ -0,0 +1,19 @@
+import { HttpStatusCode, Storyboard } from '@shared/models'
+import { AbstractCommand, OverrideCommandOptions } from '../shared'
+
+export class StoryboardCommand extends AbstractCommand {
+
+ list (options: OverrideCommandOptions & {
+ id: number | string
+ }) {
+ const path = '/api/v1/videos/' + options.id + '/storyboards'
+
+ return this.getRequestBody<{ storyboards: Storyboard[] }>({
+ ...options,
+
+ path,
+ implicitToken: true,
+ defaultExpectedStatus: HttpStatusCode.OK_200
+ })
+ }
+}
diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yaml
index ff94f802b..cd0e6ffd8 100644
--- a/support/doc/api/openapi.yaml
+++ b/support/doc/api/openapi.yaml
@@ -3668,6 +3668,27 @@ paths:
items:
$ref: '#/components/schemas/VideoBlacklist'
+ /api/v1/videos/{id}/storyboards:
+ get:
+ summary: List storyboards of a video
+ operationId: listVideoStoryboards
+ tags:
+ - Video
+ parameters:
+ - $ref: '#/components/parameters/idOrUUID'
+ responses:
+ '200':
+ description: successful operation
+ content:
+ application/json:
+ schema:
+ type: object
+ properties:
+ storyboards:
+ type: array
+ items:
+ $ref: '#/components/schemas/Storyboard'
+
/api/v1/videos/{id}/captions:
get:
summary: List captions of a video
@@ -7509,6 +7530,20 @@ components:
type: array
items:
$ref: '#/components/schemas/VideoCommentThreadTree'
+ Storyboard:
+ properties:
+ storyboardPath:
+ type: string
+ totalHeight:
+ type: integer
+ totalWidth:
+ type: integer
+ spriteHeight:
+ type: integer
+ spriteWidth:
+ type: integer
+ spriteDuration:
+ type: integer
VideoCaption:
properties:
language: