Adapt storyboard sprite ratio

packages/ffmpeg/src/ffprobe.ts

@@ -114,6 +114,12 @@ async function getVideoStreamDimensionsInfo (path: string, existingProbe?: Ffpro
     }
   }
 
+  if (videoStream.rotation === '90' || videoStream.rotation === '-90') {
+    const width = videoStream.width
+    videoStream.width = videoStream.height
+    videoStream.height = width
+  }
+
   return {
     width: videoStream.width,
     height: videoStream.height,

packages/tests/src/api/videos/video-storyboard.ts

@@ -22,10 +22,12 @@ import {
 async function checkStoryboard (options: {
   server: PeerTubeServer
   uuid: string
+  spriteHeight?: number
+  spriteWidth?: number
   tilesCount?: number
   minSize?: number
 }) {
-  const { server, uuid, tilesCount, minSize = 1000 } = options
+  const { server, uuid, tilesCount, spriteHeight = 108, spriteWidth = 192, minSize = 1000 } = options
 
   const { storyboards } = await server.storyboard.list({ id: uuid })
 
@@ -34,13 +36,13 @@ async function checkStoryboard (options: {
   const storyboard = storyboards[0]
 
   expect(storyboard.spriteDuration).to.equal(1)
-  expect(storyboard.spriteHeight).to.equal(108)
+  expect(storyboard.spriteHeight).to.equal(spriteHeight)
-  expect(storyboard.spriteWidth).to.equal(192)
+  expect(storyboard.spriteWidth).to.equal(spriteWidth)
   expect(storyboard.storyboardPath).to.exist
 
   if (tilesCount) {
-    expect(storyboard.totalWidth).to.equal(192 * Math.min(tilesCount, 10))
+    expect(storyboard.totalWidth).to.equal(spriteWidth * Math.min(tilesCount, 10))
-    expect(storyboard.totalHeight).to.equal(108 * Math.max((tilesCount / 10), 1))
+    expect(storyboard.totalHeight).to.equal(spriteHeight * Math.max((tilesCount / 10), 1))
   }
 
   const { body } = await makeGetRequest({ url: server.url, path: storyboard.storyboardPath, expectedStatus: HttpStatusCode.OK_200 })
@@ -83,7 +85,7 @@ describe('Test video storyboard', function () {
     await waitJobs(servers)
 
     for (const server of servers) {
-      await checkStoryboard({ server, uuid, tilesCount: 100 })
+      await checkStoryboard({ server, uuid, spriteHeight: 154, tilesCount: 100 })
     }
   })
 
@@ -134,7 +136,7 @@ describe('Test video storyboard', function () {
     await waitJobs(servers)
 
     for (const server of servers) {
-      await checkStoryboard({ server, uuid: video.uuid, tilesCount: 3 })
+      await checkStoryboard({ server, uuid: video.uuid, spriteHeight: 144, tilesCount: 3 })
     }
   })
 

server/core/initializers/constants.ts

@@ -864,10 +864,7 @@ const ACTOR_IMAGES_SIZE: { [key in ActorImageType_Type]: { width: number, height
 }
 
 const STORYBOARD = {
-  SPRITE_SIZE: {
+  SPRITE_MAX_SIZE: 192,
-    width: 192,
-    height: 108
-  },
   SPRITES_MAX_EDGE_COUNT: 10
 }
 

server/core/lib/job-queue/handlers/generate-storyboard.ts

@@ -13,7 +13,7 @@ import { VideoPathManager } from '@server/lib/video-path-manager.js'
 import { StoryboardModel } from '@server/models/video/storyboard.js'
 import { VideoModel } from '@server/models/video/video.js'
 import { MVideo } from '@server/types/models/index.js'
-import { FFmpegImage, isAudioFile } from '@peertube/peertube-ffmpeg'
+import { FFmpegImage, ffprobePromise, getVideoStreamDimensionsInfo, isAudioFile } from '@peertube/peertube-ffmpeg'
 import { GenerateStoryboardPayload } from '@peertube/peertube-models'
 import { getImageSizeFromWorker } from '@server/lib/worker/parent-process.js'
 
@@ -37,19 +37,32 @@ async function processGenerateStoryboard (job: Job): Promise<void> {
     const inputFile = video.getMaxQualityFile()
 
     await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {
-      const isAudio = await isAudioFile(videoPath)
+      const probe = await ffprobePromise(videoPath)
+      const isAudio = await isAudioFile(videoPath, probe)
 
       if (isAudio) {
         logger.info('Do not generate a storyboard of %s since the video does not have a video stream', payload.videoUUID, lTags)
         return
       }
 
+      const videoStreamInfo = await getVideoStreamDimensionsInfo(videoPath, probe)
+      let spriteHeight: number
+      let spriteWidth: number
+
+      if (videoStreamInfo.isPortraitMode) {
+        spriteHeight = STORYBOARD.SPRITE_MAX_SIZE
+        spriteWidth = Math.round(STORYBOARD.SPRITE_MAX_SIZE / videoStreamInfo.ratio)
+      } else {
+        spriteHeight = Math.round(STORYBOARD.SPRITE_MAX_SIZE / videoStreamInfo.ratio)
+        spriteWidth = STORYBOARD.SPRITE_MAX_SIZE
+      }
+
       const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail'))
 
       const filename = generateImageFilename()
       const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)
 
-      const totalSprites = buildTotalSprites(video)
+      const totalSprites = buildTotalSprites({ video, spriteHeight, spriteWidth })
       if (totalSprites === 0) {
         logger.info('Do not generate a storyboard of %s because the video is not long enough', payload.videoUUID, lTags)
         return
@@ -64,14 +77,17 @@ async function processGenerateStoryboard (job: Job): Promise<void> {
 
       logger.debug(
         'Generating storyboard from video of %s to %s', video.uuid, destination,
-        { ...lTags, spritesCount, spriteDuration, videoDuration: video.duration }
+        { ...lTags, spritesCount, spriteDuration, videoDuration: video.duration, spriteHeight, spriteWidth }
       )
 
       await ffmpeg.generateStoryboardFromVideo({
         destination,
         path: videoPath,
         sprites: {
-          size: STORYBOARD.SPRITE_SIZE,
+          size: {
+            height: spriteHeight,
+            width: spriteWidth
+          },
           count: spritesCount,
           duration: spriteDuration
         }
@@ -95,8 +111,8 @@ async function processGenerateStoryboard (job: Job): Promise<void> {
             filename,
             totalHeight: imageSize.height,
             totalWidth: imageSize.width,
-            spriteHeight: STORYBOARD.SPRITE_SIZE.height,
+            spriteHeight,
-            spriteWidth: STORYBOARD.SPRITE_SIZE.width,
+            spriteWidth,
             spriteDuration,
             videoId: video.id
           }, { transaction })
@@ -120,8 +136,14 @@ export {
   processGenerateStoryboard
 }
 
-function buildTotalSprites (video: MVideo) {
+function buildTotalSprites (options: {
-  const maxSprites = STORYBOARD.SPRITE_SIZE.height * STORYBOARD.SPRITE_SIZE.width
+  video: MVideo
+  spriteHeight: number
+  spriteWidth: number
+}) {
+  const { video, spriteHeight, spriteWidth } = options
+
+  const maxSprites = spriteHeight * spriteWidth
   const totalSprites = Math.min(Math.ceil(video.duration), maxSprites)
 
   // We can generate a single line