Improve storyboard grid algorithm

Try to reduce missing sprites at the end of the video

packages/tests/src/api/videos/video-storyboard.ts

@@ -26,8 +26,9 @@ async function checkStoryboard (options: {
   spriteWidth?: number
   tilesCount?: number
   minSize?: number
+  spriteDuration?: number
 }) {
-  const { server, uuid, tilesCount, spriteHeight = 108, spriteWidth = 192, minSize = 1000 } = options
+  const { server, uuid, tilesCount, spriteDuration = 1, spriteHeight = 108, spriteWidth = 192, minSize = 1000 } = options
 
   const { storyboards } = await server.storyboard.list({ id: uuid })
 
@@ -35,7 +36,7 @@ async function checkStoryboard (options: {
 
   const storyboard = storyboards[0]
 
-  expect(storyboard.spriteDuration).to.equal(1)
+  expect(storyboard.spriteDuration).to.equal(spriteDuration)
   expect(storyboard.spriteHeight).to.equal(spriteHeight)
   expect(storyboard.spriteWidth).to.equal(spriteWidth)
   expect(storyboard.storyboardPath).to.exist
@@ -85,7 +86,7 @@ describe('Test video storyboard', function () {
     await waitJobs(servers)
 
     for (const server of servers) {
-      await checkStoryboard({ server, uuid, spriteHeight: 154, tilesCount: 100 })
+      await checkStoryboard({ server, uuid, spriteDuration: 2, spriteHeight: 154, tilesCount: 60 })
     }
   })
 

server/core/lib/job-queue/handlers/generate-storyboard.ts

@@ -1,5 +1,5 @@
-import { Job } from 'bullmq'
+import { FFmpegImage, ffprobePromise, getVideoStreamDimensionsInfo, isAudioFile } from '@peertube/peertube-ffmpeg'
-import { join } from 'path'
+import { GenerateStoryboardPayload } from '@peertube/peertube-models'
 import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
 import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'
 import { generateImageFilename } from '@server/helpers/image-utils.js'
@@ -10,12 +10,12 @@ import { STORYBOARD } from '@server/initializers/constants.js'
 import { sequelizeTypescript } from '@server/initializers/database.js'
 import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
+import { getImageSizeFromWorker } from '@server/lib/worker/parent-process.js'
 import { StoryboardModel } from '@server/models/video/storyboard.js'
 import { VideoModel } from '@server/models/video/video.js'
 import { MVideo } from '@server/types/models/index.js'
-import { FFmpegImage, ffprobePromise, getVideoStreamDimensionsInfo, isAudioFile } from '@peertube/peertube-ffmpeg'
+import { Job } from 'bullmq'
-import { GenerateStoryboardPayload } from '@peertube/peertube-models'
+import { join } from 'path'
-import { getImageSizeFromWorker } from '@server/lib/worker/parent-process.js'
 
 const lTagsBase = loggerTagsFactory('storyboard')
 
@@ -62,14 +62,12 @@ async function processGenerateStoryboard (job: Job): Promise<void> {
       const filename = generateImageFilename()
       const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)
 
-      const totalSprites = buildTotalSprites({ video, spriteHeight, spriteWidth })
+      const { totalSprites, spriteDuration } = buildSpritesMetadata({ video })
       if (totalSprites === 0) {
         logger.info('Do not generate a storyboard of %s because the video is not long enough', payload.videoUUID, lTags)
         return
       }
 
-      const spriteDuration = Math.round(video.duration / totalSprites)
-
       const spritesCount = findGridSize({
         toFind: totalSprites,
         maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT
@@ -136,20 +134,22 @@ export {
   processGenerateStoryboard
 }
 
-function buildTotalSprites (options: {
+function buildSpritesMetadata (options: {
   video: MVideo
-  spriteHeight: number
-  spriteWidth: number
 }) {
-  const { video, spriteHeight, spriteWidth } = options
+  const { video } = options
 
-  const maxSprites = spriteHeight * spriteWidth
+  if (video.duration < 3) return { spriteDuration: undefined, totalSprites: 0 }
-  const totalSprites = Math.min(Math.ceil(video.duration), maxSprites)
 
-  // We can generate a single line
+  const maxSprites = Math.min(Math.ceil(video.duration), STORYBOARD.SPRITES_MAX_EDGE_COUNT * STORYBOARD.SPRITES_MAX_EDGE_COUNT)
-  if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return totalSprites
 
-  return findGridFit(totalSprites, STORYBOARD.SPRITES_MAX_EDGE_COUNT)
+  const spriteDuration = Math.ceil(video.duration / maxSprites)
+  const totalSprites = Math.ceil(video.duration / spriteDuration)
+
+  // We can generate a single line so we don't need a prime number
+  if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return { spriteDuration, totalSprites }
+
+  return { spriteDuration, totalSprites: findNearestGridPrime(totalSprites, STORYBOARD.SPRITES_MAX_EDGE_COUNT) }
 }
 
 function findGridSize (options: {
@@ -167,7 +167,7 @@ function findGridSize (options: {
   throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`)
 }
 
-function findGridFit (value: number, maxMultiplier: number) {
+function findNearestGridPrime (value: number, maxMultiplier: number) {
   for (let i = value; i--; i > 0) {
     if (!isPrimeWithin(i, maxMultiplier)) return i
   }