From 3da9fbbe396588d05c7e5168bcee633c2cff596c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 9 Nov 2023 08:58:11 +0100 Subject: [PATCH] Adapt storyboard sprite ratio --- packages/ffmpeg/src/ffprobe.ts | 6 +++ .../tests/src/api/videos/video-storyboard.ts | 16 ++++---- server/core/initializers/constants.ts | 5 +-- .../job-queue/handlers/generate-storyboard.ts | 40 ++++++++++++++----- 4 files changed, 47 insertions(+), 20 deletions(-) diff --git a/packages/ffmpeg/src/ffprobe.ts b/packages/ffmpeg/src/ffprobe.ts index 1cfee9280..657676972 100644 --- a/packages/ffmpeg/src/ffprobe.ts +++ b/packages/ffmpeg/src/ffprobe.ts @@ -114,6 +114,12 @@ async function getVideoStreamDimensionsInfo (path: string, existingProbe?: Ffpro } } + if (videoStream.rotation === '90' || videoStream.rotation === '-90') { + const width = videoStream.width + videoStream.width = videoStream.height + videoStream.height = width + } + return { width: videoStream.width, height: videoStream.height, diff --git a/packages/tests/src/api/videos/video-storyboard.ts b/packages/tests/src/api/videos/video-storyboard.ts index 39890e3ea..8ec1a0999 100644 --- a/packages/tests/src/api/videos/video-storyboard.ts +++ b/packages/tests/src/api/videos/video-storyboard.ts @@ -22,10 +22,12 @@ import { async function checkStoryboard (options: { server: PeerTubeServer uuid: string + spriteHeight?: number + spriteWidth?: number tilesCount?: number minSize?: number }) { - const { server, uuid, tilesCount, minSize = 1000 } = options + const { server, uuid, tilesCount, spriteHeight = 108, spriteWidth = 192, minSize = 1000 } = options const { storyboards } = await server.storyboard.list({ id: uuid }) @@ -34,13 +36,13 @@ async function checkStoryboard (options: { const storyboard = storyboards[0] expect(storyboard.spriteDuration).to.equal(1) - expect(storyboard.spriteHeight).to.equal(108) - expect(storyboard.spriteWidth).to.equal(192) + expect(storyboard.spriteHeight).to.equal(spriteHeight) + expect(storyboard.spriteWidth).to.equal(spriteWidth) expect(storyboard.storyboardPath).to.exist if (tilesCount) { - expect(storyboard.totalWidth).to.equal(192 * Math.min(tilesCount, 10)) - expect(storyboard.totalHeight).to.equal(108 * Math.max((tilesCount / 10), 1)) + expect(storyboard.totalWidth).to.equal(spriteWidth * Math.min(tilesCount, 10)) + expect(storyboard.totalHeight).to.equal(spriteHeight * Math.max((tilesCount / 10), 1)) } const { body } = await makeGetRequest({ url: server.url, path: storyboard.storyboardPath, expectedStatus: HttpStatusCode.OK_200 }) @@ -83,7 +85,7 @@ describe('Test video storyboard', function () { await waitJobs(servers) for (const server of servers) { - await checkStoryboard({ server, uuid, tilesCount: 100 }) + await checkStoryboard({ server, uuid, spriteHeight: 154, tilesCount: 100 }) } }) @@ -134,7 +136,7 @@ describe('Test video storyboard', function () { await waitJobs(servers) for (const server of servers) { - await checkStoryboard({ server, uuid: video.uuid, tilesCount: 3 }) + await checkStoryboard({ server, uuid: video.uuid, spriteHeight: 144, tilesCount: 3 }) } }) diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts index 76318118c..8a7d8f50f 100644 --- a/server/core/initializers/constants.ts +++ b/server/core/initializers/constants.ts @@ -864,10 +864,7 @@ const ACTOR_IMAGES_SIZE: { [key in ActorImageType_Type]: { width: number, height } const STORYBOARD = { - SPRITE_SIZE: { - width: 192, - height: 108 - }, + SPRITE_MAX_SIZE: 192, SPRITES_MAX_EDGE_COUNT: 10 } diff --git a/server/core/lib/job-queue/handlers/generate-storyboard.ts b/server/core/lib/job-queue/handlers/generate-storyboard.ts index 60ea35f19..62ae64189 100644 --- a/server/core/lib/job-queue/handlers/generate-storyboard.ts +++ b/server/core/lib/job-queue/handlers/generate-storyboard.ts @@ -13,7 +13,7 @@ import { VideoPathManager } from '@server/lib/video-path-manager.js' import { StoryboardModel } from '@server/models/video/storyboard.js' import { VideoModel } from '@server/models/video/video.js' import { MVideo } from '@server/types/models/index.js' -import { FFmpegImage, isAudioFile } from '@peertube/peertube-ffmpeg' +import { FFmpegImage, ffprobePromise, getVideoStreamDimensionsInfo, isAudioFile } from '@peertube/peertube-ffmpeg' import { GenerateStoryboardPayload } from '@peertube/peertube-models' import { getImageSizeFromWorker } from '@server/lib/worker/parent-process.js' @@ -37,19 +37,32 @@ async function processGenerateStoryboard (job: Job): Promise { const inputFile = video.getMaxQualityFile() await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => { - const isAudio = await isAudioFile(videoPath) + const probe = await ffprobePromise(videoPath) + const isAudio = await isAudioFile(videoPath, probe) if (isAudio) { logger.info('Do not generate a storyboard of %s since the video does not have a video stream', payload.videoUUID, lTags) return } + const videoStreamInfo = await getVideoStreamDimensionsInfo(videoPath, probe) + let spriteHeight: number + let spriteWidth: number + + if (videoStreamInfo.isPortraitMode) { + spriteHeight = STORYBOARD.SPRITE_MAX_SIZE + spriteWidth = Math.round(STORYBOARD.SPRITE_MAX_SIZE / videoStreamInfo.ratio) + } else { + spriteHeight = Math.round(STORYBOARD.SPRITE_MAX_SIZE / videoStreamInfo.ratio) + spriteWidth = STORYBOARD.SPRITE_MAX_SIZE + } + const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')) const filename = generateImageFilename() const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename) - const totalSprites = buildTotalSprites(video) + const totalSprites = buildTotalSprites({ video, spriteHeight, spriteWidth }) if (totalSprites === 0) { logger.info('Do not generate a storyboard of %s because the video is not long enough', payload.videoUUID, lTags) return @@ -64,14 +77,17 @@ async function processGenerateStoryboard (job: Job): Promise { logger.debug( 'Generating storyboard from video of %s to %s', video.uuid, destination, - { ...lTags, spritesCount, spriteDuration, videoDuration: video.duration } + { ...lTags, spritesCount, spriteDuration, videoDuration: video.duration, spriteHeight, spriteWidth } ) await ffmpeg.generateStoryboardFromVideo({ destination, path: videoPath, sprites: { - size: STORYBOARD.SPRITE_SIZE, + size: { + height: spriteHeight, + width: spriteWidth + }, count: spritesCount, duration: spriteDuration } @@ -95,8 +111,8 @@ async function processGenerateStoryboard (job: Job): Promise { filename, totalHeight: imageSize.height, totalWidth: imageSize.width, - spriteHeight: STORYBOARD.SPRITE_SIZE.height, - spriteWidth: STORYBOARD.SPRITE_SIZE.width, + spriteHeight, + spriteWidth, spriteDuration, videoId: video.id }, { transaction }) @@ -120,8 +136,14 @@ export { processGenerateStoryboard } -function buildTotalSprites (video: MVideo) { - const maxSprites = STORYBOARD.SPRITE_SIZE.height * STORYBOARD.SPRITE_SIZE.width +function buildTotalSprites (options: { + video: MVideo + spriteHeight: number + spriteWidth: number +}) { + const { video, spriteHeight, spriteWidth } = options + + const maxSprites = spriteHeight * spriteWidth const totalSprites = Math.min(Math.ceil(video.duration), maxSprites) // We can generate a single line