From e99e4aa93cd16b78e86b0f09886ed094c7d850fb Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 11 Jul 2024 11:29:46 +0200 Subject: [PATCH] Run transcription after studio --- .../src/api/videos/video-transcription.ts | 62 ++++++++++++++++++- packages/tests/src/shared/transcription.ts | 13 +++- .../job-queue/handlers/video-live-ending.ts | 11 +--- server/core/lib/video-captions.ts | 2 +- server/core/lib/video-jobs.ts | 2 +- server/core/lib/video-studio.ts | 24 ++++--- 6 files changed, 92 insertions(+), 22 deletions(-) diff --git a/packages/tests/src/api/videos/video-transcription.ts b/packages/tests/src/api/videos/video-transcription.ts index cfef5a8aa..96d3ab1dc 100644 --- a/packages/tests/src/api/videos/video-transcription.ts +++ b/packages/tests/src/api/videos/video-transcription.ts @@ -13,7 +13,8 @@ import { waitJobs } from '@peertube/peertube-server-commands' import { FIXTURE_URLS } from '@tests/shared/fixture-urls.js' -import { checkAutoCaption, checkLanguage, checkNoCaption, uploadForTranscription } from '@tests/shared/transcription.js' +import { checkAutoCaption, checkLanguage, checkNoCaption, getCaptionContent, uploadForTranscription } from '@tests/shared/transcription.js' +import { expect } from 'chai' import { join } from 'path' describe('Test video transcription', function () { @@ -28,7 +29,8 @@ describe('Test video transcription', function () { await setDefaultVideoChannel(servers) await doubleFollow(servers[0], servers[1]) - await waitJobs(servers) + await servers[0].config.enableTranscription() + await waitJobs(servers) }) @@ -137,13 +139,67 @@ describe('Test video transcription', function () { it('Should not run a transcription if the video does not contain audio', async function () { this.timeout(120000) - const uuid = await uploadForTranscription(servers[0], { generateTranscription: false }) + const uuid = await uploadForTranscription(servers[0], { fixture: 'video_short_no_audio.mp4' }) await waitJobs(servers) await checkNoCaption(servers, uuid) await checkLanguage(servers, uuid, null) }) + it('Should run transcription after a video edition', async function () { + this.timeout(120000) + + await servers[0].config.enableMinimumTranscoding() + await servers[0].config.enableStudio() + + const uuid = await uploadForTranscription(servers[0]) + await waitJobs(servers) + + await checkAutoCaption(servers, uuid) + const oldContent = await getCaptionContent(servers[0], uuid, 'en') + + await servers[0].videoStudio.createEditionTasks({ + videoId: uuid, + tasks: [ + { + name: 'cut' as 'cut', + options: { start: 1 } + } + ] + }) + + await waitJobs(servers) + await checkAutoCaption(servers, uuid) + + const newContent = await getCaptionContent(servers[0], uuid, 'en') + expect(oldContent).to.not.equal(newContent) + }) + + it('Should not run transcription after video edition if the subtitle has not been auto generated', async function () { + this.timeout(120000) + + const uuid = await uploadForTranscription(servers[0], { language: 'en' }) + await waitJobs(servers) + + await servers[0].captions.add({ language: 'en', videoId: uuid, fixture: 'subtitle-good1.vtt' }) + const oldContent = await getCaptionContent(servers[0], uuid, 'en') + + await servers[0].videoStudio.createEditionTasks({ + videoId: uuid, + tasks: [ + { + name: 'cut' as 'cut', + options: { start: 1 } + } + ] + }) + + await waitJobs(servers) + + const newContent = await getCaptionContent(servers[0], uuid, 'en') + expect(oldContent).to.equal(newContent) + }) + after(async function () { await cleanupTests(servers) }) diff --git a/packages/tests/src/shared/transcription.ts b/packages/tests/src/shared/transcription.ts index 5561684ad..342af9320 100644 --- a/packages/tests/src/shared/transcription.ts +++ b/packages/tests/src/shared/transcription.ts @@ -1,13 +1,14 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' -import { PeerTubeServer, VideoEdit } from '@peertube/peertube-server-commands' +import { makeGetRequest, PeerTubeServer, VideoEdit } from '@peertube/peertube-server-commands' import { downloadFile, unzip } from '@peertube/peertube-transcription-devtools' import { expect } from 'chai' import { ensureDir, pathExists } from 'fs-extra/esm' import { join } from 'path' import { testCaptionFile } from './captions.js' import { FIXTURE_URLS } from './fixture-urls.js' +import { HttpStatusCode } from '../../../models/src/http/http-status-codes.js' type CustomModelName = 'tiny.pt' | 'faster-whisper-tiny' @@ -57,6 +58,16 @@ export async function checkNoCaption (servers: PeerTubeServer[], uuid: string) { } } +export async function getCaptionContent (server: PeerTubeServer, videoId: string, language: string) { + const { data } = await server.captions.list({ videoId }) + + const caption = data.find(c => c.language.id === language) + + const { text } = await makeGetRequest({ url: server.url, path: caption.captionPath, expectedStatus: HttpStatusCode.OK_200 }) + + return text +} + // --------------------------------------------------------------------------- export async function checkLanguage (servers: PeerTubeServer[], uuid: string, expected: string | null) { diff --git a/server/core/lib/job-queue/handlers/video-live-ending.ts b/server/core/lib/job-queue/handlers/video-live-ending.ts index 91ba79a63..f743e8357 100644 --- a/server/core/lib/job-queue/handlers/video-live-ending.ts +++ b/server/core/lib/job-queue/handlers/video-live-ending.ts @@ -1,7 +1,6 @@ import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@peertube/peertube-ffmpeg' import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@peertube/peertube-models' import { peertubeTruncate } from '@server/helpers/core-utils.js' -import { CONFIG } from '@server/initializers/config.js' import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js' import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url.js' import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js' @@ -177,10 +176,7 @@ async function saveReplayToExternalVideo (options: { } await createStoryboardJob(replayVideo) - - if (CONFIG.VIDEO_TRANSCRIPTION.ENABLED === true) { - await createTranscriptionTaskIfNeeded(replayVideo) - } + await createTranscriptionTaskIfNeeded(replayVideo) await moveToNextState({ video: replayVideo, isNewVideo: true }) } @@ -251,10 +247,7 @@ async function replaceLiveByReplay (options: { await moveToNextState({ video: videoWithFiles, isNewVideo: true }) await createStoryboardJob(videoWithFiles) - - if (CONFIG.VIDEO_TRANSCRIPTION.ENABLED === true) { - await createTranscriptionTaskIfNeeded(videoWithFiles) - } + await createTranscriptionTaskIfNeeded(videoWithFiles) } async function assignReplayFilesToVideo (options: { diff --git a/server/core/lib/video-captions.ts b/server/core/lib/video-captions.ts index edc30c4fc..0741600e3 100644 --- a/server/core/lib/video-captions.ts +++ b/server/core/lib/video-captions.ts @@ -11,6 +11,7 @@ import { VideoCaptionModel } from '@server/models/video/video-caption.js' import { VideoJobInfoModel } from '@server/models/video/video-job-info.js' import { VideoModel } from '@server/models/video/video.js' import { MVideo, MVideoCaption, MVideoFullLight, MVideoUUID, MVideoUrl } from '@server/types/models/index.js' +import { MutexInterface } from 'async-mutex' import { ensureDir, remove } from 'fs-extra/esm' import { join } from 'path' import { federateVideoIfNeeded } from './activitypub/videos/federate.js' @@ -18,7 +19,6 @@ import { JobQueue } from './job-queue/job-queue.js' import { Notifier } from './notifier/notifier.js' import { TranscriptionJobHandler } from './runners/index.js' import { VideoPathManager } from './video-path-manager.js' -import { MutexInterface } from 'async-mutex' const lTags = loggerTagsFactory('video-caption') diff --git a/server/core/lib/video-jobs.ts b/server/core/lib/video-jobs.ts index 9ad05a4c4..411a9aead 100644 --- a/server/core/lib/video-jobs.ts +++ b/server/core/lib/video-jobs.ts @@ -109,7 +109,7 @@ export async function addVideoJobsAfterCreation (options: { await JobQueue.Instance.createSequentialJobFlow(...jobs) - if (generateTranscription === true && CONFIG.VIDEO_TRANSCRIPTION.ENABLED === true) { + if (generateTranscription === true) { await createTranscriptionTaskIfNeeded(video) } } diff --git a/server/core/lib/video-studio.ts b/server/core/lib/video-studio.ts index 09ccaf14b..e97ac570d 100644 --- a/server/core/lib/video-studio.ts +++ b/server/core/lib/video-studio.ts @@ -1,18 +1,20 @@ -import { move, remove } from 'fs-extra/esm' -import { join } from 'path' +import { buildAspectRatio } from '@peertube/peertube-core-utils' +import { getVideoStreamDuration } from '@peertube/peertube-ffmpeg' +import { VideoStudioEditionPayload, VideoStudioTask, VideoStudioTaskPayload } from '@peertube/peertube-models' import { logger, loggerTagsFactory } from '@server/helpers/logger.js' import { createTorrentAndSetInfoHashFromPath } from '@server/helpers/webtorrent.js' import { CONFIG } from '@server/initializers/config.js' +import { VideoCaptionModel } from '@server/models/video/video-caption.js' import { MUser, MVideo, MVideoFile, MVideoFullLight, MVideoWithAllFiles } from '@server/types/models/index.js' -import { getVideoStreamDuration } from '@peertube/peertube-ffmpeg' -import { VideoStudioEditionPayload, VideoStudioTask, VideoStudioTaskPayload } from '@peertube/peertube-models' +import { move, remove } from 'fs-extra/esm' +import { join } from 'path' import { JobQueue } from './job-queue/index.js' import { VideoStudioTranscodingJobHandler } from './runners/index.js' import { getTranscodingJobPriority } from './transcoding/transcoding-priority.js' +import { createTranscriptionTaskIfNeeded } from './video-captions.js' import { buildNewFile, removeHLSPlaylist, removeWebVideoFile } from './video-file.js' -import { VideoPathManager } from './video-path-manager.js' import { buildStoryboardJobIfNeeded } from './video-jobs.js' -import { buildAspectRatio } from '@peertube/peertube-core-utils' +import { VideoPathManager } from './video-path-manager.js' const lTags = loggerTagsFactory('video-studio') @@ -108,7 +110,7 @@ export async function onVideoStudioEnded (options: { video.aspectRatio = buildAspectRatio({ width: newFile.width, height: newFile.height }) await video.save() - return JobQueue.Instance.createSequentialJobFlow( + await JobQueue.Instance.createSequentialJobFlow( buildStoryboardJobIfNeeded({ video, federate: false }), { @@ -129,6 +131,14 @@ export async function onVideoStudioEnded (options: { } } ) + + if (video.language && CONFIG.VIDEO_TRANSCRIPTION.ENABLED) { + const caption = await VideoCaptionModel.loadByVideoIdAndLanguage(video.id, video.language) + + if (caption?.automaticallyGenerated) { + await createTranscriptionTaskIfNeeded(video) + } + } } // ---------------------------------------------------------------------------