PeerTube/server/core/lib/video-captions.ts

187 lines
6.6 KiB
TypeScript
Raw Normal View History

import { VideoFileStream } from '@peertube/peertube-models'
2024-06-13 09:23:12 +02:00
import { buildSUUID } from '@peertube/peertube-node-utils'
import { AbstractTranscriber, TranscriptionModel, WhisperBuiltinModel, transcriberFactory } from '@peertube/peertube-transcription'
2024-02-14 09:21:53 +01:00
import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils.js'
2024-06-13 09:23:12 +02:00
import { isVideoCaptionLanguageValid } from '@server/helpers/custom-validators/video-captions.js'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
import { DIRECTORIES } from '@server/initializers/constants.js'
2024-02-14 09:21:53 +01:00
import { sequelizeTypescript } from '@server/initializers/database.js'
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
2024-06-13 09:23:12 +02:00
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
import { VideoModel } from '@server/models/video/video.js'
import { MVideo, MVideoCaption, MVideoFullLight, MVideoUUID, MVideoUrl } from '@server/types/models/index.js'
2024-07-11 11:29:46 +02:00
import { MutexInterface } from 'async-mutex'
2024-06-13 09:23:12 +02:00
import { ensureDir, remove } from 'fs-extra/esm'
import { join } from 'path'
import { federateVideoIfNeeded } from './activitypub/videos/federate.js'
import { JobQueue } from './job-queue/job-queue.js'
import { Notifier } from './notifier/notifier.js'
import { TranscriptionJobHandler } from './runners/index.js'
import { VideoPathManager } from './video-path-manager.js'
const lTags = loggerTagsFactory('video-caption')
2024-02-14 09:21:53 +01:00
export async function createLocalCaption (options: {
video: MVideo
path: string
language: string
automaticallyGenerated: boolean
2024-02-14 09:21:53 +01:00
}) {
const { language, path, video, automaticallyGenerated } = options
2024-02-14 09:21:53 +01:00
const videoCaption = new VideoCaptionModel({
videoId: video.id,
filename: VideoCaptionModel.generateCaptionName(language),
language,
automaticallyGenerated
2024-02-14 09:21:53 +01:00
}) as MVideoCaption
await moveAndProcessCaptionFile({ path }, videoCaption)
await sequelizeTypescript.transaction(async t => {
await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
})
2024-06-13 09:23:12 +02:00
return Object.assign(videoCaption, { Video: video })
}
export async function createTranscriptionTaskIfNeeded (video: MVideoUUID & MVideoUrl) {
if (CONFIG.VIDEO_TRANSCRIPTION.ENABLED !== true) return
logger.info(`Creating transcription job for ${video.url}`, lTags(video.uuid))
if (CONFIG.VIDEO_TRANSCRIPTION.REMOTE_RUNNERS.ENABLED === true) {
await new TranscriptionJobHandler().create({ video })
} else {
await JobQueue.Instance.createJob({ type: 'video-transcription', payload: { videoUUID: video.uuid } })
}
await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscription')
}
// ---------------------------------------------------------------------------
// Transcription task
// ---------------------------------------------------------------------------
let transcriber: AbstractTranscriber
export async function generateSubtitle (options: {
video: MVideoUUID
}) {
const outputPath = join(CONFIG.STORAGE.TMP_DIR, 'transcription', buildSUUID())
let inputFileMutexReleaser: MutexInterface.Releaser
2024-06-13 09:23:12 +02:00
try {
await ensureDir(outputPath)
const binDirectory = join(DIRECTORIES.LOCAL_PIP_DIRECTORY, 'bin')
2024-06-13 09:23:12 +02:00
// Lazy load the transcriber
if (!transcriber) {
transcriber = transcriberFactory.createFromEngineName({
engineName: CONFIG.VIDEO_TRANSCRIPTION.ENGINE,
enginePath: CONFIG.VIDEO_TRANSCRIPTION.ENGINE_PATH,
logger,
binDirectory
})
if (!CONFIG.VIDEO_TRANSCRIPTION.ENGINE_PATH) {
logger.info(`Installing transcriber ${transcriber.engine.name} to generate subtitles`, lTags())
await transcriber.install(DIRECTORIES.LOCAL_PIP_DIRECTORY)
}
}
inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(options.video.uuid)
2024-06-13 09:23:12 +02:00
const video = await VideoModel.loadFull(options.video.uuid)
if (!video) {
logger.info('Do not process transcription, video does not exist anymore.', lTags(options.video.uuid))
return undefined
}
2024-06-13 09:23:12 +02:00
const file = video.getMaxQualityFile(VideoFileStream.AUDIO)
2024-06-13 09:23:12 +02:00
if (!file) {
logger.info(
`Do not run transcription for ${video.uuid} in ${outputPath} because it does not contain an audio stream`,
{ video, ...lTags(video.uuid) }
)
return
}
2024-06-13 09:23:12 +02:00
await VideoPathManager.Instance.makeAvailableVideoFile(file, async inputPath => {
// Release input file mutex now we are going to run the command
setTimeout(() => inputFileMutexReleaser(), 1000)
2024-06-13 09:23:12 +02:00
logger.info(`Running transcription for ${video.uuid} in ${outputPath}`, lTags(video.uuid))
const transcriptFile = await transcriber.transcribe({
mediaFilePath: inputPath,
2024-06-13 09:23:12 +02:00
model: CONFIG.VIDEO_TRANSCRIPTION.MODEL_PATH
? await TranscriptionModel.fromPath(CONFIG.VIDEO_TRANSCRIPTION.MODEL_PATH)
: new WhisperBuiltinModel(CONFIG.VIDEO_TRANSCRIPTION.MODEL),
transcriptDirectory: outputPath,
format: 'vtt'
})
await onTranscriptionEnded({ video, language: transcriptFile.language, vttPath: transcriptFile.path })
})
} finally {
if (outputPath) await remove(outputPath)
if (inputFileMutexReleaser) inputFileMutexReleaser()
2024-06-13 09:23:12 +02:00
VideoJobInfoModel.decrease(options.video.uuid, 'pendingTranscription')
.catch(err => logger.error('Cannot decrease pendingTranscription job count', { err, ...lTags(options.video.uuid) }))
2024-06-13 09:23:12 +02:00
}
}
export async function onTranscriptionEnded (options: {
video: MVideoFullLight
language: string
vttPath: string
lTags?: (string | number)[]
}) {
const { video, language, vttPath, lTags: customLTags = [] } = options
if (!isVideoCaptionLanguageValid(language)) {
logger.warn(`Invalid transcription language for video ${video.uuid}`, lTags(video.uuid))
2024-06-13 09:23:12 +02:00
return
}
if (!video.language) {
video.language = language
await video.save()
}
2024-07-12 16:20:26 +02:00
const existing = await VideoCaptionModel.loadByVideoIdAndLanguage(video.id, language)
if (existing && !existing.automaticallyGenerated) {
logger.info(
// eslint-disable-next-line max-len
`Do not replace existing caption for video ${video.uuid} after transcription (subtitle may have been added while during the transcription process)`,
lTags(video.uuid)
)
return
}
2024-06-13 09:23:12 +02:00
const caption = await createLocalCaption({
video,
language,
path: vttPath,
automaticallyGenerated: true
2024-06-13 09:23:12 +02:00
})
await sequelizeTypescript.transaction(async t => {
await federateVideoIfNeeded(video, false, t)
})
Notifier.Instance.notifyOfGeneratedVideoTranscription(caption)
logger.info(`Transcription ended for ${video.uuid}`, lTags(video.uuid, ...customLTags))
2024-02-14 09:21:53 +01:00
}