From 96b974858502fee771467d3829d1da8f535bf5bd Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 4 Jun 2024 09:08:24 +0200 Subject: [PATCH] Support CLI move of original video file --- .../server-commands/src/requests/requests.ts | 3 +- .../src/cli/create-move-video-storage-job.ts | 55 +++++++++++++------ .../job-queue/handlers/move-to-file-system.ts | 36 ++++++++++-- .../handlers/move-to-object-storage.ts | 33 ++++++++--- .../job-queue/handlers/shared/move-video.ts | 21 ++++++- server/core/lib/object-storage/videos.ts | 14 +++++ 6 files changed, 129 insertions(+), 33 deletions(-) diff --git a/packages/server-commands/src/requests/requests.ts b/packages/server-commands/src/requests/requests.ts index bd05142ce..9e77ff282 100644 --- a/packages/server-commands/src/requests/requests.ts +++ b/packages/server-commands/src/requests/requests.ts @@ -166,9 +166,10 @@ export function makePutBodyRequest (options: { // --------------------------------------------------------------------------- -export async function getRedirectionUrl (url: string) { +export async function getRedirectionUrl (url: string, token?: string) { const res = await makeRawRequest({ url, + token, redirects: 0, expectedStatus: HttpStatusCode.FOUND_302 }) diff --git a/packages/tests/src/cli/create-move-video-storage-job.ts b/packages/tests/src/cli/create-move-video-storage-job.ts index 57d5e34f3..2f72bc012 100644 --- a/packages/tests/src/cli/create-move-video-storage-job.ts +++ b/packages/tests/src/cli/create-move-video-storage-job.ts @@ -1,23 +1,26 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { join } from 'path' -import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' +import { getAllFiles } from '@peertube/peertube-core-utils' import { HttpStatusCode, VideoDetails } from '@peertube/peertube-models' +import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' import { + ObjectStorageCommand, + PeerTubeServer, cleanupTests, createMultipleServers, doubleFollow, + getRedirectionUrl, makeRawRequest, - ObjectStorageCommand, - PeerTubeServer, setAccessTokensToServers, waitJobs } from '@peertube/peertube-server-commands' -import { expectStartWith } from '../shared/checks.js' import { checkDirectoryIsEmpty } from '@tests/shared/directories.js' -import { getAllFiles } from '@peertube/peertube-core-utils' +import { join } from 'path' +import { expectStartWith } from '../shared/checks.js' async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectStorage?: ObjectStorageCommand) { + + // Web videos for (const file of video.files) { const start = objectStorage ? objectStorage.getMockWebVideosBaseUrl() @@ -28,18 +31,36 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectSt await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) } - const start = objectStorage - ? objectStorage.getMockPlaylistBaseUrl() - : origin.url + // Playlists + { + const start = objectStorage + ? objectStorage.getMockPlaylistBaseUrl() + : origin.url - const hls = video.streamingPlaylists[0] - expectStartWith(hls.playlistUrl, start) - expectStartWith(hls.segmentsSha256Url, start) + const hls = video.streamingPlaylists[0] + expectStartWith(hls.playlistUrl, start) + expectStartWith(hls.segmentsSha256Url, start) - for (const file of hls.files) { - expectStartWith(file.fileUrl, start) + for (const file of hls.files) { + expectStartWith(file.fileUrl, start) - await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + } + } + + // Original file + { + const source = await origin.videos.getSource({ id: video.uuid }) + + if (objectStorage) { + await makeRawRequest({ url: source.fileDownloadUrl, token: origin.accessToken, expectedStatus: HttpStatusCode.FOUND_302 }) + + const redirected = await getRedirectionUrl(source.fileDownloadUrl, origin.accessToken) + expectStartWith(redirected, objectStorage.getMockOriginalFileBaseUrl()) + } else { + await makeRawRequest({ url: source.fileDownloadUrl, token: origin.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + expectStartWith(source.fileDownloadUrl, origin.url) + } } } @@ -61,10 +82,10 @@ describe('Test create move video storage job', function () { await objectStorage.prepareDefaultMockBuckets() - await servers[0].config.enableTranscoding() + await servers[0].config.enableMinimumTranscoding({ keepOriginal: true }) for (let i = 0; i < 3; i++) { - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' + i } }) + const { uuid } = await servers[0].videos.quickUpload({ name: 'video' + i }) uuids.push(uuid) } diff --git a/server/core/lib/job-queue/handlers/move-to-file-system.ts b/server/core/lib/job-queue/handlers/move-to-file-system.ts index 1925db4aa..57914564c 100644 --- a/server/core/lib/job-queue/handlers/move-to-file-system.ts +++ b/server/core/lib/job-queue/handlers/move-to-file-system.ts @@ -1,20 +1,23 @@ -import { Job } from 'bullmq' -import { join } from 'path' -import { MoveStoragePayload, VideoStateType, FileStorage } from '@peertube/peertube-models' +import { FileStorage, MoveStoragePayload, VideoStateType } from '@peertube/peertube-models' import { logger, loggerTagsFactory } from '@server/helpers/logger.js' import { updateTorrentMetadata } from '@server/helpers/webtorrent.js' import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js' import { makeHLSFileAvailable, + makeOriginalFileAvailable, makeWebVideoFileAvailable, removeHLSFileObjectStorageByFilename, removeHLSObjectStorage, + removeOriginalFileObjectStorage, removeWebVideoObjectStorage } from '@server/lib/object-storage/index.js' import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js' import { VideoPathManager } from '@server/lib/video-path-manager.js' import { moveToFailedMoveToFileSystemState, moveToNextState } from '@server/lib/video-state.js' import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js' +import { MVideoSource } from '@server/types/models/video/video-source.js' +import { Job } from 'bullmq' +import { join } from 'path' import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js' const lTagsBase = loggerTagsFactory('move-file-system') @@ -30,6 +33,8 @@ export async function processMoveToFileSystem (job: Job) { moveWebVideoFiles, moveHLSFiles, + moveVideoSourceFile, + doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }), moveToFailedState: moveToFailedMoveToFileSystemState }) @@ -50,12 +55,31 @@ export async function onMoveToFileSystemFailure (job: Job, err: any) { // Private // --------------------------------------------------------------------------- +async function moveVideoSourceFile (source: MVideoSource) { + if (source.storage === FileStorage.FILE_SYSTEM) return + + await makeOriginalFileAvailable( + source.keptOriginalFilename, + VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename) + ) + + const oldFileUrl = source.fileUrl + + source.fileUrl = null + source.storage = FileStorage.FILE_SYSTEM + await source.save() + + logger.debug('Removing original video file %s because it\'s now on file system', oldFileUrl, lTagsBase()) + + await removeOriginalFileObjectStorage(source) +} + async function moveWebVideoFiles (video: MVideoWithAllFiles) { for (const file of video.VideoFiles) { if (file.storage === FileStorage.FILE_SYSTEM) continue await makeWebVideoFileAvailable(file.filename, VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)) - await onFileMoved({ + await onVideoFileMoved({ videoOrPlaylist: video, file, objetStorageRemover: () => removeWebVideoObjectStorage(file) @@ -75,7 +99,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) { await makeHLSFileAvailable(playlistWithVideo, playlistFilename, join(getHLSDirectory(video), playlistFilename)) await makeHLSFileAvailable(playlistWithVideo, file.filename, join(getHLSDirectory(video), file.filename)) - await onFileMoved({ + await onVideoFileMoved({ videoOrPlaylist: playlistWithVideo, file, objetStorageRemover: async () => { @@ -87,7 +111,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) { } } -async function onFileMoved (options: { +async function onVideoFileMoved (options: { videoOrPlaylist: MVideo | MStreamingPlaylistVideo file: MVideoFile objetStorageRemover: () => Promise diff --git a/server/core/lib/job-queue/handlers/move-to-object-storage.ts b/server/core/lib/job-queue/handlers/move-to-object-storage.ts index 7057dd0ca..a1748cc74 100644 --- a/server/core/lib/job-queue/handlers/move-to-object-storage.ts +++ b/server/core/lib/job-queue/handlers/move-to-object-storage.ts @@ -1,15 +1,16 @@ -import { Job } from 'bullmq' -import { remove } from 'fs-extra/esm' -import { join } from 'path' -import { MoveStoragePayload, VideoStateType, FileStorage } from '@peertube/peertube-models' +import { FileStorage, MoveStoragePayload, VideoStateType } from '@peertube/peertube-models' import { logger, loggerTagsFactory } from '@server/helpers/logger.js' import { updateTorrentMetadata } from '@server/helpers/webtorrent.js' import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js' -import { storeHLSFileFromFilename, storeWebVideoFile } from '@server/lib/object-storage/index.js' +import { storeHLSFileFromFilename, storeOriginalVideoFile, storeWebVideoFile } from '@server/lib/object-storage/index.js' import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js' import { VideoPathManager } from '@server/lib/video-path-manager.js' import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state.js' import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js' +import { MVideoSource } from '@server/types/models/video/video-source.js' +import { Job } from 'bullmq' +import { remove } from 'fs-extra/esm' +import { join } from 'path' import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js' const lTagsBase = loggerTagsFactory('move-object-storage') @@ -25,6 +26,7 @@ export async function processMoveToObjectStorage (job: Job) { moveWebVideoFiles, moveHLSFiles, + moveVideoSourceFile, doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }), moveToFailedState: moveToFailedMoveToObjectStorageState }) @@ -43,6 +45,21 @@ export async function onMoveToObjectStorageFailure (job: Job, err: any) { // --------------------------------------------------------------------------- +async function moveVideoSourceFile (source: MVideoSource) { + if (source.storage !== FileStorage.FILE_SYSTEM) return + + const sourcePath = VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename) + const fileUrl = await storeOriginalVideoFile(sourcePath, source.keptOriginalFilename) + + source.storage = FileStorage.OBJECT_STORAGE + source.fileUrl = fileUrl + await source.save() + + logger.debug('Removing original video file ' + sourcePath + ' because it\'s now on object storage', lTagsBase()) + + await remove(sourcePath) +} + async function moveWebVideoFiles (video: MVideoWithAllFiles) { for (const file of video.VideoFiles) { if (file.storage !== FileStorage.FILE_SYSTEM) continue @@ -50,7 +67,7 @@ async function moveWebVideoFiles (video: MVideoWithAllFiles) { const fileUrl = await storeWebVideoFile(video, file) const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file) - await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath }) + await onVideoFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath }) } } @@ -70,12 +87,12 @@ async function moveHLSFiles (video: MVideoWithAllFiles) { const oldPath = join(getHLSDirectory(video), file.filename) - await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath }) + await onVideoFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath }) } } } -async function onFileMoved (options: { +async function onVideoFileMoved (options: { videoOrPlaylist: MVideo | MStreamingPlaylistVideo file: MVideoFile fileUrl: string diff --git a/server/core/lib/job-queue/handlers/shared/move-video.ts b/server/core/lib/job-queue/handlers/shared/move-video.ts index 93c49b207..a9c90a0b4 100644 --- a/server/core/lib/job-queue/handlers/shared/move-video.ts +++ b/server/core/lib/job-queue/handlers/shared/move-video.ts @@ -1,8 +1,10 @@ import { LoggerTags, logger, loggerTagsFactory } from '@server/helpers/logger.js' import { VideoPathManager } from '@server/lib/video-path-manager.js' import { VideoJobInfoModel } from '@server/models/video/video-job-info.js' +import { VideoSourceModel } from '@server/models/video/video-source.js' import { VideoModel } from '@server/models/video/video.js' import { MVideoWithAllFiles } from '@server/types/models/index.js' +import { MVideoSource } from '@server/types/models/video/video-source.js' export async function moveToJob (options: { jobId: string @@ -11,10 +13,20 @@ export async function moveToJob (options: { moveWebVideoFiles: (video: MVideoWithAllFiles) => Promise moveHLSFiles: (video: MVideoWithAllFiles) => Promise + moveVideoSourceFile: (source: MVideoSource) => Promise moveToFailedState: (video: MVideoWithAllFiles) => Promise doAfterLastMove: (video: MVideoWithAllFiles) => Promise }) { - const { jobId, loggerTags, videoUUID, moveHLSFiles, moveWebVideoFiles, moveToFailedState, doAfterLastMove } = options + const { + jobId, + loggerTags, + videoUUID, + moveVideoSourceFile, + moveHLSFiles, + moveWebVideoFiles, + moveToFailedState, + doAfterLastMove + } = options const lTagsBase = loggerTagsFactory(...loggerTags) @@ -31,6 +43,13 @@ export async function moveToJob (options: { const lTags = lTagsBase(video.uuid, video.url) try { + const source = await VideoSourceModel.loadLatest(video.id) + if (source) { + logger.debug(`Moving video source ${source.keptOriginalFilename} file of video ${video.uuid}`, lTags) + + await moveVideoSourceFile(source) + } + if (video.VideoFiles) { logger.debug('Moving %d web video files for video %s.', video.VideoFiles.length, video.uuid, lTags) diff --git a/server/core/lib/object-storage/videos.ts b/server/core/lib/object-storage/videos.ts index 737490491..06e8b39de 100644 --- a/server/core/lib/object-storage/videos.ts +++ b/server/core/lib/object-storage/videos.ts @@ -159,6 +159,20 @@ export async function makeWebVideoFileAvailable (filename: string, destination: return destination } +export async function makeOriginalFileAvailable (keptOriginalFilename: string, destination: string) { + const key = generateOriginalVideoObjectStorageKey(keptOriginalFilename) + + logger.info('Fetching Original Video file %s from object storage to %s.', key, destination, lTags()) + + await makeAvailable({ + key, + destination, + bucketInfo: CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES + }) + + return destination +} + // --------------------------------------------------------------------------- export function getWebVideoFileReadStream (options: {