diff --git a/package.json b/package.json index 54fef1a06..c53b5a53e 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "plugin:install": "node ./dist/scripts/plugin/install.js", "plugin:uninstall": "node ./dist/scripts/plugin/uninstall.js", "reset-password": "node ./dist/scripts/reset-password.js", + "update-object-storage-url": "LOGGER_LEVEL=warn node ./dist/scripts/update-object-storage-url.js", "update-host": "node ./dist/scripts/update-host.js", "regenerate-thumbnails": "node ./dist/scripts/regenerate-thumbnails.js", "create-import-video-file-job": "node ./dist/scripts/create-import-video-file-job.js", diff --git a/packages/models/src/import-export/user-export.model.ts b/packages/models/src/import-export/user-export.model.ts index 40c18d3d3..4d7b56fa8 100644 --- a/packages/models/src/import-export/user-export.model.ts +++ b/packages/models/src/import-export/user-export.model.ts @@ -11,6 +11,7 @@ export interface UserExport { // In bytes size: number + fileUrl: string privateDownloadUrl: string createdAt: string | Date diff --git a/packages/models/src/videos/video-source.model.ts b/packages/models/src/videos/video-source.model.ts index 4dcbb35d9..40fb6c4ab 100644 --- a/packages/models/src/videos/video-source.model.ts +++ b/packages/models/src/videos/video-source.model.ts @@ -10,6 +10,7 @@ export interface VideoSource { width?: number height?: number + fileUrl: string fileDownloadUrl: string fps?: number diff --git a/packages/tests/src/cli/index.ts b/packages/tests/src/cli/index.ts index 94444ace3..56c2bad33 100644 --- a/packages/tests/src/cli/index.ts +++ b/packages/tests/src/cli/index.ts @@ -8,3 +8,4 @@ import './prune-storage' import './regenerate-thumbnails' import './reset-password' import './update-host' +import './update-object-storage-url.js' diff --git a/packages/tests/src/cli/update-object-storage-url.ts b/packages/tests/src/cli/update-object-storage-url.ts new file mode 100644 index 000000000..cb5da0750 --- /dev/null +++ b/packages/tests/src/cli/update-object-storage-url.ts @@ -0,0 +1,131 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { getHLS } from '@peertube/peertube-core-utils' +import { VideoDetails } from '@peertube/peertube-models' +import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' +import { + ObjectStorageCommand, + PeerTubeServer, + cleanupTests, + createSingleServer, + getRedirectionUrl, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' +import { expectStartWith } from '@tests/shared/checks.js' +import { expect } from 'chai' + +describe('Update object storage URL', function () { + if (areMockObjectStorageTestsDisabled()) return + + let server: PeerTubeServer + let uuid: string + const objectStorage = new ObjectStorageCommand() + + function runUpdate (from: string, to: string) { + const env = server.cli.getEnv() + const command = `echo y | ${env} npm run update-object-storage-url -- --from "${from}" --to "${to}"` + + return server.cli.execWithEnv(command, objectStorage.getDefaultMockConfig()) + } + + before(async function () { + this.timeout(360000) + + server = await createSingleServer(1, objectStorage.getDefaultMockConfig()) + await setAccessTokensToServers([ server ]) + + await objectStorage.prepareDefaultMockBuckets() + + await server.config.enableMinimumTranscoding({ keepOriginal: true }) + + const video = await server.videos.quickUpload({ name: 'video' }) + uuid = video.uuid + + await waitJobs([ server ]) + }) + + it('Should update video URLs', async function () { + this.timeout(120000) + + const check = async (options: { + baseUrl: string + newBaseUrl: string + urlGetter: (video: VideoDetails) => Promise | string[] + }) => { + const { baseUrl, newBaseUrl, urlGetter } = options + + const oldVideo = await server.videos.get({ id: uuid }) + const oldFileUrls = await urlGetter(oldVideo) + + for (const url of oldFileUrls) { + expectStartWith(url, baseUrl) + } + + await runUpdate(baseUrl, newBaseUrl) + + const newVideo = await server.videos.get({ id: uuid }) + + const shouldBe = oldFileUrls.map(f => f.replace(baseUrl, newBaseUrl)) + expect(await urlGetter(newVideo)).to.have.members(shouldBe) + } + + await check({ + baseUrl: objectStorage.getMockWebVideosBaseUrl(), + newBaseUrl: 'https://web-video.example.com/', + urlGetter: video => video.files.map(f => f.fileUrl) + }) + + await check({ + baseUrl: objectStorage.getMockPlaylistBaseUrl(), + newBaseUrl: 'https://streaming-playlists.example.com/', + urlGetter: video => { + const hls = getHLS(video) + + return [ + ...hls.files.map(f => f.fileUrl), + + hls.playlistUrl, + hls.segmentsSha256Url + ] + } + }) + + await check({ + baseUrl: objectStorage.getMockOriginalFileBaseUrl(), + newBaseUrl: 'https://original-file.example.com/', + urlGetter: async video => { + const source = await server.videos.getSource({ id: video.uuid }) + + return [ source.fileUrl ] + } + }) + }) + + it('Should update user export URLs', async function () { + this.timeout(120000) + + const user = await server.users.getMyInfo() + + await server.userExports.request({ userId: user.id, withVideoFiles: false }) + await waitJobs([ server ]) + + { + const { data } = await server.userExports.list({ userId: user.id }) + expectStartWith(data[0].fileUrl, objectStorage.getMockUserExportBaseUrl()) + } + + await runUpdate(objectStorage.getMockUserExportBaseUrl(), 'https://user-export.example.com/') + + { + const { data } = await server.userExports.list({ userId: user.id }) + expectStartWith(data[0].fileUrl, 'https://user-export.example.com/') + } + }) + + after(async function () { + await objectStorage.cleanupMock() + + await cleanupTests([ server ]) + }) +}) diff --git a/server/core/helpers/regexp.ts b/server/core/helpers/regexp.ts index ca869e280..849c4cff4 100644 --- a/server/core/helpers/regexp.ts +++ b/server/core/helpers/regexp.ts @@ -27,10 +27,6 @@ export function wordsToRegExp (words: string[]) { return new RegExp(`(?:\\P{L}|^)(?:${innerRegex})(?=\\P{L}|$)`, 'iu') } -// --------------------------------------------------------------------------- -// Private -// --------------------------------------------------------------------------- - -function escapeForRegex (value: string) { +export function escapeForRegex (value: string) { return value.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&') } diff --git a/server/core/lib/views/shared/video-viewer-counters.ts b/server/core/lib/views/shared/video-viewer-counters.ts index 9d92b09a4..c81529283 100644 --- a/server/core/lib/views/shared/video-viewer-counters.ts +++ b/server/core/lib/views/shared/video-viewer-counters.ts @@ -31,7 +31,7 @@ export class VideoViewerCounters { private processingViewerCounters = false constructor () { - setInterval(() => this.cleanViewerCounters(), VIEW_LIFETIME.VIEWER_COUNTER) + setInterval(() => this.updateVideoViewersCount(), VIEW_LIFETIME.VIEWER_COUNTER) } // --------------------------------------------------------------------------- @@ -163,11 +163,13 @@ export class VideoViewerCounters { return viewer } - private async cleanViewerCounters () { + private async updateVideoViewersCount () { if (this.processingViewerCounters) return this.processingViewerCounters = true - if (!isTestOrDevInstance()) logger.info('Cleaning video viewers.', lTags()) + if (!isTestOrDevInstance()) { + logger.debug('Updating video viewer counters.', lTags()) + } try { for (const videoId of this.viewersPerVideo.keys()) { diff --git a/server/core/models/user/user-export.ts b/server/core/models/user/user-export.ts index 973b6edee..26f693aac 100644 --- a/server/core/models/user/user-export.ts +++ b/server/core/models/user/user-export.ts @@ -219,6 +219,7 @@ export class UserExportModel extends SequelizeModel { size: this.size, + fileUrl: this.fileUrl, privateDownloadUrl: this.getFileDownloadUrl(), createdAt: this.createdAt.toISOString(), expiresOn: new Date(this.createdAt.getTime() + CONFIG.EXPORT.USERS.EXPORT_EXPIRATION).toISOString() diff --git a/server/core/models/video/video-source.ts b/server/core/models/video/video-source.ts index 124f5a5b2..c863636e4 100644 --- a/server/core/models/video/video-source.ts +++ b/server/core/models/video/video-source.ts @@ -125,6 +125,8 @@ export class VideoSourceModel extends SequelizeModel { return { filename: this.inputFilename, inputFilename: this.inputFilename, + + fileUrl: this.fileUrl, fileDownloadUrl: this.getFileDownloadUrl(), resolution: { diff --git a/server/scripts/update-object-storage-url.ts b/server/scripts/update-object-storage-url.ts new file mode 100644 index 000000000..fc93aeaa7 --- /dev/null +++ b/server/scripts/update-object-storage-url.ts @@ -0,0 +1,118 @@ +/* eslint-disable max-len */ +import { InvalidArgumentError, createCommand } from '@commander-js/extra-typings' +import { FileStorage } from '@peertube/peertube-models' +import { escapeForRegex } from '@server/helpers/regexp.js' +import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js' +import { QueryTypes } from 'sequelize' +import prompt from 'prompt' + +const program = createCommand() + .description('Update PeerTube object file URLs after an object storage migration.') + .requiredOption('-f, --from ', 'Previous object storage base URL', parseUrl) + .requiredOption('-t, --to ', 'New object storage base URL', parseUrl) + .parse(process.argv) + +const options = program.opts() + +run() + .then(() => process.exit(0)) + .catch(err => { + console.error(err) + process.exit(-1) + }) + +async function run () { + await initDatabaseModels(true) + + const fromRegexp = `^${escapeForRegex(options.from)}` + const to = options.to + + const replacements = { fromRegexp, to, storage: FileStorage.OBJECT_STORAGE } + + // Candidates + { + const queries = [ + `SELECT COUNT(*) AS "c", 'videoFile->fileUrl: ' || COUNT(*) AS "t" FROM "videoFile" WHERE "fileUrl" ~ :fromRegexp AND "storage" = :storage`, + `SELECT COUNT(*) AS "c", 'videoStreamingPlaylist->playlistUrl: ' || COUNT(*) AS "t" FROM "videoStreamingPlaylist" WHERE "playlistUrl" ~ :fromRegexp AND "storage" = :storage`, + `SELECT COUNT(*) AS "c", 'videoStreamingPlaylist->segmentsSha256Url: ' || COUNT(*) AS "t" FROM "videoStreamingPlaylist" WHERE "segmentsSha256Url" ~ :fromRegexp AND "storage" = :storage`, + `SELECT COUNT(*) AS "c", 'userExport->fileUrl: ' || COUNT(*) AS "t" FROM "userExport" WHERE "fileUrl" ~ :fromRegexp AND "storage" = :storage`, + `SELECT COUNT(*) AS "c", 'videoSource->fileUrl: ' || COUNT(*) AS "t" FROM "videoSource" WHERE "fileUrl" ~ :fromRegexp AND "storage" = :storage` + ] + + let hasResults = false + + console.log('Candidate URLs to update:') + for (const query of queries) { + const [ row ] = await sequelizeTypescript.query(query, { replacements, type: QueryTypes.SELECT as QueryTypes.SELECT }) + + if (row['c'] !== 0) hasResults = true + + console.log(` ${row['t']}`) + } + + console.log('\n') + + if (!hasResults) { + console.log('No candidate URLs found, exiting.') + process.exit(0) + } + } + + const res = await askConfirmation() + if (res !== true) { + console.log('Exiting without updating URLs.') + process.exit(0) + } + + // Execute + { + const queries = [ + `UPDATE "videoFile" SET "fileUrl" = regexp_replace("fileUrl", :fromRegexp, :to) WHERE "storage" = :storage`, + `UPDATE "videoStreamingPlaylist" SET "playlistUrl" = regexp_replace("playlistUrl", :fromRegexp, :to) WHERE "storage" = :storage`, + `UPDATE "videoStreamingPlaylist" SET "segmentsSha256Url" = regexp_replace("segmentsSha256Url", :fromRegexp, :to) WHERE "storage" = :storage`, + `UPDATE "userExport" SET "fileUrl" = regexp_replace("fileUrl", :fromRegexp, :to) WHERE "storage" = :storage`, + `UPDATE "videoSource" SET "fileUrl" = regexp_replace("fileUrl", :fromRegexp, :to) WHERE "storage" = :storage` + ] + + for (const query of queries) { + await sequelizeTypescript.query(query, { replacements }) + } + + console.log('URLs updated.') + } +} + +function parseUrl (value: string) { + if (!value || /^https?:\/\//.test(value) !== true) { + throw new InvalidArgumentError('Must be a valid URL (starting with http:// or https://).') + } + + return value +} + +async function askConfirmation () { + return new Promise((res, rej) => { + prompt.start() + + const schema = { + properties: { + confirm: { + type: 'string', + description: 'These URLs can be updated, but please check your backups first (bugs happen).' + + ' Notice PeerTube must have been stopped when your ran this script.' + + ' Can we update these URLs? (y/n)', + default: 'n', + validator: /y[es]*|n[o]?/, + warning: 'Must respond yes or no', + required: true + } + } + } + + prompt.get(schema, function (err, result) { + if (err) return rej(err) + + return res(result.confirm?.match(/y/) !== null) + }) + }) +} diff --git a/support/doc/tools.md b/support/doc/tools.md index 8b412088d..74cc5550d 100644 --- a/support/doc/tools.md +++ b/support/doc/tools.md @@ -472,6 +472,24 @@ docker compose exec -u peertube peertube npm run create-move-video-storage-job - ::: +### Update object storage URLs + +**PeerTube >= 6.2** + +Use this script after you migrated to another object storage provider so PeerTube updates its internal object URLs. + +::: code-group + +```bash [Classic installation] +cd /var/www/peertube/peertube-latest +sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run update-object-storage-url -- --from 'https://region.old-s3-provider.example.com' --to 'https://region.new-s3-provider.example.com' +``` + +```bash [Docker] +cd /var/www/peertube-docker +docker compose exec -u peertube peertube npm run update-object-storage-url -- --from 'https://region.old-s3-provider.example.com' --to 'https://region.new-s3-provider.example.com' +``` + ### Generate storyboard **PeerTube >= 6.0**