Support update object storage urls

package.json

@@ -47,6 +47,7 @@
     "plugin:install": "node ./dist/scripts/plugin/install.js",
     "plugin:uninstall": "node ./dist/scripts/plugin/uninstall.js",
     "reset-password": "node ./dist/scripts/reset-password.js",
+    "update-object-storage-url": "LOGGER_LEVEL=warn node ./dist/scripts/update-object-storage-url.js",
     "update-host": "node ./dist/scripts/update-host.js",
     "regenerate-thumbnails": "node ./dist/scripts/regenerate-thumbnails.js",
     "create-import-video-file-job": "node ./dist/scripts/create-import-video-file-job.js",

packages/models/src/import-export/user-export.model.ts

@@ -11,6 +11,7 @@ export interface UserExport {
   // In bytes
   size: number
 
+  fileUrl: string
   privateDownloadUrl: string
 
   createdAt: string | Date

packages/models/src/videos/video-source.model.ts

@@ -10,6 +10,7 @@ export interface VideoSource {
   width?: number
   height?: number
 
+  fileUrl: string
   fileDownloadUrl: string
 
   fps?: number

packages/tests/src/cli/index.ts

@@ -8,3 +8,4 @@ import './prune-storage'
 import './regenerate-thumbnails'
 import './reset-password'
 import './update-host'
+import './update-object-storage-url.js'

packages/tests/src/cli/update-object-storage-url.ts

@@ -0,0 +1,131 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import { getHLS } from '@peertube/peertube-core-utils'
+import { VideoDetails } from '@peertube/peertube-models'
+import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'
+import {
+  ObjectStorageCommand,
+  PeerTubeServer,
+  cleanupTests,
+  createSingleServer,
+  getRedirectionUrl,
+  setAccessTokensToServers,
+  waitJobs
+} from '@peertube/peertube-server-commands'
+import { expectStartWith } from '@tests/shared/checks.js'
+import { expect } from 'chai'
+
+describe('Update object storage URL', function () {
+  if (areMockObjectStorageTestsDisabled()) return
+
+  let server: PeerTubeServer
+  let uuid: string
+  const objectStorage = new ObjectStorageCommand()
+
+  function runUpdate (from: string, to: string) {
+    const env = server.cli.getEnv()
+    const command = `echo y | ${env} npm run update-object-storage-url -- --from "${from}" --to "${to}"`
+
+    return server.cli.execWithEnv(command, objectStorage.getDefaultMockConfig())
+  }
+
+  before(async function () {
+    this.timeout(360000)
+
+    server = await createSingleServer(1, objectStorage.getDefaultMockConfig())
+    await setAccessTokensToServers([ server ])
+
+    await objectStorage.prepareDefaultMockBuckets()
+
+    await server.config.enableMinimumTranscoding({ keepOriginal: true })
+
+    const video = await server.videos.quickUpload({ name: 'video' })
+    uuid = video.uuid
+
+    await waitJobs([ server ])
+  })
+
+  it('Should update video URLs', async function () {
+    this.timeout(120000)
+
+    const check = async (options: {
+      baseUrl: string
+      newBaseUrl: string
+      urlGetter: (video: VideoDetails) => Promise<string[]> | string[]
+    }) => {
+      const { baseUrl, newBaseUrl, urlGetter } = options
+
+      const oldVideo = await server.videos.get({ id: uuid })
+      const oldFileUrls = await urlGetter(oldVideo)
+
+      for (const url of oldFileUrls) {
+        expectStartWith(url, baseUrl)
+      }
+
+      await runUpdate(baseUrl, newBaseUrl)
+
+      const newVideo = await server.videos.get({ id: uuid })
+
+      const shouldBe = oldFileUrls.map(f => f.replace(baseUrl, newBaseUrl))
+      expect(await urlGetter(newVideo)).to.have.members(shouldBe)
+    }
+
+    await check({
+      baseUrl: objectStorage.getMockWebVideosBaseUrl(),
+      newBaseUrl: 'https://web-video.example.com/',
+      urlGetter: video => video.files.map(f => f.fileUrl)
+    })
+
+    await check({
+      baseUrl: objectStorage.getMockPlaylistBaseUrl(),
+      newBaseUrl: 'https://streaming-playlists.example.com/',
+      urlGetter: video => {
+        const hls = getHLS(video)
+
+        return [
+          ...hls.files.map(f => f.fileUrl),
+
+          hls.playlistUrl,
+          hls.segmentsSha256Url
+        ]
+      }
+    })
+
+    await check({
+      baseUrl: objectStorage.getMockOriginalFileBaseUrl(),
+      newBaseUrl: 'https://original-file.example.com/',
+      urlGetter: async video => {
+        const source = await server.videos.getSource({ id: video.uuid })
+
+        return [ source.fileUrl ]
+      }
+    })
+  })
+
+  it('Should update user export URLs', async function () {
+    this.timeout(120000)
+
+    const user = await server.users.getMyInfo()
+
+    await server.userExports.request({ userId: user.id, withVideoFiles: false })
+    await waitJobs([ server ])
+
+    {
+      const { data } = await server.userExports.list({ userId: user.id })
+      expectStartWith(data[0].fileUrl, objectStorage.getMockUserExportBaseUrl())
+    }
+
+    await runUpdate(objectStorage.getMockUserExportBaseUrl(), 'https://user-export.example.com/')
+
+    {
+      const { data } = await server.userExports.list({ userId: user.id })
+      expectStartWith(data[0].fileUrl, 'https://user-export.example.com/')
+    }
+  })
+
+  after(async function () {
+    await objectStorage.cleanupMock()
+
+    await cleanupTests([ server ])
+  })
+})

server/core/helpers/regexp.ts

@@ -27,10 +27,6 @@ export function wordsToRegExp (words: string[]) {
   return new RegExp(`(?:\\P{L}|^)(?:${innerRegex})(?=\\P{L}|$)`, 'iu')
 }
 
-// ---------------------------------------------------------------------------
-// Private
-// ---------------------------------------------------------------------------
-
-function escapeForRegex (value: string) {
+export function escapeForRegex (value: string) {
   return value.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&')
 }

server/core/lib/views/shared/video-viewer-counters.ts

@@ -31,7 +31,7 @@ export class VideoViewerCounters {
   private processingViewerCounters = false
 
   constructor () {
-    setInterval(() => this.cleanViewerCounters(), VIEW_LIFETIME.VIEWER_COUNTER)
+    setInterval(() => this.updateVideoViewersCount(), VIEW_LIFETIME.VIEWER_COUNTER)
   }
 
   // ---------------------------------------------------------------------------
@@ -163,11 +163,13 @@ export class VideoViewerCounters {
     return viewer
   }
 
-  private async cleanViewerCounters () {
+  private async updateVideoViewersCount () {
     if (this.processingViewerCounters) return
     this.processingViewerCounters = true
 
-    if (!isTestOrDevInstance()) logger.info('Cleaning video viewers.', lTags())
+    if (!isTestOrDevInstance()) {
+      logger.debug('Updating video viewer counters.', lTags())
+    }
 
     try {
       for (const videoId of this.viewersPerVideo.keys()) {

server/core/models/user/user-export.ts

@@ -219,6 +219,7 @@ export class UserExportModel extends SequelizeModel<UserExportModel> {
 
       size: this.size,
 
+      fileUrl: this.fileUrl,
       privateDownloadUrl: this.getFileDownloadUrl(),
       createdAt: this.createdAt.toISOString(),
       expiresOn: new Date(this.createdAt.getTime() + CONFIG.EXPORT.USERS.EXPORT_EXPIRATION).toISOString()

server/core/models/video/video-source.ts

@@ -125,6 +125,8 @@ export class VideoSourceModel extends SequelizeModel<VideoSourceModel> {
     return {
       filename: this.inputFilename,
       inputFilename: this.inputFilename,
+
+      fileUrl: this.fileUrl,
       fileDownloadUrl: this.getFileDownloadUrl(),
 
       resolution: {

server/scripts/update-object-storage-url.ts

@@ -0,0 +1,118 @@
+/* eslint-disable max-len */
+import { InvalidArgumentError, createCommand } from '@commander-js/extra-typings'
+import { FileStorage } from '@peertube/peertube-models'
+import { escapeForRegex } from '@server/helpers/regexp.js'
+import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js'
+import { QueryTypes } from 'sequelize'
+import prompt from 'prompt'
+
+const program = createCommand()
+  .description('Update PeerTube object file URLs after an object storage migration.')
+  .requiredOption('-f, --from <url>', 'Previous object storage base URL', parseUrl)
+  .requiredOption('-t, --to <url>', 'New object storage base URL', parseUrl)
+  .parse(process.argv)
+
+const options = program.opts()
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  await initDatabaseModels(true)
+
+  const fromRegexp = `^${escapeForRegex(options.from)}`
+  const to = options.to
+
+  const replacements = { fromRegexp, to, storage: FileStorage.OBJECT_STORAGE }
+
+  // Candidates
+  {
+    const queries = [
+      `SELECT COUNT(*) AS "c", 'videoFile->fileUrl: ' || COUNT(*) AS "t" FROM "videoFile" WHERE "fileUrl" ~ :fromRegexp AND "storage" = :storage`,
+      `SELECT COUNT(*) AS "c", 'videoStreamingPlaylist->playlistUrl: ' || COUNT(*) AS "t" FROM "videoStreamingPlaylist" WHERE "playlistUrl" ~ :fromRegexp AND "storage" = :storage`,
+      `SELECT COUNT(*) AS "c", 'videoStreamingPlaylist->segmentsSha256Url: ' || COUNT(*) AS "t" FROM "videoStreamingPlaylist" WHERE "segmentsSha256Url" ~ :fromRegexp AND "storage" = :storage`,
+      `SELECT COUNT(*) AS "c", 'userExport->fileUrl: ' || COUNT(*) AS "t" FROM "userExport" WHERE "fileUrl" ~ :fromRegexp AND "storage" = :storage`,
+      `SELECT COUNT(*) AS "c", 'videoSource->fileUrl: ' || COUNT(*) AS "t" FROM "videoSource" WHERE "fileUrl" ~ :fromRegexp AND "storage" = :storage`
+    ]
+
+    let hasResults = false
+
+    console.log('Candidate URLs to update:')
+    for (const query of queries) {
+      const [ row ] = await sequelizeTypescript.query(query, { replacements, type: QueryTypes.SELECT as QueryTypes.SELECT })
+
+      if (row['c'] !== 0) hasResults = true
+
+      console.log(` ${row['t']}`)
+    }
+
+    console.log('\n')
+
+    if (!hasResults) {
+      console.log('No candidate URLs found, exiting.')
+      process.exit(0)
+    }
+  }
+
+  const res = await askConfirmation()
+  if (res !== true) {
+    console.log('Exiting without updating URLs.')
+    process.exit(0)
+  }
+
+  // Execute
+  {
+    const queries = [
+      `UPDATE "videoFile" SET "fileUrl" = regexp_replace("fileUrl", :fromRegexp, :to) WHERE "storage" = :storage`,
+      `UPDATE "videoStreamingPlaylist" SET "playlistUrl" = regexp_replace("playlistUrl", :fromRegexp, :to) WHERE "storage" = :storage`,
+      `UPDATE "videoStreamingPlaylist" SET "segmentsSha256Url" = regexp_replace("segmentsSha256Url", :fromRegexp, :to) WHERE "storage" = :storage`,
+      `UPDATE "userExport" SET "fileUrl" = regexp_replace("fileUrl", :fromRegexp, :to) WHERE "storage" = :storage`,
+      `UPDATE "videoSource" SET "fileUrl" = regexp_replace("fileUrl", :fromRegexp, :to) WHERE "storage" = :storage`
+    ]
+
+    for (const query of queries) {
+      await sequelizeTypescript.query(query, { replacements })
+    }
+
+    console.log('URLs updated.')
+  }
+}
+
+function parseUrl (value: string) {
+  if (!value || /^https?:\/\//.test(value) !== true) {
+    throw new InvalidArgumentError('Must be a valid URL (starting with http:// or https://).')
+  }
+
+  return value
+}
+
+async function askConfirmation () {
+  return new Promise((res, rej) => {
+    prompt.start()
+
+    const schema = {
+      properties: {
+        confirm: {
+          type: 'string',
+          description: 'These URLs can be updated, but please check your backups first (bugs happen).' +
+            ' Notice PeerTube must have been stopped when your ran this script.' +
+            ' Can we update these URLs? (y/n)',
+          default: 'n',
+          validator: /y[es]*|n[o]?/,
+          warning: 'Must respond yes or no',
+          required: true
+        }
+      }
+    }
+
+    prompt.get(schema, function (err, result) {
+      if (err) return rej(err)
+
+      return res(result.confirm?.match(/y/) !== null)
+    })
+  })
+}

support/doc/tools.md

@@ -472,6 +472,24 @@ docker compose exec -u peertube peertube npm run create-move-video-storage-job -
 
 :::
 
+### Update object storage URLs
+
+**PeerTube >= 6.2**
+
+Use this script after you migrated to another object storage provider so PeerTube updates its internal object URLs.
+
+::: code-group
+
+```bash [Classic installation]
+cd /var/www/peertube/peertube-latest
+sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run update-object-storage-url -- --from 'https://region.old-s3-provider.example.com' --to 'https://region.new-s3-provider.example.com'
+```
+
+```bash [Docker]
+cd /var/www/peertube-docker
+docker compose exec -u peertube peertube npm run update-object-storage-url -- --from 'https://region.old-s3-provider.example.com' --to 'https://region.new-s3-provider.example.com'
+```
+
 ### Generate storyboard
 
 **PeerTube >= 6.0**