Support object storage in prune script

Also prune original files and user exports

package.json

@@ -53,7 +53,7 @@
     "create-move-video-storage-job": "node ./dist/scripts/create-move-video-storage-job.js",
     "create-generate-storyboard-job": "node ./dist/scripts/create-generate-storyboard-job.js",
     "parse-log": "node ./dist/scripts/parse-log.js",
-    "prune-storage": "node ./dist/scripts/prune-storage.js",
+    "prune-storage": "LOGGER_LEVEL=warn node ./dist/scripts/prune-storage.js",
     "test": "bash ./scripts/test.sh",
     "generate-cli-doc": "bash ./scripts/generate-cli-doc.sh",
     "generate-types-package": "tsx --conditions=peertube:tsx ./packages/types-generator/generate-package.ts",

packages/tests/src/cli/prune-storage.ts

@@ -1,77 +1,49 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
-import { expect } from 'chai'
+import { getAllFiles, wait } from '@peertube/peertube-core-utils'
-import { createFile } from 'fs-extra/esm'
+import { FileStorage, HttpStatusCode, HttpStatusCodeType, VideoPlaylistPrivacy, VideoPrivacy } from '@peertube/peertube-models'
-import { readdir } from 'fs/promises'
+import { areMockObjectStorageTestsDisabled, buildUUID } from '@peertube/peertube-node-utils'
-import { join } from 'path'
-import { wait } from '@peertube/peertube-core-utils'
-import { buildUUID } from '@peertube/peertube-node-utils'
-import { HttpStatusCode, VideoPlaylistPrivacy, VideoPrivacy } from '@peertube/peertube-models'
 import {
-  cleanupTests,
   CLICommand,
+  ObjectStorageCommand,
+  PeerTubeServer,
+  cleanupTests,
   createMultipleServers,
   doubleFollow,
   killallServers,
   makeGetRequest,
-  PeerTubeServer,
+  makeRawRequest,
   setAccessTokensToServers,
   setDefaultVideoChannel,
   waitJobs
 } from '@peertube/peertube-server-commands'
-
+import { SQLCommand } from '@tests/shared/sql-command.js'
-async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) {
+import { expect } from 'chai'
-  const files = await readdir(server.servers.buildDirectory(directory))
+import { createFile } from 'fs-extra/esm'
-
+import { readdir } from 'fs/promises'
-  for (const f of files) {
+import { join } from 'path'
-    expect(f).to.not.contain(substring)
-  }
-}
-
-async function assertCountAreOkay (servers: PeerTubeServer[]) {
-  for (const server of servers) {
-    const videosCount = await server.servers.countFiles('web-videos')
-    expect(videosCount).to.equal(9) // 2 videos with 4 resolutions + private directory
-
-    const privateVideosCount = await server.servers.countFiles('web-videos/private')
-    expect(privateVideosCount).to.equal(4)
-
-    const torrentsCount = await server.servers.countFiles('torrents')
-    expect(torrentsCount).to.equal(24)
-
-    const previewsCount = await server.servers.countFiles('previews')
-    expect(previewsCount).to.equal(3)
-
-    const thumbnailsCount = await server.servers.countFiles('thumbnails')
-    expect(thumbnailsCount).to.equal(5) // 3 local videos, 1 local playlist, 2 remotes videos (lazy downloaded) and 1 remote playlist
-
-    const avatarsCount = await server.servers.countFiles('avatars')
-    expect(avatarsCount).to.equal(8)
-
-    const hlsRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls'))
-    expect(hlsRootCount).to.equal(3) // 2 videos + private directory
-
-    const hlsPrivateRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls', 'private'))
-    expect(hlsPrivateRootCount).to.equal(1)
-  }
-}
 
 describe('Test prune storage scripts', function () {
   let servers: PeerTubeServer[]
-  const badNames: { [directory: string]: string[] } = {}
 
   before(async function () {
     this.timeout(120000)
 
-    servers = await createMultipleServers(2, { transcoding: { enabled: true } })
+    servers = await createMultipleServers(2)
+
     await setAccessTokensToServers(servers)
     await setDefaultVideoChannel(servers)
 
     for (const server of servers) {
-      await server.videos.upload({ attributes: { name: 'video 1', privacy: VideoPrivacy.PUBLIC } })
+      await server.config.enableMinimumTranscoding({ keepOriginal: true })
-      await server.videos.upload({ attributes: { name: 'video 2', privacy: VideoPrivacy.PUBLIC } })
+      await server.config.enableUserExport()
+    }
 
-      await server.videos.upload({ attributes: { name: 'video 3', privacy: VideoPrivacy.PRIVATE } })
+    for (const server of servers) {
+      await server.videos.quickUpload({ name: 'video 1', privacy: VideoPrivacy.PUBLIC })
+      await server.videos.quickUpload({ name: 'video 2', privacy: VideoPrivacy.PUBLIC })
+
+      await server.videos.quickUpload({ name: 'video 3', privacy: VideoPrivacy.PRIVATE })
 
       await server.users.updateMyAvatar({ fixture: 'avatar.png' })
 
@@ -85,6 +57,12 @@ describe('Test prune storage scripts', function () {
       })
     }
 
+    for (const server of servers) {
+      const user = await server.users.getMyInfo()
+
+      await server.userExports.request({ userId: user.id, withVideoFiles: false })
+    }
+
     await doubleFollow(servers[0], servers[1])
 
     // Lazy load the remote avatars
@@ -119,103 +97,259 @@ describe('Test prune storage scripts', function () {
     await wait(1000)
   })
 
-  it('Should have the files on the disk', async function () {
+  describe('On filesystem', function () {
-    await assertCountAreOkay(servers)
+    const badNames: { [directory: string]: string[] } = {}
-  })
 
-  it('Should create some dirty files', async function () {
+    async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) {
-    for (let i = 0; i < 2; i++) {
+      const files = await readdir(server.servers.buildDirectory(directory))
-      {
-        const basePublic = servers[0].servers.buildDirectory('web-videos')
-        const basePrivate = servers[0].servers.buildDirectory(join('web-videos', 'private'))
 
-        const n1 = buildUUID() + '.mp4'
+      for (const f of files) {
-        const n2 = buildUUID() + '.webm'
+        expect(f).to.not.contain(substring)
-
-        await createFile(join(basePublic, n1))
-        await createFile(join(basePublic, n2))
-        await createFile(join(basePrivate, n1))
-        await createFile(join(basePrivate, n2))
-
-        badNames['web-videos'] = [ n1, n2 ]
-      }
-
-      {
-        const base = servers[0].servers.buildDirectory('torrents')
-
-        const n1 = buildUUID() + '-240.torrent'
-        const n2 = buildUUID() + '-480.torrent'
-
-        await createFile(join(base, n1))
-        await createFile(join(base, n2))
-
-        badNames['torrents'] = [ n1, n2 ]
-      }
-
-      {
-        const base = servers[0].servers.buildDirectory('thumbnails')
-
-        const n1 = buildUUID() + '.jpg'
-        const n2 = buildUUID() + '.jpg'
-
-        await createFile(join(base, n1))
-        await createFile(join(base, n2))
-
-        badNames['thumbnails'] = [ n1, n2 ]
-      }
-
-      {
-        const base = servers[0].servers.buildDirectory('previews')
-
-        const n1 = buildUUID() + '.jpg'
-        const n2 = buildUUID() + '.jpg'
-
-        await createFile(join(base, n1))
-        await createFile(join(base, n2))
-
-        badNames['previews'] = [ n1, n2 ]
-      }
-
-      {
-        const base = servers[0].servers.buildDirectory('avatars')
-
-        const n1 = buildUUID() + '.png'
-        const n2 = buildUUID() + '.jpg'
-
-        await createFile(join(base, n1))
-        await createFile(join(base, n2))
-
-        badNames['avatars'] = [ n1, n2 ]
-      }
-
-      {
-        const directory = join('streaming-playlists', 'hls')
-        const basePublic = servers[0].servers.buildDirectory(directory)
-        const basePrivate = servers[0].servers.buildDirectory(join(directory, 'private'))
-
-        const n1 = buildUUID()
-        await createFile(join(basePublic, n1))
-        await createFile(join(basePrivate, n1))
-        badNames[directory] = [ n1 ]
       }
     }
-  })
 
-  it('Should run prune storage', async function () {
+    async function assertCountAreOkay () {
-    this.timeout(30000)
+      for (const server of servers) {
+        const videosCount = await server.servers.countFiles('web-videos')
+        expect(videosCount).to.equal(5) // 2 videos with 2 resolutions + private directory
 
-    const env = servers[0].cli.getEnv()
+        const privateVideosCount = await server.servers.countFiles('web-videos/private')
-    await CLICommand.exec(`echo y | ${env} npm run prune-storage`)
+        expect(privateVideosCount).to.equal(2)
-  })
 
-  it('Should have removed files', async function () {
+        const torrentsCount = await server.servers.countFiles('torrents')
-    await assertCountAreOkay(servers)
+        expect(torrentsCount).to.equal(12)
 
-    for (const directory of Object.keys(badNames)) {
+        const previewsCount = await server.servers.countFiles('previews')
-      for (const name of badNames[directory]) {
+        expect(previewsCount).to.equal(3)
-        await assertNotExists(servers[0], directory, name)
+
+        const thumbnailsCount = await server.servers.countFiles('thumbnails')
+        expect(thumbnailsCount).to.equal(5) // 3 local videos, 1 local playlist, 2 remotes videos (lazy downloaded) and 1 remote playlist
+
+        const avatarsCount = await server.servers.countFiles('avatars')
+        expect(avatarsCount).to.equal(8)
+
+        const hlsRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls'))
+        expect(hlsRootCount).to.equal(3) // 2 videos + private directory
+
+        const hlsPrivateRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls', 'private'))
+        expect(hlsPrivateRootCount).to.equal(1)
+
+        const originalVideoFilesCount = await server.servers.countFiles(join('original-video-files'))
+        expect(originalVideoFilesCount).to.equal(3)
+
+        const userExportFilesCount = await server.servers.countFiles(join('tmp-persistent'))
+        expect(userExportFilesCount).to.equal(1)
       }
     }
+
+    it('Should have the files on the disk', async function () {
+      await assertCountAreOkay()
+    })
+
+    it('Should create some dirty files', async function () {
+      for (let i = 0; i < 2; i++) {
+        {
+          const basePublic = servers[0].servers.buildDirectory('web-videos')
+          const basePrivate = servers[0].servers.buildDirectory(join('web-videos', 'private'))
+
+          const n1 = buildUUID() + '.mp4'
+          const n2 = buildUUID() + '.webm'
+
+          await createFile(join(basePublic, n1))
+          await createFile(join(basePublic, n2))
+          await createFile(join(basePrivate, n1))
+          await createFile(join(basePrivate, n2))
+
+          badNames['web-videos'] = [ n1, n2 ]
+        }
+
+        {
+          const base = servers[0].servers.buildDirectory('torrents')
+
+          const n1 = buildUUID() + '-240.torrent'
+          const n2 = buildUUID() + '-480.torrent'
+
+          await createFile(join(base, n1))
+          await createFile(join(base, n2))
+
+          badNames['torrents'] = [ n1, n2 ]
+        }
+
+        {
+          const base = servers[0].servers.buildDirectory('thumbnails')
+
+          const n1 = buildUUID() + '.jpg'
+          const n2 = buildUUID() + '.jpg'
+
+          await createFile(join(base, n1))
+          await createFile(join(base, n2))
+
+          badNames['thumbnails'] = [ n1, n2 ]
+        }
+
+        {
+          const base = servers[0].servers.buildDirectory('previews')
+
+          const n1 = buildUUID() + '.jpg'
+          const n2 = buildUUID() + '.jpg'
+
+          await createFile(join(base, n1))
+          await createFile(join(base, n2))
+
+          badNames['previews'] = [ n1, n2 ]
+        }
+
+        {
+          const base = servers[0].servers.buildDirectory('avatars')
+
+          const n1 = buildUUID() + '.png'
+          const n2 = buildUUID() + '.jpg'
+
+          await createFile(join(base, n1))
+          await createFile(join(base, n2))
+
+          badNames['avatars'] = [ n1, n2 ]
+        }
+
+        {
+          const directory = join('streaming-playlists', 'hls')
+          const basePublic = servers[0].servers.buildDirectory(directory)
+          const basePrivate = servers[0].servers.buildDirectory(join(directory, 'private'))
+
+          const n1 = buildUUID()
+          await createFile(join(basePublic, n1))
+          await createFile(join(basePrivate, n1))
+          badNames[directory] = [ n1 ]
+        }
+
+        {
+          const base = servers[0].servers.buildDirectory('original-video-files')
+
+          const n1 = buildUUID() + '.mp4'
+          await createFile(join(base, n1))
+
+          badNames['original-video-files'] = [ n1 ]
+        }
+
+        {
+          const base = servers[0].servers.buildDirectory('tmp-persistent')
+
+          const n1 = 'user-export-1.zip'
+          const n2 = 'user-export-2.zip'
+
+          await createFile(join(base, n1))
+          await createFile(join(base, n2))
+
+          badNames['tmp-persistent'] = [ n1, n2 ]
+        }
+      }
+    })
+
+    it('Should run prune storage', async function () {
+      this.timeout(30000)
+
+      const env = servers[0].cli.getEnv()
+      await CLICommand.exec(`echo y | ${env} npm run prune-storage`)
+    })
+
+    it('Should have removed files', async function () {
+      await assertCountAreOkay()
+
+      for (const directory of Object.keys(badNames)) {
+        for (const name of badNames[directory]) {
+          await assertNotExists(servers[0], directory, name)
+        }
+      }
+    })
+  })
+
+  describe('On object storage', function () {
+    if (areMockObjectStorageTestsDisabled()) return
+
+    const videos: string[] = []
+    const objectStorage = new ObjectStorageCommand()
+
+    let sqlCommand: SQLCommand
+    let rootId: number
+
+    async function checkVideosFiles (uuids: string[], expectedStatus: HttpStatusCodeType) {
+      for (const uuid of uuids) {
+        const video = await servers[0].videos.getWithToken({ id: uuid })
+
+        for (const file of getAllFiles(video)) {
+          await makeRawRequest({ url: file.fileUrl, token: servers[0].accessToken, expectedStatus })
+        }
+
+        const source = await servers[0].videos.getSource({ id: uuid })
+        await makeRawRequest({ url: source.fileDownloadUrl, redirects: 1, token: servers[0].accessToken, expectedStatus })
+      }
+    }
+
+    async function checkUserExport (expectedStatus: HttpStatusCodeType) {
+      const { data } = await servers[0].userExports.list({ userId: rootId })
+      await makeRawRequest({ url: data[0].privateDownloadUrl, redirects: 1, expectedStatus })
+    }
+
+    before(async function () {
+      this.timeout(120000)
+
+      sqlCommand = new SQLCommand(servers[0])
+
+      await objectStorage.prepareDefaultMockBuckets()
+
+      await servers[0].run(objectStorage.getDefaultMockConfig())
+
+      {
+        const { uuid } = await servers[0].videos.quickUpload({ name: 's3 video 1', privacy: VideoPrivacy.PUBLIC })
+        videos.push(uuid)
+      }
+
+      {
+        const { uuid } = await servers[0].videos.quickUpload({ name: 's3 video 2', privacy: VideoPrivacy.PUBLIC })
+        videos.push(uuid)
+      }
+
+      {
+        const { uuid } = await servers[0].videos.quickUpload({ name: 's3 video 3', privacy: VideoPrivacy.PRIVATE })
+        videos.push(uuid)
+      }
+
+      const user = await servers[0].users.getMyInfo()
+      rootId = user.id
+
+      await servers[0].userExports.deleteAllArchives({ userId: rootId })
+      await servers[0].userExports.request({ userId: rootId, withVideoFiles: false })
+
+      await waitJobs([ servers[0] ])
+    })
+
+    it('Should have the files on object storage', async function () {
+      await checkVideosFiles(videos, HttpStatusCode.OK_200)
+      await checkUserExport(HttpStatusCode.OK_200)
+    })
+
+    it('Should run prune-storage script on videos', async function () {
+      await sqlCommand.setVideoFileStorageOf(videos[1], FileStorage.FILE_SYSTEM)
+      await sqlCommand.setVideoFileStorageOf(videos[2], FileStorage.FILE_SYSTEM)
+
+      await checkVideosFiles([ videos[1], videos[2] ], HttpStatusCode.NOT_FOUND_404)
+      await checkVideosFiles([ videos[0] ], HttpStatusCode.OK_200)
+
+      await checkUserExport(HttpStatusCode.OK_200)
+    })
+
+    it('Should run prune-storage script on exports', async function () {
+      await sqlCommand.setUserExportStorageOf(rootId, FileStorage.FILE_SYSTEM)
+
+      await checkVideosFiles([ videos[1], videos[2] ], HttpStatusCode.NOT_FOUND_404)
+      await checkVideosFiles([ videos[0] ], HttpStatusCode.OK_200)
+
+      await checkUserExport(HttpStatusCode.NOT_FOUND_404)
+    })
+
+    after(async function () {
+      await sqlCommand.cleanup()
+    })
   })
 
   after(async function () {

packages/tests/src/shared/sql-command.ts

@@ -1,6 +1,7 @@
 import { QueryTypes, Sequelize } from 'sequelize'
 import { forceNumber } from '@peertube/peertube-core-utils'
 import { PeerTubeServer } from '@peertube/peertube-server-commands'
+import { FileStorageType } from '@peertube/peertube-models'
 
 export class SQLCommand {
   private sequelize: Sequelize
@@ -58,6 +59,30 @@ export class SQLCommand {
 
   // ---------------------------------------------------------------------------
 
+  async setVideoFileStorageOf (uuid: string, storage: FileStorageType) {
+    await this.updateQuery(
+      `UPDATE "videoFile" SET storage = :storage ` +
+      `WHERE "videoId" IN (SELECT id FROM "video" WHERE uuid = :uuid) OR ` +
+      // eslint-disable-next-line max-len
+      `"videoStreamingPlaylistId" IN (` +
+        `SELECT "videoStreamingPlaylist".id FROM "videoStreamingPlaylist" ` +
+        `INNER JOIN video ON video.id = "videoStreamingPlaylist"."videoId" AND "video".uuid = :uuid` +
+      `)`,
+      { storage, uuid }
+    )
+
+    await this.updateQuery(
+      `UPDATE "videoSource" SET storage = :storage WHERE "videoId" IN (SELECT id FROM "video" WHERE uuid = :uuid)`,
+      { storage, uuid }
+    )
+  }
+
+  async setUserExportStorageOf (userId: number, storage: FileStorageType) {
+    await this.updateQuery(`UPDATE "userExport" SET storage = :storage WHERE "userId" = :userId`, { storage, userId })
+  }
+
+  // ---------------------------------------------------------------------------
+
   setPluginVersion (pluginName: string, newVersion: string) {
     return this.setPluginField(pluginName, 'version', newVersion)
   }

server/core/helpers/logger.ts

@@ -1,11 +1,11 @@
+import { context } from '@opentelemetry/api'
+import { getSpanContext } from '@opentelemetry/api/build/src/trace/context-utils.js'
+import { omit } from '@peertube/peertube-core-utils'
 import { stat } from 'fs/promises'
 import { join } from 'path'
 import { format as sqlFormat } from 'sql-formatter'
 import { createLogger, format, transports } from 'winston'
 import { FileTransportOptions } from 'winston/lib/winston/transports'
-import { context } from '@opentelemetry/api'
-import { getSpanContext } from '@opentelemetry/api/build/src/trace/context-utils.js'
-import { omit } from '@peertube/peertube-core-utils'
 import { CONFIG } from '../initializers/config.js'
 import { LOG_FILENAME } from '../initializers/constants.js'
 
@@ -60,7 +60,7 @@ if (CONFIG.LOG.ROTATION.ENABLED) {
 
 function buildLogger (labelSuffix?: string) {
   return createLogger({
-    level: CONFIG.LOG.LEVEL,
+    level: process.env.LOGGER_LEVEL ?? CONFIG.LOG.LEVEL,
     defaultMeta: {
       get traceId () { return getSpanContext(context.active())?.traceId },
       get spanId () { return getSpanContext(context.active())?.spanId },
@@ -154,18 +154,10 @@ async function mtimeSortFilesDesc (files: string[], basePath: string) {
 // ---------------------------------------------------------------------------
 
 export {
-  type LoggerTagsFn,
-  type LoggerTags,
 
-  buildLogger,
+  buildLogger, bunyanLogger, consoleLoggerFormat,
-  timestampFormatter,
+  jsonLoggerFormat, labelFormatter, logger,
-  labelFormatter,
+  loggerTagsFactory, mtimeSortFilesDesc, timestampFormatter, type LoggerTags, type LoggerTagsFn
-  consoleLoggerFormat,
-  jsonLoggerFormat,
-  mtimeSortFilesDesc,
-  logger,
-  loggerTagsFactory,
-  bunyanLogger
 }
 
 // ---------------------------------------------------------------------------

server/core/initializers/constants.ts

@@ -845,6 +845,7 @@ const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = {
 // ---------------------------------------------------------------------------
 
 const USER_EXPORT_MAX_ITEMS = 1000
+const USER_EXPORT_FILE_PREFIX = 'user-export-'
 
 // ---------------------------------------------------------------------------
 
@@ -1268,6 +1269,7 @@ export {
   CONSTRAINTS_FIELDS,
   EMBED_SIZE,
   REDUNDANCY,
+  USER_EXPORT_FILE_PREFIX,
   JOB_CONCURRENCY,
   JOB_ATTEMPTS,
   AP_CLEANER,

server/core/lib/object-storage/object-storage-helpers.ts

@@ -7,9 +7,9 @@ import { createReadStream, createWriteStream } from 'fs'
 import { ensureDir } from 'fs-extra/esm'
 import { dirname } from 'path'
 import { Readable } from 'stream'
-import { getInternalUrl } from '../urls.js'
+import { getInternalUrl } from './urls.js'
-import { getClient } from './client.js'
+import { getClient } from './shared/client.js'
-import { lTags } from './logger.js'
+import { lTags } from './shared/logger.js'
 
 import type { _Object, ObjectCannedACL, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3'
 
@@ -18,7 +18,7 @@ type BucketInfo = {
   PREFIX?: string
 }
 
-async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) {
+async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo, continuationToken?: string) {
   const s3Client = await getClient()
 
   const { ListObjectsV2Command } = await import('@aws-sdk/client-s3')
@@ -26,14 +26,21 @@ async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) {
   const commandPrefix = bucketInfo.PREFIX + prefix
   const listCommand = new ListObjectsV2Command({
     Bucket: bucketInfo.BUCKET_NAME,
-    Prefix: commandPrefix
+    Prefix: commandPrefix,
+    ContinuationToken: continuationToken
   })
 
   const listedObjects = await s3Client.send(listCommand)
 
   if (isArray(listedObjects.Contents) !== true) return []
 
-  return listedObjects.Contents.map(c => c.Key)
+  let keys = listedObjects.Contents.map(c => c.Key)
+
+  if (listedObjects.IsTruncated) {
+    keys = keys.concat(await listKeysOfPrefix(prefix, bucketInfo, listedObjects.NextContinuationToken))
+  }
+
+  return keys
 }
 
 // ---------------------------------------------------------------------------
@@ -145,7 +152,7 @@ function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) {
   return removeObjectByFullKey(key, bucketInfo)
 }
 
-async function removeObjectByFullKey (fullKey: string, bucketInfo: BucketInfo) {
+async function removeObjectByFullKey (fullKey: string, bucketInfo: Pick<BucketInfo, 'BUCKET_NAME'>) {
   logger.debug('Removing file %s in bucket %s', fullKey, bucketInfo.BUCKET_NAME, lTags())
 
   const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')

server/core/lib/object-storage/shared/index.ts

@@ -1,3 +1,3 @@
 export * from './client.js'
 export * from './logger.js'
-export * from './object-storage-helpers.js'
+export * from '../object-storage-helpers.js'

server/core/models/user/user-export.ts

@@ -1,23 +1,25 @@
-import { FindOptions, Op } from 'sequelize'
+import { FileStorage, UserExportState, type FileStorageType, type UserExport, type UserExportStateType } from '@peertube/peertube-models'
-import { AllowNull, BeforeDestroy, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript'
-import { MUserAccountId, MUserExport } from '@server/types/models/index.js'
-import { UserModel } from './user.js'
-import { getSort } from '../shared/sort.js'
-import { UserExportState, type UserExport, type UserExportStateType, type FileStorageType, FileStorage } from '@peertube/peertube-models'
 import { logger } from '@server/helpers/logger.js'
-import { remove } from 'fs-extra/esm'
+import { CONFIG } from '@server/initializers/config.js'
-import { getFSUserExportFilePath } from '@server/lib/paths.js'
 import {
   JWT_TOKEN_USER_EXPORT_FILE_LIFETIME,
   STATIC_DOWNLOAD_PATHS,
+  USER_EXPORT_FILE_PREFIX,
   USER_EXPORT_STATES,
   WEBSERVER
 } from '@server/initializers/constants.js'
-import { join } from 'path'
-import jwt from 'jsonwebtoken'
-import { CONFIG } from '@server/initializers/config.js'
 import { removeUserExportObjectStorage } from '@server/lib/object-storage/user-export.js'
+import { getFSUserExportFilePath } from '@server/lib/paths.js'
+import { MUserAccountId, MUserExport } from '@server/types/models/index.js'
+import { remove } from 'fs-extra/esm'
+import jwt from 'jsonwebtoken'
+import { join } from 'path'
+import { FindOptions, Op } from 'sequelize'
+import { AllowNull, BeforeDestroy, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript'
+import { doesExist } from '../shared/query.js'
 import { SequelizeModel } from '../shared/sequelize-type.js'
+import { getSort } from '../shared/sort.js'
+import { UserModel } from './user.js'
 
 @Table({
   tableName: 'userExport',
@@ -147,11 +149,20 @@ export class UserExportModel extends SequelizeModel<UserExportModel> {
 
   // ---------------------------------------------------------------------------
 
+  static async doesOwnedFileExist (filename: string, storage: FileStorageType) {
+    const query = 'SELECT 1 FROM "userExport" ' +
+      `WHERE "filename" = $filename AND "storage" = $storage LIMIT 1`
+
+    return doesExist({ sequelize: this.sequelize, query, bind: { filename, storage } })
+  }
+
+  // ---------------------------------------------------------------------------
+
   generateAndSetFilename () {
     if (!this.userId) throw new Error('Cannot generate filename without userId')
     if (!this.createdAt) throw new Error('Cannot generate filename without createdAt')
 
-    this.filename = `user-export-${this.userId}-${this.createdAt.toISOString()}.zip`
+    this.filename = `${USER_EXPORT_FILE_PREFIX}${this.userId}-${this.createdAt.toISOString()}.zip`
   }
 
   canBeSafelyRemoved () {

server/core/models/video/video-file.ts

@@ -289,11 +289,11 @@ export class VideoFileModel extends SequelizeModel<VideoFileModel> {
     return doesExist({ sequelize: this.sequelize, query, bind: { filename } })
   }
 
-  static async doesOwnedWebVideoFileExist (filename: string) {
+  static async doesOwnedFileExist (filename: string, storage: FileStorageType) {
     const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
-                  `WHERE "filename" = $filename AND "storage" = ${FileStorage.FILE_SYSTEM} LIMIT 1`
+      `WHERE "filename" = $filename AND "storage" = $storage LIMIT 1`
 
-    return doesExist({ sequelize: this.sequelize, query, bind: { filename } })
+    return doesExist({ sequelize: this.sequelize, query, bind: { filename, storage } })
   }
 
   static loadByFilename (filename: string) {

server/core/models/video/video-source.ts

@@ -1,12 +1,12 @@
-import type { FileStorageType, VideoSource } from '@peertube/peertube-models'
+import { type FileStorageType, type VideoSource } from '@peertube/peertube-models'
 import { STATIC_DOWNLOAD_PATHS, WEBSERVER } from '@server/initializers/constants.js'
+import { MVideoSource } from '@server/types/models/video/video-source.js'
 import { join } from 'path'
 import { Transaction } from 'sequelize'
 import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript'
-import { SequelizeModel, getSort } from '../shared/index.js'
+import { SequelizeModel, doesExist, getSort } from '../shared/index.js'
 import { getResolutionLabel } from './formatter/video-api-format.js'
 import { VideoModel } from './video.js'
-import { MVideoSource } from '@server/types/models/video/video-source.js'
 
 @Table({
   tableName: 'videoSource',
@@ -103,6 +103,18 @@ export class VideoSourceModel extends SequelizeModel<VideoSourceModel> {
     })
   }
 
+  // ---------------------------------------------------------------------------
+
+  static async doesOwnedFileExist (filename: string, storage: FileStorageType) {
+    const query = 'SELECT 1 FROM "videoSource" ' +
+      'INNER JOIN "video" ON "video"."id" = "videoSource"."videoId" AND "video"."remote" IS FALSE ' +
+      `WHERE "keptOriginalFilename" = $filename AND "storage" = $storage LIMIT 1`
+
+    return doesExist({ sequelize: this.sequelize, query, bind: { filename, storage } })
+  }
+
+  // ---------------------------------------------------------------------------
+
   getFileDownloadUrl () {
     if (!this.keptOriginalFilename) return null
 

server/core/models/video/video-streaming-playlist.ts

@@ -229,13 +229,13 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPl
     return Object.assign(playlist, { Video: video })
   }
 
-  static doesOwnedHLSPlaylistExist (videoUUID: string) {
+  static doesOwnedVideoUUIDExist (videoUUID: string, storage: FileStorageType) {
     const query = `SELECT 1 FROM "videoStreamingPlaylist" ` +
       `INNER JOIN "video" ON "video"."id" = "videoStreamingPlaylist"."videoId" ` +
       `AND "video"."remote" IS FALSE AND "video"."uuid" = $videoUUID ` +
-      `AND "storage" = ${FileStorage.FILE_SYSTEM} LIMIT 1`
+      `AND "storage" = $storage LIMIT 1`
 
-    return doesExist({ sequelize: this.sequelize, query, bind: { videoUUID } })
+    return doesExist({ sequelize: this.sequelize, query, bind: { videoUUID, storage } })
   }
 
   assignP2PMediaLoaderInfoHashes (video: MVideo, files: unknown[]) {

server/scripts/prune-storage.ts

@@ -1,13 +1,16 @@
+import { uniqify } from '@peertube/peertube-core-utils'
+import { FileStorage, ThumbnailType, ThumbnailType_Type } from '@peertube/peertube-models'
+import { DIRECTORIES, USER_EXPORT_FILE_PREFIX } from '@server/initializers/constants.js'
+import { listKeysOfPrefix, removeObjectByFullKey } from '@server/lib/object-storage/object-storage-helpers.js'
+import { UserExportModel } from '@server/models/user/user-export.js'
+import { VideoFileModel } from '@server/models/video/video-file.js'
+import { VideoSourceModel } from '@server/models/video/video-source.js'
+import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
 import Bluebird from 'bluebird'
 import { remove } from 'fs-extra/esm'
 import { readdir, stat } from 'fs/promises'
-import { basename, join } from 'path'
+import { basename, dirname, join } from 'path'
 import prompt from 'prompt'
-import { uniqify } from '@peertube/peertube-core-utils'
-import { ThumbnailType, ThumbnailType_Type } from '@peertube/peertube-models'
-import { DIRECTORIES } from '@server/initializers/constants.js'
-import { VideoFileModel } from '@server/models/video/video-file.js'
-import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
 import { getUUIDFromFilename } from '../core/helpers/utils.js'
 import { CONFIG } from '../core/initializers/config.js'
 import { initDatabaseModels } from '../core/initializers/database.js'
@@ -24,141 +27,276 @@ run()
   })
 
 async function run () {
-  const dirs = Object.values(CONFIG.STORAGE)
-
-  if (uniqify(dirs).length !== dirs.length) {
-    console.error('Cannot prune storage because you put multiple storage keys in the same directory.')
-    process.exit(0)
-  }
-
   await initDatabaseModels(true)
 
-  let toDelete: string[] = []
+  await new FSPruner().prune()
 
-  console.log('Detecting files to remove, it could take a while...')
+  console.log('\n')
 
-  toDelete = toDelete.concat(
+  await new ObjectStoragePruner().prune()
-    await pruneDirectory(DIRECTORIES.WEB_VIDEOS.PUBLIC, doesWebVideoFileExist()),
+}
-    await pruneDirectory(DIRECTORIES.WEB_VIDEOS.PRIVATE, doesWebVideoFileExist()),
 
-    await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, doesHLSPlaylistExist()),
+// ---------------------------------------------------------------------------
-    await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, doesHLSPlaylistExist()),
+// Object storage
+// ---------------------------------------------------------------------------
 
-    await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()),
+class ObjectStoragePruner {
+  private readonly keysToDelete: { bucket: string, key: string }[] = []
 
-    await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
+  async prune () {
+    if (!CONFIG.OBJECT_STORAGE.ENABLED) return
 
-    await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true, ThumbnailType.PREVIEW)),
+    console.log('Pruning object storage.')
-    await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false, ThumbnailType.MINIATURE)),
 
-    await pruneDirectory(CONFIG.STORAGE.ACTOR_IMAGES_DIR, doesActorImageExist)
+    await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.WEB_VIDEOS, this.doesWebVideoFileExistFactory())
-  )
+    await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, this.doesStreamingPlaylistFileExistFactory())
+    await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES, this.doesOriginalFileExistFactory())
+    await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.USER_EXPORTS, this.doesUserExportFileExistFactory())
 
-  const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
+    if (this.keysToDelete.length === 0) {
-  toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
+      console.log('No unknown object storage files to delete.')
+      return
+    }
 
-  if (toDelete.length === 0) {
+    const formattedKeysToDelete = this.keysToDelete.map(({ bucket, key }) => ` In bucket ${bucket}: ${key}`).join('\n')
-    console.log('No files to delete.')
+    console.log(`${this.keysToDelete.length} unknown files from object storage can be deleted:\n${formattedKeysToDelete}\n`)
-    return
+
+    const res = await askConfirmation()
+    if (res !== true) {
+      console.log('Exiting without deleting object storage files.')
+      return
+    }
+
+    console.log('Deleting object storage files...\n')
+
+    for (const { bucket, key } of this.keysToDelete) {
+      await removeObjectByFullKey(key, { BUCKET_NAME: bucket })
+    }
+
+    console.log(`${this.keysToDelete.length} object storage files deleted.`)
   }
 
-  console.log('Will delete %d files:\n\n%s\n\n', toDelete.length, toDelete.join('\n'))
+  private async findFilesToDelete (
+    config: { BUCKET_NAME: string, PREFIX?: string },
+    existFun: (file: string) => Promise<boolean> | boolean
+  ) {
+    try {
+      const keys = await listKeysOfPrefix('', config)
 
-  const res = await askConfirmation()
+      await Bluebird.map(keys, async key => {
-  if (res === true) {
+        if (await existFun(key) !== true) {
-    console.log('Processing delete...\n')
+          this.keysToDelete.push({ bucket: config.BUCKET_NAME, key })
+        }
+      }, { concurrency: 20 })
+    } catch (err) {
+      const prefixMessage = config.PREFIX
+        ? ` and prefix ${config.PREFIX}`
+        : ''
 
-    for (const path of toDelete) {
+      console.error('Cannot find files to delete in bucket ' + config.BUCKET_NAME + prefixMessage)
+    }
+  }
+
+  private doesWebVideoFileExistFactory () {
+    return (key: string) => {
+      const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.WEB_VIDEOS)
+
+      return VideoFileModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
+    }
+  }
+
+  private doesStreamingPlaylistFileExistFactory () {
+    return (key: string) => {
+      const uuid = basename(dirname(this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)))
+
+      return VideoStreamingPlaylistModel.doesOwnedVideoUUIDExist(uuid, FileStorage.OBJECT_STORAGE)
+    }
+  }
+
+  private doesOriginalFileExistFactory () {
+    return (key: string) => {
+      const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES)
+
+      return VideoSourceModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
+    }
+  }
+
+  private doesUserExportFileExistFactory () {
+    return (key: string) => {
+      const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.USER_EXPORTS)
+
+      return UserExportModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
+    }
+  }
+
+  private sanitizeKey (key: string, config: { PREFIX: string }) {
+    return key.replace(new RegExp(`^${config.PREFIX}`), '')
+  }
+}
+
+// ---------------------------------------------------------------------------
+// FS
+// ---------------------------------------------------------------------------
+
+class FSPruner {
+  private pathsToDelete: string[] = []
+
+  async prune () {
+    const dirs = Object.values(CONFIG.STORAGE)
+
+    if (uniqify(dirs).length !== dirs.length) {
+      console.error('Cannot prune storage because you put multiple storage keys in the same directory.')
+      process.exit(0)
+    }
+
+    console.log('Pruning filesystem storage.')
+
+    console.log('Detecting files to remove, it can take a while...')
+
+    await this.findFilesToDelete(DIRECTORIES.WEB_VIDEOS.PUBLIC, this.doesWebVideoFileExistFactory())
+    await this.findFilesToDelete(DIRECTORIES.WEB_VIDEOS.PRIVATE, this.doesWebVideoFileExistFactory())
+
+    await this.findFilesToDelete(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, this.doesHLSPlaylistExistFactory())
+    await this.findFilesToDelete(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, this.doesHLSPlaylistExistFactory())
+
+    await this.findFilesToDelete(DIRECTORIES.ORIGINAL_VIDEOS, this.doesOriginalVideoExistFactory())
+
+    await this.findFilesToDelete(CONFIG.STORAGE.TORRENTS_DIR, this.doesTorrentFileExistFactory())
+
+    await this.findFilesToDelete(CONFIG.STORAGE.REDUNDANCY_DIR, this.doesRedundancyExistFactory())
+
+    await this.findFilesToDelete(CONFIG.STORAGE.PREVIEWS_DIR, this.doesThumbnailExistFactory(true, ThumbnailType.PREVIEW))
+    await this.findFilesToDelete(CONFIG.STORAGE.THUMBNAILS_DIR, this.doesThumbnailExistFactory(false, ThumbnailType.MINIATURE))
+
+    await this.findFilesToDelete(CONFIG.STORAGE.ACTOR_IMAGES_DIR, this.doesActorImageExistFactory())
+
+    await this.findFilesToDelete(CONFIG.STORAGE.TMP_PERSISTENT_DIR, this.doesUserExportExistFactory())
+
+    const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
+    this.pathsToDelete = [ ...this.pathsToDelete, ...tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)) ]
+
+    if (this.pathsToDelete.length === 0) {
+      console.log('No unknown filesystem files to delete.')
+      return
+    }
+
+    const formattedKeysToDelete = this.pathsToDelete.map(p => ` ${p}`).join('\n')
+    console.log(`${this.pathsToDelete.length} unknown files from filesystem can be deleted:\n${formattedKeysToDelete}\n`)
+
+    const res = await askConfirmation()
+    if (res !== true) {
+      console.log('Exiting without deleting filesystem files.')
+      return
+    }
+
+    console.log('Deleting filesystem files...\n')
+
+    for (const path of this.pathsToDelete) {
       await remove(path)
     }
 
-    console.log('Done!')
+    console.log(`${this.pathsToDelete.length} filesystem files deleted.`)
-  } else {
-    console.log('Exiting without deleting files.')
   }
-}
 
-type ExistFun = (file: string) => Promise<boolean> | boolean
+  private async findFilesToDelete (directory: string, existFun: (file: string) => Promise<boolean> | boolean) {
-async function pruneDirectory (directory: string, existFun: ExistFun) {
+    const files = await readdir(directory)
-  const files = await readdir(directory)
 
-  const toDelete: string[] = []
+    await Bluebird.map(files, async file => {
-  await Bluebird.map(files, async file => {
+      const filePath = join(directory, file)
-    const filePath = join(directory, file)
 
-    if (await existFun(filePath) !== true) {
+      if (await existFun(filePath) !== true) {
-      toDelete.push(filePath)
+        this.pathsToDelete.push(filePath)
+      }
+    }, { concurrency: 20 })
+  }
+
+  private doesWebVideoFileExistFactory () {
+    return (filePath: string) => {
+      // Don't delete private directory
+      if (filePath === DIRECTORIES.WEB_VIDEOS.PRIVATE) return true
+
+      return VideoFileModel.doesOwnedFileExist(basename(filePath), FileStorage.FILE_SYSTEM)
     }
-  }, { concurrency: 20 })
-
-  return toDelete
-}
-
-function doesWebVideoFileExist () {
-  return (filePath: string) => {
-    // Don't delete private directory
-    if (filePath === DIRECTORIES.WEB_VIDEOS.PRIVATE) return true
-
-    return VideoFileModel.doesOwnedWebVideoFileExist(basename(filePath))
   }
-}
 
-function doesHLSPlaylistExist () {
+  private doesHLSPlaylistExistFactory () {
-  return (hlsPath: string) => {
+    return (hlsPath: string) => {
-    // Don't delete private directory
+      // Don't delete private directory
-    if (hlsPath === DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) return true
+      if (hlsPath === DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) return true
 
-    return VideoStreamingPlaylistModel.doesOwnedHLSPlaylistExist(basename(hlsPath))
+      return VideoStreamingPlaylistModel.doesOwnedVideoUUIDExist(basename(hlsPath), FileStorage.FILE_SYSTEM)
-  }
-}
-
-function doesTorrentFileExist () {
-  return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
-}
-
-function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType_Type) {
-  return async (filePath: string) => {
-    const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
-    if (!thumbnail) return false
-
-    if (keepOnlyOwned) {
-      const video = await VideoModel.load(thumbnail.videoId)
-      if (video.isOwned() === false) return false
     }
-
-    return true
-  }
-}
-
-async function doesActorImageExist (filePath: string) {
-  const image = await ActorImageModel.loadByName(basename(filePath))
-
-  return !!image
-}
-
-async function doesRedundancyExist (filePath: string) {
-  const isPlaylist = (await stat(filePath)).isDirectory()
-
-  if (isPlaylist) {
-    // Don't delete HLS redundancy directory
-    if (filePath === DIRECTORIES.HLS_REDUNDANCY) return true
-
-    const uuid = getUUIDFromFilename(filePath)
-    const video = await VideoModel.loadWithFiles(uuid)
-    if (!video) return false
-
-    const p = video.getHLSPlaylist()
-    if (!p) return false
-
-    const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
-    return !!redundancy
   }
 
-  const file = await VideoFileModel.loadByFilename(basename(filePath))
+  private doesOriginalVideoExistFactory () {
-  if (!file) return false
+    return (filePath: string) => {
+      return VideoSourceModel.doesOwnedFileExist(basename(filePath), FileStorage.FILE_SYSTEM)
+    }
+  }
 
-  const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
+  private doesTorrentFileExistFactory () {
-  return !!redundancy
+    return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
+  }
+
+  private doesThumbnailExistFactory (keepOnlyOwned: boolean, type: ThumbnailType_Type) {
+    return async (filePath: string) => {
+      const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
+      if (!thumbnail) return false
+
+      if (keepOnlyOwned) {
+        const video = await VideoModel.load(thumbnail.videoId)
+        if (video.isOwned() === false) return false
+      }
+
+      return true
+    }
+  }
+
+  private doesActorImageExistFactory () {
+    return async (filePath: string) => {
+      const image = await ActorImageModel.loadByName(basename(filePath))
+
+      return !!image
+    }
+  }
+
+  private doesRedundancyExistFactory () {
+    return async (filePath: string) => {
+      const isPlaylist = (await stat(filePath)).isDirectory()
+
+      if (isPlaylist) {
+        // Don't delete HLS redundancy directory
+        if (filePath === DIRECTORIES.HLS_REDUNDANCY) return true
+
+        const uuid = getUUIDFromFilename(filePath)
+        const video = await VideoModel.loadWithFiles(uuid)
+        if (!video) return false
+
+        const p = video.getHLSPlaylist()
+        if (!p) return false
+
+        const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
+        return !!redundancy
+      }
+
+      const file = await VideoFileModel.loadByFilename(basename(filePath))
+      if (!file) return false
+
+      const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
+      return !!redundancy
+    }
+  }
+
+  private doesUserExportExistFactory () {
+    return (filePath: string) => {
+      const filename = basename(filePath)
+
+      // Only detect non-existing user export
+      if (!filename.startsWith(USER_EXPORT_FILE_PREFIX)) return true
+
+      return UserExportModel.doesOwnedFileExist(filename, FileStorage.FILE_SYSTEM)
+    }
+  }
 }
 
 async function askConfirmation () {
@@ -169,10 +307,12 @@ async function askConfirmation () {
       properties: {
         confirm: {
           type: 'string',
-          description: 'These following unused files can be deleted, but please check your backups first (bugs happen).' +
+          description: 'These unknown files can be deleted, but please check your backups first (bugs happen).' +
             ' Notice PeerTube must have been stopped when your ran this script.' +
-            ' Can we delete these files?',
+            ' Can we delete these files? (y/n)',
           default: 'n',
+          validator: /y[es]*|n[o]?/,
+          warning: 'Must respond yes or no',
           required: true
         }
       }