From 54c140c8004fad5df136fca73268c35f07249e79 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 3 Jun 2024 16:37:44 +0200 Subject: [PATCH] Support object storage in prune script Also prune original files and user exports --- package.json | 2 +- packages/tests/src/cli/prune-storage.ts | 408 ++++++++++++------ packages/tests/src/shared/sql-command.ts | 25 ++ server/core/helpers/logger.ts | 22 +- server/core/initializers/constants.ts | 2 + .../{shared => }/object-storage-helpers.ts | 21 +- .../core/lib/object-storage/shared/index.ts | 2 +- server/core/models/user/user-export.ts | 35 +- server/core/models/video/video-file.ts | 6 +- server/core/models/video/video-source.ts | 18 +- .../models/video/video-streaming-playlist.ts | 6 +- server/scripts/prune-storage.ts | 366 +++++++++++----- 12 files changed, 618 insertions(+), 295 deletions(-) rename server/core/lib/object-storage/{shared => }/object-storage-helpers.ts (95%) diff --git a/package.json b/package.json index 48323ea3c..54fef1a06 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "create-move-video-storage-job": "node ./dist/scripts/create-move-video-storage-job.js", "create-generate-storyboard-job": "node ./dist/scripts/create-generate-storyboard-job.js", "parse-log": "node ./dist/scripts/parse-log.js", - "prune-storage": "node ./dist/scripts/prune-storage.js", + "prune-storage": "LOGGER_LEVEL=warn node ./dist/scripts/prune-storage.js", "test": "bash ./scripts/test.sh", "generate-cli-doc": "bash ./scripts/generate-cli-doc.sh", "generate-types-package": "tsx --conditions=peertube:tsx ./packages/types-generator/generate-package.ts", diff --git a/packages/tests/src/cli/prune-storage.ts b/packages/tests/src/cli/prune-storage.ts index f3d7d19f2..f21979ad0 100644 --- a/packages/tests/src/cli/prune-storage.ts +++ b/packages/tests/src/cli/prune-storage.ts @@ -1,77 +1,49 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { expect } from 'chai' -import { createFile } from 'fs-extra/esm' -import { readdir } from 'fs/promises' -import { join } from 'path' -import { wait } from '@peertube/peertube-core-utils' -import { buildUUID } from '@peertube/peertube-node-utils' -import { HttpStatusCode, VideoPlaylistPrivacy, VideoPrivacy } from '@peertube/peertube-models' +import { getAllFiles, wait } from '@peertube/peertube-core-utils' +import { FileStorage, HttpStatusCode, HttpStatusCodeType, VideoPlaylistPrivacy, VideoPrivacy } from '@peertube/peertube-models' +import { areMockObjectStorageTestsDisabled, buildUUID } from '@peertube/peertube-node-utils' import { - cleanupTests, CLICommand, + ObjectStorageCommand, + PeerTubeServer, + cleanupTests, createMultipleServers, doubleFollow, killallServers, makeGetRequest, - PeerTubeServer, + makeRawRequest, setAccessTokensToServers, setDefaultVideoChannel, waitJobs } from '@peertube/peertube-server-commands' - -async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) { - const files = await readdir(server.servers.buildDirectory(directory)) - - for (const f of files) { - expect(f).to.not.contain(substring) - } -} - -async function assertCountAreOkay (servers: PeerTubeServer[]) { - for (const server of servers) { - const videosCount = await server.servers.countFiles('web-videos') - expect(videosCount).to.equal(9) // 2 videos with 4 resolutions + private directory - - const privateVideosCount = await server.servers.countFiles('web-videos/private') - expect(privateVideosCount).to.equal(4) - - const torrentsCount = await server.servers.countFiles('torrents') - expect(torrentsCount).to.equal(24) - - const previewsCount = await server.servers.countFiles('previews') - expect(previewsCount).to.equal(3) - - const thumbnailsCount = await server.servers.countFiles('thumbnails') - expect(thumbnailsCount).to.equal(5) // 3 local videos, 1 local playlist, 2 remotes videos (lazy downloaded) and 1 remote playlist - - const avatarsCount = await server.servers.countFiles('avatars') - expect(avatarsCount).to.equal(8) - - const hlsRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls')) - expect(hlsRootCount).to.equal(3) // 2 videos + private directory - - const hlsPrivateRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls', 'private')) - expect(hlsPrivateRootCount).to.equal(1) - } -} +import { SQLCommand } from '@tests/shared/sql-command.js' +import { expect } from 'chai' +import { createFile } from 'fs-extra/esm' +import { readdir } from 'fs/promises' +import { join } from 'path' describe('Test prune storage scripts', function () { let servers: PeerTubeServer[] - const badNames: { [directory: string]: string[] } = {} before(async function () { this.timeout(120000) - servers = await createMultipleServers(2, { transcoding: { enabled: true } }) + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) await setDefaultVideoChannel(servers) for (const server of servers) { - await server.videos.upload({ attributes: { name: 'video 1', privacy: VideoPrivacy.PUBLIC } }) - await server.videos.upload({ attributes: { name: 'video 2', privacy: VideoPrivacy.PUBLIC } }) + await server.config.enableMinimumTranscoding({ keepOriginal: true }) + await server.config.enableUserExport() + } - await server.videos.upload({ attributes: { name: 'video 3', privacy: VideoPrivacy.PRIVATE } }) + for (const server of servers) { + await server.videos.quickUpload({ name: 'video 1', privacy: VideoPrivacy.PUBLIC }) + await server.videos.quickUpload({ name: 'video 2', privacy: VideoPrivacy.PUBLIC }) + + await server.videos.quickUpload({ name: 'video 3', privacy: VideoPrivacy.PRIVATE }) await server.users.updateMyAvatar({ fixture: 'avatar.png' }) @@ -85,6 +57,12 @@ describe('Test prune storage scripts', function () { }) } + for (const server of servers) { + const user = await server.users.getMyInfo() + + await server.userExports.request({ userId: user.id, withVideoFiles: false }) + } + await doubleFollow(servers[0], servers[1]) // Lazy load the remote avatars @@ -119,103 +97,259 @@ describe('Test prune storage scripts', function () { await wait(1000) }) - it('Should have the files on the disk', async function () { - await assertCountAreOkay(servers) - }) + describe('On filesystem', function () { + const badNames: { [directory: string]: string[] } = {} - it('Should create some dirty files', async function () { - for (let i = 0; i < 2; i++) { - { - const basePublic = servers[0].servers.buildDirectory('web-videos') - const basePrivate = servers[0].servers.buildDirectory(join('web-videos', 'private')) + async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) { + const files = await readdir(server.servers.buildDirectory(directory)) - const n1 = buildUUID() + '.mp4' - const n2 = buildUUID() + '.webm' - - await createFile(join(basePublic, n1)) - await createFile(join(basePublic, n2)) - await createFile(join(basePrivate, n1)) - await createFile(join(basePrivate, n2)) - - badNames['web-videos'] = [ n1, n2 ] - } - - { - const base = servers[0].servers.buildDirectory('torrents') - - const n1 = buildUUID() + '-240.torrent' - const n2 = buildUUID() + '-480.torrent' - - await createFile(join(base, n1)) - await createFile(join(base, n2)) - - badNames['torrents'] = [ n1, n2 ] - } - - { - const base = servers[0].servers.buildDirectory('thumbnails') - - const n1 = buildUUID() + '.jpg' - const n2 = buildUUID() + '.jpg' - - await createFile(join(base, n1)) - await createFile(join(base, n2)) - - badNames['thumbnails'] = [ n1, n2 ] - } - - { - const base = servers[0].servers.buildDirectory('previews') - - const n1 = buildUUID() + '.jpg' - const n2 = buildUUID() + '.jpg' - - await createFile(join(base, n1)) - await createFile(join(base, n2)) - - badNames['previews'] = [ n1, n2 ] - } - - { - const base = servers[0].servers.buildDirectory('avatars') - - const n1 = buildUUID() + '.png' - const n2 = buildUUID() + '.jpg' - - await createFile(join(base, n1)) - await createFile(join(base, n2)) - - badNames['avatars'] = [ n1, n2 ] - } - - { - const directory = join('streaming-playlists', 'hls') - const basePublic = servers[0].servers.buildDirectory(directory) - const basePrivate = servers[0].servers.buildDirectory(join(directory, 'private')) - - const n1 = buildUUID() - await createFile(join(basePublic, n1)) - await createFile(join(basePrivate, n1)) - badNames[directory] = [ n1 ] + for (const f of files) { + expect(f).to.not.contain(substring) } } - }) - it('Should run prune storage', async function () { - this.timeout(30000) + async function assertCountAreOkay () { + for (const server of servers) { + const videosCount = await server.servers.countFiles('web-videos') + expect(videosCount).to.equal(5) // 2 videos with 2 resolutions + private directory - const env = servers[0].cli.getEnv() - await CLICommand.exec(`echo y | ${env} npm run prune-storage`) - }) + const privateVideosCount = await server.servers.countFiles('web-videos/private') + expect(privateVideosCount).to.equal(2) - it('Should have removed files', async function () { - await assertCountAreOkay(servers) + const torrentsCount = await server.servers.countFiles('torrents') + expect(torrentsCount).to.equal(12) - for (const directory of Object.keys(badNames)) { - for (const name of badNames[directory]) { - await assertNotExists(servers[0], directory, name) + const previewsCount = await server.servers.countFiles('previews') + expect(previewsCount).to.equal(3) + + const thumbnailsCount = await server.servers.countFiles('thumbnails') + expect(thumbnailsCount).to.equal(5) // 3 local videos, 1 local playlist, 2 remotes videos (lazy downloaded) and 1 remote playlist + + const avatarsCount = await server.servers.countFiles('avatars') + expect(avatarsCount).to.equal(8) + + const hlsRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls')) + expect(hlsRootCount).to.equal(3) // 2 videos + private directory + + const hlsPrivateRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls', 'private')) + expect(hlsPrivateRootCount).to.equal(1) + + const originalVideoFilesCount = await server.servers.countFiles(join('original-video-files')) + expect(originalVideoFilesCount).to.equal(3) + + const userExportFilesCount = await server.servers.countFiles(join('tmp-persistent')) + expect(userExportFilesCount).to.equal(1) } } + + it('Should have the files on the disk', async function () { + await assertCountAreOkay() + }) + + it('Should create some dirty files', async function () { + for (let i = 0; i < 2; i++) { + { + const basePublic = servers[0].servers.buildDirectory('web-videos') + const basePrivate = servers[0].servers.buildDirectory(join('web-videos', 'private')) + + const n1 = buildUUID() + '.mp4' + const n2 = buildUUID() + '.webm' + + await createFile(join(basePublic, n1)) + await createFile(join(basePublic, n2)) + await createFile(join(basePrivate, n1)) + await createFile(join(basePrivate, n2)) + + badNames['web-videos'] = [ n1, n2 ] + } + + { + const base = servers[0].servers.buildDirectory('torrents') + + const n1 = buildUUID() + '-240.torrent' + const n2 = buildUUID() + '-480.torrent' + + await createFile(join(base, n1)) + await createFile(join(base, n2)) + + badNames['torrents'] = [ n1, n2 ] + } + + { + const base = servers[0].servers.buildDirectory('thumbnails') + + const n1 = buildUUID() + '.jpg' + const n2 = buildUUID() + '.jpg' + + await createFile(join(base, n1)) + await createFile(join(base, n2)) + + badNames['thumbnails'] = [ n1, n2 ] + } + + { + const base = servers[0].servers.buildDirectory('previews') + + const n1 = buildUUID() + '.jpg' + const n2 = buildUUID() + '.jpg' + + await createFile(join(base, n1)) + await createFile(join(base, n2)) + + badNames['previews'] = [ n1, n2 ] + } + + { + const base = servers[0].servers.buildDirectory('avatars') + + const n1 = buildUUID() + '.png' + const n2 = buildUUID() + '.jpg' + + await createFile(join(base, n1)) + await createFile(join(base, n2)) + + badNames['avatars'] = [ n1, n2 ] + } + + { + const directory = join('streaming-playlists', 'hls') + const basePublic = servers[0].servers.buildDirectory(directory) + const basePrivate = servers[0].servers.buildDirectory(join(directory, 'private')) + + const n1 = buildUUID() + await createFile(join(basePublic, n1)) + await createFile(join(basePrivate, n1)) + badNames[directory] = [ n1 ] + } + + { + const base = servers[0].servers.buildDirectory('original-video-files') + + const n1 = buildUUID() + '.mp4' + await createFile(join(base, n1)) + + badNames['original-video-files'] = [ n1 ] + } + + { + const base = servers[0].servers.buildDirectory('tmp-persistent') + + const n1 = 'user-export-1.zip' + const n2 = 'user-export-2.zip' + + await createFile(join(base, n1)) + await createFile(join(base, n2)) + + badNames['tmp-persistent'] = [ n1, n2 ] + } + } + }) + + it('Should run prune storage', async function () { + this.timeout(30000) + + const env = servers[0].cli.getEnv() + await CLICommand.exec(`echo y | ${env} npm run prune-storage`) + }) + + it('Should have removed files', async function () { + await assertCountAreOkay() + + for (const directory of Object.keys(badNames)) { + for (const name of badNames[directory]) { + await assertNotExists(servers[0], directory, name) + } + } + }) + }) + + describe('On object storage', function () { + if (areMockObjectStorageTestsDisabled()) return + + const videos: string[] = [] + const objectStorage = new ObjectStorageCommand() + + let sqlCommand: SQLCommand + let rootId: number + + async function checkVideosFiles (uuids: string[], expectedStatus: HttpStatusCodeType) { + for (const uuid of uuids) { + const video = await servers[0].videos.getWithToken({ id: uuid }) + + for (const file of getAllFiles(video)) { + await makeRawRequest({ url: file.fileUrl, token: servers[0].accessToken, expectedStatus }) + } + + const source = await servers[0].videos.getSource({ id: uuid }) + await makeRawRequest({ url: source.fileDownloadUrl, redirects: 1, token: servers[0].accessToken, expectedStatus }) + } + } + + async function checkUserExport (expectedStatus: HttpStatusCodeType) { + const { data } = await servers[0].userExports.list({ userId: rootId }) + await makeRawRequest({ url: data[0].privateDownloadUrl, redirects: 1, expectedStatus }) + } + + before(async function () { + this.timeout(120000) + + sqlCommand = new SQLCommand(servers[0]) + + await objectStorage.prepareDefaultMockBuckets() + + await servers[0].run(objectStorage.getDefaultMockConfig()) + + { + const { uuid } = await servers[0].videos.quickUpload({ name: 's3 video 1', privacy: VideoPrivacy.PUBLIC }) + videos.push(uuid) + } + + { + const { uuid } = await servers[0].videos.quickUpload({ name: 's3 video 2', privacy: VideoPrivacy.PUBLIC }) + videos.push(uuid) + } + + { + const { uuid } = await servers[0].videos.quickUpload({ name: 's3 video 3', privacy: VideoPrivacy.PRIVATE }) + videos.push(uuid) + } + + const user = await servers[0].users.getMyInfo() + rootId = user.id + + await servers[0].userExports.deleteAllArchives({ userId: rootId }) + await servers[0].userExports.request({ userId: rootId, withVideoFiles: false }) + + await waitJobs([ servers[0] ]) + }) + + it('Should have the files on object storage', async function () { + await checkVideosFiles(videos, HttpStatusCode.OK_200) + await checkUserExport(HttpStatusCode.OK_200) + }) + + it('Should run prune-storage script on videos', async function () { + await sqlCommand.setVideoFileStorageOf(videos[1], FileStorage.FILE_SYSTEM) + await sqlCommand.setVideoFileStorageOf(videos[2], FileStorage.FILE_SYSTEM) + + await checkVideosFiles([ videos[1], videos[2] ], HttpStatusCode.NOT_FOUND_404) + await checkVideosFiles([ videos[0] ], HttpStatusCode.OK_200) + + await checkUserExport(HttpStatusCode.OK_200) + }) + + it('Should run prune-storage script on exports', async function () { + await sqlCommand.setUserExportStorageOf(rootId, FileStorage.FILE_SYSTEM) + + await checkVideosFiles([ videos[1], videos[2] ], HttpStatusCode.NOT_FOUND_404) + await checkVideosFiles([ videos[0] ], HttpStatusCode.OK_200) + + await checkUserExport(HttpStatusCode.NOT_FOUND_404) + }) + + after(async function () { + await sqlCommand.cleanup() + }) }) after(async function () { diff --git a/packages/tests/src/shared/sql-command.ts b/packages/tests/src/shared/sql-command.ts index 1c4f89351..2073efc2a 100644 --- a/packages/tests/src/shared/sql-command.ts +++ b/packages/tests/src/shared/sql-command.ts @@ -1,6 +1,7 @@ import { QueryTypes, Sequelize } from 'sequelize' import { forceNumber } from '@peertube/peertube-core-utils' import { PeerTubeServer } from '@peertube/peertube-server-commands' +import { FileStorageType } from '@peertube/peertube-models' export class SQLCommand { private sequelize: Sequelize @@ -58,6 +59,30 @@ export class SQLCommand { // --------------------------------------------------------------------------- + async setVideoFileStorageOf (uuid: string, storage: FileStorageType) { + await this.updateQuery( + `UPDATE "videoFile" SET storage = :storage ` + + `WHERE "videoId" IN (SELECT id FROM "video" WHERE uuid = :uuid) OR ` + + // eslint-disable-next-line max-len + `"videoStreamingPlaylistId" IN (` + + `SELECT "videoStreamingPlaylist".id FROM "videoStreamingPlaylist" ` + + `INNER JOIN video ON video.id = "videoStreamingPlaylist"."videoId" AND "video".uuid = :uuid` + + `)`, + { storage, uuid } + ) + + await this.updateQuery( + `UPDATE "videoSource" SET storage = :storage WHERE "videoId" IN (SELECT id FROM "video" WHERE uuid = :uuid)`, + { storage, uuid } + ) + } + + async setUserExportStorageOf (userId: number, storage: FileStorageType) { + await this.updateQuery(`UPDATE "userExport" SET storage = :storage WHERE "userId" = :userId`, { storage, userId }) + } + + // --------------------------------------------------------------------------- + setPluginVersion (pluginName: string, newVersion: string) { return this.setPluginField(pluginName, 'version', newVersion) } diff --git a/server/core/helpers/logger.ts b/server/core/helpers/logger.ts index 464f6388d..20e972c23 100644 --- a/server/core/helpers/logger.ts +++ b/server/core/helpers/logger.ts @@ -1,11 +1,11 @@ +import { context } from '@opentelemetry/api' +import { getSpanContext } from '@opentelemetry/api/build/src/trace/context-utils.js' +import { omit } from '@peertube/peertube-core-utils' import { stat } from 'fs/promises' import { join } from 'path' import { format as sqlFormat } from 'sql-formatter' import { createLogger, format, transports } from 'winston' import { FileTransportOptions } from 'winston/lib/winston/transports' -import { context } from '@opentelemetry/api' -import { getSpanContext } from '@opentelemetry/api/build/src/trace/context-utils.js' -import { omit } from '@peertube/peertube-core-utils' import { CONFIG } from '../initializers/config.js' import { LOG_FILENAME } from '../initializers/constants.js' @@ -60,7 +60,7 @@ if (CONFIG.LOG.ROTATION.ENABLED) { function buildLogger (labelSuffix?: string) { return createLogger({ - level: CONFIG.LOG.LEVEL, + level: process.env.LOGGER_LEVEL ?? CONFIG.LOG.LEVEL, defaultMeta: { get traceId () { return getSpanContext(context.active())?.traceId }, get spanId () { return getSpanContext(context.active())?.spanId }, @@ -154,18 +154,10 @@ async function mtimeSortFilesDesc (files: string[], basePath: string) { // --------------------------------------------------------------------------- export { - type LoggerTagsFn, - type LoggerTags, - buildLogger, - timestampFormatter, - labelFormatter, - consoleLoggerFormat, - jsonLoggerFormat, - mtimeSortFilesDesc, - logger, - loggerTagsFactory, - bunyanLogger + buildLogger, bunyanLogger, consoleLoggerFormat, + jsonLoggerFormat, labelFormatter, logger, + loggerTagsFactory, mtimeSortFilesDesc, timestampFormatter, type LoggerTags, type LoggerTagsFn } // --------------------------------------------------------------------------- diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts index dde4c41e4..ebb205a33 100644 --- a/server/core/initializers/constants.ts +++ b/server/core/initializers/constants.ts @@ -845,6 +845,7 @@ const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = { // --------------------------------------------------------------------------- const USER_EXPORT_MAX_ITEMS = 1000 +const USER_EXPORT_FILE_PREFIX = 'user-export-' // --------------------------------------------------------------------------- @@ -1268,6 +1269,7 @@ export { CONSTRAINTS_FIELDS, EMBED_SIZE, REDUNDANCY, + USER_EXPORT_FILE_PREFIX, JOB_CONCURRENCY, JOB_ATTEMPTS, AP_CLEANER, diff --git a/server/core/lib/object-storage/shared/object-storage-helpers.ts b/server/core/lib/object-storage/object-storage-helpers.ts similarity index 95% rename from server/core/lib/object-storage/shared/object-storage-helpers.ts rename to server/core/lib/object-storage/object-storage-helpers.ts index 88fee8eb1..119477203 100644 --- a/server/core/lib/object-storage/shared/object-storage-helpers.ts +++ b/server/core/lib/object-storage/object-storage-helpers.ts @@ -7,9 +7,9 @@ import { createReadStream, createWriteStream } from 'fs' import { ensureDir } from 'fs-extra/esm' import { dirname } from 'path' import { Readable } from 'stream' -import { getInternalUrl } from '../urls.js' -import { getClient } from './client.js' -import { lTags } from './logger.js' +import { getInternalUrl } from './urls.js' +import { getClient } from './shared/client.js' +import { lTags } from './shared/logger.js' import type { _Object, ObjectCannedACL, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3' @@ -18,7 +18,7 @@ type BucketInfo = { PREFIX?: string } -async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) { +async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo, continuationToken?: string) { const s3Client = await getClient() const { ListObjectsV2Command } = await import('@aws-sdk/client-s3') @@ -26,14 +26,21 @@ async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) { const commandPrefix = bucketInfo.PREFIX + prefix const listCommand = new ListObjectsV2Command({ Bucket: bucketInfo.BUCKET_NAME, - Prefix: commandPrefix + Prefix: commandPrefix, + ContinuationToken: continuationToken }) const listedObjects = await s3Client.send(listCommand) if (isArray(listedObjects.Contents) !== true) return [] - return listedObjects.Contents.map(c => c.Key) + let keys = listedObjects.Contents.map(c => c.Key) + + if (listedObjects.IsTruncated) { + keys = keys.concat(await listKeysOfPrefix(prefix, bucketInfo, listedObjects.NextContinuationToken)) + } + + return keys } // --------------------------------------------------------------------------- @@ -145,7 +152,7 @@ function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) { return removeObjectByFullKey(key, bucketInfo) } -async function removeObjectByFullKey (fullKey: string, bucketInfo: BucketInfo) { +async function removeObjectByFullKey (fullKey: string, bucketInfo: Pick) { logger.debug('Removing file %s in bucket %s', fullKey, bucketInfo.BUCKET_NAME, lTags()) const { DeleteObjectCommand } = await import('@aws-sdk/client-s3') diff --git a/server/core/lib/object-storage/shared/index.ts b/server/core/lib/object-storage/shared/index.ts index d6b76a3d5..042d5e4ec 100644 --- a/server/core/lib/object-storage/shared/index.ts +++ b/server/core/lib/object-storage/shared/index.ts @@ -1,3 +1,3 @@ export * from './client.js' export * from './logger.js' -export * from './object-storage-helpers.js' +export * from '../object-storage-helpers.js' diff --git a/server/core/models/user/user-export.ts b/server/core/models/user/user-export.ts index 3b6377ebe..973b6edee 100644 --- a/server/core/models/user/user-export.ts +++ b/server/core/models/user/user-export.ts @@ -1,23 +1,25 @@ -import { FindOptions, Op } from 'sequelize' -import { AllowNull, BeforeDestroy, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript' -import { MUserAccountId, MUserExport } from '@server/types/models/index.js' -import { UserModel } from './user.js' -import { getSort } from '../shared/sort.js' -import { UserExportState, type UserExport, type UserExportStateType, type FileStorageType, FileStorage } from '@peertube/peertube-models' +import { FileStorage, UserExportState, type FileStorageType, type UserExport, type UserExportStateType } from '@peertube/peertube-models' import { logger } from '@server/helpers/logger.js' -import { remove } from 'fs-extra/esm' -import { getFSUserExportFilePath } from '@server/lib/paths.js' +import { CONFIG } from '@server/initializers/config.js' import { JWT_TOKEN_USER_EXPORT_FILE_LIFETIME, STATIC_DOWNLOAD_PATHS, + USER_EXPORT_FILE_PREFIX, USER_EXPORT_STATES, WEBSERVER } from '@server/initializers/constants.js' -import { join } from 'path' -import jwt from 'jsonwebtoken' -import { CONFIG } from '@server/initializers/config.js' import { removeUserExportObjectStorage } from '@server/lib/object-storage/user-export.js' +import { getFSUserExportFilePath } from '@server/lib/paths.js' +import { MUserAccountId, MUserExport } from '@server/types/models/index.js' +import { remove } from 'fs-extra/esm' +import jwt from 'jsonwebtoken' +import { join } from 'path' +import { FindOptions, Op } from 'sequelize' +import { AllowNull, BeforeDestroy, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript' +import { doesExist } from '../shared/query.js' import { SequelizeModel } from '../shared/sequelize-type.js' +import { getSort } from '../shared/sort.js' +import { UserModel } from './user.js' @Table({ tableName: 'userExport', @@ -147,11 +149,20 @@ export class UserExportModel extends SequelizeModel { // --------------------------------------------------------------------------- + static async doesOwnedFileExist (filename: string, storage: FileStorageType) { + const query = 'SELECT 1 FROM "userExport" ' + + `WHERE "filename" = $filename AND "storage" = $storage LIMIT 1` + + return doesExist({ sequelize: this.sequelize, query, bind: { filename, storage } }) + } + + // --------------------------------------------------------------------------- + generateAndSetFilename () { if (!this.userId) throw new Error('Cannot generate filename without userId') if (!this.createdAt) throw new Error('Cannot generate filename without createdAt') - this.filename = `user-export-${this.userId}-${this.createdAt.toISOString()}.zip` + this.filename = `${USER_EXPORT_FILE_PREFIX}${this.userId}-${this.createdAt.toISOString()}.zip` } canBeSafelyRemoved () { diff --git a/server/core/models/video/video-file.ts b/server/core/models/video/video-file.ts index 9c2b03519..b44d56940 100644 --- a/server/core/models/video/video-file.ts +++ b/server/core/models/video/video-file.ts @@ -289,11 +289,11 @@ export class VideoFileModel extends SequelizeModel { return doesExist({ sequelize: this.sequelize, query, bind: { filename } }) } - static async doesOwnedWebVideoFileExist (filename: string) { + static async doesOwnedFileExist (filename: string, storage: FileStorageType) { const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' + - `WHERE "filename" = $filename AND "storage" = ${FileStorage.FILE_SYSTEM} LIMIT 1` + `WHERE "filename" = $filename AND "storage" = $storage LIMIT 1` - return doesExist({ sequelize: this.sequelize, query, bind: { filename } }) + return doesExist({ sequelize: this.sequelize, query, bind: { filename, storage } }) } static loadByFilename (filename: string) { diff --git a/server/core/models/video/video-source.ts b/server/core/models/video/video-source.ts index a6b6609b5..124f5a5b2 100644 --- a/server/core/models/video/video-source.ts +++ b/server/core/models/video/video-source.ts @@ -1,12 +1,12 @@ -import type { FileStorageType, VideoSource } from '@peertube/peertube-models' +import { type FileStorageType, type VideoSource } from '@peertube/peertube-models' import { STATIC_DOWNLOAD_PATHS, WEBSERVER } from '@server/initializers/constants.js' +import { MVideoSource } from '@server/types/models/video/video-source.js' import { join } from 'path' import { Transaction } from 'sequelize' import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript' -import { SequelizeModel, getSort } from '../shared/index.js' +import { SequelizeModel, doesExist, getSort } from '../shared/index.js' import { getResolutionLabel } from './formatter/video-api-format.js' import { VideoModel } from './video.js' -import { MVideoSource } from '@server/types/models/video/video-source.js' @Table({ tableName: 'videoSource', @@ -103,6 +103,18 @@ export class VideoSourceModel extends SequelizeModel { }) } + // --------------------------------------------------------------------------- + + static async doesOwnedFileExist (filename: string, storage: FileStorageType) { + const query = 'SELECT 1 FROM "videoSource" ' + + 'INNER JOIN "video" ON "video"."id" = "videoSource"."videoId" AND "video"."remote" IS FALSE ' + + `WHERE "keptOriginalFilename" = $filename AND "storage" = $storage LIMIT 1` + + return doesExist({ sequelize: this.sequelize, query, bind: { filename, storage } }) + } + + // --------------------------------------------------------------------------- + getFileDownloadUrl () { if (!this.keptOriginalFilename) return null diff --git a/server/core/models/video/video-streaming-playlist.ts b/server/core/models/video/video-streaming-playlist.ts index 649ed2716..4cafd10f7 100644 --- a/server/core/models/video/video-streaming-playlist.ts +++ b/server/core/models/video/video-streaming-playlist.ts @@ -229,13 +229,13 @@ export class VideoStreamingPlaylistModel extends SequelizeModel join(CONFIG.STORAGE.TMP_DIR, t))) + if (this.keysToDelete.length === 0) { + console.log('No unknown object storage files to delete.') + return + } - if (toDelete.length === 0) { - console.log('No files to delete.') - return + const formattedKeysToDelete = this.keysToDelete.map(({ bucket, key }) => ` In bucket ${bucket}: ${key}`).join('\n') + console.log(`${this.keysToDelete.length} unknown files from object storage can be deleted:\n${formattedKeysToDelete}\n`) + + const res = await askConfirmation() + if (res !== true) { + console.log('Exiting without deleting object storage files.') + return + } + + console.log('Deleting object storage files...\n') + + for (const { bucket, key } of this.keysToDelete) { + await removeObjectByFullKey(key, { BUCKET_NAME: bucket }) + } + + console.log(`${this.keysToDelete.length} object storage files deleted.`) } - console.log('Will delete %d files:\n\n%s\n\n', toDelete.length, toDelete.join('\n')) + private async findFilesToDelete ( + config: { BUCKET_NAME: string, PREFIX?: string }, + existFun: (file: string) => Promise | boolean + ) { + try { + const keys = await listKeysOfPrefix('', config) - const res = await askConfirmation() - if (res === true) { - console.log('Processing delete...\n') + await Bluebird.map(keys, async key => { + if (await existFun(key) !== true) { + this.keysToDelete.push({ bucket: config.BUCKET_NAME, key }) + } + }, { concurrency: 20 }) + } catch (err) { + const prefixMessage = config.PREFIX + ? ` and prefix ${config.PREFIX}` + : '' - for (const path of toDelete) { + console.error('Cannot find files to delete in bucket ' + config.BUCKET_NAME + prefixMessage) + } + } + + private doesWebVideoFileExistFactory () { + return (key: string) => { + const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.WEB_VIDEOS) + + return VideoFileModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE) + } + } + + private doesStreamingPlaylistFileExistFactory () { + return (key: string) => { + const uuid = basename(dirname(this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS))) + + return VideoStreamingPlaylistModel.doesOwnedVideoUUIDExist(uuid, FileStorage.OBJECT_STORAGE) + } + } + + private doesOriginalFileExistFactory () { + return (key: string) => { + const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES) + + return VideoSourceModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE) + } + } + + private doesUserExportFileExistFactory () { + return (key: string) => { + const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.USER_EXPORTS) + + return UserExportModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE) + } + } + + private sanitizeKey (key: string, config: { PREFIX: string }) { + return key.replace(new RegExp(`^${config.PREFIX}`), '') + } +} + +// --------------------------------------------------------------------------- +// FS +// --------------------------------------------------------------------------- + +class FSPruner { + private pathsToDelete: string[] = [] + + async prune () { + const dirs = Object.values(CONFIG.STORAGE) + + if (uniqify(dirs).length !== dirs.length) { + console.error('Cannot prune storage because you put multiple storage keys in the same directory.') + process.exit(0) + } + + console.log('Pruning filesystem storage.') + + console.log('Detecting files to remove, it can take a while...') + + await this.findFilesToDelete(DIRECTORIES.WEB_VIDEOS.PUBLIC, this.doesWebVideoFileExistFactory()) + await this.findFilesToDelete(DIRECTORIES.WEB_VIDEOS.PRIVATE, this.doesWebVideoFileExistFactory()) + + await this.findFilesToDelete(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, this.doesHLSPlaylistExistFactory()) + await this.findFilesToDelete(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, this.doesHLSPlaylistExistFactory()) + + await this.findFilesToDelete(DIRECTORIES.ORIGINAL_VIDEOS, this.doesOriginalVideoExistFactory()) + + await this.findFilesToDelete(CONFIG.STORAGE.TORRENTS_DIR, this.doesTorrentFileExistFactory()) + + await this.findFilesToDelete(CONFIG.STORAGE.REDUNDANCY_DIR, this.doesRedundancyExistFactory()) + + await this.findFilesToDelete(CONFIG.STORAGE.PREVIEWS_DIR, this.doesThumbnailExistFactory(true, ThumbnailType.PREVIEW)) + await this.findFilesToDelete(CONFIG.STORAGE.THUMBNAILS_DIR, this.doesThumbnailExistFactory(false, ThumbnailType.MINIATURE)) + + await this.findFilesToDelete(CONFIG.STORAGE.ACTOR_IMAGES_DIR, this.doesActorImageExistFactory()) + + await this.findFilesToDelete(CONFIG.STORAGE.TMP_PERSISTENT_DIR, this.doesUserExportExistFactory()) + + const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR) + this.pathsToDelete = [ ...this.pathsToDelete, ...tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)) ] + + if (this.pathsToDelete.length === 0) { + console.log('No unknown filesystem files to delete.') + return + } + + const formattedKeysToDelete = this.pathsToDelete.map(p => ` ${p}`).join('\n') + console.log(`${this.pathsToDelete.length} unknown files from filesystem can be deleted:\n${formattedKeysToDelete}\n`) + + const res = await askConfirmation() + if (res !== true) { + console.log('Exiting without deleting filesystem files.') + return + } + + console.log('Deleting filesystem files...\n') + + for (const path of this.pathsToDelete) { await remove(path) } - console.log('Done!') - } else { - console.log('Exiting without deleting files.') + console.log(`${this.pathsToDelete.length} filesystem files deleted.`) } -} -type ExistFun = (file: string) => Promise | boolean -async function pruneDirectory (directory: string, existFun: ExistFun) { - const files = await readdir(directory) + private async findFilesToDelete (directory: string, existFun: (file: string) => Promise | boolean) { + const files = await readdir(directory) - const toDelete: string[] = [] - await Bluebird.map(files, async file => { - const filePath = join(directory, file) + await Bluebird.map(files, async file => { + const filePath = join(directory, file) - if (await existFun(filePath) !== true) { - toDelete.push(filePath) + if (await existFun(filePath) !== true) { + this.pathsToDelete.push(filePath) + } + }, { concurrency: 20 }) + } + + private doesWebVideoFileExistFactory () { + return (filePath: string) => { + // Don't delete private directory + if (filePath === DIRECTORIES.WEB_VIDEOS.PRIVATE) return true + + return VideoFileModel.doesOwnedFileExist(basename(filePath), FileStorage.FILE_SYSTEM) } - }, { concurrency: 20 }) - - return toDelete -} - -function doesWebVideoFileExist () { - return (filePath: string) => { - // Don't delete private directory - if (filePath === DIRECTORIES.WEB_VIDEOS.PRIVATE) return true - - return VideoFileModel.doesOwnedWebVideoFileExist(basename(filePath)) } -} -function doesHLSPlaylistExist () { - return (hlsPath: string) => { - // Don't delete private directory - if (hlsPath === DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) return true + private doesHLSPlaylistExistFactory () { + return (hlsPath: string) => { + // Don't delete private directory + if (hlsPath === DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) return true - return VideoStreamingPlaylistModel.doesOwnedHLSPlaylistExist(basename(hlsPath)) - } -} - -function doesTorrentFileExist () { - return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath)) -} - -function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType_Type) { - return async (filePath: string) => { - const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type) - if (!thumbnail) return false - - if (keepOnlyOwned) { - const video = await VideoModel.load(thumbnail.videoId) - if (video.isOwned() === false) return false + return VideoStreamingPlaylistModel.doesOwnedVideoUUIDExist(basename(hlsPath), FileStorage.FILE_SYSTEM) } - - return true - } -} - -async function doesActorImageExist (filePath: string) { - const image = await ActorImageModel.loadByName(basename(filePath)) - - return !!image -} - -async function doesRedundancyExist (filePath: string) { - const isPlaylist = (await stat(filePath)).isDirectory() - - if (isPlaylist) { - // Don't delete HLS redundancy directory - if (filePath === DIRECTORIES.HLS_REDUNDANCY) return true - - const uuid = getUUIDFromFilename(filePath) - const video = await VideoModel.loadWithFiles(uuid) - if (!video) return false - - const p = video.getHLSPlaylist() - if (!p) return false - - const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id) - return !!redundancy } - const file = await VideoFileModel.loadByFilename(basename(filePath)) - if (!file) return false + private doesOriginalVideoExistFactory () { + return (filePath: string) => { + return VideoSourceModel.doesOwnedFileExist(basename(filePath), FileStorage.FILE_SYSTEM) + } + } - const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) - return !!redundancy + private doesTorrentFileExistFactory () { + return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath)) + } + + private doesThumbnailExistFactory (keepOnlyOwned: boolean, type: ThumbnailType_Type) { + return async (filePath: string) => { + const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type) + if (!thumbnail) return false + + if (keepOnlyOwned) { + const video = await VideoModel.load(thumbnail.videoId) + if (video.isOwned() === false) return false + } + + return true + } + } + + private doesActorImageExistFactory () { + return async (filePath: string) => { + const image = await ActorImageModel.loadByName(basename(filePath)) + + return !!image + } + } + + private doesRedundancyExistFactory () { + return async (filePath: string) => { + const isPlaylist = (await stat(filePath)).isDirectory() + + if (isPlaylist) { + // Don't delete HLS redundancy directory + if (filePath === DIRECTORIES.HLS_REDUNDANCY) return true + + const uuid = getUUIDFromFilename(filePath) + const video = await VideoModel.loadWithFiles(uuid) + if (!video) return false + + const p = video.getHLSPlaylist() + if (!p) return false + + const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id) + return !!redundancy + } + + const file = await VideoFileModel.loadByFilename(basename(filePath)) + if (!file) return false + + const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) + return !!redundancy + } + } + + private doesUserExportExistFactory () { + return (filePath: string) => { + const filename = basename(filePath) + + // Only detect non-existing user export + if (!filename.startsWith(USER_EXPORT_FILE_PREFIX)) return true + + return UserExportModel.doesOwnedFileExist(filename, FileStorage.FILE_SYSTEM) + } + } } async function askConfirmation () { @@ -169,10 +307,12 @@ async function askConfirmation () { properties: { confirm: { type: 'string', - description: 'These following unused files can be deleted, but please check your backups first (bugs happen).' + + description: 'These unknown files can be deleted, but please check your backups first (bugs happen).' + ' Notice PeerTube must have been stopped when your ran this script.' + - ' Can we delete these files?', + ' Can we delete these files? (y/n)', default: 'n', + validator: /y[es]*|n[o]?/, + warning: 'Must respond yes or no', required: true } }