From 2b189131fa77b110c779899746261dca7fccb83b Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 5 Jun 2024 14:43:41 +0200 Subject: [PATCH] Add house-keeping script --- package.json | 97 ++++++++++--------- .../server-commands/src/users/accounts.ts | 11 +-- .../server-commands/src/videos/channels.ts | 14 +-- .../src/cli/create-generate-storyboard-job.ts | 2 +- .../src/cli/create-import-video-file-job.ts | 2 +- .../src/cli/create-move-video-storage-job.ts | 2 +- packages/tests/src/cli/house-keeping.ts | 96 ++++++++++++++++++ packages/tests/src/cli/index.ts | 1 + packages/tests/src/cli/plugins.ts | 2 +- packages/tests/src/cli/prune-storage.ts | 2 +- .../tests/src/cli/regenerate-thumbnails.ts | 2 +- packages/tests/src/cli/reset-password.ts | 2 +- packages/tests/src/cli/update-host.ts | 2 +- .../src/cli/update-object-storage-url.ts | 2 +- server/core/controllers/lazy-static.ts | 1 - server/core/models/actor/actor-image.ts | 27 +++++- server/core/models/video/thumbnail.ts | 22 +++++ server/scripts/house-keeping.ts | 91 +++++++++++++++++ server/scripts/prune-storage.ts | 38 ++------ server/scripts/shared/common.ts | 30 ++++++ server/scripts/update-object-storage-url.ts | 36 ++----- support/doc/tools.md | 19 ++++ 22 files changed, 368 insertions(+), 133 deletions(-) create mode 100644 packages/tests/src/cli/house-keeping.ts create mode 100644 server/scripts/house-keeping.ts create mode 100644 server/scripts/shared/common.ts diff --git a/package.json b/package.json index c53b5a53e..a7ccc563f 100644 --- a/package.json +++ b/package.json @@ -24,65 +24,66 @@ "server" ], "scripts": { - "e2e:browserstack": "bash ./scripts/e2e/browserstack.sh", - "e2e:local": "bash ./scripts/e2e/local.sh", - "build": "bash ./scripts/build/index.sh", - "build:embed": "bash ./scripts/build/embed.sh", - "build:server": "bash ./scripts/build/server.sh", + "benchmark-server": "tsx --conditions=peertube:tsx ./scripts/benchmark.ts", "build:client": "bash ./scripts/build/client.sh", - "build:peertube-runner": "bash ./scripts/build/peertube-runner.sh", + "build:embed": "bash ./scripts/build/embed.sh", "build:peertube-cli": "bash ./scripts/build/peertube-cli.sh", + "build:peertube-runner": "bash ./scripts/build/peertube-runner.sh", + "build:server": "bash ./scripts/build/server.sh", "build:tests": "bash ./scripts/build/tests.sh", + "build": "bash ./scripts/build/index.sh", + "ci": "bash ./scripts/ci.sh", "clean:client": "bash ./scripts/clean/client/index.sh", "clean:server:test": "bash ./scripts/clean/server/test.sh", - "i18n:update": "bash ./scripts/i18n/update.sh", - "dev": "bash ./scripts/dev/index.sh", - "dev:server": "bash ./scripts/dev/server.sh", - "dev:embed": "bash ./scripts/dev/embed.sh", - "dev:client": "bash ./scripts/dev/client.sh", - "dev:peertube-cli": "bash ./scripts/dev/peertube-cli.sh", - "dev:peertube-runner": "bash ./scripts/dev/peertube-runner.sh", - "start": "node dist/server", - "start:server": "node dist/server --no-client", - "plugin:install": "node ./dist/scripts/plugin/install.js", - "plugin:uninstall": "node ./dist/scripts/plugin/uninstall.js", - "reset-password": "node ./dist/scripts/reset-password.js", - "update-object-storage-url": "LOGGER_LEVEL=warn node ./dist/scripts/update-object-storage-url.js", - "update-host": "node ./dist/scripts/update-host.js", - "regenerate-thumbnails": "node ./dist/scripts/regenerate-thumbnails.js", + "client-report": "bash ./scripts/client-report.sh", + "client:build-stats": "tsx --conditions=peertube:tsx ./scripts/client-build-stats.ts", + "commander": "commander", + "concurrently": "concurrently", + "create-generate-storyboard-job": "node ./dist/scripts/create-generate-storyboard-job.js", "create-import-video-file-job": "node ./dist/scripts/create-import-video-file-job.js", "create-move-video-storage-job": "node ./dist/scripts/create-move-video-storage-job.js", - "create-generate-storyboard-job": "node ./dist/scripts/create-generate-storyboard-job.js", - "parse-log": "node ./dist/scripts/parse-log.js", - "prune-storage": "LOGGER_LEVEL=warn node ./dist/scripts/prune-storage.js", - "test": "bash ./scripts/test.sh", - "generate-cli-doc": "bash ./scripts/generate-cli-doc.sh", - "generate-types-package": "tsx --conditions=peertube:tsx ./packages/types-generator/generate-package.ts", - "i18n:create-custom-files": "tsx --tsconfig ./scripts/tsconfig.json --conditions=peertube:tsx ./scripts/i18n/create-custom-files.ts", - "benchmark-server": "tsx --conditions=peertube:tsx ./scripts/benchmark.ts", - "client:build-stats": "tsx --conditions=peertube:tsx ./scripts/client-build-stats.ts", - "generate-code-contributors": "tsx --conditions=peertube:tsx ./scripts/generate-code-contributors.ts", - "simulate-many-viewers": "tsx --conditions=peertube:tsx ./scripts/simulate-many-viewers.ts", - "postinstall": "test -n \"$NOCLIENT\" || (cd client && yarn install --pure-lockfile)", - "tsc": "tsc", - "commander": "commander", - "lint": "npm run ci -- lint", - "ng": "ng", - "tsx": "tsx", + "dev:client": "bash ./scripts/dev/client.sh", + "dev:embed": "bash ./scripts/dev/embed.sh", + "dev:peertube-cli": "bash ./scripts/dev/peertube-cli.sh", + "dev:peertube-runner": "bash ./scripts/dev/peertube-runner.sh", + "dev:server": "bash ./scripts/dev/server.sh", + "dev": "bash ./scripts/dev/index.sh", + "e2e:browserstack": "bash ./scripts/e2e/browserstack.sh", + "e2e:local": "bash ./scripts/e2e/local.sh", "eslint": "eslint", - "resolve-tspaths": "resolve-tspaths", - "resolve-tspaths:server": "npm run resolve-tspaths -- --project server/tsconfig.json --src server --out dist", - "resolve-tspaths:server-lib": "npm run resolve-tspaths -- --project server/tsconfig.lib.json --src server --out server/dist", - "resolve-tspaths:tests": "npm run resolve-tspaths -- --project packages/tests/tsconfig.json --src packages/tests/src --out packages/tests/dist", - "concurrently": "concurrently", + "generate-cli-doc": "bash ./scripts/generate-cli-doc.sh", + "generate-code-contributors": "tsx --conditions=peertube:tsx ./scripts/generate-code-contributors.ts", + "generate-types-package": "tsx --conditions=peertube:tsx ./packages/types-generator/generate-package.ts", + "house-keeping": "LOGGER_LEVEL=warn node ./dist/scripts/house-keeping.js", + "i18n:create-custom-files": "tsx --tsconfig ./scripts/tsconfig.json --conditions=peertube:tsx ./scripts/i18n/create-custom-files.ts", + "i18n:update": "bash ./scripts/i18n/update.sh", + "lint": "npm run ci -- lint", "mocha": "mocha", - "ci": "bash ./scripts/ci.sh", - "release": "bash ./scripts/release.sh", - "release-embed-api": "bash ./scripts/release-embed-api.sh", + "ng": "ng", "nightly": "bash ./scripts/nightly.sh", "openapi-clients": "bash ./scripts/openapi-clients.sh", - "client-report": "bash ./scripts/client-report.sh", - "swagger-cli": "swagger-cli" + "parse-log": "node ./dist/scripts/parse-log.js", + "plugin:install": "node ./dist/scripts/plugin/install.js", + "plugin:uninstall": "node ./dist/scripts/plugin/uninstall.js", + "postinstall": "test -n \"$NOCLIENT\" || (cd client && yarn install --pure-lockfile)", + "prune-storage": "LOGGER_LEVEL=warn node ./dist/scripts/prune-storage.js", + "regenerate-thumbnails": "node ./dist/scripts/regenerate-thumbnails.js", + "release-embed-api": "bash ./scripts/release-embed-api.sh", + "release": "bash ./scripts/release.sh", + "reset-password": "node ./dist/scripts/reset-password.js", + "resolve-tspaths:server-lib": "npm run resolve-tspaths -- --project server/tsconfig.lib.json --src server --out server/dist", + "resolve-tspaths:server": "npm run resolve-tspaths -- --project server/tsconfig.json --src server --out dist", + "resolve-tspaths:tests": "npm run resolve-tspaths -- --project packages/tests/tsconfig.json --src packages/tests/src --out packages/tests/dist", + "resolve-tspaths": "resolve-tspaths", + "simulate-many-viewers": "tsx --conditions=peertube:tsx ./scripts/simulate-many-viewers.ts", + "start:server": "node dist/server --no-client", + "start": "node dist/server", + "swagger-cli": "swagger-cli", + "test": "bash ./scripts/test.sh", + "tsc": "tsc", + "tsx": "tsx", + "update-host": "node ./dist/scripts/update-host.js", + "update-object-storage-url": "LOGGER_LEVEL=warn node ./dist/scripts/update-object-storage-url.js" }, "dependencies": { "@aws-sdk/client-s3": "^3.190.0", diff --git a/packages/server-commands/src/users/accounts.ts b/packages/server-commands/src/users/accounts.ts index 3b8b9d36a..15ad0ef52 100644 --- a/packages/server-commands/src/users/accounts.ts +++ b/packages/server-commands/src/users/accounts.ts @@ -1,15 +1,10 @@ +import { arrayify } from '@peertube/peertube-core-utils' import { PeerTubeServer } from '../server/server.js' -async function setDefaultAccountAvatar (serversArg: PeerTubeServer | PeerTubeServer[], token?: string) { - const servers = Array.isArray(serversArg) - ? serversArg - : [ serversArg ] +export async function setDefaultAccountAvatar (serversArg: PeerTubeServer | PeerTubeServer[], token?: string) { + const servers = arrayify(serversArg) for (const server of servers) { await server.users.updateMyAvatar({ fixture: 'avatar.png', token }) } } - -export { - setDefaultAccountAvatar -} diff --git a/packages/server-commands/src/videos/channels.ts b/packages/server-commands/src/videos/channels.ts index e3487d024..52f3a2265 100644 --- a/packages/server-commands/src/videos/channels.ts +++ b/packages/server-commands/src/videos/channels.ts @@ -1,6 +1,7 @@ +import { arrayify } from '@peertube/peertube-core-utils' import { PeerTubeServer } from '../server/server.js' -function setDefaultVideoChannel (servers: PeerTubeServer[]) { +export function setDefaultVideoChannel (servers: PeerTubeServer[]) { const tasks: Promise[] = [] for (const server of servers) { @@ -13,17 +14,10 @@ function setDefaultVideoChannel (servers: PeerTubeServer[]) { return Promise.all(tasks) } -async function setDefaultChannelAvatar (serversArg: PeerTubeServer | PeerTubeServer[], channelName: string = 'root_channel') { - const servers = Array.isArray(serversArg) - ? serversArg - : [ serversArg ] +export async function setDefaultChannelAvatar (serversArg: PeerTubeServer | PeerTubeServer[], channelName: string = 'root_channel') { + const servers = arrayify(serversArg) for (const server of servers) { await server.channels.updateImage({ channelName, fixture: 'avatar.png', type: 'avatar' }) } } - -export { - setDefaultVideoChannel, - setDefaultChannelAvatar -} diff --git a/packages/tests/src/cli/create-generate-storyboard-job.ts b/packages/tests/src/cli/create-generate-storyboard-job.ts index 5a1c61ef1..157738664 100644 --- a/packages/tests/src/cli/create-generate-storyboard-job.ts +++ b/packages/tests/src/cli/create-generate-storyboard-job.ts @@ -22,7 +22,7 @@ function listStoryboardFiles (server: PeerTubeServer) { return readdir(storage) } -describe('Test create generate storyboard job', function () { +describe('Test create generate storyboard job CLI', function () { let servers: PeerTubeServer[] = [] const uuids: string[] = [] let sql: SQLCommand diff --git a/packages/tests/src/cli/create-import-video-file-job.ts b/packages/tests/src/cli/create-import-video-file-job.ts index fa934510c..b670abf50 100644 --- a/packages/tests/src/cli/create-import-video-file-job.ts +++ b/packages/tests/src/cli/create-import-video-file-job.ts @@ -154,7 +154,7 @@ function runTests (enableObjectStorage: boolean) { }) } -describe('Test create import video jobs', function () { +describe('Test create import video jobs CLI', function () { describe('On filesystem', function () { runTests(false) diff --git a/packages/tests/src/cli/create-move-video-storage-job.ts b/packages/tests/src/cli/create-move-video-storage-job.ts index 2f72bc012..b0ccbf069 100644 --- a/packages/tests/src/cli/create-move-video-storage-job.ts +++ b/packages/tests/src/cli/create-move-video-storage-job.ts @@ -64,7 +64,7 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectSt } } -describe('Test create move video storage job', function () { +describe('Test create move video storage job CLI', function () { if (areMockObjectStorageTestsDisabled()) return let servers: PeerTubeServer[] = [] diff --git a/packages/tests/src/cli/house-keeping.ts b/packages/tests/src/cli/house-keeping.ts new file mode 100644 index 000000000..329d90d5a --- /dev/null +++ b/packages/tests/src/cli/house-keeping.ts @@ -0,0 +1,96 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { HttpStatusCode } from '@peertube/peertube-models' +import { + PeerTubeServer, + cleanupTests, + createMultipleServers, + doubleFollow, + makeGetRequest, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultChannelAvatar, + waitJobs +} from '@peertube/peertube-server-commands' +import { expect } from 'chai' + +describe('House keeping CLI', function () { + let servers: PeerTubeServer[] + + function runHouseKeeping (option: string) { + const env = servers[0].cli.getEnv() + const command = `echo y | ${env} npm run house-keeping -- ${option}` + + return servers[0].cli.execWithEnv(command) + } + + async function fetchRemoteData () { + { + const { data } = await servers[0].videos.list() + for (const video of data) { + await makeGetRequest({ url: servers[0].url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) + await makeGetRequest({ url: servers[0].url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 }) + } + } + + { + const { data: accounts } = await servers[0].accounts.list() + const { data: channels } = await servers[0].channels.list() + + for (const { avatars } of [ ...accounts, ...channels ]) { + for (const avatar of avatars) { + await makeGetRequest({ url: servers[0].url, path: avatar.path, expectedStatus: HttpStatusCode.OK_200 }) + } + } + } + } + + before(async function () { + this.timeout(360000) + + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + + await setDefaultAccountAvatar(servers) + await setDefaultChannelAvatar(servers) + + await servers[1].config.enableMinimumTranscoding() + + for (const server of servers) { + await server.videos.quickUpload({ name: 'video' }) + } + + await waitJobs(servers) + + await doubleFollow(servers[0], servers[1]) + }) + + it('Should have remote files locally', async function () { + this.timeout(120000) + + await fetchRemoteData() + + expect(await servers[0].servers.countFiles('thumbnails')).to.equal(2) + expect(await servers[0].servers.countFiles('avatars')).to.equal((2 + 2) * 4) // 2 accounts and 2 channels in 4 versions + }) + + it('Should remove remote files', async function () { + this.timeout(60000) + + await servers[0].kill() + await runHouseKeeping('--delete-remote-files') + await servers[0].run() + + expect(await servers[0].servers.countFiles('thumbnails')).to.equal(1) + expect(await servers[0].servers.countFiles('avatars')).to.equal((1 + 1) * 4) // 1 account and 1 channel in 4 versions + + await fetchRemoteData() + + expect(await servers[0].servers.countFiles('thumbnails')).to.equal(2) + expect(await servers[0].servers.countFiles('avatars')).to.equal((2 + 2) * 4) // 2 accounts and 2 channels in 4 versions + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/cli/index.ts b/packages/tests/src/cli/index.ts index 56c2bad33..fb3bb39af 100644 --- a/packages/tests/src/cli/index.ts +++ b/packages/tests/src/cli/index.ts @@ -2,6 +2,7 @@ import './create-import-video-file-job' import './create-generate-storyboard-job' import './create-move-video-storage-job' +import './house-keeping.js' import './peertube' import './plugins' import './prune-storage' diff --git a/packages/tests/src/cli/plugins.ts b/packages/tests/src/cli/plugins.ts index ab7f7dd85..c63371ee9 100644 --- a/packages/tests/src/cli/plugins.ts +++ b/packages/tests/src/cli/plugins.ts @@ -10,7 +10,7 @@ import { setAccessTokensToServers } from '@peertube/peertube-server-commands' -describe('Test plugin scripts', function () { +describe('Test plugin CLI', function () { let server: PeerTubeServer before(async function () { diff --git a/packages/tests/src/cli/prune-storage.ts b/packages/tests/src/cli/prune-storage.ts index f21979ad0..b36653036 100644 --- a/packages/tests/src/cli/prune-storage.ts +++ b/packages/tests/src/cli/prune-storage.ts @@ -23,7 +23,7 @@ import { createFile } from 'fs-extra/esm' import { readdir } from 'fs/promises' import { join } from 'path' -describe('Test prune storage scripts', function () { +describe('Test prune storage CLI', function () { let servers: PeerTubeServer[] before(async function () { diff --git a/packages/tests/src/cli/regenerate-thumbnails.ts b/packages/tests/src/cli/regenerate-thumbnails.ts index 1448e5cfc..976774915 100644 --- a/packages/tests/src/cli/regenerate-thumbnails.ts +++ b/packages/tests/src/cli/regenerate-thumbnails.ts @@ -26,7 +26,7 @@ async function testThumbnail (server: PeerTubeServer, videoId: number | string) } } -describe('Test regenerate thumbnails script', function () { +describe('Test regenerate thumbnails CLI', function () { let servers: PeerTubeServer[] let video1: Video diff --git a/packages/tests/src/cli/reset-password.ts b/packages/tests/src/cli/reset-password.ts index 62e1a37a0..30cf5e594 100644 --- a/packages/tests/src/cli/reset-password.ts +++ b/packages/tests/src/cli/reset-password.ts @@ -1,6 +1,6 @@ import { cleanupTests, CLICommand, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@peertube/peertube-server-commands' -describe('Test reset password scripts', function () { +describe('Test reset password CLI', function () { let server: PeerTubeServer before(async function () { diff --git a/packages/tests/src/cli/update-host.ts b/packages/tests/src/cli/update-host.ts index 38b160d2c..30a7b17e4 100644 --- a/packages/tests/src/cli/update-host.ts +++ b/packages/tests/src/cli/update-host.ts @@ -15,7 +15,7 @@ import { import { parseTorrentVideo } from '@tests/shared/webtorrent.js' import { VideoPlaylistPrivacy } from '@peertube/peertube-models' -describe('Test update host scripts', function () { +describe('Test update host CLI', function () { let server: PeerTubeServer before(async function () { diff --git a/packages/tests/src/cli/update-object-storage-url.ts b/packages/tests/src/cli/update-object-storage-url.ts index a1b550ed8..522f7f60c 100644 --- a/packages/tests/src/cli/update-object-storage-url.ts +++ b/packages/tests/src/cli/update-object-storage-url.ts @@ -14,7 +14,7 @@ import { import { expectStartWith } from '@tests/shared/checks.js' import { expect } from 'chai' -describe('Update object storage URL', function () { +describe('Update object storage URL CLI', function () { if (areMockObjectStorageTestsDisabled()) return let server: PeerTubeServer diff --git a/server/core/controllers/lazy-static.ts b/server/core/controllers/lazy-static.ts index 69aa549a7..9a1dc8f28 100644 --- a/server/core/controllers/lazy-static.ts +++ b/server/core/controllers/lazy-static.ts @@ -123,6 +123,5 @@ async function getTorrent (req: express.Request, res: express.Response) { const result = await VideoTorrentsSimpleFileCache.Instance.getFilePath(req.params.filename) if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end() - // Torrents still use the old naming convention (video uuid + .torrent) return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.SERVER }) } diff --git a/server/core/models/actor/actor-image.ts b/server/core/models/actor/actor-image.ts index d7e7b163e..bcc9853cd 100644 --- a/server/core/models/actor/actor-image.ts +++ b/server/core/models/actor/actor-image.ts @@ -3,6 +3,7 @@ import { getLowercaseExtension } from '@peertube/peertube-node-utils' import { MActorId, MActorImage, MActorImageFormattable } from '@server/types/models/index.js' import { remove } from 'fs-extra/esm' import { join } from 'path' +import { Op } from 'sequelize' import { AfterDestroy, AllowNull, @@ -76,10 +77,10 @@ export class ActorImageModel extends SequelizeModel { }, onDelete: 'CASCADE' }) - Actor: Awaited // Remove awaited: https://github.com/sequelize/sequelize-typescript/issues/825 + Actor: Awaited // TODO: Remove awaited: https://github.com/sequelize/sequelize-typescript/issues/825 @AfterDestroy - static removeFilesAndSendDelete (instance: ActorImageModel) { + static removeFile (instance: ActorImageModel) { logger.info('Removing actor image file %s.', instance.filename) // Don't block the transaction @@ -128,12 +129,34 @@ export class ActorImageModel extends SequelizeModel { return { avatars, banners } } + static listRemoteOnDisk () { + return this.findAll({ + where: { + onDisk: true + }, + include: [ + { + attributes: [ 'id' ], + model: ActorModel.unscoped(), + required: true, + where: { + serverId: { + [Op.ne]: null + } + } + } + ] + }) + } + static getImageUrl (image: MActorImage) { if (!image) return undefined return WEBSERVER.URL + image.getStaticPath() } + // --------------------------------------------------------------------------- + toFormattedJSON (this: MActorImageFormattable): ActorImage { return { width: this.width, diff --git a/server/core/models/video/thumbnail.ts b/server/core/models/video/thumbnail.ts index 86faa63a9..51a9971ef 100644 --- a/server/core/models/video/thumbnail.ts +++ b/server/core/models/video/thumbnail.ts @@ -160,12 +160,34 @@ export class ThumbnailModel extends SequelizeModel { return ThumbnailModel.findOne(query) } + static listRemoteOnDisk () { + return this.findAll({ + where: { + onDisk: true + }, + include: [ + { + attributes: [ 'id' ], + model: VideoModel.unscoped(), + required: true, + where: { + remote: true + } + } + ] + }) + } + + // --------------------------------------------------------------------------- + static buildPath (type: ThumbnailType_Type, filename: string) { const directory = ThumbnailModel.types[type].directory return join(directory, filename) } + // --------------------------------------------------------------------------- + getOriginFileUrl (videoOrPlaylist: MVideo | MVideoPlaylist) { const staticPath = ThumbnailModel.types[this.type].staticPath + this.filename diff --git a/server/scripts/house-keeping.ts b/server/scripts/house-keeping.ts new file mode 100644 index 000000000..2b6d43e11 --- /dev/null +++ b/server/scripts/house-keeping.ts @@ -0,0 +1,91 @@ +import { createCommand } from '@commander-js/extra-typings' +import { initDatabaseModels } from '@server/initializers/database.js' +import { ActorImageModel } from '@server/models/actor/actor-image.js' +import { ThumbnailModel } from '@server/models/video/thumbnail.js' +import { askConfirmation, displayPeerTubeMustBeStoppedWarning } from './shared/common.js' + +const program = createCommand() + .description('Remove unused objects from database or remote files') + .option('--delete-remote-files', 'Remove remote files (avatars, banners, thumbnails...)') + .parse(process.argv) + +const options = program.opts() + +if (!options.deleteRemoteFiles) { + console.log('At least one option must be set (for example --delete-remote-files).') + process.exit(0) +} + +run() + .then(() => process.exit(0)) + .catch(err => { + console.error(err) + process.exit(-1) + }) + +async function run () { + await initDatabaseModels(true) + + displayPeerTubeMustBeStoppedWarning() + + if (options.deleteRemoteFiles) { + return deleteRemoteFiles() + } +} + +async function deleteRemoteFiles () { + console.log('Detecting remote files that can be deleted...') + + const thumbnails = await ThumbnailModel.listRemoteOnDisk() + const actorImages = await ActorImageModel.listRemoteOnDisk() + + if (thumbnails.length === 0 && actorImages.length === 0) { + console.log('No remote files to delete detected.') + process.exit(0) + } + + const res = await askConfirmation( + `${thumbnails.length} thumbnails and ${actorImages.length} avatars/banners can be locally deleted. ` + + `PeerTube will download them again on-demand.` + + `Do you want to delete these remote files?` + ) + + if (res !== true) { + console.log('Exiting without delete remote files.') + process.exit(0) + } + + // --------------------------------------------------------------------------- + + console.log('Deleting remote thumbnails...') + + for (const thumbnail of thumbnails) { + if (!thumbnail.fileUrl) { + console.log(`Skipping thumbnail removal of ${thumbnail.getPath()} as we don't have its remote file URL in the database.`) + continue + } + + await thumbnail.removeThumbnail() + + thumbnail.onDisk = false + await thumbnail.save() + } + + // --------------------------------------------------------------------------- + + console.log('Deleting remote avatars/banners...') + + for (const actorImage of actorImages) { + if (!actorImage.fileUrl) { + console.log(`Skipping avatar/banner removal of ${actorImage.getPath()} as we don't have its remote file URL in the database.`) + continue + } + + await actorImage.removeImage() + + actorImage.onDisk = false + await actorImage.save() + } + + console.log('Remote files deleted!') +} diff --git a/server/scripts/prune-storage.ts b/server/scripts/prune-storage.ts index dc3c3a4e7..c3dafcc69 100755 --- a/server/scripts/prune-storage.ts +++ b/server/scripts/prune-storage.ts @@ -10,7 +10,6 @@ import Bluebird from 'bluebird' import { remove } from 'fs-extra/esm' import { readdir, stat } from 'fs/promises' import { basename, dirname, join } from 'path' -import prompt from 'prompt' import { getUUIDFromFilename } from '../core/helpers/utils.js' import { CONFIG } from '../core/initializers/config.js' import { initDatabaseModels } from '../core/initializers/database.js' @@ -18,6 +17,7 @@ import { ActorImageModel } from '../core/models/actor/actor-image.js' import { VideoRedundancyModel } from '../core/models/redundancy/video-redundancy.js' import { ThumbnailModel } from '../core/models/video/thumbnail.js' import { VideoModel } from '../core/models/video/video.js' +import { askConfirmation, displayPeerTubeMustBeStoppedWarning } from './shared/common.js' run() .then(() => process.exit(0)) @@ -29,6 +29,8 @@ run() async function run () { await initDatabaseModels(true) + displayPeerTubeMustBeStoppedWarning() + await new FSPruner().prune() console.log('\n') @@ -61,7 +63,7 @@ class ObjectStoragePruner { const formattedKeysToDelete = this.keysToDelete.map(({ bucket, key }) => ` In bucket ${bucket}: ${key}`).join('\n') console.log(`${this.keysToDelete.length} unknown files from object storage can be deleted:\n${formattedKeysToDelete}\n`) - const res = await askConfirmation() + const res = await askPruneConfirmation() if (res !== true) { console.log('Exiting without deleting object storage files.') return @@ -183,7 +185,7 @@ class FSPruner { const formattedKeysToDelete = this.pathsToDelete.map(p => ` ${p}`).join('\n') console.log(`${this.pathsToDelete.length} unknown files from filesystem can be deleted:\n${formattedKeysToDelete}\n`) - const res = await askConfirmation() + const res = await askPruneConfirmation() if (res !== true) { console.log('Exiting without deleting filesystem files.') return @@ -299,29 +301,9 @@ class FSPruner { } } -async function askConfirmation () { - return new Promise((res, rej) => { - prompt.start() - - const schema = { - properties: { - confirm: { - type: 'string', - description: 'These unknown files can be deleted, but please check your backups first (bugs happen).' + - ' Notice PeerTube must have been stopped when your ran this script.' + - ' Can we delete these files? (y/n)', - default: 'n', - validator: /y[es]*|n[o]?/, - warning: 'Must respond yes or no', - required: true - } - } - } - - prompt.get(schema, function (err, result) { - if (err) return rej(err) - - return res(result.confirm?.match(/y/) !== null) - }) - }) +async function askPruneConfirmation () { + return askConfirmation( + 'These unknown files can be deleted, but please check your backups first (bugs happen). ' + + 'Can we delete these files?' + ) } diff --git a/server/scripts/shared/common.ts b/server/scripts/shared/common.ts new file mode 100644 index 000000000..4bb28c002 --- /dev/null +++ b/server/scripts/shared/common.ts @@ -0,0 +1,30 @@ +import prompt from 'prompt' + +export async function askConfirmation (message: string) { + return new Promise((res, rej) => { + prompt.start() + + const schema = { + properties: { + confirm: { + type: 'string', + description: message + ' (y/n)', + default: 'n', + validator: /y[es]*|n[o]?/, + warning: 'Must respond yes or no', + required: true + } + } + } + + prompt.get(schema, function (err, result) { + if (err) return rej(err) + + return res(result.confirm?.match(/y/) !== null) + }) + }) +} + +export function displayPeerTubeMustBeStoppedWarning () { + console.log(`/!\\ PeerTube must be stopped before running this script /!\\\n`) +} diff --git a/server/scripts/update-object-storage-url.ts b/server/scripts/update-object-storage-url.ts index fc93aeaa7..76687c18f 100644 --- a/server/scripts/update-object-storage-url.ts +++ b/server/scripts/update-object-storage-url.ts @@ -4,7 +4,7 @@ import { FileStorage } from '@peertube/peertube-models' import { escapeForRegex } from '@server/helpers/regexp.js' import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js' import { QueryTypes } from 'sequelize' -import prompt from 'prompt' +import { askConfirmation, displayPeerTubeMustBeStoppedWarning } from './shared/common.js' const program = createCommand() .description('Update PeerTube object file URLs after an object storage migration.') @@ -24,6 +24,8 @@ run() async function run () { await initDatabaseModels(true) + displayPeerTubeMustBeStoppedWarning() + const fromRegexp = `^${escapeForRegex(options.from)}` const to = options.to @@ -58,7 +60,7 @@ async function run () { } } - const res = await askConfirmation() + const res = await askUpdateConfirmation() if (res !== true) { console.log('Exiting without updating URLs.') process.exit(0) @@ -90,29 +92,9 @@ function parseUrl (value: string) { return value } -async function askConfirmation () { - return new Promise((res, rej) => { - prompt.start() - - const schema = { - properties: { - confirm: { - type: 'string', - description: 'These URLs can be updated, but please check your backups first (bugs happen).' + - ' Notice PeerTube must have been stopped when your ran this script.' + - ' Can we update these URLs? (y/n)', - default: 'n', - validator: /y[es]*|n[o]?/, - warning: 'Must respond yes or no', - required: true - } - } - } - - prompt.get(schema, function (err, result) { - if (err) return rej(err) - - return res(result.confirm?.match(/y/) !== null) - }) - }) +async function askUpdateConfirmation () { + return askConfirmation( + 'These URLs can be updated, but please check your backups first (bugs happen). ' + + 'Can we update these URLs?' + ) } diff --git a/support/doc/tools.md b/support/doc/tools.md index 49dac76be..0ecbd83bc 100644 --- a/support/doc/tools.md +++ b/support/doc/tools.md @@ -494,6 +494,25 @@ docker compose exec -u peertube peertube npm run update-object-storage-url -- -- ::: +### Cleanup remote files + +**PeerTube >= 6.2** + +Use this script to recover disk space by removing remote files (thumbnails, avatars...) that can be re-fetched later by your PeerTube instance on-demand: + +```bash [Classic installation] +cd /var/www/peertube/peertube-latest +sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run house-keeping -- --delete-remote-files +``` + +```bash [Docker] +cd /var/www/peertube-docker +docker compose exec -u peertube peertube npm run house-keeping -- --delete-remote-files +``` + +::: + + ### Generate storyboard **PeerTube >= 6.0**