PeerTube/server/tests/cli/prune-storage.ts

215 lines
5.7 KiB
TypeScript
Raw Normal View History

2020-01-31 16:56:52 +01:00
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2019-08-09 15:04:36 +02:00
import 'mocha'
import * as chai from 'chai'
import { createFile, readdir } from 'fs-extra'
import { join } from 'path'
import { buildUUID } from '@server/helpers/uuid'
2019-08-09 15:04:36 +02:00
import {
cleanupTests,
2021-07-05 16:37:50 +02:00
CLICommand,
2021-07-16 09:47:51 +02:00
createMultipleServers,
2021-07-16 14:27:30 +02:00
doubleFollow,
2021-06-17 11:09:54 +02:00
killallServers,
2020-01-31 16:56:52 +01:00
makeGetRequest,
2021-07-16 09:47:51 +02:00
PeerTubeServer,
2020-01-31 16:56:52 +01:00
setAccessTokensToServers,
setDefaultVideoChannel,
2021-07-07 13:38:26 +02:00
wait,
waitJobs
} from '@shared/extra-utils'
2021-07-16 14:27:30 +02:00
import { HttpStatusCode, VideoPlaylistPrivacy } from '@shared/models'
2019-08-09 15:04:36 +02:00
const expect = chai.expect
2021-07-16 09:47:51 +02:00
async function countFiles (server: PeerTubeServer, directory: string) {
2021-07-16 09:04:35 +02:00
const files = await readdir(server.servers.buildDirectory(directory))
2019-08-09 15:04:36 +02:00
return files.length
}
2021-07-16 09:47:51 +02:00
async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) {
2021-07-16 09:04:35 +02:00
const files = await readdir(server.servers.buildDirectory(directory))
2019-08-09 15:04:36 +02:00
for (const f of files) {
expect(f).to.not.contain(substring)
}
}
2021-12-10 10:28:46 +01:00
async function assertCountAreOkay (servers: PeerTubeServer[]) {
2019-08-09 15:04:36 +02:00
for (const server of servers) {
2021-07-13 09:43:59 +02:00
const videosCount = await countFiles(server, 'videos')
2019-08-09 15:04:36 +02:00
expect(videosCount).to.equal(8)
2021-07-13 09:43:59 +02:00
const torrentsCount = await countFiles(server, 'torrents')
2019-11-21 12:16:27 +01:00
expect(torrentsCount).to.equal(16)
2019-08-09 15:04:36 +02:00
2021-07-13 09:43:59 +02:00
const previewsCount = await countFiles(server, 'previews')
2019-08-09 15:04:36 +02:00
expect(previewsCount).to.equal(2)
2021-07-13 09:43:59 +02:00
const thumbnailsCount = await countFiles(server, 'thumbnails')
2019-08-09 15:04:36 +02:00
expect(thumbnailsCount).to.equal(6)
2021-07-13 09:43:59 +02:00
const avatarsCount = await countFiles(server, 'avatars')
2019-08-09 15:04:36 +02:00
expect(avatarsCount).to.equal(2)
2021-07-23 11:20:00 +02:00
2021-12-10 10:28:46 +01:00
const hlsRootCount = await countFiles(server, 'streaming-playlists/hls')
expect(hlsRootCount).to.equal(2)
}
2019-08-09 15:04:36 +02:00
}
describe('Test prune storage scripts', function () {
2021-07-16 09:47:51 +02:00
let servers: PeerTubeServer[]
2020-01-31 16:56:52 +01:00
const badNames: { [directory: string]: string[] } = {}
2019-08-09 15:04:36 +02:00
before(async function () {
this.timeout(120000)
2021-07-16 09:47:51 +02:00
servers = await createMultipleServers(2, { transcoding: { enabled: true } })
2019-08-09 15:04:36 +02:00
await setAccessTokensToServers(servers)
await setDefaultVideoChannel(servers)
for (const server of servers) {
2021-07-16 09:04:35 +02:00
await server.videos.upload({ attributes: { name: 'video 1' } })
2021-12-10 10:28:46 +01:00
await server.videos.upload({ attributes: { name: 'video 2' } })
2019-08-09 15:04:36 +02:00
2021-07-16 09:04:35 +02:00
await server.users.updateMyAvatar({ fixture: 'avatar.png' })
2019-08-09 15:04:36 +02:00
2021-07-16 09:04:35 +02:00
await server.playlists.create({
2021-07-08 15:54:39 +02:00
attributes: {
2019-08-09 15:04:36 +02:00
displayName: 'playlist',
privacy: VideoPlaylistPrivacy.PUBLIC,
2021-07-16 09:04:35 +02:00
videoChannelId: server.store.channel.id,
2019-08-09 15:04:36 +02:00
thumbnailfile: 'thumbnail.jpg'
}
})
}
await doubleFollow(servers[0], servers[1])
// Lazy load the remote avatar
{
2021-07-16 09:04:35 +02:00
const account = await servers[0].accounts.get({ accountName: 'root@localhost:' + servers[1].port })
2019-11-21 12:16:27 +01:00
await makeGetRequest({
2020-01-31 16:56:52 +01:00
url: servers[0].url,
2019-11-21 12:16:27 +01:00
path: account.avatar.path,
2021-07-16 10:42:24 +02:00
expectedStatus: HttpStatusCode.OK_200
2019-11-21 12:16:27 +01:00
})
2019-08-09 15:04:36 +02:00
}
{
2021-07-16 09:04:35 +02:00
const account = await servers[1].accounts.get({ accountName: 'root@localhost:' + servers[0].port })
2019-11-21 12:16:27 +01:00
await makeGetRequest({
2020-01-31 16:56:52 +01:00
url: servers[1].url,
2019-11-21 12:16:27 +01:00
path: account.avatar.path,
2021-07-16 10:42:24 +02:00
expectedStatus: HttpStatusCode.OK_200
2019-11-21 12:16:27 +01:00
})
2019-08-09 15:04:36 +02:00
}
await wait(1000)
await waitJobs(servers)
2021-07-09 15:37:43 +02:00
await killallServers(servers)
2021-06-17 11:09:54 +02:00
await wait(1000)
2019-08-09 15:04:36 +02:00
})
it('Should have the files on the disk', async function () {
2021-12-10 10:28:46 +01:00
await assertCountAreOkay(servers)
2019-08-09 15:04:36 +02:00
})
it('Should create some dirty files', async function () {
for (let i = 0; i < 2; i++) {
{
2021-07-16 09:04:35 +02:00
const base = servers[0].servers.buildDirectory('videos')
2019-08-09 15:04:36 +02:00
const n1 = buildUUID() + '.mp4'
const n2 = buildUUID() + '.webm'
2019-08-09 15:04:36 +02:00
await createFile(join(base, n1))
await createFile(join(base, n2))
badNames['videos'] = [ n1, n2 ]
}
{
2021-07-16 09:04:35 +02:00
const base = servers[0].servers.buildDirectory('torrents')
2019-08-09 15:04:36 +02:00
const n1 = buildUUID() + '-240.torrent'
const n2 = buildUUID() + '-480.torrent'
2019-08-09 15:04:36 +02:00
await createFile(join(base, n1))
await createFile(join(base, n2))
badNames['torrents'] = [ n1, n2 ]
}
{
2021-07-16 09:04:35 +02:00
const base = servers[0].servers.buildDirectory('thumbnails')
2019-08-09 15:04:36 +02:00
const n1 = buildUUID() + '.jpg'
const n2 = buildUUID() + '.jpg'
2019-08-09 15:04:36 +02:00
await createFile(join(base, n1))
await createFile(join(base, n2))
badNames['thumbnails'] = [ n1, n2 ]
}
{
2021-07-16 09:04:35 +02:00
const base = servers[0].servers.buildDirectory('previews')
2019-08-09 15:04:36 +02:00
const n1 = buildUUID() + '.jpg'
const n2 = buildUUID() + '.jpg'
2019-08-09 15:04:36 +02:00
await createFile(join(base, n1))
await createFile(join(base, n2))
badNames['previews'] = [ n1, n2 ]
}
{
2021-07-16 09:04:35 +02:00
const base = servers[0].servers.buildDirectory('avatars')
2019-08-09 15:04:36 +02:00
const n1 = buildUUID() + '.png'
const n2 = buildUUID() + '.jpg'
2019-08-09 15:04:36 +02:00
await createFile(join(base, n1))
await createFile(join(base, n2))
badNames['avatars'] = [ n1, n2 ]
}
2021-07-23 11:20:00 +02:00
2021-12-10 10:28:46 +01:00
{
const directory = join('streaming-playlists', 'hls')
const base = servers[0].servers.buildDirectory(directory)
2021-07-23 11:20:00 +02:00
2021-12-10 10:28:46 +01:00
const n1 = buildUUID()
await createFile(join(base, n1))
badNames[directory] = [ n1 ]
}
2019-08-09 15:04:36 +02:00
}
})
it('Should run prune storage', async function () {
this.timeout(30000)
2021-07-16 09:04:35 +02:00
const env = servers[0].cli.getEnv()
2021-07-05 16:37:50 +02:00
await CLICommand.exec(`echo y | ${env} npm run prune-storage`)
2019-08-09 15:04:36 +02:00
})
it('Should have removed files', async function () {
2021-12-10 10:28:46 +01:00
await assertCountAreOkay(servers)
2019-08-09 15:04:36 +02:00
for (const directory of Object.keys(badNames)) {
for (const name of badNames[directory]) {
2021-07-13 09:43:59 +02:00
await assertNotExists(servers[0], directory, name)
2019-08-09 15:04:36 +02:00
}
}
})
after(async function () {
await cleanupTests(servers)
})
})