mirror of https://github.com/Chocobozzz/PeerTube
Fix broken object storage playlist on file removal
Chocobozzz
parent
bd60f178af
commit
b2bb45cf91
|
|
@ -1,16 +1,18 @@
|
|||
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
|
||||
|
||||
import { expect } from 'chai'
|
||||
import { HttpStatusCode } from '@peertube/peertube-models'
|
||||
import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'
|
||||
import {
|
||||
cleanupTests,
|
||||
createMultipleServers,
|
||||
doubleFollow,
|
||||
makeRawRequest,
|
||||
ObjectStorageCommand,
|
||||
PeerTubeServer,
|
||||
setAccessTokensToServers,
|
||||
waitJobs
|
||||
} from '@peertube/peertube-server-commands'
|
||||
import { expect } from 'chai'
|
||||
|
||||
describe('Test videos files', function () {
|
||||
let servers: PeerTubeServer[]
|
||||
|
|
@ -28,173 +30,199 @@ describe('Test videos files', function () {
|
|||
await servers[0].config.enableTranscoding({ hls: true, webVideo: true, resolutions: 'max' })
|
||||
})
|
||||
|
||||
describe('When deleting all files', function () {
|
||||
let validId1: string
|
||||
let validId2: string
|
||||
function runTests (objectStorage?: ObjectStorageCommand) {
|
||||
|
||||
before(async function () {
|
||||
this.timeout(360_000)
|
||||
describe('When deleting all files', function () {
|
||||
let validId1: string
|
||||
let validId2: string
|
||||
|
||||
{
|
||||
const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
|
||||
validId1 = uuid
|
||||
}
|
||||
before(async function () {
|
||||
this.timeout(360_000)
|
||||
|
||||
{
|
||||
const { uuid } = await servers[0].videos.quickUpload({ name: 'video 2' })
|
||||
validId2 = uuid
|
||||
}
|
||||
{
|
||||
const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
|
||||
validId1 = uuid
|
||||
}
|
||||
|
||||
await waitJobs(servers)
|
||||
{
|
||||
const { uuid } = await servers[0].videos.quickUpload({ name: 'video 2' })
|
||||
validId2 = uuid
|
||||
}
|
||||
|
||||
await waitJobs(servers)
|
||||
})
|
||||
|
||||
it('Should delete web video files', async function () {
|
||||
this.timeout(30_000)
|
||||
|
||||
await servers[0].videos.removeAllWebVideoFiles({ videoId: validId1 })
|
||||
|
||||
await waitJobs(servers)
|
||||
|
||||
for (const server of servers) {
|
||||
const video = await server.videos.get({ id: validId1 })
|
||||
|
||||
expect(video.files).to.have.lengthOf(0)
|
||||
expect(video.streamingPlaylists).to.have.lengthOf(1)
|
||||
}
|
||||
})
|
||||
|
||||
it('Should delete HLS files', async function () {
|
||||
this.timeout(30_000)
|
||||
|
||||
await servers[0].videos.removeHLSPlaylist({ videoId: validId2 })
|
||||
|
||||
await waitJobs(servers)
|
||||
|
||||
for (const server of servers) {
|
||||
const video = await server.videos.get({ id: validId2 })
|
||||
|
||||
expect(video.files).to.have.length.above(0)
|
||||
expect(video.streamingPlaylists).to.have.lengthOf(0)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('Should delete web video files', async function () {
|
||||
this.timeout(30_000)
|
||||
describe('When deleting a specific file', function () {
|
||||
let webVideoId: string
|
||||
let hlsId: string
|
||||
|
||||
await servers[0].videos.removeAllWebVideoFiles({ videoId: validId1 })
|
||||
before(async function () {
|
||||
this.timeout(120_000)
|
||||
|
||||
await waitJobs(servers)
|
||||
{
|
||||
const { uuid } = await servers[0].videos.quickUpload({ name: 'web-video' })
|
||||
webVideoId = uuid
|
||||
}
|
||||
|
||||
for (const server of servers) {
|
||||
const video = await server.videos.get({ id: validId1 })
|
||||
{
|
||||
const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' })
|
||||
hlsId = uuid
|
||||
}
|
||||
|
||||
expect(video.files).to.have.lengthOf(0)
|
||||
expect(video.streamingPlaylists).to.have.lengthOf(1)
|
||||
}
|
||||
await waitJobs(servers)
|
||||
})
|
||||
|
||||
it('Shoulde delete a web video file', async function () {
|
||||
this.timeout(30_000)
|
||||
|
||||
const video = await servers[0].videos.get({ id: webVideoId })
|
||||
const files = video.files
|
||||
|
||||
const toDelete = files[0]
|
||||
await servers[0].videos.removeWebVideoFile({ videoId: webVideoId, fileId: toDelete.id })
|
||||
|
||||
await waitJobs(servers)
|
||||
|
||||
for (const server of servers) {
|
||||
const video = await server.videos.get({ id: webVideoId })
|
||||
|
||||
expect(video.files).to.have.lengthOf(files.length - 1)
|
||||
expect(video.files.find(f => f.resolution.id === toDelete.resolution.id)).to.not.exist
|
||||
}
|
||||
})
|
||||
|
||||
it('Should delete all web video files', async function () {
|
||||
this.timeout(30_000)
|
||||
|
||||
const video = await servers[0].videos.get({ id: webVideoId })
|
||||
const files = video.files
|
||||
|
||||
for (const file of files) {
|
||||
await servers[0].videos.removeWebVideoFile({ videoId: webVideoId, fileId: file.id })
|
||||
}
|
||||
|
||||
await waitJobs(servers)
|
||||
|
||||
for (const server of servers) {
|
||||
const video = await server.videos.get({ id: webVideoId })
|
||||
|
||||
expect(video.files).to.have.lengthOf(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('Should delete a hls file', async function () {
|
||||
this.timeout(30_000)
|
||||
|
||||
const video = await servers[0].videos.get({ id: hlsId })
|
||||
const files = video.streamingPlaylists[0].files
|
||||
const toDelete = files[0]
|
||||
|
||||
await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: toDelete.id })
|
||||
|
||||
await waitJobs(servers)
|
||||
|
||||
for (const server of servers) {
|
||||
const video = await server.videos.get({ id: hlsId })
|
||||
|
||||
expect(video.streamingPlaylists[0].files).to.have.lengthOf(files.length - 1)
|
||||
expect(video.streamingPlaylists[0].files.find(f => f.resolution.id === toDelete.resolution.id)).to.not.exist
|
||||
|
||||
const m3u8Content = await servers[0].streamingPlaylists.get({ url: video.streamingPlaylists[0].playlistUrl })
|
||||
await makeRawRequest({ url: video.streamingPlaylists[0].segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
|
||||
|
||||
expect(m3u8Content.includes(`-${toDelete.resolution.id}.m3u8`)).to.be.false
|
||||
expect(m3u8Content.includes(`-${video.streamingPlaylists[0].files[0].resolution.id}.m3u8`)).to.be.true
|
||||
}
|
||||
})
|
||||
|
||||
it('Should delete all hls files', async function () {
|
||||
this.timeout(30_000)
|
||||
|
||||
const video = await servers[0].videos.get({ id: hlsId })
|
||||
const files = video.streamingPlaylists[0].files
|
||||
|
||||
for (const file of files) {
|
||||
await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: file.id })
|
||||
}
|
||||
|
||||
await waitJobs(servers)
|
||||
|
||||
for (const server of servers) {
|
||||
const video = await server.videos.get({ id: hlsId })
|
||||
|
||||
expect(video.streamingPlaylists).to.have.lengthOf(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('Should not delete last file of a video', async function () {
|
||||
this.timeout(60_000)
|
||||
|
||||
const webVideoOnly = await servers[0].videos.get({ id: hlsId })
|
||||
const hlsOnly = await servers[0].videos.get({ id: webVideoId })
|
||||
|
||||
for (let i = 0; i < 4; i++) {
|
||||
await servers[0].videos.removeWebVideoFile({ videoId: webVideoOnly.id, fileId: webVideoOnly.files[i].id })
|
||||
await servers[0].videos.removeHLSFile({ videoId: hlsOnly.id, fileId: hlsOnly.streamingPlaylists[0].files[i].id })
|
||||
}
|
||||
|
||||
const expectedStatus = HttpStatusCode.BAD_REQUEST_400
|
||||
await servers[0].videos.removeWebVideoFile({ videoId: webVideoOnly.id, fileId: webVideoOnly.files[4].id, expectedStatus })
|
||||
await servers[0].videos.removeHLSFile({ videoId: hlsOnly.id, fileId: hlsOnly.streamingPlaylists[0].files[4].id, expectedStatus })
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
it('Should delete HLS files', async function () {
|
||||
this.timeout(30_000)
|
||||
|
||||
await servers[0].videos.removeHLSPlaylist({ videoId: validId2 })
|
||||
|
||||
await waitJobs(servers)
|
||||
|
||||
for (const server of servers) {
|
||||
const video = await server.videos.get({ id: validId2 })
|
||||
|
||||
expect(video.files).to.have.length.above(0)
|
||||
expect(video.streamingPlaylists).to.have.lengthOf(0)
|
||||
}
|
||||
})
|
||||
describe('Using filesystem', function () {
|
||||
runTests()
|
||||
})
|
||||
|
||||
describe('When deleting a specific file', function () {
|
||||
let webVideoId: string
|
||||
let hlsId: string
|
||||
describe('Using object storage', function () {
|
||||
if (areMockObjectStorageTestsDisabled()) return
|
||||
|
||||
before(async function () {
|
||||
this.timeout(120_000)
|
||||
this.timeout(120000)
|
||||
|
||||
{
|
||||
const { uuid } = await servers[0].videos.quickUpload({ name: 'web-video' })
|
||||
webVideoId = uuid
|
||||
}
|
||||
const configOverride = objectStorage.getDefaultMockConfig()
|
||||
await objectStorage.prepareDefaultMockBuckets()
|
||||
|
||||
{
|
||||
const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' })
|
||||
hlsId = uuid
|
||||
}
|
||||
|
||||
await waitJobs(servers)
|
||||
await servers[0].kill()
|
||||
await servers[0].run(configOverride)
|
||||
})
|
||||
|
||||
it('Shoulde delete a web video file', async function () {
|
||||
this.timeout(30_000)
|
||||
const objectStorage = new ObjectStorageCommand()
|
||||
|
||||
const video = await servers[0].videos.get({ id: webVideoId })
|
||||
const files = video.files
|
||||
|
||||
const toDelete = files[0]
|
||||
await servers[0].videos.removeWebVideoFile({ videoId: webVideoId, fileId: toDelete.id })
|
||||
|
||||
await waitJobs(servers)
|
||||
|
||||
for (const server of servers) {
|
||||
const video = await server.videos.get({ id: webVideoId })
|
||||
|
||||
expect(video.files).to.have.lengthOf(files.length - 1)
|
||||
expect(video.files.find(f => f.resolution.id === toDelete.resolution.id)).to.not.exist
|
||||
}
|
||||
})
|
||||
|
||||
it('Should delete all web video files', async function () {
|
||||
this.timeout(30_000)
|
||||
|
||||
const video = await servers[0].videos.get({ id: webVideoId })
|
||||
const files = video.files
|
||||
|
||||
for (const file of files) {
|
||||
await servers[0].videos.removeWebVideoFile({ videoId: webVideoId, fileId: file.id })
|
||||
}
|
||||
|
||||
await waitJobs(servers)
|
||||
|
||||
for (const server of servers) {
|
||||
const video = await server.videos.get({ id: webVideoId })
|
||||
|
||||
expect(video.files).to.have.lengthOf(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('Should delete a hls file', async function () {
|
||||
this.timeout(30_000)
|
||||
|
||||
const video = await servers[0].videos.get({ id: hlsId })
|
||||
const files = video.streamingPlaylists[0].files
|
||||
const toDelete = files[0]
|
||||
|
||||
await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: toDelete.id })
|
||||
|
||||
await waitJobs(servers)
|
||||
|
||||
for (const server of servers) {
|
||||
const video = await server.videos.get({ id: hlsId })
|
||||
|
||||
expect(video.streamingPlaylists[0].files).to.have.lengthOf(files.length - 1)
|
||||
expect(video.streamingPlaylists[0].files.find(f => f.resolution.id === toDelete.resolution.id)).to.not.exist
|
||||
|
||||
const { text } = await makeRawRequest({ url: video.streamingPlaylists[0].playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
|
||||
|
||||
expect(text.includes(`-${toDelete.resolution.id}.m3u8`)).to.be.false
|
||||
expect(text.includes(`-${video.streamingPlaylists[0].files[0].resolution.id}.m3u8`)).to.be.true
|
||||
}
|
||||
})
|
||||
|
||||
it('Should delete all hls files', async function () {
|
||||
this.timeout(30_000)
|
||||
|
||||
const video = await servers[0].videos.get({ id: hlsId })
|
||||
const files = video.streamingPlaylists[0].files
|
||||
|
||||
for (const file of files) {
|
||||
await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: file.id })
|
||||
}
|
||||
|
||||
await waitJobs(servers)
|
||||
|
||||
for (const server of servers) {
|
||||
const video = await server.videos.get({ id: hlsId })
|
||||
|
||||
expect(video.streamingPlaylists).to.have.lengthOf(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('Should not delete last file of a video', async function () {
|
||||
this.timeout(60_000)
|
||||
|
||||
const webVideoOnly = await servers[0].videos.get({ id: hlsId })
|
||||
const hlsOnly = await servers[0].videos.get({ id: webVideoId })
|
||||
|
||||
for (let i = 0; i < 4; i++) {
|
||||
await servers[0].videos.removeWebVideoFile({ videoId: webVideoOnly.id, fileId: webVideoOnly.files[i].id })
|
||||
await servers[0].videos.removeHLSFile({ videoId: hlsOnly.id, fileId: hlsOnly.streamingPlaylists[0].files[i].id })
|
||||
}
|
||||
|
||||
const expectedStatus = HttpStatusCode.BAD_REQUEST_400
|
||||
await servers[0].videos.removeWebVideoFile({ videoId: webVideoOnly.id, fileId: webVideoOnly.files[4].id, expectedStatus })
|
||||
await servers[0].videos.removeHLSFile({ videoId: hlsOnly.id, fileId: hlsOnly.streamingPlaylists[0].files[4].id, expectedStatus })
|
||||
})
|
||||
runTests(objectStorage)
|
||||
})
|
||||
|
||||
after(async function () {
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers
|
|||
import { sequelizeTypescript } from '../initializers/database.js'
|
||||
import { VideoFileModel } from '../models/video/video-file.js'
|
||||
import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist.js'
|
||||
import { storeHLSFileFromFilename } from './object-storage/index.js'
|
||||
import { storeHLSFileFromContent } from './object-storage/index.js'
|
||||
import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths.js'
|
||||
import { VideoPathManager } from './video-path-manager.js'
|
||||
|
||||
|
|
@ -121,14 +121,17 @@ export function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingP
|
|||
}
|
||||
playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive)
|
||||
|
||||
const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename)
|
||||
await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
|
||||
|
||||
logger.info('Updating %s master playlist file of video %s', masterPlaylistPath, video.uuid, lTags(video.uuid))
|
||||
const masterPlaylistContent = masterPlaylists.join('\n') + '\n'
|
||||
|
||||
if (playlist.storage === FileStorage.OBJECT_STORAGE) {
|
||||
playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename)
|
||||
await remove(masterPlaylistPath)
|
||||
playlist.playlistUrl = await storeHLSFileFromContent(playlist, playlist.playlistFilename, masterPlaylistContent)
|
||||
|
||||
logger.info(`Updated master playlist file of video ${video.uuid} to object storage ${playlist.playlistUrl}`, lTags(video.uuid))
|
||||
} else {
|
||||
const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename)
|
||||
await writeFile(masterPlaylistPath, masterPlaylistContent)
|
||||
|
||||
logger.info(`Updated master playlist file ${masterPlaylistPath} of video ${video.uuid}`, lTags(video.uuid))
|
||||
}
|
||||
|
||||
return playlist.save()
|
||||
|
|
@ -174,12 +177,11 @@ export function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingP
|
|||
}
|
||||
playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive)
|
||||
|
||||
const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
|
||||
await outputJSON(outputPath, json)
|
||||
|
||||
if (playlist.storage === FileStorage.OBJECT_STORAGE) {
|
||||
playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlist, playlist.segmentsSha256Filename)
|
||||
await remove(outputPath)
|
||||
playlist.segmentsSha256Url = await storeHLSFileFromContent(playlist, playlist.segmentsSha256Filename, JSON.stringify(json))
|
||||
} else {
|
||||
const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
|
||||
await outputJSON(outputPath, json)
|
||||
}
|
||||
|
||||
return playlist.save()
|
||||
|
|
|
|||
|
|
@ -62,14 +62,13 @@ async function storeObject (options: {
|
|||
|
||||
async function storeContent (options: {
|
||||
content: string
|
||||
inputPath: string
|
||||
objectStorageKey: string
|
||||
bucketInfo: BucketInfo
|
||||
isPrivate: boolean
|
||||
}): Promise<string> {
|
||||
const { content, objectStorageKey, bucketInfo, inputPath, isPrivate } = options
|
||||
const { content, objectStorageKey, bucketInfo, isPrivate } = options
|
||||
|
||||
logger.debug('Uploading %s content to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags())
|
||||
logger.debug('Uploading %s content to %s%s in bucket %s', content, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags())
|
||||
|
||||
return uploadToStorage({ objectStorageKey, content, bucketInfo, isPrivate })
|
||||
}
|
||||
|
|
|
|||
|
|
@ -49,11 +49,10 @@ export function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: s
|
|||
})
|
||||
}
|
||||
|
||||
export function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, path: string, content: string) {
|
||||
export function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, pathOrFilename: string, content: string) {
|
||||
return storeContent({
|
||||
content,
|
||||
inputPath: path,
|
||||
objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)),
|
||||
objectStorageKey: generateHLSObjectStorageKey(playlist, basename(pathOrFilename)),
|
||||
bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
|
||||
isPrivate: playlist.Video.hasPrivateStaticPath()
|
||||
})
|
||||
|
|
|
|||
Loading…
Reference in New Issue