OpenCloud
/
PeerTube
mirror of https://github.com/Chocobozzz/PeerTube
1
0
Fork 0
Code Issues Releases Wiki Activity

Prevent object storage mock conflicts 

When running tests in parallel
pull/5817/head
Chocobozzz 2023-05-23 10:49:45 +02:00
parent 41cde76bbf
commit f89189907b
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
20 changed files with 277 additions and 201 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server/tests/api/live/live-fast-restream.ts
View File

@ -65,8 +65,7 @@ describe('Fast restream in live', function () {
server,
videoUUID: liveVideoUUID,
segment: 1,
playlistNumber: 0,
objectStorage: false
playlistNumber: 0
})
return { ffmpegCommand, liveVideoUUID }

8
server/tests/api/live/live.ts
View File

@ -423,7 +423,6 @@ describe('Test live', function () {
servers,
liveVideoId,
resolutions: [ 720 ],
objectStorage: false,
transcoded: true
})
@ -459,7 +458,6 @@ describe('Test live', function () {
servers,
liveVideoId,
resolutions: resolutions.concat([ 720 ]),
objectStorage: false,
transcoded: true
})
@ -512,7 +510,6 @@ describe('Test live', function () {
servers,
liveVideoId,
resolutions,
objectStorage: false,
transcoded: true
})
@ -609,7 +606,6 @@ describe('Test live', function () {
servers,
liveVideoId,
resolutions,
objectStorage: false,
transcoded: true
})
@ -646,7 +642,6 @@ describe('Test live', function () {
servers,
liveVideoId,
resolutions: [ 720 ],
objectStorage: false,
transcoded: true
})
@ -720,8 +715,7 @@ describe('Test live', function () {
server: servers[0],
videoUUID,
playlistNumber: 0,
segment: 2,
objectStorage: false
segment: 2
})
}

58
server/tests/api/object-storage/live.ts
View File

@ -36,7 +36,14 @@ async function createLive (server: PeerTubeServer, permanent: boolean) {
return uuid
}
async function checkFilesExist (servers: PeerTubeServer[], videoUUID: string, numberOfFiles: number) {
async function checkFilesExist (options: {
servers: PeerTubeServer[]
videoUUID: string
numberOfFiles: number
objectStorage: ObjectStorageCommand
}) {
const { servers, videoUUID, numberOfFiles, objectStorage } = options
for (const server of servers) {
const video = await server.videos.get({ id: videoUUID })
@ -47,14 +54,21 @@ async function checkFilesExist (servers: PeerTubeServer[], videoUUID: string, nu
expect(files).to.have.lengthOf(numberOfFiles)
for (const file of files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
}
}
async function checkFilesCleanup (server: PeerTubeServer, videoUUID: string, resolutions: number[]) {
async function checkFilesCleanup (options: {
server: PeerTubeServer
videoUUID: string
resolutions: number[]
objectStorage: ObjectStorageCommand
}) {
const { server, videoUUID, resolutions, objectStorage } = options
const resolutionFiles = resolutions.map((_value, i) => `${i}.m3u8`)
for (const playlistName of [ 'master.m3u8' ].concat(resolutionFiles)) {
@ -62,7 +76,7 @@ async function checkFilesCleanup (server: PeerTubeServer, videoUUID: string, res
videoUUID,
playlistName,
expectedStatus: HttpStatusCode.NOT_FOUND_404,
objectStorage: true
objectStorage
})
}
@ -70,7 +84,7 @@ async function checkFilesCleanup (server: PeerTubeServer, videoUUID: string, res
videoUUID,
playlistNumber: 0,
segment: 0,
objectStorage: true,
objectStorage,
expectedStatus: HttpStatusCode.NOT_FOUND_404
})
}
@ -80,13 +94,13 @@ describe('Object storage for lives', function () {
let servers: PeerTubeServer[]
let sqlCommandServer1: SQLCommand
const objectStorage = new ObjectStorageCommand()
before(async function () {
this.timeout(120000)
await ObjectStorageCommand.prepareDefaultMockBuckets()
servers = await createMultipleServers(2, ObjectStorageCommand.getDefaultMockConfig())
await objectStorage.prepareDefaultMockBuckets()
servers = await createMultipleServers(2, objectStorage.getDefaultMockConfig())
await setAccessTokensToServers(servers)
await setDefaultVideoChannel(servers)
@ -119,7 +133,7 @@ describe('Object storage for lives', function () {
liveVideoId: videoUUID,
resolutions: [ 720 ],
transcoded: false,
objectStorage: true
objectStorage
})
await stopFfmpeg(ffmpegCommand)
@ -131,11 +145,11 @@ describe('Object storage for lives', function () {
await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUID)
await waitJobs(servers)
await checkFilesExist(servers, videoUUID, 1)
await checkFilesExist({ servers, videoUUID, numberOfFiles: 1, objectStorage })
})
it('Should have cleaned up live files from object storage', async function () {
await checkFilesCleanup(servers[0], videoUUID, [ 720 ])
await checkFilesCleanup({ server: servers[0], videoUUID, resolutions: [ 720 ], objectStorage })
})
})
@ -166,7 +180,7 @@ describe('Object storage for lives', function () {
liveVideoId: videoUUIDNonPermanent,
resolutions,
transcoded: true,
objectStorage: true
objectStorage
})
await stopFfmpeg(ffmpegCommand)
@ -178,11 +192,11 @@ describe('Object storage for lives', function () {
await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUIDNonPermanent)
await waitJobs(servers)
await checkFilesExist(servers, videoUUIDNonPermanent, 5)
await checkFilesExist({ servers, videoUUID: videoUUIDNonPermanent, numberOfFiles: 5, objectStorage })
})
it('Should have cleaned up live files from object storage', async function () {
await checkFilesCleanup(servers[0], videoUUIDNonPermanent, resolutions)
await checkFilesCleanup({ server: servers[0], videoUUID: videoUUIDNonPermanent, resolutions, objectStorage })
})
})
@ -206,7 +220,7 @@ describe('Object storage for lives', function () {
liveVideoId: videoUUIDPermanent,
resolutions,
transcoded: true,
objectStorage: true
objectStorage
})
await stopFfmpeg(ffmpegCommand)
@ -221,11 +235,11 @@ describe('Object storage for lives', function () {
const videoLiveDetails = await servers[0].videos.get({ id: videoUUIDPermanent })
const replay = await findExternalSavedVideo(servers[0], videoLiveDetails)
await checkFilesExist(servers, replay.uuid, 5)
await checkFilesExist({ servers, videoUUID: replay.uuid, numberOfFiles: 5, objectStorage })
})
it('Should have cleaned up live files from object storage', async function () {
await checkFilesCleanup(servers[0], videoUUIDPermanent, resolutions)
await checkFilesCleanup({ server: servers[0], videoUUID: videoUUIDPermanent, resolutions, objectStorage })
})
})
})
@ -238,9 +252,10 @@ describe('Object storage for lives', function () {
this.timeout(120000)
const port = await mockObjectStorageProxy.initialize()
baseMockUrl = `http://127.0.0.1:${port}/streaming-playlists`
const bucketName = objectStorage.getMockStreamingPlaylistsBucketName()
baseMockUrl = `http://127.0.0.1:${port}/${bucketName}`
await ObjectStorageCommand.createMockBucket('streaming-playlists')
await objectStorage.prepareDefaultMockBuckets()
const config = {
object_storage: {
@ -251,7 +266,7 @@ describe('Object storage for lives', function () {
credentials: ObjectStorageCommand.getMockCredentialsConfig(),
streaming_playlists: {
bucket_name: 'streaming-playlists',
bucket_name: bucketName,
prefix: '',
base_url: baseMockUrl
}
@ -279,7 +294,7 @@ describe('Object storage for lives', function () {
liveVideoId: videoUUIDPermanent,
resolutions: [ 720 ],
transcoded: true,
objectStorage: true,
objectStorage,
objectStorageBaseUrl: baseMockUrl
})
@ -289,6 +304,7 @@ describe('Object storage for lives', function () {
after(async function () {
await sqlCommandServer1.cleanup()
await objectStorage.cleanupMock()
await cleanupTests(servers)
})

13
server/tests/api/object-storage/video-imports.ts
View File

@ -32,13 +32,14 @@ describe('Object storage for video import', function () {
if (areMockObjectStorageTestsDisabled()) return
let server: PeerTubeServer
const objectStorage = new ObjectStorageCommand()
before(async function () {
this.timeout(120000)
await ObjectStorageCommand.prepareDefaultMockBuckets()
await objectStorage.prepareDefaultMockBuckets()
server = await createSingleServer(1, ObjectStorageCommand.getDefaultMockConfig())
server = await createSingleServer(1, objectStorage.getDefaultMockConfig())
await setAccessTokensToServers([ server ])
await setDefaultVideoChannel([ server ])
@ -64,7 +65,7 @@ describe('Object storage for video import', function () {
expect(video.streamingPlaylists).to.have.lengthOf(0)
const fileUrl = video.files[0].fileUrl
expectStartWith(fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
expectStartWith(fileUrl, objectStorage.getMockWebVideosBaseUrl())
await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.OK_200 })
})
@ -89,13 +90,13 @@ describe('Object storage for video import', function () {
expect(video.streamingPlaylists[0].files).to.have.lengthOf(5)
for (const file of video.files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
for (const file of video.streamingPlaylists[0].files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
@ -103,6 +104,8 @@ describe('Object storage for video import', function () {
})
after(async function () {
await objectStorage.cleanupMock()
await cleanupTests([ server ])
})
})

50
server/tests/api/object-storage/videos.ts
View File

@ -145,6 +145,7 @@ function runTestSuite (options: {
let servers: PeerTubeServer[]
let sqlCommands: SQLCommand[] = []
const objectStorage = new ObjectStorageCommand()
let keptUrls: string[] = []
@ -159,8 +160,8 @@ function runTestSuite (options: {
? `http://127.0.0.1:${port}`
: undefined
await ObjectStorageCommand.createMockBucket(options.playlistBucket)
await ObjectStorageCommand.createMockBucket(options.webtorrentBucket)
await objectStorage.createMockBucket(options.playlistBucket)
await objectStorage.createMockBucket(options.webtorrentBucket)
const config = {
object_storage: {
@ -275,6 +276,7 @@ function runTestSuite (options: {
after(async function () {
await mockObjectStorageProxy.terminate()
await objectStorage.cleanupMock()
for (const sqlCommand of sqlCommands) {
await sqlCommand.cleanup()
@ -287,26 +289,12 @@ function runTestSuite (options: {
describe('Object storage for videos', function () {
if (areMockObjectStorageTestsDisabled()) return
const objectStorage = new ObjectStorageCommand()
describe('Test config', function () {
let server: PeerTubeServer
const baseConfig = {
object_storage: {
enabled: true,
endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
region: ObjectStorageCommand.getMockRegion(),
credentials: ObjectStorageCommand.getMockCredentialsConfig(),
streaming_playlists: {
bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_MOCK_BUCKET
},
videos: {
bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_MOCK_BUCKET
}
}
}
const baseConfig = objectStorage.getDefaultMockConfig()
const badCredentials = {
access_key_id: 'AKIAIOSFODNN7EXAMPLE',
@ -334,7 +322,7 @@ describe('Object storage for videos', function () {
it('Should fail with bad credentials', async function () {
this.timeout(60000)
await ObjectStorageCommand.prepareDefaultMockBuckets()
await objectStorage.prepareDefaultMockBuckets()
const config = merge({}, baseConfig, {
object_storage: {
@ -358,7 +346,7 @@ describe('Object storage for videos', function () {
it('Should succeed with credentials from env', async function () {
this.timeout(60000)
await ObjectStorageCommand.prepareDefaultMockBuckets()
await objectStorage.prepareDefaultMockBuckets()
const config = merge({}, baseConfig, {
object_storage: {
@ -385,25 +373,27 @@ describe('Object storage for videos', function () {
await waitJobs([ server ], { skipDelayed: true })
const video = await server.videos.get({ id: uuid })
expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
expectStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl())
})
after(async function () {
await objectStorage.cleanupMock()
await cleanupTests([ server ])
})
})
describe('Test simple object storage', function () {
runTestSuite({
playlistBucket: 'streaming-playlists',
webtorrentBucket: 'videos'
playlistBucket: objectStorage.getMockBucketName('streaming-playlists'),
webtorrentBucket: objectStorage.getMockBucketName('videos')
})
})
describe('Test object storage with prefix', function () {
runTestSuite({
playlistBucket: 'mybucket',
webtorrentBucket: 'mybucket',
playlistBucket: objectStorage.getMockBucketName('mybucket'),
webtorrentBucket: objectStorage.getMockBucketName('mybucket'),
playlistPrefix: 'streaming-playlists_',
webtorrentPrefix: 'webtorrent_'
@ -412,8 +402,8 @@ describe('Object storage for videos', function () {
describe('Test object storage with prefix and base URL', function () {
runTestSuite({
playlistBucket: 'mybucket',
webtorrentBucket: 'mybucket',
playlistBucket: objectStorage.getMockBucketName('mybucket'),
webtorrentBucket: objectStorage.getMockBucketName('mybucket'),
playlistPrefix: 'streaming-playlists/',
webtorrentPrefix: 'webtorrent/',
@ -440,8 +430,8 @@ describe('Object storage for videos', function () {
runTestSuite({
maxUploadPart,
playlistBucket: 'streaming-playlists',
webtorrentBucket: 'videos',
playlistBucket: objectStorage.getMockBucketName('streaming-playlists'),
webtorrentBucket: objectStorage.getMockBucketName('videos'),
fixture
})
})

2
server/tests/api/server/follows.ts
View File

@ -279,7 +279,7 @@ describe('Test follows', function () {
})
it('Should upload a video on server 2 and 3 and propagate only the video of server 2', async function () {
this.timeout(120000)
this.timeout(160000)
await servers[1].videos.upload({ attributes: { name: 'server2' } })
await servers[2].videos.upload({ attributes: { name: 'server3' } })

16
server/tests/api/server/proxy.ts
View File

@ -122,38 +122,44 @@ describe('Test proxy', function () {
describe('Object storage', function () {
if (areMockObjectStorageTestsDisabled()) return
const objectStorage = new ObjectStorageCommand()
before(async function () {
this.timeout(30000)
await ObjectStorageCommand.prepareDefaultMockBuckets()
await objectStorage.prepareDefaultMockBuckets()
})
it('Should succeed to upload to object storage with the appropriate proxy config', async function () {
this.timeout(120000)
await servers[0].kill()
await servers[0].run(ObjectStorageCommand.getDefaultMockConfig(), { env: goodEnv })
await servers[0].run(objectStorage.getDefaultMockConfig(), { env: goodEnv })
const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
await waitJobs(servers)
const video = await servers[0].videos.get({ id: uuid })
expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
expectStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl())
})
it('Should fail to upload to object storage with a wrong proxy config', async function () {
this.timeout(120000)
await servers[0].kill()
await servers[0].run(ObjectStorageCommand.getDefaultMockConfig(), { env: badEnv })
await servers[0].run(objectStorage.getDefaultMockConfig(), { env: badEnv })
const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
await waitJobs(servers, { skipDelayed: true })
const video = await servers[0].videos.get({ id: uuid })
expectNotStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
expectNotStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl())
})
after(async function () {
await objectStorage.cleanupMock()
})
})

35
server/tests/api/transcoding/create-transcoding.ts
View File

@ -17,9 +17,9 @@ import {
waitJobs
} from '@shared/server-commands'
async function checkFilesInObjectStorage (video: VideoDetails) {
async function checkFilesInObjectStorage (objectStorage: ObjectStorageCommand, video: VideoDetails) {
for (const file of video.files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
@ -27,29 +27,30 @@ async function checkFilesInObjectStorage (video: VideoDetails) {
const hlsPlaylist = video.streamingPlaylists[0]
for (const file of hlsPlaylist.files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
expectStartWith(hlsPlaylist.playlistUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
expectStartWith(hlsPlaylist.playlistUrl, objectStorage.getMockPlaylistBaseUrl())
await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
expectStartWith(hlsPlaylist.segmentsSha256Url, ObjectStorageCommand.getMockPlaylistBaseUrl())
expectStartWith(hlsPlaylist.segmentsSha256Url, objectStorage.getMockPlaylistBaseUrl())
await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
}
function runTests (objectStorage: boolean) {
function runTests (enableObjectStorage: boolean) {
let servers: PeerTubeServer[] = []
let videoUUID: string
let publishedAt: string
let shouldBeDeleted: string[]
const objectStorage = new ObjectStorageCommand()
before(async function () {
this.timeout(120000)
const config = objectStorage
? ObjectStorageCommand.getDefaultMockConfig()
const config = enableObjectStorage
? objectStorage.getDefaultMockConfig()
: {}
// Run server 2 to have transcoding enabled
@ -60,7 +61,7 @@ function runTests (objectStorage: boolean) {
await doubleFollow(servers[0], servers[1])
if (objectStorage) await ObjectStorageCommand.prepareDefaultMockBuckets()
if (enableObjectStorage) await objectStorage.prepareDefaultMockBuckets()
const { shortUUID } = await servers[0].videos.quickUpload({ name: 'video' })
videoUUID = shortUUID
@ -91,7 +92,7 @@ function runTests (objectStorage: boolean) {
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5)
if (objectStorage) await checkFilesInObjectStorage(videoDetails)
if (enableObjectStorage) await checkFilesInObjectStorage(objectStorage, videoDetails)
}
})
@ -112,7 +113,7 @@ function runTests (objectStorage: boolean) {
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5)
if (objectStorage) await checkFilesInObjectStorage(videoDetails)
if (enableObjectStorage) await checkFilesInObjectStorage(objectStorage, videoDetails)
}
})
@ -132,7 +133,7 @@ function runTests (objectStorage: boolean) {
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5)
if (objectStorage) await checkFilesInObjectStorage(videoDetails)
if (enableObjectStorage) await checkFilesInObjectStorage(objectStorage, videoDetails)
}
})
@ -151,7 +152,7 @@ function runTests (objectStorage: boolean) {
expect(videoDetails.files).to.have.lengthOf(5)
expect(videoDetails.streamingPlaylists).to.have.lengthOf(0)
if (objectStorage) await checkFilesInObjectStorage(videoDetails)
if (enableObjectStorage) await checkFilesInObjectStorage(objectStorage, videoDetails)
}
})
@ -185,7 +186,7 @@ function runTests (objectStorage: boolean) {
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(1)
if (objectStorage) await checkFilesInObjectStorage(videoDetails)
if (enableObjectStorage) await checkFilesInObjectStorage(objectStorage, videoDetails)
shouldBeDeleted = [
videoDetails.streamingPlaylists[0].files[0].fileUrl,
@ -219,8 +220,8 @@ function runTests (objectStorage: boolean) {
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5)
if (objectStorage) {
await checkFilesInObjectStorage(videoDetails)
if (enableObjectStorage) {
await checkFilesInObjectStorage(objectStorage, videoDetails)
const hlsPlaylist = videoDetails.streamingPlaylists[0]
const resolutions = hlsPlaylist.files.map(f => f.resolution.id)
@ -245,6 +246,8 @@ function runTests (objectStorage: boolean) {
})
after(async function () {
if (objectStorage) await objectStorage.cleanupMock()
await cleanupTests(servers)
})
}

12
server/tests/api/transcoding/hls.ts
View File

@ -150,17 +150,23 @@ describe('Test HLS videos', function () {
describe('With object storage enabled', function () {
if (areMockObjectStorageTestsDisabled()) return
const objectStorage = new ObjectStorageCommand()
before(async function () {
this.timeout(120000)
const configOverride = ObjectStorageCommand.getDefaultMockConfig()
await ObjectStorageCommand.prepareDefaultMockBuckets()
const configOverride = objectStorage.getDefaultMockConfig()
await objectStorage.prepareDefaultMockBuckets()
await servers[0].kill()
await servers[0].run(configOverride)
})
runTestSuite(true, ObjectStorageCommand.getMockPlaylistBaseUrl())
runTestSuite(true, objectStorage.getMockPlaylistBaseUrl())
after(async function () {
await objectStorage.cleanupMock()
})
})
after(async function () {

12
server/tests/api/transcoding/update-while-transcoding.ts
View File

@ -135,17 +135,23 @@ describe('Test update video privacy while transcoding', function () {
describe('With object storage enabled', function () {
if (areMockObjectStorageTestsDisabled()) return
const objectStorage = new ObjectStorageCommand()
before(async function () {
this.timeout(120000)
const configOverride = ObjectStorageCommand.getDefaultMockConfig()
await ObjectStorageCommand.prepareDefaultMockBuckets()
const configOverride = objectStorage.getDefaultMockConfig()
await objectStorage.prepareDefaultMockBuckets()
await servers[0].kill()
await servers[0].run(configOverride)
})
runTestSuite(true, ObjectStorageCommand.getMockPlaylistBaseUrl())
runTestSuite(true, objectStorage.getMockPlaylistBaseUrl())
after(async function () {
await objectStorage.cleanupMock()
})
})
after(async function () {

14
server/tests/api/transcoding/video-studio.ts
View File

@ -326,11 +326,13 @@ describe('Test video studio', function () {
describe('Object storage studio edition', function () {
if (areMockObjectStorageTestsDisabled()) return
const objectStorage = new ObjectStorageCommand()
before(async function () {
await ObjectStorageCommand.prepareDefaultMockBuckets()
await objectStorage.prepareDefaultMockBuckets()
await servers[0].kill()
await servers[0].run(ObjectStorageCommand.getDefaultMockConfig())
await servers[0].run(objectStorage.getDefaultMockConfig())
await servers[0].config.enableMinimumTranscoding()
})
@ -353,16 +355,20 @@ describe('Test video studio', function () {
}
for (const webtorrentFile of video.files) {
expectStartWith(webtorrentFile.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
expectStartWith(webtorrentFile.fileUrl, objectStorage.getMockWebVideosBaseUrl())
}
for (const hlsFile of video.streamingPlaylists[0].files) {
expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
expectStartWith(hlsFile.fileUrl, objectStorage.getMockPlaylistBaseUrl())
}
await checkVideoDuration(server, videoUUID, 9)
}
})
after(async function () {
await objectStorage.cleanupMock()
})
})
after(async function () {

22
server/tests/cli/create-import-video-file-job.ts
View File

@ -25,25 +25,27 @@ function assertVideoProperties (video: VideoFile, resolution: number, extname: s
if (size) expect(video.size).to.equal(size)
}
async function checkFiles (video: VideoDetails, objectStorage: boolean) {
async function checkFiles (video: VideoDetails, objectStorage: ObjectStorageCommand) {
for (const file of video.files) {
if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
if (objectStorage) expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
}
function runTests (objectStorage: boolean) {
function runTests (enableObjectStorage: boolean) {
let video1ShortId: string
let video2UUID: string
let servers: PeerTubeServer[] = []
const objectStorage = new ObjectStorageCommand()
before(async function () {
this.timeout(90000)
const config = objectStorage
? ObjectStorageCommand.getDefaultMockConfig()
const config = enableObjectStorage
? objectStorage.getDefaultMockConfig()
: {}
// Run server 2 to have transcoding enabled
@ -52,7 +54,7 @@ function runTests (objectStorage: boolean) {
await doubleFollow(servers[0], servers[1])
if (objectStorage) await ObjectStorageCommand.prepareDefaultMockBuckets()
if (enableObjectStorage) await objectStorage.prepareDefaultMockBuckets()
// Upload two videos for our needs
{
@ -90,7 +92,7 @@ function runTests (objectStorage: boolean) {
assertVideoProperties(originalVideo, 720, 'webm', 218910)
assertVideoProperties(transcodedVideo, 480, 'webm', 69217)
await checkFiles(videoDetails, objectStorage)
await checkFiles(videoDetails, enableObjectStorage && objectStorage)
}
})
@ -114,7 +116,7 @@ function runTests (objectStorage: boolean) {
assertVideoProperties(transcodedVideo320, 360, 'mp4')
assertVideoProperties(transcodedVideo240, 240, 'mp4')
await checkFiles(videoDetails, objectStorage)
await checkFiles(videoDetails, enableObjectStorage && objectStorage)
}
})
@ -136,7 +138,7 @@ function runTests (objectStorage: boolean) {
assertVideoProperties(video720, 720, 'webm', 942961)
assertVideoProperties(video480, 480, 'webm', 69217)
await checkFiles(videoDetails, objectStorage)
await checkFiles(videoDetails, enableObjectStorage && objectStorage)
}
})
@ -146,6 +148,8 @@ function runTests (objectStorage: boolean) {
})
after(async function () {
await objectStorage.cleanupMock()
await cleanupTests(servers)
})
}

25
server/tests/cli/create-move-video-storage-job.ts
View File

@ -15,10 +15,10 @@ import {
} from '@shared/server-commands'
import { checkDirectoryIsEmpty, expectStartWith } from '../shared'
async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObjectStorage: boolean) {
async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectStorage?: ObjectStorageCommand) {
for (const file of video.files) {
const start = inObjectStorage
? ObjectStorageCommand.getMockWebTorrentBaseUrl()
const start = objectStorage
? objectStorage.getMockWebVideosBaseUrl()
: origin.url
expectStartWith(file.fileUrl, start)
@ -26,8 +26,8 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObject
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
const start = inObjectStorage
? ObjectStorageCommand.getMockPlaylistBaseUrl()
const start = objectStorage
? objectStorage.getMockPlaylistBaseUrl()
: origin.url
const hls = video.streamingPlaylists[0]
@ -46,6 +46,7 @@ describe('Test create move video storage job', function () {
let servers: PeerTubeServer[] = []
const uuids: string[] = []
const objectStorage = new ObjectStorageCommand()
before(async function () {
this.timeout(360000)
@ -56,7 +57,7 @@ describe('Test create move video storage job', function () {
await doubleFollow(servers[0], servers[1])
await ObjectStorageCommand.prepareDefaultMockBuckets()
await objectStorage.prepareDefaultMockBuckets()
await servers[0].config.enableTranscoding()
@ -68,25 +69,25 @@ describe('Test create move video storage job', function () {
await waitJobs(servers)
await servers[0].kill()
await servers[0].run(ObjectStorageCommand.getDefaultMockConfig())
await servers[0].run(objectStorage.getDefaultMockConfig())
})
it('Should move only one file', async function () {
this.timeout(120000)
const command = `npm run create-move-video-storage-job -- --to-object-storage -v ${uuids[1]}`
await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultMockConfig())
await servers[0].cli.execWithEnv(command, objectStorage.getDefaultMockConfig())
await waitJobs(servers)
for (const server of servers) {
const video = await server.videos.get({ id: uuids[1] })
await checkFiles(servers[0], video, true)
await checkFiles(servers[0], video, objectStorage)
for (const id of [ uuids[0], uuids[2] ]) {
const video = await server.videos.get({ id })
await checkFiles(servers[0], video, false)
await checkFiles(servers[0], video)
}
}
})
@ -95,14 +96,14 @@ describe('Test create move video storage job', function () {
this.timeout(120000)
const command = `npm run create-move-video-storage-job -- --to-object-storage --all-videos`
await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultMockConfig())
await servers[0].cli.execWithEnv(command, objectStorage.getDefaultMockConfig())
await waitJobs(servers)
for (const server of servers) {
for (const id of [ uuids[0], uuids[2] ]) {
const video = await server.videos.get({ id })
await checkFiles(servers[0], video, true)
await checkFiles(servers[0], video, objectStorage)
}
}
})

20
server/tests/peertube-runner/live-transcoding.ts
View File

@ -31,8 +31,8 @@ describe('Test Live transcoding in peertube-runner program', function () {
let sqlCommandServer1: SQLCommand
function runSuite (options: {
objectStorage: boolean
}) {
objectStorage?: ObjectStorageCommand
} = {}) {
const { objectStorage } = options
it('Should enable transcoding without additional resolutions', async function () {
@ -117,7 +117,7 @@ describe('Test Live transcoding in peertube-runner program', function () {
for (const file of files) {
if (objectStorage) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl())
}
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
@ -155,24 +155,30 @@ describe('Test Live transcoding in peertube-runner program', function () {
await servers[0].config.enableTranscoding(true, false, true)
})
runSuite({ objectStorage: false })
runSuite()
})
describe('With lives on object storage', function () {
if (areMockObjectStorageTestsDisabled()) return
const objectStorage = new ObjectStorageCommand()
before(async function () {
await ObjectStorageCommand.prepareDefaultMockBuckets()
await objectStorage.prepareDefaultMockBuckets()
await servers[0].kill()
await servers[0].run(ObjectStorageCommand.getDefaultMockConfig())
await servers[0].run(objectStorage.getDefaultMockConfig())
// Wait for peertube runner socket reconnection
await wait(1500)
})
runSuite({ objectStorage: true })
runSuite({ objectStorage })
after(async function () {
await objectStorage.cleanupMock()
})
})
describe('Check cleanup', function () {

22
server/tests/peertube-runner/studio-transcoding.ts
View File

@ -19,8 +19,8 @@ describe('Test studio transcoding in peertube-runner program', function () {
let peertubeRunner: PeerTubeRunnerProcess
function runSuite (options: {
objectStorage: boolean
}) {
objectStorage?: ObjectStorageCommand
} = {}) {
const { objectStorage } = options
it('Should run a complex studio transcoding', async function () {
@ -45,11 +45,11 @@ describe('Test studio transcoding in peertube-runner program', function () {
if (objectStorage) {
for (const webtorrentFile of video.files) {
expectStartWith(webtorrentFile.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
expectStartWith(webtorrentFile.fileUrl, objectStorage.getMockWebVideosBaseUrl())
}
for (const hlsFile of video.streamingPlaylists[0].files) {
expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
expectStartWith(hlsFile.fileUrl, objectStorage.getMockPlaylistBaseUrl())
}
}
@ -80,24 +80,30 @@ describe('Test studio transcoding in peertube-runner program', function () {
})
describe('With videos on local filesystem storage', function () {
runSuite({ objectStorage: false })
runSuite()
})
describe('With videos on object storage', function () {
if (areMockObjectStorageTestsDisabled()) return
const objectStorage = new ObjectStorageCommand()
before(async function () {
await ObjectStorageCommand.prepareDefaultMockBuckets()
await objectStorage.prepareDefaultMockBuckets()
await servers[0].kill()
await servers[0].run(ObjectStorageCommand.getDefaultMockConfig())
await servers[0].run(objectStorage.getDefaultMockConfig())
// Wait for peertube runner socket reconnection
await wait(1500)
})
runSuite({ objectStorage: true })
runSuite({ objectStorage })
after(async function () {
await objectStorage.cleanupMock()
})
})
describe('Check cleanup', function () {

28
server/tests/peertube-runner/vod-transcoding.ts
View File

@ -26,16 +26,16 @@ describe('Test VOD transcoding in peertube-runner program', function () {
function runSuite (options: {
webtorrentEnabled: boolean
hlsEnabled: boolean
objectStorage: boolean
objectStorage?: ObjectStorageCommand
}) {
const { webtorrentEnabled, hlsEnabled, objectStorage } = options
const objectStorageBaseUrlWebTorrent = objectStorage
? ObjectStorageCommand.getMockWebTorrentBaseUrl()
? objectStorage.getMockWebVideosBaseUrl()
: undefined
const objectStorageBaseUrlHLS = objectStorage
? ObjectStorageCommand.getMockPlaylistBaseUrl()
? objectStorage.getMockPlaylistBaseUrl()
: undefined
it('Should upload a classic video mp4 and transcode it', async function () {
@ -262,7 +262,7 @@ describe('Test VOD transcoding in peertube-runner program', function () {
await servers[0].config.enableTranscoding(true, false, true)
})
runSuite({ webtorrentEnabled: true, hlsEnabled: false, objectStorage: false })
runSuite({ webtorrentEnabled: true, hlsEnabled: false })
})
describe('HLS videos only enabled', function () {
@ -271,7 +271,7 @@ describe('Test VOD transcoding in peertube-runner program', function () {
await servers[0].config.enableTranscoding(false, true, true)
})
runSuite({ webtorrentEnabled: false, hlsEnabled: true, objectStorage: false })
runSuite({ webtorrentEnabled: false, hlsEnabled: true })
})
describe('Web video & HLS enabled', function () {
@ -280,19 +280,21 @@ describe('Test VOD transcoding in peertube-runner program', function () {
await servers[0].config.enableTranscoding(true, true, true)
})
runSuite({ webtorrentEnabled: true, hlsEnabled: true, objectStorage: false })
runSuite({ webtorrentEnabled: true, hlsEnabled: true })
})
})
describe('With videos on object storage', function () {
if (areMockObjectStorageTestsDisabled()) return
const objectStorage = new ObjectStorageCommand()
before(async function () {
await ObjectStorageCommand.prepareDefaultMockBuckets()
await objectStorage.prepareDefaultMockBuckets()
await servers[0].kill()
await servers[0].run(ObjectStorageCommand.getDefaultMockConfig())
await servers[0].run(objectStorage.getDefaultMockConfig())
// Wait for peertube runner socket reconnection
await wait(1500)
@ -304,7 +306,7 @@ describe('Test VOD transcoding in peertube-runner program', function () {
await servers[0].config.enableTranscoding(true, false, true)
})
runSuite({ webtorrentEnabled: true, hlsEnabled: false, objectStorage: true })
runSuite({ webtorrentEnabled: true, hlsEnabled: false, objectStorage })
})
describe('HLS videos only enabled', function () {
@ -313,7 +315,7 @@ describe('Test VOD transcoding in peertube-runner program', function () {
await servers[0].config.enableTranscoding(false, true, true)
})
runSuite({ webtorrentEnabled: false, hlsEnabled: true, objectStorage: true })
runSuite({ webtorrentEnabled: false, hlsEnabled: true, objectStorage })
})
describe('Web video & HLS enabled', function () {
@ -322,7 +324,11 @@ describe('Test VOD transcoding in peertube-runner program', function () {
await servers[0].config.enableTranscoding(true, true, true)
})
runSuite({ webtorrentEnabled: true, hlsEnabled: true, objectStorage: true })
runSuite({ webtorrentEnabled: true, hlsEnabled: true, objectStorage })
})
after(async function () {
await objectStorage.cleanupMock()
})
})

10
server/tests/shared/live.ts
View File

@ -46,7 +46,7 @@ async function testLiveVideoResolutions (options: {
resolutions: number[]
transcoded: boolean
objectStorage: boolean
objectStorage?: ObjectStorageCommand
objectStorageBaseUrl?: string
}) {
const {
@ -57,7 +57,7 @@ async function testLiveVideoResolutions (options: {
resolutions,
transcoded,
objectStorage,
objectStorageBaseUrl = ObjectStorageCommand.getMockPlaylistBaseUrl()
objectStorageBaseUrl = objectStorage?.getMockPlaylistBaseUrl()
} = options
for (const server of servers) {
@ -76,7 +76,7 @@ async function testLiveVideoResolutions (options: {
playlistUrl: hlsPlaylist.playlistUrl,
resolutions,
transcoded,
withRetry: objectStorage
withRetry: !!objectStorage
})
if (objectStorage) {
@ -105,7 +105,7 @@ async function testLiveVideoResolutions (options: {
const subPlaylist = await originServer.streamingPlaylists.get({
url: `${baseUrl}/${video.uuid}/${i}.m3u8`,
withRetry: objectStorage // With object storage, the request may fail because of inconsistent data in S3
withRetry: !!objectStorage // With object storage, the request may fail because of inconsistent data in S3
})
expect(subPlaylist).to.contain(segmentName)
@ -116,7 +116,7 @@ async function testLiveVideoResolutions (options: {
videoUUID: video.uuid,
segmentName,
hlsPlaylist,
withRetry: objectStorage // With object storage, the request may fail because of inconsistent data in S3
withRetry: !!objectStorage // With object storage, the request may fail because of inconsistent data in S3
})
if (originServer.internalServerNumber === server.internalServerNumber) {

105
shared/server-commands/server/object-storage-command.ts
View File

@ -1,34 +1,17 @@
import { randomInt } from 'crypto'
import { HttpStatusCode } from '@shared/models'
import { makePostBodyRequest } from '../requests'
import { AbstractCommand } from '../shared'
export class ObjectStorageCommand extends AbstractCommand {
static readonly DEFAULT_PLAYLIST_MOCK_BUCKET = 'streaming-playlists'
static readonly DEFAULT_WEBTORRENT_MOCK_BUCKET = 'videos'
export class ObjectStorageCommand {
static readonly DEFAULT_SCALEWAY_BUCKET = 'peertube-ci-test'
private readonly bucketsCreated: string[] = []
private readonly seed: number
// ---------------------------------------------------------------------------
static getDefaultMockConfig () {
return {
object_storage: {
enabled: true,
endpoint: 'http://' + this.getMockEndpointHost(),
region: this.getMockRegion(),
credentials: this.getMockCredentialsConfig(),
streaming_playlists: {
bucket_name: this.DEFAULT_PLAYLIST_MOCK_BUCKET
},
videos: {
bucket_name: this.DEFAULT_WEBTORRENT_MOCK_BUCKET
}
}
}
constructor () {
this.seed = randomInt(0, 10000)
}
static getMockCredentialsConfig () {
@ -46,39 +29,83 @@ export class ObjectStorageCommand extends AbstractCommand {
return 'us-east-1'
}
static getMockWebTorrentBaseUrl () {
return `http://${this.DEFAULT_WEBTORRENT_MOCK_BUCKET}.${this.getMockEndpointHost()}/`
getDefaultMockConfig () {
return {
object_storage: {
enabled: true,
endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
region: ObjectStorageCommand.getMockRegion(),
credentials: ObjectStorageCommand.getMockCredentialsConfig(),
streaming_playlists: {
bucket_name: this.getMockStreamingPlaylistsBucketName()
},
videos: {
bucket_name: this.getMockWebVideosBucketName()
}
}
}
}
static getMockPlaylistBaseUrl () {
return `http://${this.DEFAULT_PLAYLIST_MOCK_BUCKET}.${this.getMockEndpointHost()}/`
getMockWebVideosBaseUrl () {
return `http://${this.getMockWebVideosBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/`
}
static async prepareDefaultMockBuckets () {
await this.createMockBucket(this.DEFAULT_PLAYLIST_MOCK_BUCKET)
await this.createMockBucket(this.DEFAULT_WEBTORRENT_MOCK_BUCKET)
getMockPlaylistBaseUrl () {
return `http://${this.getMockStreamingPlaylistsBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/`
}
static async createMockBucket (name: string) {
await makePostBodyRequest({
url: this.getMockEndpointHost(),
path: '/ui/' + name + '?delete',
expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
})
async prepareDefaultMockBuckets () {
await this.createMockBucket(this.getMockStreamingPlaylistsBucketName())
await this.createMockBucket(this.getMockWebVideosBucketName())
}
async createMockBucket (name: string) {
this.bucketsCreated.push(name)
await this.deleteMockBucket(name)
await makePostBodyRequest({
url: this.getMockEndpointHost(),
url: ObjectStorageCommand.getMockEndpointHost(),
path: '/ui/' + name + '?create',
expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
})
await makePostBodyRequest({
url: this.getMockEndpointHost(),
url: ObjectStorageCommand.getMockEndpointHost(),
path: '/ui/' + name + '?make-public',
expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
})
}
async cleanupMock () {
for (const name of this.bucketsCreated) {
await this.deleteMockBucket(name)
}
}
getMockStreamingPlaylistsBucketName (name = 'streaming-playlists') {
return this.getMockBucketName(name)
}
getMockWebVideosBucketName (name = 'web-videos') {
return this.getMockBucketName(name)
}
getMockBucketName (name: string) {
return `${this.seed}-${name}`
}
private async deleteMockBucket (name: string) {
await makePostBodyRequest({
url: ObjectStorageCommand.getMockEndpointHost(),
path: '/ui/' + name + '?delete',
expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
})
}
// ---------------------------------------------------------------------------
static getDefaultScalewayConfig (options: {

3
shared/server-commands/server/server.ts
View File

@ -48,7 +48,6 @@ import { DebugCommand } from './debug-command'
import { FollowsCommand } from './follows-command'
import { JobsCommand } from './jobs-command'
import { MetricsCommand } from './metrics-command'
import { ObjectStorageCommand } from './object-storage-command'
import { PluginsCommand } from './plugins-command'
import { RedundancyCommand } from './redundancy-command'
import { ServersCommand } from './servers-command'
@ -140,7 +139,6 @@ export class PeerTubeServer {
servers?: ServersCommand
login?: LoginCommand
users?: UsersCommand
objectStorage?: ObjectStorageCommand
videoStudio?: VideoStudioCommand
videos?: VideosCommand
videoStats?: VideoStatsCommand
@ -429,7 +427,6 @@ export class PeerTubeServer {
this.login = new LoginCommand(this)
this.users = new UsersCommand(this)
this.videos = new VideosCommand(this)
this.objectStorage = new ObjectStorageCommand(this)
this.videoStudio = new VideoStudioCommand(this)
this.videoStats = new VideoStatsCommand(this)
this.views = new ViewsCommand(this)

20
shared/server-commands/videos/live-command.ts
View File

@ -192,7 +192,7 @@ export class LiveCommand extends AbstractCommand {
videoUUID: string
playlistNumber: number
segment: number
objectStorage: boolean
objectStorage?: ObjectStorageCommand
objectStorageBaseUrl?: string
}) {
const {
@ -201,12 +201,12 @@ export class LiveCommand extends AbstractCommand {
playlistNumber,
segment,
videoUUID,
objectStorageBaseUrl = ObjectStorageCommand.getMockPlaylistBaseUrl()
objectStorageBaseUrl
} = options
const segmentName = `${playlistNumber}-00000${segment}.ts`
const baseUrl = objectStorage
? join(objectStorageBaseUrl, 'hls')
? join(objectStorageBaseUrl || objectStorage.getMockPlaylistBaseUrl(), 'hls')
: server.url + '/static/streaming-playlists/hls'
let error = true
@ -226,7 +226,7 @@ export class LiveCommand extends AbstractCommand {
const hlsPlaylist = video.streamingPlaylists[0]
// Check SHA generation
const shaBody = await server.streamingPlaylists.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url, withRetry: objectStorage })
const shaBody = await server.streamingPlaylists.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url, withRetry: !!objectStorage })
if (!shaBody[segmentName]) {
throw new Error('Segment SHA does not exist')
}
@ -261,13 +261,13 @@ export class LiveCommand extends AbstractCommand {
videoUUID: string
playlistNumber: number
segment: number
objectStorage?: boolean // default false
objectStorage?: ObjectStorageCommand
}) {
const { playlistNumber, segment, videoUUID, objectStorage = false } = options
const { playlistNumber, segment, videoUUID, objectStorage } = options
const segmentName = `${playlistNumber}-00000${segment}.ts`
const baseUrl = objectStorage
? ObjectStorageCommand.getMockPlaylistBaseUrl()
? objectStorage.getMockPlaylistBaseUrl()
: `${this.server.url}/static/streaming-playlists/hls`
const url = `${baseUrl}/${videoUUID}/${segmentName}`
@ -284,12 +284,12 @@ export class LiveCommand extends AbstractCommand {
getPlaylistFile (options: OverrideCommandOptions & {
videoUUID: string
playlistName: string
objectStorage?: boolean // default false
objectStorage?: ObjectStorageCommand
}) {
const { playlistName, videoUUID, objectStorage = false } = options
const { playlistName, videoUUID, objectStorage } = options
const baseUrl = objectStorage
? ObjectStorageCommand.getMockPlaylistBaseUrl()
? objectStorage.getMockPlaylistBaseUrl()
: `${this.server.url}/static/streaming-playlists/hls`
const url = `${baseUrl}/${videoUUID}/${playlistName}`