OpenCloud
/
PeerTube
mirror of https://github.com/Chocobozzz/PeerTube
1
0
Fork 0
Code Issues Releases Wiki Activity

Introduce redundancy command

pull/4271/head
Chocobozzz 2021-07-07 10:56:45 +02:00
parent ae2abfd3ae
commit dab047092b
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
7 changed files with 166 additions and 247 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

167
server/tests/api/redundancy/manage-redundancy.ts
View File

@ -1,21 +1,21 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import * as chai from 'chai'
import 'mocha'
import * as chai from 'chai'
import {
cleanupTests,
doubleFollow,
flushAndRunMultipleServers,
getLocalIdByUUID,
RedundancyCommand,
ServerInfo,
setAccessTokensToServers,
uploadVideo,
uploadVideoAndGetId,
waitJobs,
waitUntilLog
} from '../../../../shared/extra-utils'
import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
import { addVideoRedundancy, listVideoRedundancies, removeVideoRedundancy, updateRedundancy } from '@shared/extra-utils/server/redundancy'
import { VideoPrivacy, VideoRedundanciesTarget, VideoRedundancy } from '@shared/models'
} from '@shared/extra-utils'
import { VideoPrivacy, VideoRedundanciesTarget } from '@shared/models'
const expect = chai.expect
@ -27,6 +27,8 @@ describe('Test manage videos redundancy', function () {
let video2Server2UUID: string
let redundanciesToRemove: number[] = []
let commands: RedundancyCommand[]
before(async function () {
this.timeout(120000)
@ -55,6 +57,8 @@ describe('Test manage videos redundancy', function () {
// Get the access tokens
await setAccessTokensToServers(servers)
commands = servers.map(s => s.redundancyCommand)
{
const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 1 server 2' })
video1Server2UUID = res.body.video.uuid
@ -69,21 +73,17 @@ describe('Test manage videos redundancy', function () {
// Server 1 and server 2 follow each other
await doubleFollow(servers[0], servers[1])
await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true)
await commands[0].updateRedundancy({ host: servers[1].host, redundancyAllowed: true })
await waitJobs(servers)
})
it('Should not have redundancies on server 3', async function () {
for (const target of targets) {
const res = await listVideoRedundancies({
url: servers[2].url,
accessToken: servers[2].accessToken,
target
})
const body = await commands[2].listVideos({ target })
expect(res.body.total).to.equal(0)
expect(res.body.data).to.have.lengthOf(0)
expect(body.total).to.equal(0)
expect(body.data).to.have.lengthOf(0)
}
})
@ -94,28 +94,19 @@ describe('Test manage videos redundancy', function () {
await waitUntilLog(servers[0], 'Duplicated ', 10)
await waitJobs(servers)
const res = await listVideoRedundancies({
url: servers[1].url,
accessToken: servers[1].accessToken,
target: 'remote-videos'
})
const body = await commands[1].listVideos({ target: 'remote-videos' })
expect(res.body.total).to.equal(0)
expect(res.body.data).to.have.lengthOf(0)
expect(body.total).to.equal(0)
expect(body.data).to.have.lengthOf(0)
})
it('Should have "my-videos" redundancies on server 2', async function () {
this.timeout(120000)
const res = await listVideoRedundancies({
url: servers[1].url,
accessToken: servers[1].accessToken,
target: 'my-videos'
})
const body = await commands[1].listVideos({ target: 'my-videos' })
expect(body.total).to.equal(2)
expect(res.body.total).to.equal(2)
const videos = res.body.data as VideoRedundancy[]
const videos = body.data
expect(videos).to.have.lengthOf(2)
const videos1 = videos.find(v => v.uuid === video1Server2UUID)
@ -139,28 +130,19 @@ describe('Test manage videos redundancy', function () {
})
it('Should not have "my-videos" redundancies on server 1', async function () {
const res = await listVideoRedundancies({
url: servers[0].url,
accessToken: servers[0].accessToken,
target: 'my-videos'
})
const body = await commands[0].listVideos({ target: 'my-videos' })
expect(res.body.total).to.equal(0)
expect(res.body.data).to.have.lengthOf(0)
expect(body.total).to.equal(0)
expect(body.data).to.have.lengthOf(0)
})
it('Should have "remote-videos" redundancies on server 1', async function () {
this.timeout(120000)
const res = await listVideoRedundancies({
url: servers[0].url,
accessToken: servers[0].accessToken,
target: 'remote-videos'
})
const body = await commands[0].listVideos({ target: 'remote-videos' })
expect(body.total).to.equal(2)
expect(res.body.total).to.equal(2)
const videos = res.body.data as VideoRedundancy[]
const videos = body.data
expect(videos).to.have.lengthOf(2)
const videos1 = videos.find(v => v.uuid === video1Server2UUID)
@ -185,47 +167,40 @@ describe('Test manage videos redundancy', function () {
it('Should correctly paginate and sort results', async function () {
{
const res = await listVideoRedundancies({
url: servers[0].url,
accessToken: servers[0].accessToken,
const body = await commands[0].listVideos({
target: 'remote-videos',
sort: 'name',
start: 0,
count: 2
})
const videos = res.body.data
const videos = body.data
expect(videos[0].name).to.equal('video 1 server 2')
expect(videos[1].name).to.equal('video 2 server 2')
}
{
const res = await listVideoRedundancies({
url: servers[0].url,
accessToken: servers[0].accessToken,
const body = await commands[0].listVideos({
target: 'remote-videos',
sort: '-name',
start: 0,
count: 2
})
const videos = res.body.data
const videos = body.data
expect(videos[0].name).to.equal('video 2 server 2')
expect(videos[1].name).to.equal('video 1 server 2')
}
{
const res = await listVideoRedundancies({
url: servers[0].url,
accessToken: servers[0].accessToken,
const body = await commands[0].listVideos({
target: 'remote-videos',
sort: '-name',
start: 1,
count: 1
})
const videos = res.body.data
expect(videos[0].name).to.equal('video 1 server 2')
expect(body.data[0].name).to.equal('video 1 server 2')
}
})
@ -236,30 +211,23 @@ describe('Test manage videos redundancy', function () {
await waitJobs(servers)
const videoId = await getLocalIdByUUID(servers[0].url, uuid)
await addVideoRedundancy({
url: servers[0].url,
accessToken: servers[0].accessToken,
videoId
})
await commands[0].addVideo({ videoId })
await waitJobs(servers)
await waitUntilLog(servers[0], 'Duplicated ', 15)
await waitJobs(servers)
{
const res = await listVideoRedundancies({
url: servers[0].url,
accessToken: servers[0].accessToken,
const body = await commands[0].listVideos({
target: 'remote-videos',
sort: '-name',
start: 0,
count: 5
})
const videos = res.body.data
expect(videos[0].name).to.equal('video 3 server 2')
const video = body.data[0]
const video = videos[0]
expect(video.name).to.equal('video 3 server 2')
expect(video.redundancies.files).to.have.lengthOf(4)
expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1)
@ -276,19 +244,15 @@ describe('Test manage videos redundancy', function () {
}
}
const res = await listVideoRedundancies({
url: servers[1].url,
accessToken: servers[1].accessToken,
const body = await commands[1].listVideos({
target: 'my-videos',
sort: '-name',
start: 0,
count: 5
})
const videos = res.body.data
expect(videos[0].name).to.equal('video 3 server 2')
const video = videos[0]
const video = body.data[0]
expect(video.name).to.equal('video 3 server 2')
expect(video.redundancies.files).to.have.lengthOf(4)
expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1)
@ -307,64 +271,47 @@ describe('Test manage videos redundancy', function () {
this.timeout(120000)
for (const redundancyId of redundanciesToRemove) {
await removeVideoRedundancy({
url: servers[0].url,
accessToken: servers[0].accessToken,
redundancyId
})
await commands[0].removeVideo({ redundancyId })
}
{
const res = await listVideoRedundancies({
url: servers[0].url,
accessToken: servers[0].accessToken,
const body = await commands[0].listVideos({
target: 'remote-videos',
sort: '-name',
start: 0,
count: 5
})
const videos = res.body.data
const videos = body.data
expect(videos).to.have.lengthOf(2)
expect(videos[0].name).to.equal('video 2 server 2')
redundanciesToRemove = []
const video = videos[0]
expect(video.name).to.equal('video 2 server 2')
expect(video.redundancies.files).to.have.lengthOf(4)
expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1)
const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists)
for (const r of redundancies) {
redundanciesToRemove.push(r.id)
}
redundanciesToRemove = redundancies.map(r => r.id)
}
})
it('Should remove another (auto) redundancy', async function () {
{
for (const redundancyId of redundanciesToRemove) {
await removeVideoRedundancy({
url: servers[0].url,
accessToken: servers[0].accessToken,
redundancyId
})
}
const res = await listVideoRedundancies({
url: servers[0].url,
accessToken: servers[0].accessToken,
target: 'remote-videos',
sort: '-name',
start: 0,
count: 5
})
const videos = res.body.data
expect(videos[0].name).to.equal('video 1 server 2')
expect(videos).to.have.lengthOf(1)
for (const redundancyId of redundanciesToRemove) {
await commands[0].removeVideo({ redundancyId })
}
const body = await commands[0].listVideos({
target: 'remote-videos',
sort: '-name',
start: 0,
count: 5
})
const videos = body.data
expect(videos).to.have.lengthOf(1)
expect(videos[0].name).to.equal('video 1 server 2')
})
after(async function () {

28
server/tests/api/redundancy/redundancy-constraints.ts
View File

@ -1,9 +1,6 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import * as chai from 'chai'
import { listVideoRedundancies, updateRedundancy } from '@shared/extra-utils/server/redundancy'
import { VideoPrivacy } from '@shared/models'
import {
cleanupTests,
flushAndRunServer,
@ -13,9 +10,10 @@ import {
setAccessTokensToServers,
updateVideo,
uploadVideo,
waitJobs,
waitUntilLog
} from '../../../../shared/extra-utils'
import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
} from '@shared/extra-utils'
import { VideoPrivacy } from '@shared/models'
const expect = chai.expect
@ -50,23 +48,15 @@ describe('Test redundancy constraints', function () {
}
async function getTotalRedundanciesLocalServer () {
const res = await listVideoRedundancies({
url: localServer.url,
accessToken: localServer.accessToken,
target: 'my-videos'
})
const body = await localServer.redundancyCommand.listVideos({ target: 'my-videos' })
return res.body.total
return body.total
}
async function getTotalRedundanciesRemoteServer () {
const res = await listVideoRedundancies({
url: remoteServer.url,
accessToken: remoteServer.accessToken,
target: 'remote-videos'
})
const body = await remoteServer.redundancyCommand.listVideos({ target: 'remote-videos' })
return res.body.total
return body.total
}
before(async function () {
@ -99,7 +89,7 @@ describe('Test redundancy constraints', function () {
// Server 1 and server 2 follow each other
await remoteServer.followsCommand.follow({ targets: [ localServer.url ] })
await waitJobs(servers)
await updateRedundancy(remoteServer.url, remoteServer.accessToken, localServer.host, true)
await remoteServer.redundancyCommand.updateRedundancy({ host: localServer.host, redundancyAllowed: true })
await waitJobs(servers)
})
@ -161,7 +151,7 @@ describe('Test redundancy constraints', function () {
}
}
}
await killallServers([ localServer ])
killallServers([ localServer ])
await reRunServer(localServer, config)
await uploadWrapper('video 3 server 2')

49
server/tests/api/redundancy/redundancy.ts
View File

@ -5,7 +5,8 @@ import * as chai from 'chai'
import { readdir } from 'fs-extra'
import * as magnetUtil from 'magnet-uri'
import { join } from 'path'
import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
import { removeVideoRedundancy } from '@server/lib/redundancy'
import { HttpStatusCode } from '@shared/core-utils'
import {
checkSegmentHash,
checkVideoFilesWereRemoved,
@ -26,19 +27,18 @@ import {
uploadVideo,
viewVideo,
wait,
waitJobs,
waitUntilLog
} from '../../../../shared/extra-utils'
import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
} from '@shared/extra-utils'
import { getStats } from '@shared/extra-utils/server/stats'
import {
addVideoRedundancy,
listVideoRedundancies,
removeVideoRedundancy,
updateRedundancy
} from '../../../../shared/extra-utils/server/redundancy'
import { getStats } from '../../../../shared/extra-utils/server/stats'
import { VideoRedundancy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../../shared/models/redundancy'
import { ServerStats } from '../../../../shared/models/server/server-stats.model'
import { VideoDetails, VideoPrivacy } from '../../../../shared/models/videos'
ServerStats,
VideoDetails,
VideoPrivacy,
VideoRedundancy,
VideoRedundancyStrategy,
VideoRedundancyStrategyWithManual
} from '@shared/models'
const expect = chai.expect
@ -279,7 +279,7 @@ async function findServerFollows () {
}
async function enableRedundancyOnServer1 () {
await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true)
await servers[0].redundancyCommand.updateRedundancy({ host: servers[1].host, redundancyAllowed: true })
const { server2, server3 } = await findServerFollows()
@ -291,7 +291,7 @@ async function enableRedundancyOnServer1 () {
}
async function disableRedundancyOnServer1 () {
await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, false)
await servers[0].redundancyCommand.updateRedundancy({ host: servers[1].host, redundancyAllowed: false })
const { server2, server3 } = await findServerFollows()
@ -551,11 +551,7 @@ describe('Test videos redundancy', function () {
})
it('Should create a redundancy on first video', async function () {
await addVideoRedundancy({
url: servers[0].url,
accessToken: servers[0].accessToken,
videoId: video1Server2Id
})
await servers[0].redundancyCommand.addVideo({ videoId: video1Server2Id })
})
it('Should have 2 webseeds on the first video', async function () {
@ -573,22 +569,15 @@ describe('Test videos redundancy', function () {
it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
this.timeout(80000)
const res = await listVideoRedundancies({
url: servers[0].url,
accessToken: servers[0].accessToken,
target: 'remote-videos'
})
const body = await servers[0].redundancyCommand.listVideos({ target: 'remote-videos' })
const videos = res.body.data as VideoRedundancy[]
const videos = body.data
expect(videos).to.have.lengthOf(1)
const video = videos[0]
for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) {
await removeVideoRedundancy({
url: servers[0].url,
accessToken: servers[0].accessToken,
redundancyId: r.id
})
await servers[0].redundancyCommand.removeVideo({ redundancyId: r.id })
}
await waitJobs(servers)

1
shared/extra-utils/server/index.ts
View File

@ -6,3 +6,4 @@ export * from './jobs'
export * from './jobs-command'
export * from './plugins-command'
export * from './plugins'
export * from './redundancy-command'

77
shared/extra-utils/server/redundancy-command.ts Normal file
View File

@ -0,0 +1,77 @@
import { ResultList, VideoRedundanciesTarget, VideoRedundancy } from '@shared/models'
import { HttpStatusCode } from '../../core-utils/miscs/http-error-codes'
import { AbstractCommand, OverrideCommandOptions } from '../shared'
export class RedundancyCommand extends AbstractCommand {
updateRedundancy (options: OverrideCommandOptions & {
host: string
redundancyAllowed: boolean
}) {
const { host, redundancyAllowed } = options
const path = '/api/v1/server/redundancy/' + host
return this.putBodyRequest({
...options,
path,
fields: { redundancyAllowed },
defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
})
}
listVideos (options: OverrideCommandOptions & {
target: VideoRedundanciesTarget
start?: number
count?: number
sort?: string
}) {
const path = '/api/v1/server/redundancy/videos'
const { target, start, count, sort } = options
return this.getRequestBody<ResultList<VideoRedundancy>>({
...options,
path,
query: {
start: start ?? 0,
count: count ?? 5,
sort: sort ?? 'name',
target
},
defaultExpectedStatus: HttpStatusCode.OK_200
})
}
addVideo (options: OverrideCommandOptions & {
videoId: number
}) {
const path = '/api/v1/server/redundancy/videos'
const { videoId } = options
return this.postBodyRequest({
...options,
path,
fields: { videoId },
defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
})
}
removeVideo (options: OverrideCommandOptions & {
redundancyId: number
}) {
const { redundancyId } = options
const path = '/api/v1/server/redundancy/videos/' + redundancyId
return this.deleteRequest({
...options,
path,
defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
})
}
}

88
shared/extra-utils/server/redundancy.ts
View File

@ -1,88 +0,0 @@
import { makeDeleteRequest, makeGetRequest, makePostBodyRequest, makePutBodyRequest } from '../requests/requests'
import { VideoRedundanciesTarget } from '@shared/models'
import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
function updateRedundancy (
url: string,
accessToken: string,
host: string,
redundancyAllowed: boolean,
expectedStatus = HttpStatusCode.NO_CONTENT_204
) {
const path = '/api/v1/server/redundancy/' + host
return makePutBodyRequest({
url,
path,
token: accessToken,
fields: { redundancyAllowed },
statusCodeExpected: expectedStatus
})
}
function listVideoRedundancies (options: {
url: string
accessToken: string
target: VideoRedundanciesTarget
start?: number
count?: number
sort?: string
statusCodeExpected?: HttpStatusCode
}) {
const path = '/api/v1/server/redundancy/videos'
const { url, accessToken, target, statusCodeExpected, start, count, sort } = options
return makeGetRequest({
url,
token: accessToken,
path,
query: {
start: start ?? 0,
count: count ?? 5,
sort: sort ?? 'name',
target
},
statusCodeExpected: statusCodeExpected || HttpStatusCode.OK_200
})
}
function addVideoRedundancy (options: {
url: string
accessToken: string
videoId: number
}) {
const path = '/api/v1/server/redundancy/videos'
const { url, accessToken, videoId } = options
return makePostBodyRequest({
url,
token: accessToken,
path,
fields: { videoId },
statusCodeExpected: HttpStatusCode.NO_CONTENT_204
})
}
function removeVideoRedundancy (options: {
url: string
accessToken: string
redundancyId: number
}) {
const { url, accessToken, redundancyId } = options
const path = '/api/v1/server/redundancy/videos/' + redundancyId
return makeDeleteRequest({
url,
token: accessToken,
path,
statusCodeExpected: HttpStatusCode.NO_CONTENT_204
})
}
export {
updateRedundancy,
listVideoRedundancies,
addVideoRedundancy,
removeVideoRedundancy
}

3
shared/extra-utils/server/servers.ts
View File

@ -21,6 +21,7 @@ import { DebugCommand } from './debug-command'
import { FollowsCommand } from './follows-command'
import { JobsCommand } from './jobs-command'
import { PluginsCommand } from './plugins-command'
import { RedundancyCommand } from './redundancy-command'
interface ServerInfo {
app: ChildProcess
@ -87,6 +88,7 @@ interface ServerInfo {
followsCommand?: FollowsCommand
jobsCommand?: JobsCommand
pluginsCommand?: PluginsCommand
redundancyCommand?: RedundancyCommand
}
function parallelTests () {
@ -305,6 +307,7 @@ async function runServer (server: ServerInfo, configOverrideArg?: any, args = []
server.followsCommand = new FollowsCommand(server)
server.jobsCommand = new JobsCommand(server)
server.pluginsCommand = new PluginsCommand(server)
server.redundancyCommand = new RedundancyCommand(server)
res(server)
})