mirror of https://github.com/Chocobozzz/PeerTube
Add redundancy stats
parent
cfc16a6db8
commit
4b5384f6e7
|
@ -21,6 +21,16 @@ export { overviewsRouter }
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const buildSamples = memoizee(async function () {
|
||||||
|
const [ categories, channels, tags ] = await Promise.all([
|
||||||
|
VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT),
|
||||||
|
VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT),
|
||||||
|
TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT)
|
||||||
|
])
|
||||||
|
|
||||||
|
return { categories, channels, tags }
|
||||||
|
}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE })
|
||||||
|
|
||||||
// This endpoint could be quite long, but we cache it
|
// This endpoint could be quite long, but we cache it
|
||||||
async function getVideosOverview (req: express.Request, res: express.Response) {
|
async function getVideosOverview (req: express.Request, res: express.Response) {
|
||||||
const attributes = await buildSamples()
|
const attributes = await buildSamples()
|
||||||
|
@ -45,16 +55,6 @@ async function getVideosOverview (req: express.Request, res: express.Response) {
|
||||||
return res.json(result)
|
return res.json(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
const buildSamples = memoizee(async function () {
|
|
||||||
const [ categories, channels, tags ] = await Promise.all([
|
|
||||||
VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT),
|
|
||||||
VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT),
|
|
||||||
TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT)
|
|
||||||
])
|
|
||||||
|
|
||||||
return { categories, channels, tags }
|
|
||||||
}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE })
|
|
||||||
|
|
||||||
async function getVideosByTag (tag: string, res: express.Response) {
|
async function getVideosByTag (tag: string, res: express.Response) {
|
||||||
const videos = await getVideos(res, { tagsOneOf: [ tag ] })
|
const videos = await getVideos(res, { tagsOneOf: [ tag ] })
|
||||||
|
|
||||||
|
|
|
@ -5,10 +5,14 @@ import { UserModel } from '../../../models/account/user'
|
||||||
import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
|
import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
|
||||||
import { VideoModel } from '../../../models/video/video'
|
import { VideoModel } from '../../../models/video/video'
|
||||||
import { VideoCommentModel } from '../../../models/video/video-comment'
|
import { VideoCommentModel } from '../../../models/video/video-comment'
|
||||||
|
import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
|
||||||
|
import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../../initializers/constants'
|
||||||
|
import { cacheRoute } from '../../../middlewares/cache'
|
||||||
|
|
||||||
const statsRouter = express.Router()
|
const statsRouter = express.Router()
|
||||||
|
|
||||||
statsRouter.get('/stats',
|
statsRouter.get('/stats',
|
||||||
|
asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.STATS)),
|
||||||
asyncMiddleware(getStats)
|
asyncMiddleware(getStats)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -18,6 +22,13 @@ async function getStats (req: express.Request, res: express.Response, next: expr
|
||||||
const { totalUsers } = await UserModel.getStats()
|
const { totalUsers } = await UserModel.getStats()
|
||||||
const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats()
|
const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats()
|
||||||
|
|
||||||
|
const videosRedundancyStats = await Promise.all(
|
||||||
|
CONFIG.REDUNDANCY.VIDEOS.map(r => {
|
||||||
|
return VideoRedundancyModel.getStats(r.strategy)
|
||||||
|
.then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size }))
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
const data: ServerStats = {
|
const data: ServerStats = {
|
||||||
totalLocalVideos,
|
totalLocalVideos,
|
||||||
totalLocalVideoViews,
|
totalLocalVideoViews,
|
||||||
|
@ -26,7 +37,8 @@ async function getStats (req: express.Request, res: express.Response, next: expr
|
||||||
totalVideoComments,
|
totalVideoComments,
|
||||||
totalUsers,
|
totalUsers,
|
||||||
totalInstanceFollowers,
|
totalInstanceFollowers,
|
||||||
totalInstanceFollowing
|
totalInstanceFollowing,
|
||||||
|
videosRedundancy: videosRedundancyStats
|
||||||
}
|
}
|
||||||
|
|
||||||
return res.json(data).end()
|
return res.json(data).end()
|
||||||
|
|
|
@ -66,7 +66,8 @@ const ROUTE_CACHE_LIFETIME = {
|
||||||
},
|
},
|
||||||
ACTIVITY_PUB: {
|
ACTIVITY_PUB: {
|
||||||
VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example
|
VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example
|
||||||
}
|
},
|
||||||
|
STATS: '4 hours'
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
|
@ -245,6 +245,37 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
|
||||||
.findAll(query)
|
.findAll(query)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static async getStats (strategy: VideoRedundancyStrategy) {
|
||||||
|
const actor = await getServerActor()
|
||||||
|
|
||||||
|
const query = {
|
||||||
|
raw: true,
|
||||||
|
attributes: [
|
||||||
|
[ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoFile.size')), '0'), 'totalUsed' ],
|
||||||
|
[ Sequelize.fn('COUNT', Sequelize.fn('DISTINCT', 'videoId')), 'totalVideos' ],
|
||||||
|
[ Sequelize.fn('COUNT', 'videoFileId'), 'totalVideoFiles' ]
|
||||||
|
],
|
||||||
|
where: {
|
||||||
|
strategy,
|
||||||
|
actorId: actor.id
|
||||||
|
},
|
||||||
|
include: [
|
||||||
|
{
|
||||||
|
attributes: [],
|
||||||
|
model: VideoFileModel,
|
||||||
|
required: true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
return VideoRedundancyModel.find(query as any) // FIXME: typings
|
||||||
|
.then((r: any) => ({
|
||||||
|
totalUsed: parseInt(r.totalUsed.toString(), 10),
|
||||||
|
totalVideos: r.totalVideos,
|
||||||
|
totalVideoFiles: r.totalVideoFiles
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
toActivityPubObject (): CacheFileObject {
|
toActivityPubObject (): CacheFileObject {
|
||||||
return {
|
return {
|
||||||
id: this.url,
|
id: this.url,
|
||||||
|
|
|
@ -23,6 +23,8 @@ import { ActorFollow } from '../../../../shared/models/actors'
|
||||||
import { readdir } from 'fs-extra'
|
import { readdir } from 'fs-extra'
|
||||||
import { join } from 'path'
|
import { join } from 'path'
|
||||||
import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
|
import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
|
||||||
|
import { getStats } from '../../utils/server/stats'
|
||||||
|
import { ServerStats } from '../../../../shared/models/server/server-stats.model'
|
||||||
|
|
||||||
const expect = chai.expect
|
const expect = chai.expect
|
||||||
|
|
||||||
|
@ -79,16 +81,32 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams:
|
||||||
await waitJobs(servers)
|
await waitJobs(servers)
|
||||||
}
|
}
|
||||||
|
|
||||||
async function check1WebSeed () {
|
async function check1WebSeed (strategy: VideoRedundancyStrategy) {
|
||||||
const webseeds = [
|
const webseeds = [
|
||||||
'http://localhost:9002/static/webseed/' + video1Server2UUID
|
'http://localhost:9002/static/webseed/' + video1Server2UUID
|
||||||
]
|
]
|
||||||
|
|
||||||
for (const server of servers) {
|
for (const server of servers) {
|
||||||
const res = await getVideo(server.url, video1Server2UUID)
|
{
|
||||||
|
const res = await getVideo(server.url, video1Server2UUID)
|
||||||
|
|
||||||
const video: VideoDetails = res.body
|
const video: VideoDetails = res.body
|
||||||
video.files.forEach(f => checkMagnetWebseeds(f, webseeds))
|
video.files.forEach(f => checkMagnetWebseeds(f, webseeds))
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
const res = await getStats(server.url)
|
||||||
|
const data: ServerStats = res.body
|
||||||
|
|
||||||
|
expect(data.videosRedundancy).to.have.lengthOf(1)
|
||||||
|
|
||||||
|
const stat = data.videosRedundancy[0]
|
||||||
|
expect(stat.strategy).to.equal(strategy)
|
||||||
|
expect(stat.totalSize).to.equal(102400)
|
||||||
|
expect(stat.totalUsed).to.equal(0)
|
||||||
|
expect(stat.totalVideoFiles).to.equal(0)
|
||||||
|
expect(stat.totalVideos).to.equal(0)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -107,7 +125,7 @@ async function enableRedundancy () {
|
||||||
expect(server2.following.hostRedundancyAllowed).to.be.true
|
expect(server2.following.hostRedundancyAllowed).to.be.true
|
||||||
}
|
}
|
||||||
|
|
||||||
async function check2Webseeds () {
|
async function check2Webseeds (strategy: VideoRedundancyStrategy) {
|
||||||
await waitJobs(servers)
|
await waitJobs(servers)
|
||||||
await wait(15000)
|
await wait(15000)
|
||||||
await waitJobs(servers)
|
await waitJobs(servers)
|
||||||
|
@ -118,12 +136,14 @@ async function check2Webseeds () {
|
||||||
]
|
]
|
||||||
|
|
||||||
for (const server of servers) {
|
for (const server of servers) {
|
||||||
const res = await getVideo(server.url, video1Server2UUID)
|
{
|
||||||
|
const res = await getVideo(server.url, video1Server2UUID)
|
||||||
|
|
||||||
const video: VideoDetails = res.body
|
const video: VideoDetails = res.body
|
||||||
|
|
||||||
for (const file of video.files) {
|
for (const file of video.files) {
|
||||||
checkMagnetWebseeds(file, webseeds)
|
checkMagnetWebseeds(file, webseeds)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -133,6 +153,20 @@ async function check2Webseeds () {
|
||||||
for (const resolution of [ 240, 360, 480, 720 ]) {
|
for (const resolution of [ 240, 360, 480, 720 ]) {
|
||||||
expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined
|
expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined
|
||||||
}
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
const res = await getStats(servers[0].url)
|
||||||
|
const data: ServerStats = res.body
|
||||||
|
|
||||||
|
expect(data.videosRedundancy).to.have.lengthOf(1)
|
||||||
|
const stat = data.videosRedundancy[0]
|
||||||
|
|
||||||
|
expect(stat.strategy).to.equal(strategy)
|
||||||
|
expect(stat.totalSize).to.equal(102400)
|
||||||
|
expect(stat.totalUsed).to.be.at.least(1).and.below(102401)
|
||||||
|
expect(stat.totalVideoFiles).to.equal(4)
|
||||||
|
expect(stat.totalVideos).to.equal(1)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function cleanServers () {
|
async function cleanServers () {
|
||||||
|
@ -142,15 +176,16 @@ async function cleanServers () {
|
||||||
describe('Test videos redundancy', function () {
|
describe('Test videos redundancy', function () {
|
||||||
|
|
||||||
describe('With most-views strategy', function () {
|
describe('With most-views strategy', function () {
|
||||||
|
const strategy = 'most-views'
|
||||||
|
|
||||||
before(function () {
|
before(function () {
|
||||||
this.timeout(120000)
|
this.timeout(120000)
|
||||||
|
|
||||||
return runServers('most-views')
|
return runServers(strategy)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Should have 1 webseed on the first video', function () {
|
it('Should have 1 webseed on the first video', function () {
|
||||||
return check1WebSeed()
|
return check1WebSeed(strategy)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Should enable redundancy on server 1', function () {
|
it('Should enable redundancy on server 1', function () {
|
||||||
|
@ -160,7 +195,7 @@ describe('Test videos redundancy', function () {
|
||||||
it('Should have 2 webseed on the first video', function () {
|
it('Should have 2 webseed on the first video', function () {
|
||||||
this.timeout(40000)
|
this.timeout(40000)
|
||||||
|
|
||||||
return check2Webseeds()
|
return check2Webseeds(strategy)
|
||||||
})
|
})
|
||||||
|
|
||||||
after(function () {
|
after(function () {
|
||||||
|
@ -169,15 +204,16 @@ describe('Test videos redundancy', function () {
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('With trending strategy', function () {
|
describe('With trending strategy', function () {
|
||||||
|
const strategy = 'trending'
|
||||||
|
|
||||||
before(function () {
|
before(function () {
|
||||||
this.timeout(120000)
|
this.timeout(120000)
|
||||||
|
|
||||||
return runServers('trending')
|
return runServers(strategy)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Should have 1 webseed on the first video', function () {
|
it('Should have 1 webseed on the first video', function () {
|
||||||
return check1WebSeed()
|
return check1WebSeed(strategy)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Should enable redundancy on server 1', function () {
|
it('Should enable redundancy on server 1', function () {
|
||||||
|
@ -187,7 +223,7 @@ describe('Test videos redundancy', function () {
|
||||||
it('Should have 2 webseed on the first video', function () {
|
it('Should have 2 webseed on the first video', function () {
|
||||||
this.timeout(40000)
|
this.timeout(40000)
|
||||||
|
|
||||||
return check2Webseeds()
|
return check2Webseeds(strategy)
|
||||||
})
|
})
|
||||||
|
|
||||||
after(function () {
|
after(function () {
|
||||||
|
@ -196,15 +232,16 @@ describe('Test videos redundancy', function () {
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('With recently added strategy', function () {
|
describe('With recently added strategy', function () {
|
||||||
|
const strategy = 'recently-added'
|
||||||
|
|
||||||
before(function () {
|
before(function () {
|
||||||
this.timeout(120000)
|
this.timeout(120000)
|
||||||
|
|
||||||
return runServers('recently-added', { minViews: 3 })
|
return runServers(strategy, { minViews: 3 })
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Should have 1 webseed on the first video', function () {
|
it('Should have 1 webseed on the first video', function () {
|
||||||
return check1WebSeed()
|
return check1WebSeed(strategy)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Should enable redundancy on server 1', function () {
|
it('Should enable redundancy on server 1', function () {
|
||||||
|
@ -218,7 +255,7 @@ describe('Test videos redundancy', function () {
|
||||||
await wait(15000)
|
await wait(15000)
|
||||||
await waitJobs(servers)
|
await waitJobs(servers)
|
||||||
|
|
||||||
return check1WebSeed()
|
return check1WebSeed(strategy)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Should view 2 times the first video', async function () {
|
it('Should view 2 times the first video', async function () {
|
||||||
|
@ -234,7 +271,7 @@ describe('Test videos redundancy', function () {
|
||||||
it('Should have 2 webseed on the first video', function () {
|
it('Should have 2 webseed on the first video', function () {
|
||||||
this.timeout(40000)
|
this.timeout(40000)
|
||||||
|
|
||||||
return check2Webseeds()
|
return check2Webseeds(strategy)
|
||||||
})
|
})
|
||||||
|
|
||||||
after(function () {
|
after(function () {
|
||||||
|
|
|
@ -21,7 +21,7 @@ import { waitJobs } from '../../utils/server/jobs'
|
||||||
|
|
||||||
const expect = chai.expect
|
const expect = chai.expect
|
||||||
|
|
||||||
describe('Test stats', function () {
|
describe('Test stats (excluding redundancy)', function () {
|
||||||
let servers: ServerInfo[] = []
|
let servers: ServerInfo[] = []
|
||||||
|
|
||||||
before(async function () {
|
before(async function () {
|
||||||
|
@ -65,6 +65,7 @@ describe('Test stats', function () {
|
||||||
expect(data.totalVideos).to.equal(1)
|
expect(data.totalVideos).to.equal(1)
|
||||||
expect(data.totalInstanceFollowers).to.equal(2)
|
expect(data.totalInstanceFollowers).to.equal(2)
|
||||||
expect(data.totalInstanceFollowing).to.equal(1)
|
expect(data.totalInstanceFollowing).to.equal(1)
|
||||||
|
expect(data.videosRedundancy).to.have.lengthOf(0)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Should have the correct stats on instance 2', async function () {
|
it('Should have the correct stats on instance 2', async function () {
|
||||||
|
@ -79,6 +80,7 @@ describe('Test stats', function () {
|
||||||
expect(data.totalVideos).to.equal(1)
|
expect(data.totalVideos).to.equal(1)
|
||||||
expect(data.totalInstanceFollowers).to.equal(1)
|
expect(data.totalInstanceFollowers).to.equal(1)
|
||||||
expect(data.totalInstanceFollowing).to.equal(1)
|
expect(data.totalInstanceFollowing).to.equal(1)
|
||||||
|
expect(data.videosRedundancy).to.have.lengthOf(0)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Should have the correct stats on instance 3', async function () {
|
it('Should have the correct stats on instance 3', async function () {
|
||||||
|
@ -93,6 +95,7 @@ describe('Test stats', function () {
|
||||||
expect(data.totalVideos).to.equal(1)
|
expect(data.totalVideos).to.equal(1)
|
||||||
expect(data.totalInstanceFollowing).to.equal(1)
|
expect(data.totalInstanceFollowing).to.equal(1)
|
||||||
expect(data.totalInstanceFollowers).to.equal(0)
|
expect(data.totalInstanceFollowers).to.equal(0)
|
||||||
|
expect(data.videosRedundancy).to.have.lengthOf(0)
|
||||||
})
|
})
|
||||||
|
|
||||||
after(async function () {
|
after(async function () {
|
||||||
|
|
|
@ -1,11 +1,16 @@
|
||||||
import { makeGetRequest } from '../'
|
import { makeGetRequest } from '../'
|
||||||
|
|
||||||
function getStats (url: string) {
|
function getStats (url: string, useCache = false) {
|
||||||
const path = '/api/v1/server/stats'
|
const path = '/api/v1/server/stats'
|
||||||
|
|
||||||
|
const query = {
|
||||||
|
t: useCache ? undefined : new Date().getTime()
|
||||||
|
}
|
||||||
|
|
||||||
return makeGetRequest({
|
return makeGetRequest({
|
||||||
url,
|
url,
|
||||||
path,
|
path,
|
||||||
|
query,
|
||||||
statusCodeExpected: 200
|
statusCodeExpected: 200
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
import { VideoRedundancyStrategy } from '../redundancy'
|
||||||
|
|
||||||
export interface ServerStats {
|
export interface ServerStats {
|
||||||
totalUsers: number
|
totalUsers: number
|
||||||
totalLocalVideos: number
|
totalLocalVideos: number
|
||||||
|
@ -9,4 +11,12 @@ export interface ServerStats {
|
||||||
|
|
||||||
totalInstanceFollowers: number
|
totalInstanceFollowers: number
|
||||||
totalInstanceFollowing: number
|
totalInstanceFollowing: number
|
||||||
|
|
||||||
|
videosRedundancy: {
|
||||||
|
strategy: VideoRedundancyStrategy
|
||||||
|
totalSize: number
|
||||||
|
totalUsed: number
|
||||||
|
totalVideoFiles: number
|
||||||
|
totalVideos: number
|
||||||
|
}[]
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue