diff --git a/server/controllers/api/remote/videos.ts b/server/controllers/api/remote/videos.ts index f7ee0356a..9dd4afdb5 100644 --- a/server/controllers/api/remote/videos.ts +++ b/server/controllers/api/remote/videos.ts @@ -351,11 +351,9 @@ function updateRemoteVideo (videoAttributesToUpdate: RemoteVideoUpdateData, from videoInstance.set('language', videoAttributesToUpdate.language) videoInstance.set('nsfw', videoAttributesToUpdate.nsfw) videoInstance.set('description', videoAttributesToUpdate.description) - videoInstance.set('infoHash', videoAttributesToUpdate.infoHash) videoInstance.set('duration', videoAttributesToUpdate.duration) videoInstance.set('createdAt', videoAttributesToUpdate.createdAt) videoInstance.set('updatedAt', videoAttributesToUpdate.updatedAt) - videoInstance.set('extname', videoAttributesToUpdate.extname) videoInstance.set('views', videoAttributesToUpdate.views) videoInstance.set('likes', videoAttributesToUpdate.likes) videoInstance.set('dislikes', videoAttributesToUpdate.dislikes) diff --git a/server/tests/api/request-schedulers.ts b/server/tests/api/request-schedulers.ts index 2358ed8f7..326ed3468 100644 --- a/server/tests/api/request-schedulers.ts +++ b/server/tests/api/request-schedulers.ts @@ -12,7 +12,9 @@ import { makeFriends, wait, setAccessTokensToServers, - flushAndRunMultipleServers + flushAndRunMultipleServers, + getRequestsStats, + killallServers } from '../utils' describe('Test requests schedulers stats', function () { @@ -28,14 +30,6 @@ describe('Test requests schedulers stats', function () { return uploadVideo(server.url, server.accessToken, videoAttributes) } - function getRequestsStats (server: ServerInfo) { - return request(server.url) - .get(path) - .set('Accept', 'application/json') - .set('Authorization', 'Bearer ' + server.accessToken) - .expect(200) - } - // --------------------------------------------------------------- before(async function () { @@ -80,7 +74,7 @@ describe('Test requests schedulers stats', function () { }) after(async function () { - process.kill(-servers[0].app.pid) + killallServers(servers) if (this['ok']) { await flushTests() diff --git a/server/tests/real-world/real-world.js b/server/tests/real-world/real-world.js deleted file mode 100644 index ea189c5f2..000000000 --- a/server/tests/real-world/real-world.js +++ /dev/null @@ -1,367 +0,0 @@ -'use strict' - -const each = require('async/each') -const isEqual = require('lodash/isEqual') -const differenceWith = require('lodash/differenceWith') -const program = require('commander') -const series = require('async/series') - -process.env.NODE_ENV = 'test' -const constants = require('../../initializers/constants') - -const loginUtils = require('../utils/login') -const podsUtils = require('../utils/pods') -const serversUtils = require('../utils/servers') -const videosUtils = require('../utils/videos') -const requestSchedulersUtils = require('../utils/request-schedulers') - -program - .option('-c, --create [weight]', 'Weight for creating videos') - .option('-r, --remove [weight]', 'Weight for removing videos') - .option('-u, --update [weight]', 'Weight for updating videos') - .option('-v, --view [weight]', 'Weight for viewing videos') - .option('-l, --like [weight]', 'Weight for liking videos') - .option('-s, --dislike [weight]', 'Weight for disliking videos') - .option('-p, --pods [n]', 'Number of pods to run (3 or 6)', /^3|6$/, 3) - .option('-a, --action [interval]', 'Interval in ms for an action') - .option('-i, --integrity [interval]', 'Interval in ms for an integrity check') - .option('-f, --flush', 'Flush datas on exit') - .option('-d, --difference', 'Display difference if integrity is not okay') - .parse(process.argv) - -const createWeight = program.create !== undefined ? parseInt(program.create) : 5 -const removeWeight = program.remove !== undefined ? parseInt(program.remove) : 4 -const updateWeight = program.update !== undefined ? parseInt(program.update) : 4 -const viewWeight = program.view !== undefined ? parseInt(program.view) : 4 -const likeWeight = program.like !== undefined ? parseInt(program.like) : 4 -const dislikeWeight = program.dislike !== undefined ? parseInt(program.dislike) : 4 -const flushAtExit = program.flush || false -const actionInterval = program.action !== undefined ? parseInt(program.action) : 500 -const integrityInterval = program.integrity !== undefined ? parseInt(program.integrity) : 60000 -const displayDiffOnFail = program.integrity || false - -const numberOfPods = 6 - -console.log('Create weight: %d, update weight: %d, remove weight: %d, view weight: %d, like weight: %d, dislike weight: %d.', createWeight, updateWeight, removeWeight, viewWeight, likeWeight, dislikeWeight) -if (flushAtExit) { - console.log('Program will flush data on exit.') -} else { - console.log('Program will not flush data on exit.') -} -if (displayDiffOnFail) { - console.log('Program will display diff on failure.') -} else { - console.log('Program will not display diff on failure') -} -console.log('Interval in ms for each action: %d.', actionInterval) -console.log('Interval in ms for each integrity check: %d.', integrityInterval) - -console.log('Run servers...') -runServers(numberOfPods, function (err, servers) { - if (err) throw err - - process.on('exit', function () { - exitServers(servers, flushAtExit) - }) - process.on('SIGINT', goodbye) - process.on('SIGTERM', goodbye) - - console.log('Servers runned') - initializeRequestsPerServer(servers) - - let checking = false - - setInterval(function () { - if (checking === true) return - - const rand = getRandomInt(0, createWeight + updateWeight + removeWeight + viewWeight + likeWeight + dislikeWeight) - - const numServer = getRandomNumServer(servers) - servers[numServer].requestsNumber++ - - if (rand < createWeight) { - upload(servers, numServer) - } else if (rand < createWeight + updateWeight) { - update(servers, numServer) - } else if (rand < createWeight + updateWeight + removeWeight) { - remove(servers, numServer) - } else if (rand < createWeight + updateWeight + removeWeight + viewWeight) { - view(servers, numServer) - } else if (rand < createWeight + updateWeight + removeWeight + viewWeight + likeWeight) { - like(servers, numServer) - } else { - dislike(servers, numServer) - } - }, actionInterval) - - // The function will check the consistency between servers (should have the same videos with same attributes...) - setInterval(function () { - if (checking === true) return - - console.log('Checking integrity...') - checking = true - - const waitingInterval = setInterval(function () { - isThereAwaitingRequests(servers, function (err, res) { - if (err) throw err - - if (res === true) { - console.log('A server has awaiting requests, waiting...') - return - } - - checkIntegrity(servers, function () { - initializeRequestsPerServer(servers) - checking = false - clearInterval(waitingInterval) - }) - }) - }, constants.REQUESTS_INTERVAL) - }, integrityInterval) -}) - -// ---------------------------------------------------------------------------- - -function initializeRequestsPerServer (servers) { - servers.forEach(function (server) { - server.requestsNumber = 0 - }) -} - -function getRandomInt (min, max) { - return Math.floor(Math.random() * (max - min)) + min -} - -function getRandomNumServer (servers) { - return getRandomInt(0, servers.length) -} - -function runServers (numberOfPods, callback) { - let servers = null - - series([ - // Run servers - function (next) { - serversUtils.flushAndRunMultipleServers(numberOfPods, function (serversRun) { - servers = serversRun - next() - }) - }, - // Get the access tokens - function (next) { - each(servers, function (server, callbackEach) { - loginUtils.loginAndGetAccessToken(server, function (err, accessToken) { - if (err) return callbackEach(err) - - server.accessToken = accessToken - callbackEach() - }) - }, next) - }, - function (next) { - const server = servers[1] - podsUtils.makeFriends(server.url, server.accessToken, next) - }, - function (next) { - const server = servers[0] - podsUtils.makeFriends(server.url, server.accessToken, next) - }, - function (next) { - setTimeout(next, 1000) - }, - function (next) { - const server = servers[3] - podsUtils.makeFriends(server.url, server.accessToken, next) - }, - function (next) { - const server = servers[5] - podsUtils.makeFriends(server.url, server.accessToken, next) - }, - function (next) { - const server = servers[4] - podsUtils.makeFriends(server.url, server.accessToken, next) - }, - function (next) { - setTimeout(next, 1000) - } - ], function (err) { - return callback(err, servers) - }) -} - -function exitServers (servers, callback) { - if (!callback) callback = function () {} - - servers.forEach(function (server) { - if (server.app) process.kill(-server.app.pid) - }) - - if (flushAtExit) serversUtils.flushTests(callback) -} - -function upload (servers, numServer, callback) { - if (!callback) callback = function () {} - - console.log('Uploading video to server ' + numServer) - - const videoAttributes = { - name: Date.now() + ' name', - category: 4, - nsfw: false, - licence: 2, - language: 1, - description: Date.now() + ' description', - tags: [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ], - fixture: 'video_short1.webm' - } - videosUtils.uploadVideo(servers[numServer].url, servers[numServer].accessToken, videoAttributes, callback) -} - -function update (servers, numServer, callback) { - if (!callback) callback = function () {} - - videosUtils.getVideosList(servers[numServer].url, function (err, res) { - if (err) throw err - - const videos = res.body.data.filter(function (video) { return video.isLocal }) - if (videos.length === 0) return callback() - - const toUpdate = videos[getRandomInt(0, videos.length)].id - const attributes = { - name: Date.now() + ' name', - description: Date.now() + ' description', - tags: [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ] - } - - console.log('Updating video of server ' + numServer) - - videosUtils.updateVideo(servers[numServer].url, servers[numServer].accessToken, toUpdate, attributes, callback) - }) -} - -function remove (servers, numServer, callback) { - if (!callback) callback = function () {} - - videosUtils.getVideosList(servers[numServer].url, function (err, res) { - if (err) throw err - - const videos = res.body.data - if (videos.length === 0) return callback() - - const toRemove = videos[getRandomInt(0, videos.length)].id - - console.log('Removing video from server ' + numServer) - videosUtils.removeVideo(servers[numServer].url, servers[numServer].accessToken, toRemove, callback) - }) -} - -function view (servers, numServer, callback) { - if (!callback) callback = function () {} - - videosUtils.getVideosList(servers[numServer].url, function (err, res) { - if (err) throw err - - const videos = res.body.data - if (videos.length === 0) return callback() - - const toView = videos[getRandomInt(0, videos.length)].id - - console.log('Viewing video from server ' + numServer) - videosUtils.getVideo(servers[numServer].url, toView, callback) - }) -} - -function like (servers, numServer, callback) { - rate(servers, numServer, 'like', callback) -} - -function dislike (servers, numServer, callback) { - rate(servers, numServer, 'dislike', callback) -} - -function rate (servers, numServer, rating, callback) { - if (!callback) callback = function () {} - - videosUtils.getVideosList(servers[numServer].url, function (err, res) { - if (err) throw err - - const videos = res.body.data - if (videos.length === 0) return callback() - - const toRate = videos[getRandomInt(0, videos.length)].id - - console.log('Rating (%s) video from server %d', rating, numServer) - videosUtils.getVideo(servers[numServer].url, toRate, callback) - }) -} - -function checkIntegrity (servers, callback) { - const videos = [] - each(servers, function (server, callback) { - videosUtils.getAllVideosListBy(server.url, function (err, res) { - if (err) throw err - const serverVideos = res.body.data - for (const serverVideo of serverVideos) { - delete serverVideo.id - delete serverVideo.isLocal - delete serverVideo.thumbnailPath - delete serverVideo.updatedAt - delete serverVideo.views - } - - videos.push(serverVideos) - callback() - }) - }, function () { - let i = 0 - - for (const video of videos) { - if (!isEqual(video, videos[0])) { - console.error('Integrity not ok with server %d!', i + 1) - - if (displayDiffOnFail) { - console.log(differenceWith(videos[0], video, isEqual)) - console.log(differenceWith(video, videos[0], isEqual)) - } - - process.exit(-1) - } - - i++ - } - - console.log('Integrity ok.') - return callback() - }) -} - -function goodbye () { - return process.exit(-1) -} - -function isThereAwaitingRequests (servers, callback) { - let noRequests = true - - // Check is each server has awaiting requestq - each(servers, function (server, callbackEach) { - requestSchedulersUtils.getRequestsStats(server, server.accessToken, function (err, res) { - if (err) throw err - - const stats = res.body - - if ( - stats.requestScheduler.totalRequests !== 0 || - stats.requestVideoEventScheduler.totalRequests !== 0 || - stats.requestVideoQaduScheduler.totalRequests !== 0 - ) { - noRequests = false - } - - callbackEach() - }) - }, function (err) { - if (err) throw err - - return callback(null, noRequests === false) - }) -} diff --git a/server/tests/real-world/real-world.ts b/server/tests/real-world/real-world.ts new file mode 100644 index 000000000..a200bd02d --- /dev/null +++ b/server/tests/real-world/real-world.ts @@ -0,0 +1,327 @@ +import * as program from 'commander' +import { isEqual, differenceWith } from 'lodash' + +// /!\ Before imports /!\ +process.env.NODE_ENV = 'test' + +import { REQUESTS_INTERVAL } from '../../initializers/constants' +import { Video, VideoRateType } from '../../../shared' +import { + ServerInfo as DefaultServerInfo, + flushAndRunMultipleServers, + setAccessTokensToServers, + makeFriends, + wait, + killallServers, + flushTests, + uploadVideo, + getVideosList, + updateVideo, + removeVideo, + getVideo, + getAllVideosListBy, + getRequestsStats +} from '../utils' + +interface ServerInfo extends DefaultServerInfo { + requestsNumber: number +} + +program + .option('-c, --create [weight]', 'Weight for creating videos') + .option('-r, --remove [weight]', 'Weight for removing videos') + .option('-u, --update [weight]', 'Weight for updating videos') + .option('-v, --view [weight]', 'Weight for viewing videos') + .option('-l, --like [weight]', 'Weight for liking videos') + .option('-s, --dislike [weight]', 'Weight for disliking videos') + .option('-p, --pods [n]', 'Number of pods to run (3 or 6)', /^3|6$/, 3) + .option('-i, --interval-action [interval]', 'Interval in ms for an action') + .option('-I, --interval-integrity [interval]', 'Interval in ms for an integrity check') + .option('-f, --flush', 'Flush datas on exit') + .option('-d, --difference', 'Display difference if integrity is not okay') + .parse(process.argv) + +const createWeight = program['create'] !== undefined ? parseInt(program['create'], 10) : 5 +const removeWeight = program['remove'] !== undefined ? parseInt(program['remove'], 10) : 4 +const updateWeight = program['update'] !== undefined ? parseInt(program['update'], 10) : 4 +const viewWeight = program['view'] !== undefined ? parseInt(program['view'], 10) : 4 +const likeWeight = program['like'] !== undefined ? parseInt(program['like'], 10) : 4 +const dislikeWeight = program['dislike'] !== undefined ? parseInt(program['dislike'], 10) : 4 +const flushAtExit = program['flush'] || false +const actionInterval = program['intervalAction'] !== undefined ? parseInt(program['intervalAction'], 10) : 500 +const integrityInterval = program['intervalIntegrity'] !== undefined ? parseInt(program['intervalIntegrity'], 10) : 60000 +const displayDiffOnFail = program['difference'] || false + +const numberOfPods = 6 + +console.log( + 'Create weight: %d, update weight: %d, remove weight: %d, view weight: %d, like weight: %d, dislike weight: %d.', + createWeight, updateWeight, removeWeight, viewWeight, likeWeight, dislikeWeight +) + +if (flushAtExit) { + console.log('Program will flush data on exit.') +} else { + console.log('Program will not flush data on exit.') +} +if (displayDiffOnFail) { + console.log('Program will display diff on failure.') +} else { + console.log('Program will not display diff on failure') +} +console.log('Interval in ms for each action: %d.', actionInterval) +console.log('Interval in ms for each integrity check: %d.', integrityInterval) + +console.log('Run servers...') + +start() + +// ---------------------------------------------------------------------------- + +async function start () { + const servers = await runServers(numberOfPods) + + process.on('exit', async () => await exitServers(servers, flushAtExit)) + process.on('SIGINT', goodbye) + process.on('SIGTERM', goodbye) + + console.log('Servers runned') + initializeRequestsPerServer(servers) + + let checking = false + + setInterval(async () => { + if (checking === true) return + + const rand = getRandomInt(0, createWeight + updateWeight + removeWeight + viewWeight + likeWeight + dislikeWeight) + + const numServer = getRandomNumServer(servers) + servers[numServer].requestsNumber++ + + if (rand < createWeight) { + await upload(servers, numServer) + } else if (rand < createWeight + updateWeight) { + await update(servers, numServer) + } else if (rand < createWeight + updateWeight + removeWeight) { + await remove(servers, numServer) + } else if (rand < createWeight + updateWeight + removeWeight + viewWeight) { + await view(servers, numServer) + } else if (rand < createWeight + updateWeight + removeWeight + viewWeight + likeWeight) { + await like(servers, numServer) + } else { + await dislike(servers, numServer) + } + }, actionInterval) + + // The function will check the consistency between servers (should have the same videos with same attributes...) + setInterval(function () { + if (checking === true) return + + console.log('Checking integrity...') + checking = true + + const waitingInterval = setInterval(async () => { + const awaitingRequests = await isThereAwaitingRequests(servers) + if (awaitingRequests === true) { + console.log('A server has awaiting requests, waiting...') + return + } + + await checkIntegrity(servers) + + initializeRequestsPerServer(servers) + checking = false + clearInterval(waitingInterval) + }, REQUESTS_INTERVAL) + }, integrityInterval) +} + +function initializeRequestsPerServer (servers: ServerInfo[]) { + servers.forEach(server => server.requestsNumber = 0) +} + +function getRandomInt (min, max) { + return Math.floor(Math.random() * (max - min)) + min +} + +function getRandomNumServer (servers) { + return getRandomInt(0, servers.length) +} + +async function runServers (numberOfPods: number) { + let servers = null + + // Run servers + servers = await flushAndRunMultipleServers(numberOfPods) + + // Get the access tokens + await setAccessTokensToServers(servers) + + await makeFriends(servers[1].url, servers[1].accessToken) + await makeFriends(servers[0].url, servers[0].accessToken) + await wait(1000) + + await makeFriends(servers[3].url, servers[3].accessToken) + await makeFriends(servers[5].url, servers[5].accessToken) + await makeFriends(servers[4].url, servers[4].accessToken) + + await wait(1000) + + return servers +} + +async function exitServers (servers: ServerInfo[], flushAtExit: boolean) { + killallServers(servers) + + if (flushAtExit) await flushTests() +} + +function upload (servers: ServerInfo[], numServer: number) { + console.log('Uploading video to server ' + numServer) + + const videoAttributes = { + name: Date.now() + ' name', + category: 4, + nsfw: false, + licence: 2, + language: 1, + description: Date.now() + ' description', + tags: [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ], + fixture: 'video_short1.webm' + } + return uploadVideo(servers[numServer].url, servers[numServer].accessToken, videoAttributes) +} + +async function update (servers: ServerInfo[], numServer: number) { + const res = await getVideosList(servers[numServer].url) + + const videos = res.body.data.filter(video => video.isLocal === true) + if (videos.length === 0) return undefined + + const toUpdate = videos[getRandomInt(0, videos.length)].id + const attributes = { + name: Date.now() + ' name', + description: Date.now() + ' description', + tags: [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ] + } + + console.log('Updating video of server ' + numServer) + + return updateVideo(servers[numServer].url, servers[numServer].accessToken, toUpdate, attributes) +} + +async function remove (servers: ServerInfo[], numServer: number) { + const res = await getVideosList(servers[numServer].url) + const videos = res.body.data + if (videos.length === 0) return undefined + + const toRemove = videos[getRandomInt(0, videos.length)].id + + console.log('Removing video from server ' + numServer) + return removeVideo(servers[numServer].url, servers[numServer].accessToken, toRemove) +} + +async function view (servers: ServerInfo[], numServer: number) { + const res = await getVideosList(servers[numServer].url) + + const videos = res.body.data + if (videos.length === 0) return undefined + + const toView = videos[getRandomInt(0, videos.length)].id + + console.log('Viewing video from server ' + numServer) + return getVideo(servers[numServer].url, toView) +} + +function like (servers: ServerInfo[], numServer: number) { + return rate(servers, numServer, 'like') +} + +function dislike (servers: ServerInfo[], numServer: number) { + return rate(servers, numServer, 'dislike') +} + +async function rate (servers: ServerInfo[], numServer: number, rating: VideoRateType) { + const res = await getVideosList(servers[numServer].url) + + const videos = res.body.data + if (videos.length === 0) return undefined + + const toRate = videos[getRandomInt(0, videos.length)].id + + console.log('Rating (%s) video from server %d', rating, numServer) + return getVideo(servers[numServer].url, toRate) +} + +async function checkIntegrity (servers: ServerInfo[]) { + const videos: Video[][] = [] + const tasks: Promise[] = [] + + // Fetch all videos and remove some fields that can differ between pods + for (const server of servers) { + const p = getAllVideosListBy(server.url).then(res => { + const serverVideos = res.body.data + for (const serverVideo of serverVideos) { + delete serverVideo.id + delete serverVideo.isLocal + delete serverVideo.thumbnailPath + delete serverVideo.updatedAt + delete serverVideo.views + } + + videos.push(serverVideos) + }) + + tasks.push(p) + } + + await Promise.all(tasks) + + let i = 0 + for (const video of videos) { + if (!isEqual(video, videos[0])) { + console.error('Integrity not ok with server %d!', i + 1) + + if (displayDiffOnFail) { + console.log(differenceWith(videos[0], video, isEqual)) + console.log(differenceWith(video, videos[0], isEqual)) + } + + process.exit(-1) + } + + i++ + } + + console.log('Integrity ok.') +} + +function goodbye () { + return process.exit(-1) +} + +async function isThereAwaitingRequests (servers: ServerInfo[]) { + const tasks: Promise[] = [] + let awaitingRequests = false + + // Check if each server has awaiting request + for (const server of servers) { + const p = getRequestsStats(server).then(res => { + const stats = res.body + + if ( + stats.requestScheduler.totalRequests !== 0 || + stats.requestVideoEventScheduler.totalRequests !== 0 || + stats.requestVideoQaduScheduler.totalRequests !== 0 + ) { + awaitingRequests = true + } + }) + + tasks.push(p) + } + + await Promise.all(tasks) + + return awaitingRequests +} diff --git a/server/tests/utils/request-schedulers.ts b/server/tests/utils/request-schedulers.ts index ae8227b79..f100f6d99 100644 --- a/server/tests/utils/request-schedulers.ts +++ b/server/tests/utils/request-schedulers.ts @@ -1,12 +1,14 @@ import * as request from 'supertest' -function getRequestsStats (server: { url: string }, accessToken: string) { - const path = '/api/v1/requests/stats' +import { ServerInfo } from '../utils' + +function getRequestsStats (server: ServerInfo) { + const path = '/api/v1/request-schedulers/stats' return request(server.url) .get(path) .set('Accept', 'application/json') - .set('Authorization', 'Bearer ' + accessToken) + .set('Authorization', 'Bearer ' + server.accessToken) .expect(200) .expect('Content-Type', /json/) }