From f148e5edb62c914eff12d7d4a5f879fedcfc64be Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 19 Mar 2017 09:16:33 +0100 Subject: [PATCH] Better real world test --- server/controllers/api/remote/videos.js | 6 +- server/tests/real-world/real-world.js | 135 ++++++++++++++++++++---- server/tests/utils/requests-stats.js | 25 +++++ 3 files changed, 147 insertions(+), 19 deletions(-) create mode 100644 server/tests/utils/requests-stats.js diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js index 98891c99e..cbd58e0e4 100644 --- a/server/controllers/api/remote/videos.js +++ b/server/controllers/api/remote/videos.js @@ -196,6 +196,8 @@ function quickAndDirtyUpdateVideoRetryWrapper (videoData, fromPod, finalCallback } function quickAndDirtyUpdateVideo (videoData, fromPod, finalCallback) { + let videoName + waterfall([ databaseUtils.startSerializableTransaction, @@ -208,6 +210,8 @@ function quickAndDirtyUpdateVideo (videoData, fromPod, finalCallback) { function updateVideoIntoDB (t, videoInstance, callback) { const options = { transaction: t } + videoName = videoInstance.name + if (videoData.views) { videoInstance.set('views', videoData.views) } @@ -233,7 +237,7 @@ function quickAndDirtyUpdateVideo (videoData, fromPod, finalCallback) { return databaseUtils.rollbackTransaction(err, t, finalCallback) } - logger.info('Remote video %s quick and dirty updated', videoData.name) + logger.info('Remote video %s quick and dirty updated', videoName) return finalCallback(null) }) } diff --git a/server/tests/real-world/real-world.js b/server/tests/real-world/real-world.js index 12ab06d6d..239b790ae 100644 --- a/server/tests/real-world/real-world.js +++ b/server/tests/real-world/real-world.js @@ -13,11 +13,15 @@ const loginUtils = require('../utils/login') const podsUtils = require('../utils/pods') const serversUtils = require('../utils/servers') const videosUtils = require('../utils/videos') +const requestStatsUtils = require('../utils/requests-stats') program .option('-c, --create [weight]', 'Weight for creating videos') .option('-r, --remove [weight]', 'Weight for removing videos') .option('-u, --update [weight]', 'Weight for updating videos') + .option('-v, --view [weight]', 'Weight for viewing videos') + .option('-l, --like [weight]', 'Weight for liking videos') + .option('-s --dislike [weight]', 'Weight for disliking videos') .option('-p, --pods [n]', 'Number of pods to run (3 or 6)', /^3|6$/, 3) .option('-a, --action [interval]', 'Interval in ms for an action') .option('-i, --integrity [interval]', 'Interval in ms for an integrity check') @@ -28,6 +32,9 @@ program const createWeight = program.create !== undefined ? parseInt(program.create) : 5 const removeWeight = program.remove !== undefined ? parseInt(program.remove) : 4 const updateWeight = program.update !== undefined ? parseInt(program.update) : 4 +const viewWeight = program.view !== undefined ? parseInt(program.view) : 4 +const likeWeight = program.like !== undefined ? parseInt(program.like) : 4 +const dislikeWeight = program.dislike !== undefined ? parseInt(program.dislike) : 4 const flushAtExit = program.flush || false const actionInterval = program.action !== undefined ? parseInt(program.action) : 500 const integrityInterval = program.integrity !== undefined ? parseInt(program.integrity) : 60000 @@ -35,13 +42,7 @@ const displayDiffOnFail = program.integrity || false const numberOfPods = 6 -// Wait requests between pods -const baseRequestInterval = integrityInterval < constants.REQUESTS_INTERVAL ? constants.REQUESTS_INTERVAL : integrityInterval -const requestsMaxPerInterval = baseRequestInterval / actionInterval -const intervalsToMakeAllRequests = Math.ceil(requestsMaxPerInterval / constants.REQUESTS_LIMIT_PER_POD) -const waitForBeforeIntegrityCheck = (intervalsToMakeAllRequests * constants.REQUESTS_INTERVAL) - integrityInterval + 1000 - -console.log('Create weight: %d, update weight: %d, remove weight: %d.', createWeight, updateWeight, removeWeight) +console.log('Create weight: %d, update weight: %d, remove weight: %d, view weight: %d, like weight: %d, dislike weight: %d.', createWeight, updateWeight, removeWeight, viewWeight, likeWeight, dislikeWeight) if (flushAtExit) { console.log('Program will flush data on exit.') } else { @@ -54,7 +55,6 @@ if (displayDiffOnFail) { } console.log('Interval in ms for each action: %d.', actionInterval) console.log('Interval in ms for each integrity check: %d.', integrityInterval) -console.log('Will wait %d ms before an integrity check.', waitForBeforeIntegrityCheck) console.log('Run servers...') runServers(numberOfPods, function (err, servers) { @@ -67,39 +67,65 @@ runServers(numberOfPods, function (err, servers) { process.on('SIGTERM', goodbye) console.log('Servers runned') + initializeRequestsPerServer(servers) let checking = false setInterval(function () { if (checking === true) return - const rand = getRandomInt(0, createWeight + updateWeight + removeWeight) + const rand = getRandomInt(0, createWeight + updateWeight + removeWeight + viewWeight + likeWeight + dislikeWeight) + + const numServer = getRandomNumServer(servers) + servers[numServer].requestsNumber++ if (rand < createWeight) { - upload(servers, getRandomNumServer(servers)) + upload(servers, numServer) } else if (rand < createWeight + updateWeight) { - update(servers, getRandomNumServer(servers)) + update(servers, numServer) + } else if (rand < createWeight + updateWeight + removeWeight) { + remove(servers, numServer) + } else if (rand < createWeight + updateWeight + removeWeight + viewWeight) { + view(servers, numServer) + } else if (rand < createWeight + updateWeight + removeWeight + viewWeight + likeWeight) { + like(servers, numServer) } else { - remove(servers, getRandomNumServer(servers)) + dislike(servers, numServer) } }, actionInterval) + // The function will check the consistency between servers (should have the same videos with same attributes...) setInterval(function () { if (checking === true) return console.log('Checking integrity...') checking = true - setTimeout(function () { - checkIntegrity(servers, function () { - checking = false + const waitingInterval = setInterval(function () { + isThereAwaitingRequests(servers, function (res) { + if (res === true) { + console.log('A server has awaiting requests, waiting...') + return + } + + checkIntegrity(servers, function () { + initializeRequestsPerServer(servers) + checking = false + clearInterval(waitingInterval) + }) }) - }, waitForBeforeIntegrityCheck) + }, constants.REQUESTS_INTERVAL) }, integrityInterval) }) // ---------------------------------------------------------------------------- +function initializeRequestsPerServer (servers) { + servers.forEach(function (server) { + server.requestsNumber = 0 + }) +} + function getRandomInt (min, max) { return Math.floor(Math.random() * (max - min)) + min } @@ -179,7 +205,7 @@ function upload (servers, numServer, callback) { const tags = [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ] const file = 'video_short1.webm' - console.log('Upload video to server ' + numServer) + console.log('Uploading video to server ' + numServer) videosUtils.uploadVideo(servers[numServer].url, servers[numServer].accessToken, name, description, tags, file, callback) } @@ -220,6 +246,46 @@ function remove (servers, numServer, callback) { }) } +function view (servers, numServer, callback) { + if (!callback) callback = function () {} + + videosUtils.getVideosList(servers[numServer].url, function (err, res) { + if (err) throw err + + const videos = res.body.data + if (videos.length === 0) return callback() + + const toView = videos[getRandomInt(0, videos.length)].id + + console.log('Viewing video from server ' + numServer) + videosUtils.getVideo(servers[numServer].url, toView, callback) + }) +} + +function like (servers, numServer, callback) { + rate(servers, numServer, 'like', callback) +} + +function dislike (servers, numServer, callback) { + rate(servers, numServer, 'dislike', callback) +} + +function rate (servers, numServer, rating, callback) { + if (!callback) callback = function () {} + + videosUtils.getVideosList(servers[numServer].url, function (err, res) { + if (err) throw err + + const videos = res.body.data + if (videos.length === 0) return callback() + + const toRate = videos[getRandomInt(0, videos.length)].id + + console.log('Rating (%s) video from server %d', rating, numServer) + videosUtils.getVideo(servers[numServer].url, toRate, callback) + }) +} + function checkIntegrity (servers, callback) { const videos = [] each(servers, function (server, callback) { @@ -231,22 +297,28 @@ function checkIntegrity (servers, callback) { delete serverVideo.isLocal delete serverVideo.thumbnailPath delete serverVideo.updatedAt + delete serverVideo.views } videos.push(serverVideos) callback() }) }, function () { + let i = 0 + for (const video of videos) { if (!isEqual(video, videos[0])) { - console.error('Integrity not ok!') + console.error('Integrity not ok with server %d!', i + 1) if (displayDiffOnFail) { console.log(differenceWith(videos[0], video, isEqual)) + console.log(differenceWith(video, videos[0], isEqual)) } process.exit(-1) } + + i++ } console.log('Integrity ok.') @@ -257,3 +329,30 @@ function checkIntegrity (servers, callback) { function goodbye () { return process.exit(-1) } + +function isThereAwaitingRequests (servers, callback) { + let noRequests = true + + // Check is each server has awaiting requestq + each(servers, function (server, callbackEach) { + requestStatsUtils.getRequestsStats(server, server.accessToken, function (err, res) { + if (err) throw err + + const stats = res.body + + if ( + stats.requestScheduler.totalRequests !== 0 || + stats.requestVideoEventScheduler.totalRequests !== 0 || + stats.requestVideoQaduScheduler.totalRequests !== 0 + ) { + noRequests = false + } + + callbackEach() + }) + }, function (err) { + if (err) throw err + + return callback(noRequests === false) + }) +} diff --git a/server/tests/utils/requests-stats.js b/server/tests/utils/requests-stats.js new file mode 100644 index 000000000..16835ce47 --- /dev/null +++ b/server/tests/utils/requests-stats.js @@ -0,0 +1,25 @@ +'use strict' + +const request = require('supertest') + +const requestsStatsUtils = { + getRequestsStats +} + +// ---------------------- Export functions -------------------- + +function getRequestsStats (server, accessToken, callback) { + const path = '/api/v1/requests/stats' + + request(server.url) + .get(path) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + accessToken) + .expect(200) + .expect('Content-Type', /json/) + .end(callback) +} + +// --------------------------------------------------------------------------- + +module.exports = requestsStatsUtils