Add follow tests

pull/128/head
Chocobozzz 2017-11-21 13:43:29 +01:00
parent 81de19482b
commit 0f91ae62df
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
23 changed files with 736 additions and 522 deletions

View File

@ -10,6 +10,7 @@
<p-column field="follower.host" header="Host"></p-column>
<p-column field="email" header="Email"></p-column>
<p-column field="follower.score" header="Score"></p-column>
<p-column field="state" header="State"></p-column>
<p-column field="createdAt" header="Created date" [sortable]="true"></p-column>
</p-dataTable>
</div>

View File

@ -9,7 +9,7 @@
<p-column field="id" header="ID"></p-column>
<p-column field="following.host" header="Host"></p-column>
<p-column field="email" header="Email"></p-column>
<p-column field="following.score" header="Score"></p-column>
<p-column field="state" header="State"></p-column>
<p-column field="createdAt" header="Created date" [sortable]="true"></p-column>
<p-column header="Unfollow" styleClass="action-cell">
<ng-template pTemplate="body" let-following="rowData">

View File

@ -16,6 +16,9 @@ import { followersSortValidator, followingSortValidator } from '../../../middlew
import { AccountFollowInstance } from '../../../models/index'
import { sendFollow } from '../../../lib/index'
import { sendUndoFollow } from '../../../lib/activitypub/send/send-undo'
import { AccountInstance } from '../../../models/account/account-interface'
import { retryTransactionWrapper } from '../../../helpers/database-utils'
import { saveAccountAndServerIfNotExist } from '../../../lib/activitypub/account'
const serverFollowsRouter = express.Router()
@ -32,7 +35,7 @@ serverFollowsRouter.post('/following',
ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW),
followValidator,
setBodyHostsPort,
asyncMiddleware(follow)
asyncMiddleware(followRetry)
)
serverFollowsRouter.delete('/following/:accountId',
@ -72,7 +75,7 @@ async function listFollowers (req: express.Request, res: express.Response, next:
return res.json(getFormattedObjects(resultList.data, resultList.total))
}
async function follow (req: express.Request, res: express.Response, next: express.NextFunction) {
async function followRetry (req: express.Request, res: express.Response, next: express.NextFunction) {
const hosts = req.body.hosts as string[]
const fromAccount = await getServerAccount()
@ -88,31 +91,12 @@ async function follow (req: express.Request, res: express.Response, next: expres
.then(accountResult => {
let targetAccount = accountResult.account
return db.sequelize.transaction(async t => {
if (accountResult.loadedFromDB === false) {
targetAccount = await targetAccount.save({ transaction: t })
}
const options = {
arguments: [ fromAccount, targetAccount, accountResult.loadedFromDB ],
errorMessage: 'Cannot follow with many retries.'
}
const [ accountFollow ] = await db.AccountFollow.findOrCreate({
where: {
accountId: fromAccount.id,
targetAccountId: targetAccount.id
},
defaults: {
state: 'pending',
accountId: fromAccount.id,
targetAccountId: targetAccount.id
},
transaction: t
})
accountFollow.AccountFollowing = targetAccount
accountFollow.AccountFollower = fromAccount
// Send a notification to remote server
if (accountFollow.state === 'pending') {
await sendFollow(accountFollow, t)
}
})
return retryTransactionWrapper(follow, options)
})
.catch(err => logger.warn('Cannot follow server %s.', `${accountName}@${host}`, err))
@ -121,19 +105,51 @@ async function follow (req: express.Request, res: express.Response, next: expres
// Don't make the client wait the tasks
Promise.all(tasks)
.catch(err => {
logger.error('Error in follow.', err)
})
.catch(err => logger.error('Error in follow.', err))
return res.status(204).end()
}
async function follow (fromAccount: AccountInstance, targetAccount: AccountInstance, targetAlreadyInDB: boolean) {
try {
await db.sequelize.transaction(async t => {
if (targetAlreadyInDB === false) {
await saveAccountAndServerIfNotExist(targetAccount, t)
}
const [ accountFollow ] = await db.AccountFollow.findOrCreate({
where: {
accountId: fromAccount.id,
targetAccountId: targetAccount.id
},
defaults: {
state: 'pending',
accountId: fromAccount.id,
targetAccountId: targetAccount.id
},
transaction: t
})
accountFollow.AccountFollowing = targetAccount
accountFollow.AccountFollower = fromAccount
// Send a notification to remote server
if (accountFollow.state === 'pending') {
await sendFollow(accountFollow, t)
}
})
} catch (err) {
// Reset target account
targetAccount.isNewRecord = !targetAlreadyInDB
throw err
}
}
async function removeFollow (req: express.Request, res: express.Response, next: express.NextFunction) {
const following: AccountFollowInstance = res.locals.following
const follow: AccountFollowInstance = res.locals.follow
await db.sequelize.transaction(async t => {
await sendUndoFollow(following, t)
await following.destroy({ transaction: t })
await sendUndoFollow(follow, t)
await follow.destroy({ transaction: t })
})
return res.status(204).end()

View File

@ -4,12 +4,15 @@ import * as Bluebird from 'bluebird'
import { logger } from './logger'
type RetryTransactionWrapperOptions = { errorMessage: string, arguments?: any[] }
function retryTransactionWrapper (functionToRetry: (...args) => Promise<any> | Bluebird<any>, options: RetryTransactionWrapperOptions) {
function retryTransactionWrapper <T> (
functionToRetry: (...args) => Promise<T> | Bluebird<T>,
options: RetryTransactionWrapperOptions
): Promise<T> {
const args = options.arguments ? options.arguments : []
return transactionRetryer(callback => {
return transactionRetryer<T>(callback => {
functionToRetry.apply(this, args)
.then(result => callback(null, result))
.then((result: T) => callback(null, result))
.catch(err => callback(err))
})
.catch(err => {
@ -18,8 +21,8 @@ function retryTransactionWrapper (functionToRetry: (...args) => Promise<any> | B
})
}
function transactionRetryer (func: Function) {
return new Promise((res, rej) => {
function transactionRetryer <T> (func: (err: any, data: T) => any) {
return new Promise<T>((res, rej) => {
retry({
times: 5,

View File

@ -1,9 +1,9 @@
import * as WebFinger from 'webfinger.js'
import { WebFingerData } from '../../shared'
import { fetchRemoteAccount } from '../lib/activitypub/account'
import { isTestInstance } from './core-utils'
import { isActivityPubUrlValid } from './custom-validators'
import { fetchRemoteAccountAndCreateServer } from '../lib/activitypub/account'
const webfinger = new WebFinger({
webfist_fallback: false,
@ -22,10 +22,10 @@ async function getAccountFromWebfinger (nameWithHost: string) {
throw new Error('Cannot find self link or href is not a valid URL.')
}
const res = await fetchRemoteAccountAndCreateServer(selfLink.href)
if (res === undefined) throw new Error('Cannot fetch and create server of remote account ' + selfLink.href)
const account = await fetchRemoteAccount(selfLink.href)
if (account === undefined) throw new Error('Cannot fetch remote account ' + selfLink.href)
return res.account
return account
}
// ---------------------------------------------------------------------------

View File

@ -323,7 +323,7 @@ const OPENGRAPH_AND_OEMBED_COMMENT = '<!-- open graph and oembed tags -->'
if (isTestInstance() === true) {
CONSTRAINTS_FIELDS.VIDEOS.DURATION.max = 14
FRIEND_SCORE.BASE = 20
JOBS_FETCHING_INTERVAL = 2000
JOBS_FETCHING_INTERVAL = 1000
REMOTE_SCHEME.HTTP = 'http'
REMOTE_SCHEME.WS = 'ws'
STATIC_MAX_AGE = '0'

View File

@ -1,27 +1,65 @@
import * as Bluebird from 'bluebird'
import * as url from 'url'
import { ActivityPubActor } from '../../../shared/models/activitypub/activitypub-actor'
import { isRemoteAccountValid } from '../../helpers/custom-validators/activitypub/account'
import { retryTransactionWrapper } from '../../helpers/database-utils'
import { logger } from '../../helpers/logger'
import { doRequest } from '../../helpers/requests'
import { ACTIVITY_PUB } from '../../initializers/constants'
import { database as db } from '../../initializers/database'
import { AccountInstance } from '../../models/account/account-interface'
import { Transaction } from 'sequelize'
async function getOrCreateAccount (accountUrl: string) {
async function getOrCreateAccountAndServer (accountUrl: string) {
let account = await db.Account.loadByUrl(accountUrl)
// We don't have this account in our database, fetch it on remote
if (!account) {
const res = await fetchRemoteAccountAndCreateServer(accountUrl)
if (res === undefined) throw new Error('Cannot fetch remote account.')
account = await fetchRemoteAccount(accountUrl)
if (account === undefined) throw new Error('Cannot fetch remote account.')
// Save our new account in database
account = await res.account.save()
const options = {
arguments: [ account ],
errorMessage: 'Cannot save account and server with many retries.'
}
account = await retryTransactionWrapper(saveAccountAndServerIfNotExist, options)
}
return account
}
async function fetchRemoteAccountAndCreateServer (accountUrl: string) {
function saveAccountAndServerIfNotExist (account: AccountInstance, t?: Transaction): Bluebird<AccountInstance> | Promise<AccountInstance> {
if (t !== undefined) {
return save(t)
} else {
return db.sequelize.transaction(t => {
return save(t)
})
}
async function save (t: Transaction) {
const accountHost = url.parse(account.url).host
const serverOptions = {
where: {
host: accountHost
},
defaults: {
host: accountHost
},
transaction: t
}
const [ server ] = await db.Server.findOrCreate(serverOptions)
// Save our new account in database
account.set('serverId', server.id)
account = await account.save({ transaction: t })
return account
}
}
async function fetchRemoteAccount (accountUrl: string) {
const options = {
uri: accountUrl,
method: 'GET',
@ -64,24 +102,13 @@ async function fetchRemoteAccountAndCreateServer (accountUrl: string) {
followingUrl: accountJSON.following
})
const accountHost = url.parse(account.url).host
const serverOptions = {
where: {
host: accountHost
},
defaults: {
host: accountHost
}
}
const [ server ] = await db.Server.findOrCreate(serverOptions)
account.set('serverId', server.id)
return { account, server }
return account
}
export {
getOrCreateAccount,
fetchRemoteAccountAndCreateServer
getOrCreateAccountAndServer,
fetchRemoteAccount,
saveAccountAndServerIfNotExist
}
// ---------------------------------------------------------------------------

View File

@ -6,7 +6,7 @@ import { logger } from '../../../helpers/logger'
import { database as db } from '../../../initializers'
import { AccountInstance } from '../../../models/account/account-interface'
import { VideoChannelInstance } from '../../../models/video/video-channel-interface'
import { getOrCreateAccount } from '../account'
import { getOrCreateAccountAndServer } from '../account'
import { getOrCreateVideoChannel } from '../video-channels'
import { generateThumbnailFromUrl } from '../videos'
import { videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc'
@ -14,7 +14,7 @@ import { videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes }
async function processAddActivity (activity: ActivityAdd) {
const activityObject = activity.object
const activityType = activityObject.type
const account = await getOrCreateAccount(activity.actor)
const account = await getOrCreateAccountAndServer(activity.actor)
if (activityType === 'Video') {
const videoChannelUrl = activity.target

View File

@ -5,11 +5,11 @@ import { VideoInstance } from '../../../models/index'
import { VideoChannelInstance } from '../../../models/video/video-channel-interface'
import { processAddActivity } from './process-add'
import { processCreateActivity } from './process-create'
import { getOrCreateAccount } from '../account'
import { getOrCreateAccountAndServer } from '../account'
async function processAnnounceActivity (activity: ActivityAnnounce) {
const announcedActivity = activity.object
const accountAnnouncer = await getOrCreateAccount(activity.actor)
const accountAnnouncer = await getOrCreateAccountAndServer(activity.actor)
if (announcedActivity.type === 'Create' && announcedActivity.object.type === 'VideoChannel') {
// Add share entry

View File

@ -3,14 +3,14 @@ import { VideoAbuseObject } from '../../../../shared/models/activitypub/objects/
import { logger, retryTransactionWrapper } from '../../../helpers'
import { database as db } from '../../../initializers'
import { AccountInstance } from '../../../models/account/account-interface'
import { getOrCreateAccount } from '../account'
import { getOrCreateAccountAndServer } from '../account'
import { getVideoChannelActivityPubUrl } from '../url'
import { videoChannelActivityObjectToDBAttributes } from './misc'
async function processCreateActivity (activity: ActivityCreate) {
const activityObject = activity.object
const activityType = activityObject.type
const account = await getOrCreateAccount(activity.actor)
const account = await getOrCreateAccountAndServer(activity.actor)
if (activityType === 'VideoChannel') {
return processCreateVideoChannel(account, activityObject as VideoChannelObject)

View File

@ -5,10 +5,10 @@ import { database as db } from '../../../initializers'
import { AccountInstance } from '../../../models/account/account-interface'
import { VideoChannelInstance } from '../../../models/video/video-channel-interface'
import { VideoInstance } from '../../../models/video/video-interface'
import { getOrCreateAccount } from '../account'
import { getOrCreateAccountAndServer } from '../account'
async function processDeleteActivity (activity: ActivityDelete) {
const account = await getOrCreateAccount(activity.actor)
const account = await getOrCreateAccountAndServer(activity.actor)
if (account.url === activity.id) {
return processDeleteAccount(account)

View File

@ -4,11 +4,11 @@ import { database as db } from '../../../initializers'
import { AccountInstance } from '../../../models/account/account-interface'
import { logger } from '../../../helpers/logger'
import { sendAccept } from '../send/send-accept'
import { getOrCreateAccount } from '../account'
import { getOrCreateAccountAndServer } from '../account'
async function processFollowActivity (activity: ActivityFollow) {
const activityObject = activity.object
const account = await getOrCreateAccount(activity.actor)
const account = await getOrCreateAccountAndServer(activity.actor)
return processFollow(account, activityObject)
}

View File

@ -8,10 +8,10 @@ import { AccountInstance } from '../../../models/account/account-interface'
import { VideoInstance } from '../../../models/video/video-interface'
import { videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc'
import Bluebird = require('bluebird')
import { getOrCreateAccount } from '../account'
import { getOrCreateAccountAndServer } from '../account'
async function processUpdateActivity (activity: ActivityUpdate) {
const account = await getOrCreateAccount(activity.actor)
const account = await getOrCreateAccountAndServer(activity.actor)
if (activity.object.type === 'Video') {
return processUpdateVideo(account, activity.object)

View File

@ -4,7 +4,7 @@ import { ActivityPubSignature } from '../../shared'
import { isSignatureVerified, logger } from '../helpers'
import { database as db } from '../initializers'
import { ACTIVITY_PUB } from '../initializers/constants'
import { fetchRemoteAccountAndCreateServer } from '../lib/activitypub/account'
import { fetchRemoteAccount, saveAccountAndServerIfNotExist } from '../lib/activitypub/account'
async function checkSignature (req: Request, res: Response, next: NextFunction) {
const signatureObject: ActivityPubSignature = req.body.signature
@ -15,15 +15,14 @@ async function checkSignature (req: Request, res: Response, next: NextFunction)
// We don't have this account in our database, fetch it on remote
if (!account) {
const accountResult = await fetchRemoteAccountAndCreateServer(signatureObject.creator)
account = await fetchRemoteAccount(signatureObject.creator)
if (!accountResult) {
if (!account) {
return res.sendStatus(403)
}
// Save our new account in database
account = accountResult.account
await account.save()
// Save our new account and its server in database
await saveAccountAndServerIfNotExist(account)
}
const verified = await isSignatureVerified(account, req.body)

View File

@ -31,19 +31,19 @@ const removeFollowingValidator = [
param('accountId').custom(isIdOrUUIDValid).withMessage('Should have a valid account id'),
(req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking follow parameters', { parameters: req.body })
logger.debug('Checking unfollow parameters', { parameters: req.params })
checkErrors(req, res, async () => {
try {
const serverAccount = await getServerAccount()
const following = await db.AccountFollow.loadByAccountAndTarget(serverAccount.id, req.params.accountId)
const follow = await db.AccountFollow.loadByAccountAndTarget(serverAccount.id, req.params.accountId)
if (!following) {
if (!follow) {
return res.status(404)
.end()
}
res.locals.following = following
res.locals.follow = follow
return next()
} catch (err) {

View File

@ -166,47 +166,49 @@ describe('Test server follows API validators', function () {
})
describe('When removing following', function () {
// it('Should fail with an invalid token', async function () {
// await request(server.url)
// .delete(path + '/1')
// .set('Authorization', 'Bearer faketoken')
// .set('Accept', 'application/json')
// .expect(401)
// })
//
// it('Should fail if the user is not an administrator', async function () {
// await request(server.url)
// .delete(path + '/1')
// .set('Authorization', 'Bearer ' + userAccessToken)
// .set('Accept', 'application/json')
// .expect(403)
// })
//
// it('Should fail with an undefined id', async function () {
// await request(server.url)
// .delete(path + '/' + undefined)
// .set('Authorization', 'Bearer ' + server.accessToken)
// .set('Accept', 'application/json')
// .expect(400)
// })
//
// it('Should fail with an invalid id', async function () {
// await request(server.url)
// .delete(path + '/foobar')
// .set('Authorization', 'Bearer ' + server.accessToken)
// .set('Accept', 'application/json')
// .expect(400)
// })
//
// it('Should fail we do not follow this server', async function () {
// await request(server.url)
// .delete(path + '/-1')
// .set('Authorization', 'Bearer ' + server.accessToken)
// .set('Accept', 'application/json')
// .expect(404)
// })
//
// it('Should succeed with the correct parameters')
const path = '/api/v1/server/following'
it('Should fail with an invalid token', async function () {
await request(server.url)
.delete(path + '/1')
.set('Authorization', 'Bearer faketoken')
.set('Accept', 'application/json')
.expect(401)
})
it('Should fail if the user is not an administrator', async function () {
await request(server.url)
.delete(path + '/1')
.set('Authorization', 'Bearer ' + userAccessToken)
.set('Accept', 'application/json')
.expect(403)
})
it('Should fail with an undefined id', async function () {
await request(server.url)
.delete(path + '/' + undefined)
.set('Authorization', 'Bearer ' + server.accessToken)
.set('Accept', 'application/json')
.expect(400)
})
it('Should fail with an invalid id', async function () {
await request(server.url)
.delete(path + '/foobar')
.set('Authorization', 'Bearer ' + server.accessToken)
.set('Accept', 'application/json')
.expect(400)
})
it('Should fail we do not follow this server', async function () {
await request(server.url)
.delete(path + '/-1')
.set('Authorization', 'Bearer ' + server.accessToken)
.set('Accept', 'application/json')
.expect(404)
})
it('Should succeed with the correct parameters')
})
})

174
server/tests/api/follows.ts Normal file
View File

@ -0,0 +1,174 @@
/* tslint:disable:no-unused-expression */
import * as chai from 'chai'
import 'mocha'
import {
flushAndRunMultipleServers,
flushTests,
getVideosList,
killallServers,
ServerInfo,
setAccessTokensToServers,
uploadVideo,
wait
} from '../utils'
import { follow, getFollowersListPaginationAndSort, getFollowingListPaginationAndSort, unfollow } from '../utils/follows'
const expect = chai.expect
describe('Test follows', function () {
let servers: ServerInfo[] = []
let server3Id: number
before(async function () {
this.timeout(120000)
servers = await flushAndRunMultipleServers(3)
// Get the access tokens
await setAccessTokensToServers(servers)
})
it('Should not have followers', async function () {
for (const server of servers) {
const res = await getFollowersListPaginationAndSort(server.url, 0, 5, 'createdAt')
const follows = res.body.data
expect(res.body.total).to.equal(0)
expect(follows).to.be.an('array')
expect(follows.length).to.equal(0)
}
})
it('Should not have following', async function () {
for (const server of servers) {
const res = await getFollowingListPaginationAndSort(server.url, 0, 5, 'createdAt')
const follows = res.body.data
expect(res.body.total).to.equal(0)
expect(follows).to.be.an('array')
expect(follows.length).to.equal(0)
}
})
it('Should have server 1 following server 2 and 3', async function () {
this.timeout(10000)
await follow(servers[0].url, [ servers[1].url, servers[2].url ], servers[0].accessToken)
await wait(7000)
})
it('Should have 2 followings on server 1', async function () {
let res = await getFollowingListPaginationAndSort(servers[0].url, 0, 1, 'createdAt')
let follows = res.body.data
expect(res.body.total).to.equal(2)
expect(follows).to.be.an('array')
expect(follows.length).to.equal(1)
res = await getFollowingListPaginationAndSort(servers[0].url, 1, 1, 'createdAt')
follows = follows.concat(res.body.data)
const server2Follow = follows.find(f => f.following.host === 'localhost:9002')
const server3Follow = follows.find(f => f.following.host === 'localhost:9003')
expect(server2Follow).to.not.be.undefined
expect(server3Follow).to.not.be.undefined
expect(server2Follow.state).to.equal('accepted')
expect(server3Follow.state).to.equal('accepted')
server3Id = server3Follow.following.id
})
it('Should have 0 followings on server 1 and 2', async function () {
for (const server of [ servers[1], servers[2] ]) {
const res = await getFollowingListPaginationAndSort(server.url, 0, 5, 'createdAt')
const follows = res.body.data
expect(res.body.total).to.equal(0)
expect(follows).to.be.an('array')
expect(follows.length).to.equal(0)
}
})
it('Should have 1 followers on server 2 and 3', async function () {
for (const server of [ servers[1], servers[2] ]) {
let res = await getFollowersListPaginationAndSort(server.url, 0, 1, 'createdAt')
let follows = res.body.data
expect(res.body.total).to.equal(1)
expect(follows).to.be.an('array')
expect(follows.length).to.equal(1)
expect(follows[0].follower.host).to.equal('localhost:9001')
}
})
it('Should have 0 followers on server 1', async function () {
const res = await getFollowersListPaginationAndSort(servers[0].url, 0, 5, 'createdAt')
const follows = res.body.data
expect(res.body.total).to.equal(0)
expect(follows).to.be.an('array')
expect(follows.length).to.equal(0)
})
it('Should unfollow server 3 on server 1', async function () {
this.timeout(5000)
await unfollow(servers[0].url, servers[0].accessToken, server3Id)
await wait(3000)
})
it('Should not follow server 3 on server 1 anymore', async function () {
const res = await getFollowingListPaginationAndSort(servers[0].url, 0, 2, 'createdAt')
let follows = res.body.data
expect(res.body.total).to.equal(1)
expect(follows).to.be.an('array')
expect(follows.length).to.equal(1)
expect(follows[0].following.host).to.equal('localhost:9002')
})
it('Should not have server 1 as follower on server 3 anymore', async function () {
const res = await getFollowersListPaginationAndSort(servers[2].url, 0, 1, 'createdAt')
let follows = res.body.data
expect(res.body.total).to.equal(0)
expect(follows).to.be.an('array')
expect(follows.length).to.equal(0)
})
it('Should upload a video on server 2 ans 3 and propagate only the video of server 2', async function () {
this.timeout(10000)
await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'server2' })
await uploadVideo(servers[2].url, servers[2].accessToken, { name: 'server3' })
await wait(5000)
let res = await getVideosList(servers[0].url)
expect(res.body.total).to.equal(1)
expect(res.body.data[0].name).to.equal('server2')
res = await getVideosList(servers[1].url)
expect(res.body.total).to.equal(1)
expect(res.body.data[0].name).to.equal('server2')
res = await getVideosList(servers[2].url)
expect(res.body.total).to.equal(1)
expect(res.body.data[0].name).to.equal('server3')
})
after(async function () {
killallServers(servers)
// Keep the logs if the test failed
if (this['ok']) {
await flushTests()
}
})
})

View File

@ -1,3 +1,4 @@
// Order of the tests we want to execute
// import './multiple-servers'
import './video-transcoder'
import './follows'

View File

@ -26,7 +26,7 @@ describe('Test reset password scripts', function () {
})
it('Should change the user password from CLI', async function () {
this.timeout(30000)
this.timeout(60000)
const env = getEnvCli(server)
await execCLI(`echo coucou | ${env} npm run reset-password -- -u user_1`)

View File

@ -1,373 +1,372 @@
// /!\ Before imports /!\
process.env.NODE_ENV = 'test'
import * as program from 'commander'
import { Video, VideoFile, VideoRateType } from '../../../shared'
import {
flushAndRunMultipleServers,
flushTests,
getAllVideosListBy,
getRequestsStats,
getVideo,
getVideosList,
killallServers,
removeVideo,
ServerInfo as DefaultServerInfo,
setAccessTokensToServers,
updateVideo,
uploadVideo,
wait
} from '../utils'
import { follow } from '../utils/follows'
interface ServerInfo extends DefaultServerInfo {
requestsNumber: number
}
program
.option('-c, --create [weight]', 'Weight for creating videos')
.option('-r, --remove [weight]', 'Weight for removing videos')
.option('-u, --update [weight]', 'Weight for updating videos')
.option('-v, --view [weight]', 'Weight for viewing videos')
.option('-l, --like [weight]', 'Weight for liking videos')
.option('-s, --dislike [weight]', 'Weight for disliking videos')
.option('-p, --servers [n]', 'Number of servers to run (3 or 6)', /^3|6$/, 3)
.option('-i, --interval-action [interval]', 'Interval in ms for an action')
.option('-I, --interval-integrity [interval]', 'Interval in ms for an integrity check')
.option('-f, --flush', 'Flush datas on exit')
.option('-d, --difference', 'Display difference if integrity is not okay')
.parse(process.argv)
const createWeight = program['create'] !== undefined ? parseInt(program['create'], 10) : 5
const removeWeight = program['remove'] !== undefined ? parseInt(program['remove'], 10) : 4
const updateWeight = program['update'] !== undefined ? parseInt(program['update'], 10) : 4
const viewWeight = program['view'] !== undefined ? parseInt(program['view'], 10) : 4
const likeWeight = program['like'] !== undefined ? parseInt(program['like'], 10) : 4
const dislikeWeight = program['dislike'] !== undefined ? parseInt(program['dislike'], 10) : 4
const flushAtExit = program['flush'] || false
const actionInterval = program['intervalAction'] !== undefined ? parseInt(program['intervalAction'], 10) : 500
const integrityInterval = program['intervalIntegrity'] !== undefined ? parseInt(program['intervalIntegrity'], 10) : 60000
const displayDiffOnFail = program['difference'] || false
const numberOfServers = 6
console.log(
'Create weight: %d, update weight: %d, remove weight: %d, view weight: %d, like weight: %d, dislike weight: %d.',
createWeight, updateWeight, removeWeight, viewWeight, likeWeight, dislikeWeight
)
if (flushAtExit) {
console.log('Program will flush data on exit.')
} else {
console.log('Program will not flush data on exit.')
}
if (displayDiffOnFail) {
console.log('Program will display diff on failure.')
} else {
console.log('Program will not display diff on failure')
}
console.log('Interval in ms for each action: %d.', actionInterval)
console.log('Interval in ms for each integrity check: %d.', integrityInterval)
console.log('Run servers...')
start()
// ----------------------------------------------------------------------------
async function start () {
const servers = await runServers(numberOfServers)
process.on('exit', async () => {
await exitServers(servers, flushAtExit)
return
})
process.on('SIGINT', goodbye)
process.on('SIGTERM', goodbye)
console.log('Servers ran')
initializeRequestsPerServer(servers)
let checking = false
setInterval(async () => {
if (checking === true) return
const rand = getRandomInt(0, createWeight + updateWeight + removeWeight + viewWeight + likeWeight + dislikeWeight)
const numServer = getRandomNumServer(servers)
servers[numServer].requestsNumber++
if (rand < createWeight) {
await upload(servers, numServer)
} else if (rand < createWeight + updateWeight) {
await update(servers, numServer)
} else if (rand < createWeight + updateWeight + removeWeight) {
await remove(servers, numServer)
} else if (rand < createWeight + updateWeight + removeWeight + viewWeight) {
await view(servers, numServer)
} else if (rand < createWeight + updateWeight + removeWeight + viewWeight + likeWeight) {
await like(servers, numServer)
} else {
await dislike(servers, numServer)
}
}, actionInterval)
// The function will check the consistency between servers (should have the same videos with same attributes...)
setInterval(function () {
if (checking === true) return
console.log('Checking integrity...')
checking = true
const waitingInterval = setInterval(async () => {
const pendingRequests = await isTherePendingRequests(servers)
if (pendingRequests === true) {
console.log('A server has pending requests, waiting...')
return
}
// Even if there are no pending request, wait some potential processes
await wait(2000)
await checkIntegrity(servers)
initializeRequestsPerServer(servers)
checking = false
clearInterval(waitingInterval)
}, 10000)
}, integrityInterval)
}
function initializeRequestsPerServer (servers: ServerInfo[]) {
servers.forEach(server => server.requestsNumber = 0)
}
function getRandomInt (min, max) {
return Math.floor(Math.random() * (max - min)) + min
}
function getRandomNumServer (servers) {
return getRandomInt(0, servers.length)
}
async function runServers (numberOfServers: number) {
const servers: ServerInfo[] = (await flushAndRunMultipleServers(numberOfServers))
.map(s => Object.assign({ requestsNumber: 0 }, s))
// Get the access tokens
await setAccessTokensToServers(servers)
for (let i = 0; i < numberOfServers; i++) {
for (let j = 0; j < numberOfServers; j++) {
if (i === j) continue
await follow(servers[i].url, [ servers[j].url ], servers[i].accessToken)
}
}
return servers
}
async function exitServers (servers: ServerInfo[], flushAtExit: boolean) {
killallServers(servers)
if (flushAtExit) await flushTests()
}
function upload (servers: ServerInfo[], numServer: number) {
console.log('Uploading video to server ' + numServer)
const videoAttributes = {
name: Date.now() + ' name',
category: 4,
nsfw: false,
licence: 2,
language: 1,
description: Date.now() + ' description',
tags: [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ],
fixture: 'video_short1.webm'
}
return uploadVideo(servers[numServer].url, servers[numServer].accessToken, videoAttributes)
}
async function update (servers: ServerInfo[], numServer: number) {
const res = await getVideosList(servers[numServer].url)
const videos = res.body.data.filter(video => video.isLocal === true)
if (videos.length === 0) return undefined
const toUpdate = videos[getRandomInt(0, videos.length)].id
const attributes = {
name: Date.now() + ' name',
description: Date.now() + ' description',
tags: [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ]
}
console.log('Updating video of server ' + numServer)
return updateVideo(servers[numServer].url, servers[numServer].accessToken, toUpdate, attributes)
}
async function remove (servers: ServerInfo[], numServer: number) {
const res = await getVideosList(servers[numServer].url)
const videos = res.body.data.filter(video => video.isLocal === true)
if (videos.length === 0) return undefined
const toRemove = videos[getRandomInt(0, videos.length)].id
console.log('Removing video from server ' + numServer)
return removeVideo(servers[numServer].url, servers[numServer].accessToken, toRemove)
}
async function view (servers: ServerInfo[], numServer: number) {
const res = await getVideosList(servers[numServer].url)
const videos = res.body.data
if (videos.length === 0) return undefined
const toView = videos[getRandomInt(0, videos.length)].id
console.log('Viewing video from server ' + numServer)
return getVideo(servers[numServer].url, toView)
}
function like (servers: ServerInfo[], numServer: number) {
return rate(servers, numServer, 'like')
}
function dislike (servers: ServerInfo[], numServer: number) {
return rate(servers, numServer, 'dislike')
}
async function rate (servers: ServerInfo[], numServer: number, rating: VideoRateType) {
const res = await getVideosList(servers[numServer].url)
const videos = res.body.data
if (videos.length === 0) return undefined
const toRate = videos[getRandomInt(0, videos.length)].id
console.log('Rating (%s) video from server %d', rating, numServer)
return getVideo(servers[numServer].url, toRate)
}
async function checkIntegrity (servers: ServerInfo[]) {
const videos: Video[][] = []
const tasks: Promise<any>[] = []
// Fetch all videos and remove some fields that can differ between servers
for (const server of servers) {
const p = getAllVideosListBy(server.url).then(res => videos.push(res.body.data))
tasks.push(p)
}
await Promise.all(tasks)
let i = 0
for (const video of videos) {
const differences = areDifferences(video, videos[0])
if (differences !== undefined) {
console.error('Integrity not ok with server %d!', i + 1)
if (displayDiffOnFail) {
console.log(differences)
}
process.exit(-1)
}
i++
}
console.log('Integrity ok.')
}
function areDifferences (videos1: Video[], videos2: Video[]) {
// Remove some keys we don't want to compare
videos1.concat(videos2).forEach(video => {
delete video.id
delete video.isLocal
delete video.thumbnailPath
delete video.updatedAt
delete video.views
})
if (videos1.length !== videos2.length) {
return `Videos length are different (${videos1.length}/${videos2.length}).`
}
for (const video1 of videos1) {
const video2 = videos2.find(video => video.uuid === video1.uuid)
if (!video2) return 'Video ' + video1.uuid + ' is missing.'
for (const videoKey of Object.keys(video1)) {
const attribute1 = video1[videoKey]
const attribute2 = video2[videoKey]
if (videoKey === 'tags') {
if (attribute1.length !== attribute2.length) {
return 'Tags are different.'
}
attribute1.forEach(tag1 => {
if (attribute2.indexOf(tag1) === -1) {
return 'Tag ' + tag1 + ' is missing.'
}
})
} else if (videoKey === 'files') {
if (attribute1.length !== attribute2.length) {
return 'Video files are different.'
}
attribute1.forEach((videoFile1: VideoFile) => {
const videoFile2: VideoFile = attribute2.find(videoFile => videoFile.magnetUri === videoFile1.magnetUri)
if (!videoFile2) {
return `Video ${video1.uuid} has missing video file ${videoFile1.magnetUri}.`
}
if (videoFile1.size !== videoFile2.size || videoFile1.resolutionLabel !== videoFile2.resolutionLabel) {
return `Video ${video1.uuid} has different video file ${videoFile1.magnetUri}.`
}
})
} else {
if (attribute1 !== attribute2) {
return `Video ${video1.uuid} has different value for attribute ${videoKey}.`
}
}
}
}
return undefined
}
function goodbye () {
return process.exit(-1)
}
async function isTherePendingRequests (servers: ServerInfo[]) {
const tasks: Promise<any>[] = []
let pendingRequests = false
// Check if each server has pending request
for (const server of servers) {
const p = getRequestsStats(server).then(res => {
const stats = res.body
if (
stats.requestScheduler.totalRequests !== 0 ||
stats.requestVideoEventScheduler.totalRequests !== 0 ||
stats.requestVideoQaduScheduler.totalRequests !== 0
) {
pendingRequests = true
}
})
tasks.push(p)
}
await Promise.all(tasks)
return pendingRequests
}
// // /!\ Before imports /!\
// process.env.NODE_ENV = 'test'
//
// import * as program from 'commander'
// import { Video, VideoFile, VideoRateType } from '../../../shared'
// import {
// flushAndRunMultipleServers,
// flushTests,
// getAllVideosListBy,
// getVideo,
// getVideosList,
// killallServers,
// removeVideo,
// ServerInfo as DefaultServerInfo,
// setAccessTokensToServers,
// updateVideo,
// uploadVideo,
// wait
// } from '../utils'
// import { follow } from '../utils/follows'
//
// interface ServerInfo extends DefaultServerInfo {
// requestsNumber: number
// }
//
// program
// .option('-c, --create [weight]', 'Weight for creating videos')
// .option('-r, --remove [weight]', 'Weight for removing videos')
// .option('-u, --update [weight]', 'Weight for updating videos')
// .option('-v, --view [weight]', 'Weight for viewing videos')
// .option('-l, --like [weight]', 'Weight for liking videos')
// .option('-s, --dislike [weight]', 'Weight for disliking videos')
// .option('-p, --servers [n]', 'Number of servers to run (3 or 6)', /^3|6$/, 3)
// .option('-i, --interval-action [interval]', 'Interval in ms for an action')
// .option('-I, --interval-integrity [interval]', 'Interval in ms for an integrity check')
// .option('-f, --flush', 'Flush datas on exit')
// .option('-d, --difference', 'Display difference if integrity is not okay')
// .parse(process.argv)
//
// const createWeight = program['create'] !== undefined ? parseInt(program['create'], 10) : 5
// const removeWeight = program['remove'] !== undefined ? parseInt(program['remove'], 10) : 4
// const updateWeight = program['update'] !== undefined ? parseInt(program['update'], 10) : 4
// const viewWeight = program['view'] !== undefined ? parseInt(program['view'], 10) : 4
// const likeWeight = program['like'] !== undefined ? parseInt(program['like'], 10) : 4
// const dislikeWeight = program['dislike'] !== undefined ? parseInt(program['dislike'], 10) : 4
// const flushAtExit = program['flush'] || false
// const actionInterval = program['intervalAction'] !== undefined ? parseInt(program['intervalAction'], 10) : 500
// const integrityInterval = program['intervalIntegrity'] !== undefined ? parseInt(program['intervalIntegrity'], 10) : 60000
// const displayDiffOnFail = program['difference'] || false
//
// const numberOfServers = 6
//
// console.log(
// 'Create weight: %d, update weight: %d, remove weight: %d, view weight: %d, like weight: %d, dislike weight: %d.',
// createWeight, updateWeight, removeWeight, viewWeight, likeWeight, dislikeWeight
// )
//
// if (flushAtExit) {
// console.log('Program will flush data on exit.')
// } else {
// console.log('Program will not flush data on exit.')
// }
// if (displayDiffOnFail) {
// console.log('Program will display diff on failure.')
// } else {
// console.log('Program will not display diff on failure')
// }
// console.log('Interval in ms for each action: %d.', actionInterval)
// console.log('Interval in ms for each integrity check: %d.', integrityInterval)
//
// console.log('Run servers...')
//
// start()
//
// // ----------------------------------------------------------------------------
//
// async function start () {
// const servers = await runServers(numberOfServers)
//
// process.on('exit', async () => {
// await exitServers(servers, flushAtExit)
//
// return
// })
// process.on('SIGINT', goodbye)
// process.on('SIGTERM', goodbye)
//
// console.log('Servers ran')
// initializeRequestsPerServer(servers)
//
// let checking = false
//
// setInterval(async () => {
// if (checking === true) return
//
// const rand = getRandomInt(0, createWeight + updateWeight + removeWeight + viewWeight + likeWeight + dislikeWeight)
//
// const numServer = getRandomNumServer(servers)
// servers[numServer].requestsNumber++
//
// if (rand < createWeight) {
// await upload(servers, numServer)
// } else if (rand < createWeight + updateWeight) {
// await update(servers, numServer)
// } else if (rand < createWeight + updateWeight + removeWeight) {
// await remove(servers, numServer)
// } else if (rand < createWeight + updateWeight + removeWeight + viewWeight) {
// await view(servers, numServer)
// } else if (rand < createWeight + updateWeight + removeWeight + viewWeight + likeWeight) {
// await like(servers, numServer)
// } else {
// await dislike(servers, numServer)
// }
// }, actionInterval)
//
// // The function will check the consistency between servers (should have the same videos with same attributes...)
// setInterval(function () {
// if (checking === true) return
//
// console.log('Checking integrity...')
// checking = true
//
// const waitingInterval = setInterval(async () => {
// const pendingRequests = await isTherePendingRequests(servers)
// if (pendingRequests === true) {
// console.log('A server has pending requests, waiting...')
// return
// }
//
// // Even if there are no pending request, wait some potential processes
// await wait(2000)
// await checkIntegrity(servers)
//
// initializeRequestsPerServer(servers)
// checking = false
// clearInterval(waitingInterval)
// }, 10000)
// }, integrityInterval)
// }
//
// function initializeRequestsPerServer (servers: ServerInfo[]) {
// servers.forEach(server => server.requestsNumber = 0)
// }
//
// function getRandomInt (min, max) {
// return Math.floor(Math.random() * (max - min)) + min
// }
//
// function getRandomNumServer (servers) {
// return getRandomInt(0, servers.length)
// }
//
// async function runServers (numberOfServers: number) {
// const servers: ServerInfo[] = (await flushAndRunMultipleServers(numberOfServers))
// .map(s => Object.assign({ requestsNumber: 0 }, s))
//
// // Get the access tokens
// await setAccessTokensToServers(servers)
//
// for (let i = 0; i < numberOfServers; i++) {
// for (let j = 0; j < numberOfServers; j++) {
// if (i === j) continue
//
// await follow(servers[i].url, [ servers[j].url ], servers[i].accessToken)
// }
// }
//
// return servers
// }
//
// async function exitServers (servers: ServerInfo[], flushAtExit: boolean) {
// killallServers(servers)
//
// if (flushAtExit) await flushTests()
// }
//
// function upload (servers: ServerInfo[], numServer: number) {
// console.log('Uploading video to server ' + numServer)
//
// const videoAttributes = {
// name: Date.now() + ' name',
// category: 4,
// nsfw: false,
// licence: 2,
// language: 1,
// description: Date.now() + ' description',
// tags: [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ],
// fixture: 'video_short1.webm'
// }
// return uploadVideo(servers[numServer].url, servers[numServer].accessToken, videoAttributes)
// }
//
// async function update (servers: ServerInfo[], numServer: number) {
// const res = await getVideosList(servers[numServer].url)
//
// const videos = res.body.data.filter(video => video.isLocal === true)
// if (videos.length === 0) return undefined
//
// const toUpdate = videos[getRandomInt(0, videos.length)].id
// const attributes = {
// name: Date.now() + ' name',
// description: Date.now() + ' description',
// tags: [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ]
// }
//
// console.log('Updating video of server ' + numServer)
//
// return updateVideo(servers[numServer].url, servers[numServer].accessToken, toUpdate, attributes)
// }
//
// async function remove (servers: ServerInfo[], numServer: number) {
// const res = await getVideosList(servers[numServer].url)
// const videos = res.body.data.filter(video => video.isLocal === true)
// if (videos.length === 0) return undefined
//
// const toRemove = videos[getRandomInt(0, videos.length)].id
//
// console.log('Removing video from server ' + numServer)
// return removeVideo(servers[numServer].url, servers[numServer].accessToken, toRemove)
// }
//
// async function view (servers: ServerInfo[], numServer: number) {
// const res = await getVideosList(servers[numServer].url)
//
// const videos = res.body.data
// if (videos.length === 0) return undefined
//
// const toView = videos[getRandomInt(0, videos.length)].id
//
// console.log('Viewing video from server ' + numServer)
// return getVideo(servers[numServer].url, toView)
// }
//
// function like (servers: ServerInfo[], numServer: number) {
// return rate(servers, numServer, 'like')
// }
//
// function dislike (servers: ServerInfo[], numServer: number) {
// return rate(servers, numServer, 'dislike')
// }
//
// async function rate (servers: ServerInfo[], numServer: number, rating: VideoRateType) {
// const res = await getVideosList(servers[numServer].url)
//
// const videos = res.body.data
// if (videos.length === 0) return undefined
//
// const toRate = videos[getRandomInt(0, videos.length)].id
//
// console.log('Rating (%s) video from server %d', rating, numServer)
// return getVideo(servers[numServer].url, toRate)
// }
//
// async function checkIntegrity (servers: ServerInfo[]) {
// const videos: Video[][] = []
// const tasks: Promise<any>[] = []
//
// // Fetch all videos and remove some fields that can differ between servers
// for (const server of servers) {
// const p = getAllVideosListBy(server.url).then(res => videos.push(res.body.data))
// tasks.push(p)
// }
//
// await Promise.all(tasks)
//
// let i = 0
// for (const video of videos) {
// const differences = areDifferences(video, videos[0])
// if (differences !== undefined) {
// console.error('Integrity not ok with server %d!', i + 1)
//
// if (displayDiffOnFail) {
// console.log(differences)
// }
//
// process.exit(-1)
// }
//
// i++
// }
//
// console.log('Integrity ok.')
// }
//
// function areDifferences (videos1: Video[], videos2: Video[]) {
// // Remove some keys we don't want to compare
// videos1.concat(videos2).forEach(video => {
// delete video.id
// delete video.isLocal
// delete video.thumbnailPath
// delete video.updatedAt
// delete video.views
// })
//
// if (videos1.length !== videos2.length) {
// return `Videos length are different (${videos1.length}/${videos2.length}).`
// }
//
// for (const video1 of videos1) {
// const video2 = videos2.find(video => video.uuid === video1.uuid)
//
// if (!video2) return 'Video ' + video1.uuid + ' is missing.'
//
// for (const videoKey of Object.keys(video1)) {
// const attribute1 = video1[videoKey]
// const attribute2 = video2[videoKey]
//
// if (videoKey === 'tags') {
// if (attribute1.length !== attribute2.length) {
// return 'Tags are different.'
// }
//
// attribute1.forEach(tag1 => {
// if (attribute2.indexOf(tag1) === -1) {
// return 'Tag ' + tag1 + ' is missing.'
// }
// })
// } else if (videoKey === 'files') {
// if (attribute1.length !== attribute2.length) {
// return 'Video files are different.'
// }
//
// attribute1.forEach((videoFile1: VideoFile) => {
// const videoFile2: VideoFile = attribute2.find(videoFile => videoFile.magnetUri === videoFile1.magnetUri)
// if (!videoFile2) {
// return `Video ${video1.uuid} has missing video file ${videoFile1.magnetUri}.`
// }
//
// if (videoFile1.size !== videoFile2.size || videoFile1.resolutionLabel !== videoFile2.resolutionLabel) {
// return `Video ${video1.uuid} has different video file ${videoFile1.magnetUri}.`
// }
// })
// } else {
// if (attribute1 !== attribute2) {
// return `Video ${video1.uuid} has different value for attribute ${videoKey}.`
// }
// }
// }
// }
//
// return undefined
// }
//
// function goodbye () {
// return process.exit(-1)
// }
//
// async function isTherePendingRequests (servers: ServerInfo[]) {
// const tasks: Promise<any>[] = []
// let pendingRequests = false
//
// // Check if each server has pending request
// for (const server of servers) {
// const p = getRequestsStats(server).then(res => {
// const stats = res.body
//
// if (
// stats.requestScheduler.totalRequests !== 0 ||
// stats.requestVideoEventScheduler.totalRequests !== 0 ||
// stats.requestVideoQaduScheduler.totalRequests !== 0
// ) {
// pendingRequests = true
// }
// })
//
// tasks.push(p)
// }
//
// await Promise.all(tasks)
//
// return pendingRequests
// }

View File

@ -42,6 +42,18 @@ async function follow (follower: string, following: string[], accessToken: strin
return res
}
async function unfollow (url: string, accessToken: string, id: number, expectedStatus = 204) {
const path = '/api/v1/server/following/' + id
const res = await request(url)
.delete(path)
.set('Accept', 'application/json')
.set('Authorization', 'Bearer ' + accessToken)
.expect(expectedStatus)
return res
}
async function doubleFollow (server1: ServerInfo, server2: ServerInfo) {
await Promise.all([
follow(server1.url, [ server2.url ], server1.accessToken),
@ -59,6 +71,7 @@ async function doubleFollow (server1: ServerInfo, server2: ServerInfo) {
export {
getFollowersListPaginationAndSort,
getFollowingListPaginationAndSort,
unfollow,
follow,
doubleFollow
}

View File

@ -4,7 +4,6 @@ export * from './config'
export * from './login'
export * from './miscs'
export * from './follows'
export * from './request-schedulers'
export * from './requests'
export * from './servers'
export * from './services'

View File

@ -1,20 +0,0 @@
import * as request from 'supertest'
import { ServerInfo } from '../utils'
function getRequestsStats (server: ServerInfo) {
const path = '/api/v1/request-schedulers/stats'
return request(server.url)
.get(path)
.set('Accept', 'application/json')
.set('Authorization', 'Bearer ' + server.accessToken)
.expect(200)
.expect('Content-Type', /json/)
}
// ---------------------------------------------------------------------------
export {
getRequestsStats
}