From afffe98839db7ccbfa9fb8b7d1413b97900fdc73 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 17 Nov 2017 11:35:10 +0100 Subject: [PATCH] Speed up activity pub http requests --- scripts/parse-log.ts | 8 +- server.ts | 4 +- server/helpers/activitypub.ts | 11 +- server/initializers/constants.ts | 2 +- server/lib/activitypub/send-request.ts | 172 +++++----- .../activitypub-http-broadcast-handler.ts | 43 +++ .../activitypub-http-job-scheduler.ts | 23 ++ .../activitypub-http-unicast-handler.ts | 40 +++ .../activitypub-http-job-scheduler/index.ts | 1 + .../http-request-broadcast-handler.ts | 36 -- .../http-request-job-scheduler.ts | 22 -- .../http-request-unicast-handler.ts | 34 -- .../jobs/http-request-job-scheduler/index.ts | 1 - server/lib/jobs/index.ts | 2 +- server/tests/api/index-fast.ts | 1 - server/tests/api/multiple-pods.ts | 6 +- server/tests/api/request-schedulers.ts | 82 ----- server/tests/api/services.ts | 5 +- server/tests/api/single-pod.ts | 47 +-- server/tests/api/users.ts | 307 ++++++++++++------ server/tests/api/video-abuse.ts | 60 ++-- server/tests/utils/follows.ts | 25 +- server/tests/utils/servers.ts | 2 +- shared/models/activitypub/activity.ts | 4 +- shared/models/job.model.ts | 2 +- yarn.lock | 14 +- 26 files changed, 517 insertions(+), 437 deletions(-) create mode 100644 server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts create mode 100644 server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts create mode 100644 server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts create mode 100644 server/lib/jobs/activitypub-http-job-scheduler/index.ts delete mode 100644 server/lib/jobs/http-request-job-scheduler/http-request-broadcast-handler.ts delete mode 100644 server/lib/jobs/http-request-job-scheduler/http-request-job-scheduler.ts delete mode 100644 server/lib/jobs/http-request-job-scheduler/http-request-unicast-handler.ts delete mode 100644 server/lib/jobs/http-request-job-scheduler/index.ts delete mode 100644 server/tests/api/request-schedulers.ts diff --git a/scripts/parse-log.ts b/scripts/parse-log.ts index 8aac6fbda..24a09c885 100755 --- a/scripts/parse-log.ts +++ b/scripts/parse-log.ts @@ -38,5 +38,11 @@ const rl = createInterface({ rl.on('line', line => { const log = JSON.parse(line) - logLevels[log.level](log.message, log.stack) + const additionalInfo: any = {} + + Object.keys(log).forEach(logKey => { + if (logKey !== 'message' && logKey !== 'level') additionalInfo[logKey] = log[logKey] + }) + + logLevels[log.level](log.message, additionalInfo) }) diff --git a/server.ts b/server.ts index 62219f7df..84535c7a5 100644 --- a/server.ts +++ b/server.ts @@ -46,7 +46,7 @@ db.init(false).then(() => onDatabaseInitDone()) // ----------- PeerTube modules ----------- import { migrate, installApplication } from './server/initializers' -import { httpRequestJobScheduler, transcodingJobScheduler, VideosPreviewCache } from './server/lib' +import { activitypubHttpJobScheduler, transcodingJobScheduler, VideosPreviewCache } from './server/lib' import { apiRouter, clientsRouter, staticRouter, servicesRouter, webfingerRouter, activityPubRouter } from './server/controllers' // ----------- Command line ----------- @@ -154,7 +154,7 @@ function onDatabaseInitDone () { // ----------- Make the server listening ----------- server.listen(port, () => { VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE) - httpRequestJobScheduler.activate() + activitypubHttpJobScheduler.activate() transcodingJobScheduler.activate() logger.info('Server listening on port %d', port) diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index c710117cd..338698652 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts @@ -3,6 +3,7 @@ import * as request from 'request' import * as Sequelize from 'sequelize' import * as url from 'url' import { ActivityIconObject } from '../../shared/index' +import { Activity } from '../../shared/models/activitypub/activity' import { ActivityPubActor } from '../../shared/models/activitypub/activitypub-actor' import { VideoChannelObject } from '../../shared/models/activitypub/objects/video-channel-object' import { ResultList } from '../../shared/models/result-list.model' @@ -17,6 +18,7 @@ import { VideoInstance } from '../models/video/video-interface' import { isRemoteAccountValid } from './custom-validators' import { isVideoChannelObjectValid } from './custom-validators/activitypub/videos' import { logger } from './logger' +import { signObject } from './peertube-crypto' import { doRequest, doRequestAndSaveToFile } from './requests' import { getServerAccount } from './utils' @@ -239,6 +241,12 @@ function activityPubCollectionPagination (url: string, page: number, result: Res return activityPubContextify(obj) } +function buildSignedActivity (byAccount: AccountInstance, data: Object) { + const activity = activityPubContextify(data) + + return signObject(byAccount, activity) as Promise +} + // --------------------------------------------------------------------------- export { @@ -252,7 +260,8 @@ export { fetchRemoteVideoDescription, shareVideoChannelByServer, shareVideoByServer, - getOrCreateVideoChannel + getOrCreateVideoChannel, + buildSignedActivity } // --------------------------------------------------------------------------- diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index f0a569410..48d7b5b98 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -260,7 +260,7 @@ const JOB_STATES: { [ id: string ]: JobState } = { } const JOB_CATEGORIES: { [ id: string ]: JobCategory } = { TRANSCODING: 'transcoding', - HTTP_REQUEST: 'http-request' + ACTIVITYPUB_HTTP: 'activitypub-http' } // How many maximum jobs we fetch from the database per cycle const JOBS_FETCH_LIMIT_PER_CYCLE = { diff --git a/server/lib/activitypub/send-request.ts b/server/lib/activitypub/send-request.ts index abc1b598d..8d013fa87 100644 --- a/server/lib/activitypub/send-request.ts +++ b/server/lib/activitypub/send-request.ts @@ -1,116 +1,124 @@ -import * as Sequelize from 'sequelize' - -import { database as db } from '../../initializers' +import { Transaction } from 'sequelize' import { - AccountInstance, - VideoInstance, - VideoChannelInstance -} from '../../models' -import { httpRequestJobScheduler } from '../jobs' -import { signObject, activityPubContextify } from '../../helpers' -import { Activity, VideoAbuseObject } from '../../../shared' -import { VideoAbuseInstance } from '../../models/video/video-abuse-interface' + ActivityAccept, + ActivityAdd, + ActivityCreate, + ActivityDelete, + ActivityFollow, + ActivityUpdate +} from '../../../shared/models/activitypub/activity' import { getActivityPubUrl } from '../../helpers/activitypub' import { logger } from '../../helpers/logger' +import { database as db } from '../../initializers' +import { AccountInstance, VideoChannelInstance, VideoInstance } from '../../models' +import { VideoAbuseInstance } from '../../models/video/video-abuse-interface' +import { activitypubHttpJobScheduler } from '../jobs' + +async function sendCreateVideoChannel (videoChannel: VideoChannelInstance, t: Transaction) { + const byAccount = videoChannel.Account -async function sendCreateVideoChannel (videoChannel: VideoChannelInstance, t: Sequelize.Transaction) { const videoChannelObject = videoChannel.toActivityPubObject() - const data = await createActivityData(videoChannel.url, videoChannel.Account, videoChannelObject) + const data = await createActivityData(videoChannel.url, byAccount, videoChannelObject) - return broadcastToFollowers(data, [ videoChannel.Account ], t) + return broadcastToFollowers(data, byAccount, [ byAccount ], t) } -async function sendUpdateVideoChannel (videoChannel: VideoChannelInstance, t: Sequelize.Transaction) { +async function sendUpdateVideoChannel (videoChannel: VideoChannelInstance, t: Transaction) { + const byAccount = videoChannel.Account + const videoChannelObject = videoChannel.toActivityPubObject() - const data = await updateActivityData(videoChannel.url, videoChannel.Account, videoChannelObject) + const data = await updateActivityData(videoChannel.url, byAccount, videoChannelObject) const accountsInvolved = await db.VideoChannelShare.loadAccountsByShare(videoChannel.id) - accountsInvolved.push(videoChannel.Account) + accountsInvolved.push(byAccount) - return broadcastToFollowers(data, accountsInvolved, t) + return broadcastToFollowers(data, byAccount, accountsInvolved, t) } -async function sendDeleteVideoChannel (videoChannel: VideoChannelInstance, t: Sequelize.Transaction) { - const data = await deleteActivityData(videoChannel.url, videoChannel.Account) +async function sendDeleteVideoChannel (videoChannel: VideoChannelInstance, t: Transaction) { + const byAccount = videoChannel.Account + + const data = await deleteActivityData(videoChannel.url, byAccount) const accountsInvolved = await db.VideoChannelShare.loadAccountsByShare(videoChannel.id) - accountsInvolved.push(videoChannel.Account) + accountsInvolved.push(byAccount) - return broadcastToFollowers(data, accountsInvolved, t) + return broadcastToFollowers(data, byAccount, accountsInvolved, t) } -async function sendAddVideo (video: VideoInstance, t: Sequelize.Transaction) { - const videoObject = video.toActivityPubObject() - const data = await addActivityData(video.url, video.VideoChannel.Account, video.VideoChannel.url, videoObject) +async function sendAddVideo (video: VideoInstance, t: Transaction) { + const byAccount = video.VideoChannel.Account - return broadcastToFollowers(data, [ video.VideoChannel.Account ], t) + const videoObject = video.toActivityPubObject() + const data = await addActivityData(video.url, byAccount, video.VideoChannel.url, videoObject) + + return broadcastToFollowers(data, byAccount, [ byAccount ], t) } -async function sendUpdateVideo (video: VideoInstance, t: Sequelize.Transaction) { +async function sendUpdateVideo (video: VideoInstance, t: Transaction) { + const byAccount = video.VideoChannel.Account + const videoObject = video.toActivityPubObject() - const data = await updateActivityData(video.url, video.VideoChannel.Account, videoObject) + const data = await updateActivityData(video.url, byAccount, videoObject) const accountsInvolved = await db.VideoShare.loadAccountsByShare(video.id) - accountsInvolved.push(video.VideoChannel.Account) + accountsInvolved.push(byAccount) - return broadcastToFollowers(data, accountsInvolved, t) + return broadcastToFollowers(data, byAccount, accountsInvolved, t) } -async function sendDeleteVideo (video: VideoInstance, t: Sequelize.Transaction) { - const data = await deleteActivityData(video.url, video.VideoChannel.Account) +async function sendDeleteVideo (video: VideoInstance, t: Transaction) { + const byAccount = video.VideoChannel.Account + + const data = await deleteActivityData(video.url, byAccount) const accountsInvolved = await db.VideoShare.loadAccountsByShare(video.id) - accountsInvolved.push(video.VideoChannel.Account) + accountsInvolved.push(byAccount) - return broadcastToFollowers(data, accountsInvolved, t) + return broadcastToFollowers(data, byAccount, accountsInvolved, t) } -async function sendDeleteAccount (account: AccountInstance, t: Sequelize.Transaction) { +async function sendDeleteAccount (account: AccountInstance, t: Transaction) { const data = await deleteActivityData(account.url, account) - return broadcastToFollowers(data, [ account ], t) + return broadcastToFollowers(data, account, [ account ], t) } -async function sendVideoChannelAnnounce (byAccount: AccountInstance, videoChannel: VideoChannelInstance, t: Sequelize.Transaction) { +async function sendVideoChannelAnnounce (byAccount: AccountInstance, videoChannel: VideoChannelInstance, t: Transaction) { const url = getActivityPubUrl('videoChannel', videoChannel.uuid) + '#announce' - const announcedActivity = await createActivityData(url, videoChannel.Account, videoChannel.toActivityPubObject(), true) + const announcedActivity = await createActivityData(url, videoChannel.Account, videoChannel.toActivityPubObject()) const data = await announceActivityData(url, byAccount, announcedActivity) - return broadcastToFollowers(data, [ byAccount ], t) + return broadcastToFollowers(data, byAccount, [ byAccount ], t) } -async function sendVideoAnnounce (byAccount: AccountInstance, video: VideoInstance, t: Sequelize.Transaction) { +async function sendVideoAnnounce (byAccount: AccountInstance, video: VideoInstance, t: Transaction) { const url = getActivityPubUrl('video', video.uuid) + '#announce' const videoChannel = video.VideoChannel - const announcedActivity = await addActivityData(url, videoChannel.Account, videoChannel.url, video.toActivityPubObject(), true) + const announcedActivity = await addActivityData(url, videoChannel.Account, videoChannel.url, video.toActivityPubObject()) const data = await announceActivityData(url, byAccount, announcedActivity) - return broadcastToFollowers(data, [ byAccount ], t) + return broadcastToFollowers(data, byAccount, [ byAccount ], t) } -async function sendVideoAbuse ( - fromAccount: AccountInstance, - videoAbuse: VideoAbuseInstance, - video: VideoInstance, - t: Sequelize.Transaction -) { +async function sendVideoAbuse (byAccount: AccountInstance, videoAbuse: VideoAbuseInstance, video: VideoInstance, t: Transaction) { const url = getActivityPubUrl('videoAbuse', videoAbuse.id.toString()) - const data = await createActivityData(url, fromAccount, videoAbuse.toActivityPubObject()) + const data = await createActivityData(url, byAccount, videoAbuse.toActivityPubObject()) - return unicastTo(data, video.VideoChannel.Account.sharedInboxUrl, t) + return unicastTo(data, byAccount, video.VideoChannel.Account.sharedInboxUrl, t) } -async function sendAccept (fromAccount: AccountInstance, toAccount: AccountInstance, t: Sequelize.Transaction) { - const data = await acceptActivityData(fromAccount) +async function sendAccept (byAccount: AccountInstance, toAccount: AccountInstance, t: Transaction) { + const data = await acceptActivityData(byAccount) - return unicastTo(data, toAccount.inboxUrl, t) + return unicastTo(data, byAccount, toAccount.inboxUrl, t) } -async function sendFollow (fromAccount: AccountInstance, toAccount: AccountInstance, t: Sequelize.Transaction) { - const data = await followActivityData(toAccount.url, fromAccount) +async function sendFollow (byAccount: AccountInstance, toAccount: AccountInstance, t: Transaction) { + const data = await followActivityData(toAccount.url, byAccount) - return unicastTo(data, toAccount.inboxUrl, t) + return unicastTo(data, byAccount, toAccount.inboxUrl, t) } // --------------------------------------------------------------------------- @@ -132,7 +140,7 @@ export { // --------------------------------------------------------------------------- -async function broadcastToFollowers (data: any, toAccountFollowers: AccountInstance[], t: Sequelize.Transaction) { +async function broadcastToFollowers (data: any, byAccount: AccountInstance, toAccountFollowers: AccountInstance[], t: Transaction) { const toAccountFollowerIds = toAccountFollowers.map(a => a.id) const result = await db.AccountFollow.listAcceptedFollowerSharedInboxUrls(toAccountFollowerIds) if (result.data.length === 0) { @@ -142,25 +150,21 @@ async function broadcastToFollowers (data: any, toAccountFollowers: AccountInsta const jobPayload = { uris: result.data, + signatureAccountId: byAccount.id, body: data } - return httpRequestJobScheduler.createJob(t, 'httpRequestBroadcastHandler', jobPayload) + return activitypubHttpJobScheduler.createJob(t, 'activitypubHttpBroadcastHandler', jobPayload) } -async function unicastTo (data: any, toAccountUrl: string, t: Sequelize.Transaction) { +async function unicastTo (data: any, byAccount: AccountInstance, toAccountUrl: string, t: Transaction) { const jobPayload = { uris: [ toAccountUrl ], + signatureAccountId: byAccount.id, body: data } - return httpRequestJobScheduler.createJob(t, 'httpRequestUnicastHandler', jobPayload) -} - -function buildSignedActivity (byAccount: AccountInstance, data: Object) { - const activity = activityPubContextify(data) - - return signObject(byAccount, activity) as Promise + return activitypubHttpJobScheduler.createJob(t, 'activitypubHttpUnicastHandler', jobPayload) } async function getPublicActivityTo (account: AccountInstance) { @@ -169,9 +173,9 @@ async function getPublicActivityTo (account: AccountInstance) { return inboxUrls.concat('https://www.w3.org/ns/activitystreams#Public') } -async function createActivityData (url: string, byAccount: AccountInstance, object: any, raw = false) { +async function createActivityData (url: string, byAccount: AccountInstance, object: any) { const to = await getPublicActivityTo(byAccount) - const base = { + const activity: ActivityCreate = { type: 'Create', id: url, actor: byAccount.url, @@ -179,14 +183,12 @@ async function createActivityData (url: string, byAccount: AccountInstance, obje object } - if (raw === true) return base - - return buildSignedActivity(byAccount, base) + return activity } async function updateActivityData (url: string, byAccount: AccountInstance, object: any) { const to = await getPublicActivityTo(byAccount) - const base = { + const activity: ActivityUpdate = { type: 'Update', id: url, actor: byAccount.url, @@ -194,22 +196,22 @@ async function updateActivityData (url: string, byAccount: AccountInstance, obje object } - return buildSignedActivity(byAccount, base) + return activity } async function deleteActivityData (url: string, byAccount: AccountInstance) { - const base = { + const activity: ActivityDelete = { type: 'Delete', id: url, actor: byAccount.url } - return buildSignedActivity(byAccount, base) + return activity } -async function addActivityData (url: string, byAccount: AccountInstance, target: string, object: any, raw = false) { +async function addActivityData (url: string, byAccount: AccountInstance, target: string, object: any) { const to = await getPublicActivityTo(byAccount) - const base = { + const activity: ActivityAdd = { type: 'Add', id: url, actor: byAccount.url, @@ -218,39 +220,37 @@ async function addActivityData (url: string, byAccount: AccountInstance, target: target } - if (raw === true) return base - - return buildSignedActivity(byAccount, base) + return activity } async function announceActivityData (url: string, byAccount: AccountInstance, object: any) { - const base = { + const activity = { type: 'Announce', id: url, actor: byAccount.url, object } - return buildSignedActivity(byAccount, base) + return activity } async function followActivityData (url: string, byAccount: AccountInstance) { - const base = { + const activity: ActivityFollow = { type: 'Follow', id: byAccount.url, actor: byAccount.url, object: url } - return buildSignedActivity(byAccount, base) + return activity } async function acceptActivityData (byAccount: AccountInstance) { - const base = { + const activity: ActivityAccept = { type: 'Accept', id: byAccount.url, actor: byAccount.url } - return buildSignedActivity(byAccount, base) + return activity } diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts new file mode 100644 index 000000000..111fc88a4 --- /dev/null +++ b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts @@ -0,0 +1,43 @@ +import { logger } from '../../../helpers' +import { buildSignedActivity } from '../../../helpers/activitypub' +import { doRequest } from '../../../helpers/requests' +import { database as db } from '../../../initializers' +import { ActivityPubHttpPayload } from './activitypub-http-job-scheduler' + +async function process (payload: ActivityPubHttpPayload, jobId: number) { + logger.info('Processing ActivityPub broadcast in job %d.', jobId) + + const accountSignature = await db.Account.load(payload.signatureAccountId) + if (!accountSignature) throw new Error('Unknown signature account id.') + + const signedBody = await buildSignedActivity(accountSignature, payload.body) + + const options = { + method: 'POST', + uri: '', + json: signedBody + } + + for (const uri of payload.uris) { + options.uri = uri + await doRequest(options) + } +} + +function onError (err: Error, jobId: number) { + logger.error('Error when broadcasting ActivityPub request in job %d.', jobId, err) + return Promise.resolve() +} + +function onSuccess (jobId: number) { + logger.info('Job %d is a success.', jobId) + return Promise.resolve() +} + +// --------------------------------------------------------------------------- + +export { + process, + onError, + onSuccess +} diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts new file mode 100644 index 000000000..e4f6c94a5 --- /dev/null +++ b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts @@ -0,0 +1,23 @@ +import { JobScheduler, JobHandler } from '../job-scheduler' + +import * as activitypubHttpBroadcastHandler from './activitypub-http-broadcast-handler' +import * as activitypubHttpUnicastHandler from './activitypub-http-unicast-handler' +import { JobCategory } from '../../../../shared' + +type ActivityPubHttpPayload = { + uris: string[] + signatureAccountId: number + body: any +} +const jobHandlers: { [ handlerName: string ]: JobHandler } = { + activitypubHttpBroadcastHandler, + activitypubHttpUnicastHandler +} +const jobCategory: JobCategory = 'activitypub-http' + +const activitypubHttpJobScheduler = new JobScheduler(jobCategory, jobHandlers) + +export { + ActivityPubHttpPayload, + activitypubHttpJobScheduler +} diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts new file mode 100644 index 000000000..8d3b755ad --- /dev/null +++ b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts @@ -0,0 +1,40 @@ +import { logger } from '../../../helpers' +import { doRequest } from '../../../helpers/requests' +import { ActivityPubHttpPayload } from './activitypub-http-job-scheduler' +import { database as db } from '../../../initializers/database' +import { buildSignedActivity } from '../../../helpers/activitypub' + +async function process (payload: ActivityPubHttpPayload, jobId: number) { + logger.info('Processing ActivityPub unicast in job %d.', jobId) + + const accountSignature = await db.Account.load(payload.signatureAccountId) + if (!accountSignature) throw new Error('Unknown signature account id.') + + const signedBody = await buildSignedActivity(accountSignature, payload.body) + const uri = payload.uris[0] + const options = { + method: 'POST', + uri, + json: signedBody + } + + await doRequest(options) +} + +function onError (err: Error, jobId: number) { + logger.error('Error when sending ActivityPub request in job %d.', jobId, err) + return Promise.resolve() +} + +function onSuccess (jobId: number) { + logger.info('Job %d is a success.', jobId) + return Promise.resolve() +} + +// --------------------------------------------------------------------------- + +export { + process, + onError, + onSuccess +} diff --git a/server/lib/jobs/activitypub-http-job-scheduler/index.ts b/server/lib/jobs/activitypub-http-job-scheduler/index.ts new file mode 100644 index 000000000..ad8f527b4 --- /dev/null +++ b/server/lib/jobs/activitypub-http-job-scheduler/index.ts @@ -0,0 +1 @@ +export * from './activitypub-http-job-scheduler' diff --git a/server/lib/jobs/http-request-job-scheduler/http-request-broadcast-handler.ts b/server/lib/jobs/http-request-job-scheduler/http-request-broadcast-handler.ts deleted file mode 100644 index ccb008e4d..000000000 --- a/server/lib/jobs/http-request-job-scheduler/http-request-broadcast-handler.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { logger } from '../../../helpers' -import { doRequest } from '../../../helpers/requests' -import { HTTPRequestPayload } from './http-request-job-scheduler' - -async function process (payload: HTTPRequestPayload, jobId: number) { - logger.info('Processing broadcast in job %d.', jobId) - - const options = { - method: 'POST', - uri: '', - json: payload.body - } - - for (const uri of payload.uris) { - options.uri = uri - await doRequest(options) - } -} - -function onError (err: Error, jobId: number) { - logger.error('Error when broadcasting request in job %d.', jobId, err) - return Promise.resolve() -} - -function onSuccess (jobId: number) { - logger.info('Job %d is a success.', jobId) - return Promise.resolve() -} - -// --------------------------------------------------------------------------- - -export { - process, - onError, - onSuccess -} diff --git a/server/lib/jobs/http-request-job-scheduler/http-request-job-scheduler.ts b/server/lib/jobs/http-request-job-scheduler/http-request-job-scheduler.ts deleted file mode 100644 index ad3349866..000000000 --- a/server/lib/jobs/http-request-job-scheduler/http-request-job-scheduler.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { JobScheduler, JobHandler } from '../job-scheduler' - -import * as httpRequestBroadcastHandler from './http-request-broadcast-handler' -import * as httpRequestUnicastHandler from './http-request-unicast-handler' -import { JobCategory } from '../../../../shared' - -type HTTPRequestPayload = { - uris: string[] - body: any -} -const jobHandlers: { [ handlerName: string ]: JobHandler } = { - httpRequestBroadcastHandler, - httpRequestUnicastHandler -} -const jobCategory: JobCategory = 'http-request' - -const httpRequestJobScheduler = new JobScheduler(jobCategory, jobHandlers) - -export { - HTTPRequestPayload, - httpRequestJobScheduler -} diff --git a/server/lib/jobs/http-request-job-scheduler/http-request-unicast-handler.ts b/server/lib/jobs/http-request-job-scheduler/http-request-unicast-handler.ts deleted file mode 100644 index 9e4e73891..000000000 --- a/server/lib/jobs/http-request-job-scheduler/http-request-unicast-handler.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { logger } from '../../../helpers' -import { doRequest } from '../../../helpers/requests' -import { HTTPRequestPayload } from './http-request-job-scheduler' - -async function process (payload: HTTPRequestPayload, jobId: number) { - logger.info('Processing unicast in job %d.', jobId) - - const uri = payload.uris[0] - const options = { - method: 'POST', - uri, - json: payload.body - } - - await doRequest(options) -} - -function onError (err: Error, jobId: number) { - logger.error('Error when sending request in job %d.', jobId, err) - return Promise.resolve() -} - -function onSuccess (jobId: number) { - logger.info('Job %d is a success.', jobId) - return Promise.resolve() -} - -// --------------------------------------------------------------------------- - -export { - process, - onError, - onSuccess -} diff --git a/server/lib/jobs/http-request-job-scheduler/index.ts b/server/lib/jobs/http-request-job-scheduler/index.ts deleted file mode 100644 index 4d2573296..000000000 --- a/server/lib/jobs/http-request-job-scheduler/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './http-request-job-scheduler' diff --git a/server/lib/jobs/index.ts b/server/lib/jobs/index.ts index a92743707..394264ec1 100644 --- a/server/lib/jobs/index.ts +++ b/server/lib/jobs/index.ts @@ -1,2 +1,2 @@ -export * from './http-request-job-scheduler' +export * from './activitypub-http-job-scheduler' export * from './transcoding-job-scheduler' diff --git a/server/tests/api/index-fast.ts b/server/tests/api/index-fast.ts index f13d8155d..ced973bc2 100644 --- a/server/tests/api/index-fast.ts +++ b/server/tests/api/index-fast.ts @@ -10,4 +10,3 @@ import './video-blacklist-management' import './video-description' import './video-privacy' import './services' -import './request-schedulers' diff --git a/server/tests/api/multiple-pods.ts b/server/tests/api/multiple-pods.ts index 8e89da97a..3c6b3f650 100644 --- a/server/tests/api/multiple-pods.ts +++ b/server/tests/api/multiple-pods.ts @@ -113,7 +113,7 @@ describe('Test multiple pods', function () { expect(video.tags).to.deep.equal([ 'tag1p1', 'tag2p1' ]) expect(dateIsValid(video.createdAt)).to.be.true expect(dateIsValid(video.updatedAt)).to.be.true - expect(video.author).to.equal('root') + expect(video.account).to.equal('root') const res2 = await getVideo(server.url, video.uuid) const videoDetails = res2.body @@ -202,7 +202,7 @@ describe('Test multiple pods', function () { expect(video.tags).to.deep.equal([ 'tag1p2', 'tag2p2', 'tag3p2' ]) expect(dateIsValid(video.createdAt)).to.be.true expect(dateIsValid(video.updatedAt)).to.be.true - expect(video.author).to.equal('user1') + expect(video.account).to.equal('user1') if (server.url !== 'http://localhost:9002') { expect(video.isLocal).to.be.false @@ -696,7 +696,7 @@ describe('Test multiple pods', function () { expect(baseVideo.licence).to.equal(video.licence) expect(baseVideo.category).to.equal(video.category) expect(baseVideo.nsfw).to.equal(video.nsfw) - expect(baseVideo.author).to.equal(video.author) + expect(baseVideo.author).to.equal(video.account) expect(baseVideo.tags).to.deep.equal(video.tags) } }) diff --git a/server/tests/api/request-schedulers.ts b/server/tests/api/request-schedulers.ts deleted file mode 100644 index c136d1cea..000000000 --- a/server/tests/api/request-schedulers.ts +++ /dev/null @@ -1,82 +0,0 @@ -/* tslint:disable:no-unused-expression */ - -import 'mocha' -import * as chai from 'chai' -const expect = chai.expect - -import { - ServerInfo, - flushTests, - uploadVideo, - makeFriends, - wait, - setAccessTokensToServers, - flushAndRunMultipleServers, - getRequestsStats, - killallServers -} from '../utils' - -describe('Test requests schedulers stats', function () { - const requestSchedulerNames = [ 'requestScheduler', 'requestVideoQaduScheduler', 'requestVideoEventScheduler' ] - let servers: ServerInfo[] = [] - - function uploadVideoWrapper (server: ServerInfo) { - const videoAttributes = { - tags: [ 'tag1', 'tag2' ] - } - - return uploadVideo(server.url, server.accessToken, videoAttributes) - } - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(120000) - - servers = await flushAndRunMultipleServers(2) - - await setAccessTokensToServers(servers) - - await makeFriends(servers[0].url, servers[0].accessToken) - }) - - it('Should have a correct timer', async function () { - const server = servers[0] - - const res = await getRequestsStats(server) - - const requestSchedulers = res.body - for (const requestSchedulerName of requestSchedulerNames) { - const requestScheduler = requestSchedulers[requestSchedulerName] - - expect(requestScheduler.remainingMilliSeconds).to.be.at.least(0) - expect(requestScheduler.remainingMilliSeconds).to.be.at.most(10000) - } - }) - - it('Should have the correct total request', async function () { - this.timeout(15000) - - const server = servers[0] - // Ensure the requests of pod 1 won't be made - servers[1].app.kill() - - await uploadVideoWrapper(server) - - await wait(1000) - - const res = await getRequestsStats(server) - const requestSchedulers = res.body - const requestScheduler = requestSchedulers.requestScheduler - expect(requestScheduler.totalRequests).to.equal(3) - }) - - after(async function () { - // Server 1 has already been killed - killallServers([ servers[0] ]) - - if (this['ok']) { - await flushTests() - } - }) -}) diff --git a/server/tests/api/services.ts b/server/tests/api/services.ts index 76911fdc5..c34c51f66 100644 --- a/server/tests/api/services.ts +++ b/server/tests/api/services.ts @@ -14,6 +14,7 @@ import { getOEmbed } from '../utils' import { runServer } from '../utils/servers' +import { Video } from '../../../client/src/app/videos/shared/video.model' describe('Test services', function () { let server: ServerInfo = null @@ -46,7 +47,7 @@ describe('Test services', function () { expect(res.body.html).to.equal(expectedHtml) expect(res.body.title).to.equal(server.video.name) - expect(res.body.author_name).to.equal(server.video.author) + expect(res.body.author_name).to.equal(server.video.account) expect(res.body.width).to.equal(560) expect(res.body.height).to.equal(315) expect(res.body.thumbnail_url).to.equal(expectedThumbnailUrl) @@ -66,7 +67,7 @@ describe('Test services', function () { expect(res.body.html).to.equal(expectedHtml) expect(res.body.title).to.equal(server.video.name) - expect(res.body.author_name).to.equal(server.video.author) + expect(res.body.author_name).to.equal(server.video.account) expect(res.body.height).to.equal(50) expect(res.body.width).to.equal(50) expect(res.body).to.not.have.property('thumbnail_url') diff --git a/server/tests/api/single-pod.ts b/server/tests/api/single-pod.ts index 3a05d0727..0a917f2ae 100644 --- a/server/tests/api/single-pod.ts +++ b/server/tests/api/single-pod.ts @@ -125,8 +125,8 @@ describe('Test a single pod', function () { expect(video.languageLabel).to.equal('Mandarin') expect(video.nsfw).to.be.ok expect(video.description).to.equal('my super description') - expect(video.podHost).to.equal('localhost:9001') - expect(video.author).to.equal('root') + expect(video.serverHost).to.equal('localhost:9001') + expect(video.account).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) expect(dateIsValid(video.createdAt)).to.be.true @@ -174,8 +174,8 @@ describe('Test a single pod', function () { expect(video.languageLabel).to.equal('Mandarin') expect(video.nsfw).to.be.ok expect(video.description).to.equal('my super description') - expect(video.podHost).to.equal('localhost:9001') - expect(video.author).to.equal('root') + expect(video.serverHost).to.equal('localhost:9001') + expect(video.account).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) expect(dateIsValid(video.createdAt)).to.be.true @@ -237,8 +237,8 @@ describe('Test a single pod', function () { expect(video.languageLabel).to.equal('Mandarin') expect(video.nsfw).to.be.ok expect(video.description).to.equal('my super description') - expect(video.podHost).to.equal('localhost:9001') - expect(video.author).to.equal('root') + expect(video.serverHost).to.equal('localhost:9001') + expect(video.account).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) expect(dateIsValid(video.createdAt)).to.be.true @@ -249,7 +249,7 @@ describe('Test a single pod', function () { }) // Not implemented yet - // it('Should search the video by podHost', async function () { + // it('Should search the video by serverHost', async function () { // const res = await videosUtils.searchVideo(server.url, '9001', 'host') // expect(res.body.total).to.equal(1) @@ -259,7 +259,7 @@ describe('Test a single pod', function () { // const video = res.body.data[0] // expect(video.name).to.equal('my super name') // expect(video.description).to.equal('my super description') - // expect(video.podHost).to.equal('localhost:9001') + // expect(video.serverHost).to.equal('localhost:9001') // expect(video.author).to.equal('root') // expect(video.isLocal).to.be.true // expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) @@ -291,8 +291,8 @@ describe('Test a single pod', function () { expect(video.languageLabel).to.equal('Mandarin') expect(video.nsfw).to.be.ok expect(video.description).to.equal('my super description') - expect(video.podHost).to.equal('localhost:9001') - expect(video.author).to.equal('root') + expect(video.serverHost).to.equal('localhost:9001') + expect(video.account).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) expect(dateIsValid(video.createdAt)).to.be.true @@ -311,7 +311,7 @@ describe('Test a single pod', function () { }) it('Should not find a search by author', async function () { - const res = await searchVideo(server.url, 'hello', 'author') + const res = await searchVideo(server.url, 'hello', 'account') expect(res.body.total).to.equal(0) expect(res.body.data).to.be.an('array') @@ -352,7 +352,7 @@ describe('Test a single pod', function () { 'video_short1.webm', 'video_short2.webm', 'video_short3.webm' ] - const tasks: Promise[] = [] + // const tasks: Promise[] = [] for (const video of videos) { const videoAttributes = { name: video + ' name', @@ -366,10 +366,13 @@ describe('Test a single pod', function () { } const p = uploadVideo(server.url, server.accessToken, videoAttributes) - tasks.push(p) + await p } - - await Promise.all(tasks) + // FIXME: concurrent uploads does not work :( + // tasks.push(p) + // } + // + // await Promise.all(tasks) }) it('Should have the correct durations', async function () { @@ -462,7 +465,7 @@ describe('Test a single pod', function () { }) it('Should search all the root author videos', async function () { - const res = await searchVideoWithPagination(server.url, 'root', 'author', 0, 15) + const res = await searchVideoWithPagination(server.url, 'root', 'account', 0, 15) const videos = res.body.data expect(res.body.total).to.equal(6) @@ -550,8 +553,8 @@ describe('Test a single pod', function () { expect(video.languageLabel).to.equal('Arabic') expect(video.nsfw).to.be.ok expect(video.description).to.equal('my super description updated') - expect(video.podHost).to.equal('localhost:9001') - expect(video.author).to.equal('root') + expect(video.serverHost).to.equal('localhost:9001') + expect(video.account).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tagup1', 'tagup2' ]) expect(dateIsValid(video.createdAt)).to.be.true @@ -599,8 +602,8 @@ describe('Test a single pod', function () { expect(video.languageLabel).to.equal('Arabic') expect(video.nsfw).to.be.ok expect(video.description).to.equal('my super description updated') - expect(video.podHost).to.equal('localhost:9001') - expect(video.author).to.equal('root') + expect(video.serverHost).to.equal('localhost:9001') + expect(video.account).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'supertag' ]) expect(dateIsValid(video.createdAt)).to.be.true @@ -639,8 +642,8 @@ describe('Test a single pod', function () { expect(video.languageLabel).to.equal('Arabic') expect(video.nsfw).to.be.ok expect(video.description).to.equal('hello everybody') - expect(video.podHost).to.equal('localhost:9001') - expect(video.author).to.equal('root') + expect(video.serverHost).to.equal('localhost:9001') + expect(video.account).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'supertag' ]) expect(dateIsValid(video.createdAt)).to.be.true diff --git a/server/tests/api/users.ts b/server/tests/api/users.ts index 6f40ca3c9..055dade04 100644 --- a/server/tests/api/users.ts +++ b/server/tests/api/users.ts @@ -1,38 +1,36 @@ /* tslint:disable:no-unused-expression */ - -import 'mocha' import * as chai from 'chai' -const expect = chai.expect - +import 'mocha' +import { UserRole } from '../../../shared' import { - ServerInfo, - flushTests, - runServer, - login, - uploadVideo, - makeFriends, - quitFriends, - getVideosList, - rateVideo, - getUserVideoRating, - removeVideo, - makePutBodyRequest, createUser, - loginAndGetAccessToken, + flushTests, + getBlacklistedVideosList, getMyUserInformation, + getUserInformation, getUsersList, getUsersListPaginationAndSort, - updateUser, - updateMyUser, + getUserVideoRating, + getVideosList, + killallServers, + login, + loginAndGetAccessToken, + makePutBodyRequest, + rateVideo, registerUser, removeUser, - killallServers, - getUserInformation, - getBlacklistedVideosList + removeVideo, + runServer, + ServerInfo, + updateMyUser, + updateUser, + uploadVideo } from '../utils' -import { UserRole } from '../../../shared' +import { follow } from '../utils/follows' import { getMyVideos } from '../utils/videos' +const expect = chai.expect + describe('Test users', function () { let server: ServerInfo let accessToken: string @@ -57,28 +55,36 @@ describe('Test users', function () { const client = { id: 'client', secret: server.client.secret } const res = await login(server.url, client, server.user, 400) - expect(res.body.error).to.equal('invalid_client') + expect(res.body.error) + .to + .equal('invalid_client') }) it('Should not login with an invalid client secret', async function () { const client = { id: server.client.id, secret: 'coucou' } const res = await login(server.url, client, server.user, 400) - expect(res.body.error).to.equal('invalid_client') + expect(res.body.error) + .to + .equal('invalid_client') }) it('Should not login with an invalid username', async function () { const user = { username: 'captain crochet', password: server.user.password } const res = await login(server.url, server.client, user, 400) - expect(res.body.error).to.equal('invalid_grant') + expect(res.body.error) + .to + .equal('invalid_grant') }) it('Should not login with an invalid password', async function () { const user = { username: server.user.username, password: 'mew_three' } const res = await login(server.url, server.client, user, 400) - expect(res.body.error).to.equal('invalid_grant') + expect(res.body.error) + .to + .equal('invalid_grant') }) it('Should not be able to upload a video', async function () { @@ -88,15 +94,12 @@ describe('Test users', function () { await uploadVideo(server.url, accessToken, videoAttributes, 401) }) - it('Should not be able to make friends', async function () { + it('Should not be able to follow', async function () { accessToken = 'my_super_token' - await makeFriends(server.url, accessToken, 401) + await follow(server.url, [ 'http://example.com' ], accessToken, 401) }) - it('Should not be able to quit friends', async function () { - accessToken = 'my_super_token' - await quitFriends(server.url, accessToken, 401) - }) + it('Should not be able to unfollow') it('Should be able to login', async function () { const res = await login(server.url, server.client, server.user, 200) @@ -108,9 +111,11 @@ describe('Test users', function () { const videoAttributes = {} await uploadVideo(server.url, accessToken, videoAttributes, 204) const res = await getVideosList(server.url) - const video = res.body.data[0] + const video = res.body.data[ 0 ] - expect(video.author).to.equal('root') + expect(video.account) + .to + .equal('root') videoId = video.id }) @@ -124,8 +129,12 @@ describe('Test users', function () { const res = await getUserVideoRating(server.url, accessToken, videoId) const rating = res.body - expect(rating.videoId).to.equal(videoId) - expect(rating.rating).to.equal('like') + expect(rating.videoId) + .to + .equal(videoId) + expect(rating.rating) + .to + .equal('like') }) it('Should not be able to remove the video with an incorrect token', async function () { @@ -187,12 +196,23 @@ describe('Test users', function () { const res = await getMyUserInformation(server.url, accessTokenUser) const user = res.body - expect(user.username).to.equal('user_1') - expect(user.email).to.equal('user_1@example.com') + expect(user.username) + .to + .equal('user_1') + expect(user.email) + .to + .equal('user_1@example.com') expect(user.displayNSFW).to.be.false - expect(user.videoQuota).to.equal(2 * 1024 * 1024) - expect(user.roleLabel).to.equal('User') - expect(user.id).to.be.a('number') + expect(user.videoQuota) + .to + .equal(2 * 1024 * 1024) + expect(user.roleLabel) + .to + .equal('User') + expect(user.id) + .to + .be + .a('number') }) it('Should be able to upload a video with this user', async function () { @@ -206,12 +226,19 @@ describe('Test users', function () { it('Should be able to list my videos', async function () { const res = await getMyVideos(server.url, accessTokenUser, 0, 5) - expect(res.body.total).to.equal(1) + expect(res.body.total) + .to + .equal(1) const videos = res.body.data - expect(videos).to.have.lengthOf(1) + expect(videos) + .to + .have + .lengthOf(1) - expect(videos[0].name).to.equal('super user video') + expect(videos[ 0 ].name) + .to + .equal('super user video') }) it('Should list all the users', async function () { @@ -220,18 +247,33 @@ describe('Test users', function () { const total = result.total const users = result.data - expect(total).to.equal(2) - expect(users).to.be.an('array') - expect(users.length).to.equal(2) + expect(total) + .to + .equal(2) + expect(users) + .to + .be + .an('array') + expect(users.length) + .to + .equal(2) - const user = users[0] - expect(user.username).to.equal('user_1') - expect(user.email).to.equal('user_1@example.com') + const user = users[ 0 ] + expect(user.username) + .to + .equal('user_1') + expect(user.email) + .to + .equal('user_1@example.com') expect(user.displayNSFW).to.be.false - const rootUser = users[1] - expect(rootUser.username).to.equal('root') - expect(rootUser.email).to.equal('admin1@example.com') + const rootUser = users[ 1 ] + expect(rootUser.username) + .to + .equal('root') + expect(rootUser.email) + .to + .equal('admin1@example.com') expect(rootUser.displayNSFW).to.be.false userId = user.id @@ -244,13 +286,23 @@ describe('Test users', function () { const total = result.total const users = result.data - expect(total).to.equal(2) - expect(users.length).to.equal(1) + expect(total) + .to + .equal(2) + expect(users.length) + .to + .equal(1) - const user = users[0] - expect(user.username).to.equal('root') - expect(user.email).to.equal('admin1@example.com') - expect(user.roleLabel).to.equal('Administrator') + const user = users[ 0 ] + expect(user.username) + .to + .equal('root') + expect(user.email) + .to + .equal('admin1@example.com') + expect(user.roleLabel) + .to + .equal('Administrator') expect(user.displayNSFW).to.be.false }) @@ -260,12 +312,20 @@ describe('Test users', function () { const total = result.total const users = result.data - expect(total).to.equal(2) - expect(users.length).to.equal(1) + expect(total) + .to + .equal(2) + expect(users.length) + .to + .equal(1) - const user = users[0] - expect(user.username).to.equal('user_1') - expect(user.email).to.equal('user_1@example.com') + const user = users[ 0 ] + expect(user.username) + .to + .equal('user_1') + expect(user.email) + .to + .equal('user_1@example.com') expect(user.displayNSFW).to.be.false }) @@ -275,12 +335,20 @@ describe('Test users', function () { const total = result.total const users = result.data - expect(total).to.equal(2) - expect(users.length).to.equal(1) + expect(total) + .to + .equal(2) + expect(users.length) + .to + .equal(1) - const user = users[0] - expect(user.username).to.equal('user_1') - expect(user.email).to.equal('user_1@example.com') + const user = users[ 0 ] + expect(user.username) + .to + .equal('user_1') + expect(user.email) + .to + .equal('user_1@example.com') expect(user.displayNSFW).to.be.false }) @@ -290,16 +358,28 @@ describe('Test users', function () { const total = result.total const users = result.data - expect(total).to.equal(2) - expect(users.length).to.equal(2) + expect(total) + .to + .equal(2) + expect(users.length) + .to + .equal(2) - expect(users[0].username).to.equal('root') - expect(users[0].email).to.equal('admin1@example.com') - expect(users[0].displayNSFW).to.be.false + expect(users[ 0 ].username) + .to + .equal('root') + expect(users[ 0 ].email) + .to + .equal('admin1@example.com') + expect(users[ 0 ].displayNSFW).to.be.false - expect(users[1].username).to.equal('user_1') - expect(users[1].email).to.equal('user_1@example.com') - expect(users[1].displayNSFW).to.be.false + expect(users[ 1 ].username) + .to + .equal('user_1') + expect(users[ 1 ].email) + .to + .equal('user_1@example.com') + expect(users[ 1 ].displayNSFW).to.be.false }) it('Should update my password', async function () { @@ -315,11 +395,20 @@ describe('Test users', function () { const res = await getMyUserInformation(server.url, accessTokenUser) const user = res.body - expect(user.username).to.equal('user_1') - expect(user.email).to.equal('user_1@example.com') + expect(user.username) + .to + .equal('user_1') + expect(user.email) + .to + .equal('user_1@example.com') expect(user.displayNSFW).to.be.ok - expect(user.videoQuota).to.equal(2 * 1024 * 1024) - expect(user.id).to.be.a('number') + expect(user.videoQuota) + .to + .equal(2 * 1024 * 1024) + expect(user.id) + .to + .be + .a('number') }) it('Should be able to change the email display attribute', async function () { @@ -328,11 +417,20 @@ describe('Test users', function () { const res = await getMyUserInformation(server.url, accessTokenUser) const user = res.body - expect(user.username).to.equal('user_1') - expect(user.email).to.equal('updated@example.com') + expect(user.username) + .to + .equal('user_1') + expect(user.email) + .to + .equal('updated@example.com') expect(user.displayNSFW).to.be.ok - expect(user.videoQuota).to.equal(2 * 1024 * 1024) - expect(user.id).to.be.a('number') + expect(user.videoQuota) + .to + .equal(2 * 1024 * 1024) + expect(user.id) + .to + .be + .a('number') }) it('Should be able to update another user', async function () { @@ -341,12 +439,23 @@ describe('Test users', function () { const res = await getUserInformation(server.url, accessToken, userId) const user = res.body - expect(user.username).to.equal('user_1') - expect(user.email).to.equal('updated2@example.com') + expect(user.username) + .to + .equal('user_1') + expect(user.email) + .to + .equal('updated2@example.com') expect(user.displayNSFW).to.be.ok - expect(user.videoQuota).to.equal(42) - expect(user.roleLabel).to.equal('Moderator') - expect(user.id).to.be.a('number') + expect(user.videoQuota) + .to + .equal(42) + expect(user.roleLabel) + .to + .equal('Moderator') + expect(user.id) + .to + .be + .a('number') }) it('Should not be able to delete a user by a moderator', async function () { @@ -369,10 +478,14 @@ describe('Test users', function () { it('Should not have videos of this user', async function () { const res = await getVideosList(server.url) - expect(res.body.total).to.equal(1) + expect(res.body.total) + .to + .equal(1) - const video = res.body.data[0] - expect(video.author).to.equal('root') + const video = res.body.data[ 0 ] + expect(video.account) + .to + .equal('root') }) it('Should register a new user', async function () { @@ -392,14 +505,16 @@ describe('Test users', function () { const res = await getMyUserInformation(server.url, accessToken) const user = res.body - expect(user.videoQuota).to.equal(5 * 1024 * 1024) + expect(user.videoQuota) + .to + .equal(5 * 1024 * 1024) }) after(async function () { killallServers([ server ]) // Keep the logs if the test failed - if (this['ok']) { + if (this[ 'ok' ]) { await flushTests() } }) diff --git a/server/tests/api/video-abuse.ts b/server/tests/api/video-abuse.ts index f2a2c322a..bc21ee59b 100644 --- a/server/tests/api/video-abuse.ts +++ b/server/tests/api/video-abuse.ts @@ -1,22 +1,22 @@ /* tslint:disable:no-unused-expression */ -import 'mocha' import * as chai from 'chai' -const expect = chai.expect - +import 'mocha' import { - ServerInfo, flushAndRunMultipleServers, - uploadVideo, - makeFriends, - getVideosList, - wait, - setAccessTokensToServers, + flushTests, getVideoAbusesList, - reportVideoAbuse, + getVideosList, killallServers, - flushTests + reportVideoAbuse, + ServerInfo, + setAccessTokensToServers, + uploadVideo, + wait } from '../utils' +import { doubleFollow } from '../utils/follows' + +const expect = chai.expect describe('Test video abuses', function () { let servers: ServerInfo[] = [] @@ -30,32 +30,32 @@ describe('Test video abuses', function () { // Get the access tokens await setAccessTokensToServers(servers) - // Pod 1 makes friend with pod 2 - await makeFriends(servers[0].url, servers[0].accessToken) + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) - // Upload some videos on each pods + // Upload some videos on each servers const video1Attributes = { - name: 'my super name for pod 1', - description: 'my super description for pod 1' + name: 'my super name for server 1', + description: 'my super description for server 1' } await uploadVideo(servers[0].url, servers[0].accessToken, video1Attributes) const video2Attributes = { - name: 'my super name for pod 2', - description: 'my super description for pod 2' + name: 'my super name for server 2', + description: 'my super description for server 2' } await uploadVideo(servers[1].url, servers[1].accessToken, video2Attributes) // Wait videos propagation - await wait(22000) + await wait(25000) const res = await getVideosList(servers[0].url) const videos = res.body.data expect(videos.length).to.equal(2) - servers[0].video = videos.find(video => video.name === 'my super name for pod 1') - servers[1].video = videos.find(video => video.name === 'my super name for pod 2') + servers[0].video = videos.find(video => video.name === 'my super name for server 1') + servers[1].video = videos.find(video => video.name === 'my super name for server 2') }) it('Should not have video abuses', async function () { @@ -72,11 +72,11 @@ describe('Test video abuses', function () { const reason = 'my super bad reason' await reportVideoAbuse(servers[0].url, servers[0].accessToken, servers[0].video.id, reason) - // We wait requests propagation, even if the pod 1 is not supposed to make a request to pod 2 + // We wait requests propagation, even if the server 1 is not supposed to make a request to server 2 await wait(11000) }) - it('Should have 1 video abuses on pod 1 and 0 on pod 2', async function () { + it('Should have 1 video abuses on server 1 and 0 on server 2', async function () { const res1 = await getVideoAbusesList(servers[0].url, servers[0].accessToken) expect(res1.body.total).to.equal(1) @@ -86,7 +86,7 @@ describe('Test video abuses', function () { const abuse = res1.body.data[0] expect(abuse.reason).to.equal('my super bad reason') expect(abuse.reporterUsername).to.equal('root') - expect(abuse.reporterPodHost).to.equal('localhost:9001') + expect(abuse.reporterServerHost).to.equal('localhost:9001') expect(abuse.videoId).to.equal(servers[0].video.id) const res2 = await getVideoAbusesList(servers[1].url, servers[1].accessToken) @@ -96,16 +96,16 @@ describe('Test video abuses', function () { }) it('Should report abuse on a remote video', async function () { - this.timeout(15000) + this.timeout(25000) const reason = 'my super bad reason 2' await reportVideoAbuse(servers[0].url, servers[0].accessToken, servers[1].video.id, reason) // We wait requests propagation - await wait(11000) + await wait(15000) }) - it('Should have 2 video abuse on pod 1 and 1 on pod 2', async function () { + it('Should have 2 video abuse on server 1 and 1 on server 2', async function () { const res1 = await getVideoAbusesList(servers[0].url, servers[0].accessToken) expect(res1.body.total).to.equal(2) expect(res1.body.data).to.be.an('array') @@ -114,13 +114,13 @@ describe('Test video abuses', function () { const abuse1 = res1.body.data[0] expect(abuse1.reason).to.equal('my super bad reason') expect(abuse1.reporterUsername).to.equal('root') - expect(abuse1.reporterPodHost).to.equal('localhost:9001') + expect(abuse1.reporterServerHost).to.equal('localhost:9001') expect(abuse1.videoId).to.equal(servers[0].video.id) const abuse2 = res1.body.data[1] expect(abuse2.reason).to.equal('my super bad reason 2') expect(abuse2.reporterUsername).to.equal('root') - expect(abuse2.reporterPodHost).to.equal('localhost:9001') + expect(abuse2.reporterServerHost).to.equal('localhost:9001') expect(abuse2.videoId).to.equal(servers[1].video.id) const res2 = await getVideoAbusesList(servers[1].url, servers[1].accessToken) @@ -131,7 +131,7 @@ describe('Test video abuses', function () { const abuse3 = res2.body.data[0] expect(abuse3.reason).to.equal('my super bad reason 2') expect(abuse3.reporterUsername).to.equal('root') - expect(abuse3.reporterPodHost).to.equal('localhost:9001') + expect(abuse3.reporterServerHost).to.equal('localhost:9001') }) after(async function () { diff --git a/server/tests/utils/follows.ts b/server/tests/utils/follows.ts index 9ad1ca7f4..b454fe2f8 100644 --- a/server/tests/utils/follows.ts +++ b/server/tests/utils/follows.ts @@ -1,9 +1,9 @@ import * as request from 'supertest' - import { wait } from './miscs' +import { ServerInfo } from './servers' function getFollowersListPaginationAndSort (url: string, start: number, count: number, sort: string) { - const path = '/api/v1/servers/followers' + const path = '/api/v1/server/followers' return request(url) .get(path) @@ -16,7 +16,7 @@ function getFollowersListPaginationAndSort (url: string, start: number, count: n } function getFollowingListPaginationAndSort (url: string, start: number, count: number, sort: string) { - const path = '/api/v1/servers/following' + const path = '/api/v1/server/following' return request(url) .get(path) @@ -29,25 +29,36 @@ function getFollowingListPaginationAndSort (url: string, start: number, count: n } async function follow (follower: string, following: string[], accessToken: string, expectedStatus = 204) { - const path = '/api/v1/servers/follow' + const path = '/api/v1/server/follow' + const followingHosts = following.map(f => f.replace(/^http:\/\//, '')) const res = await request(follower) .post(path) .set('Accept', 'application/json') .set('Authorization', 'Bearer ' + accessToken) - .send({ 'hosts': following }) + .send({ 'hosts': followingHosts }) .expect(expectedStatus) // Wait request propagation - await wait(1000) + await wait(20000) return res } +async function doubleFollow (server1: ServerInfo, server2: ServerInfo) { + await Promise.all([ + follow(server1.url, [ server2.url ], server1.accessToken), + follow(server2.url, [ server1.url ], server2.accessToken) + ]) + + return true +} + // --------------------------------------------------------------------------- export { getFollowersListPaginationAndSort, getFollowingListPaginationAndSort, - follow + follow, + doubleFollow } diff --git a/server/tests/utils/servers.ts b/server/tests/utils/servers.ts index f042a9e53..faa2f19ff 100644 --- a/server/tests/utils/servers.ts +++ b/server/tests/utils/servers.ts @@ -24,7 +24,7 @@ interface ServerInfo { id: number uuid: string name: string - author: string + account: string } remoteVideo?: { diff --git a/shared/models/activitypub/activity.ts b/shared/models/activitypub/activity.ts index f8e982fbb..6a05a1c39 100644 --- a/shared/models/activitypub/activity.ts +++ b/shared/models/activitypub/activity.ts @@ -11,10 +11,10 @@ export type ActivityType = 'Create' | 'Add' | 'Update' | 'Flag' | 'Delete' | 'Fo export interface BaseActivity { '@context'?: any[] id: string - to: string[] + to?: string[] actor: string type: ActivityType - signature: ActivityPubSignature + signature?: ActivityPubSignature } export interface ActivityCreate extends BaseActivity { diff --git a/shared/models/job.model.ts b/shared/models/job.model.ts index ab723084a..10696e3f8 100644 --- a/shared/models/job.model.ts +++ b/shared/models/job.model.ts @@ -1,2 +1,2 @@ export type JobState = 'pending' | 'processing' | 'error' | 'success' -export type JobCategory = 'transcoding' | 'http-request' +export type JobCategory = 'transcoding' | 'activitypub-http' diff --git a/yarn.lock b/yarn.lock index 773ff7350..695a610d7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -107,7 +107,11 @@ dependencies: "@types/express" "*" -"@types/node@*", "@types/node@^8.0.3": +"@types/node@*": + version "8.0.53" + resolved "https://registry.yarnpkg.com/@types/node/-/node-8.0.53.tgz#396b35af826fa66aad472c8cb7b8d5e277f4e6d8" + +"@types/node@^8.0.3": version "8.0.47" resolved "https://registry.yarnpkg.com/@types/node/-/node-8.0.47.tgz#968e596f91acd59069054558a00708c445ca30c2" @@ -2717,8 +2721,8 @@ moment-timezone@^0.5.4: moment ">= 2.9.0" "moment@>= 2.9.0", moment@^2.13.0: - version "2.19.1" - resolved "https://registry.yarnpkg.com/moment/-/moment-2.19.1.tgz#56da1a2d1cbf01d38b7e1afc31c10bcfa1929167" + version "2.19.2" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.19.2.tgz#8a7f774c95a64550b4c7ebd496683908f9419dbe" morgan@^1.5.3: version "1.9.0" @@ -3647,8 +3651,8 @@ send@0.16.1: statuses "~1.3.1" sequelize@^4.7.5: - version "4.22.5" - resolved "https://registry.yarnpkg.com/sequelize/-/sequelize-4.22.5.tgz#5771f8dc2173c61366d77b9fb89aeb34b0522435" + version "4.22.7" + resolved "https://registry.yarnpkg.com/sequelize/-/sequelize-4.22.7.tgz#9425ad640f9813455cdc49cbeaf54aece141d76e" dependencies: bluebird "^3.4.6" cls-bluebird "^2.0.1"