From 28dfb44b145c537aba07ae73cb1287f25532022a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 8 Jun 2021 17:29:45 +0200 Subject: [PATCH] Try to speed up AP update transaction --- server/helpers/database-utils.ts | 25 +++++++++----- server/lib/activitypub/actors/get.ts | 2 +- server/lib/activitypub/actors/updater.ts | 26 +++++++-------- .../lib/activitypub/process/process-view.ts | 9 +++-- .../videos/shared/abstract-builder.ts | 2 +- .../lib/activitypub/videos/shared/trackers.ts | 2 +- server/lib/activitypub/videos/updater.ts | 33 ++++++++++--------- server/models/video/video.ts | 2 +- 8 files changed, 56 insertions(+), 45 deletions(-) diff --git a/server/helpers/database-utils.ts b/server/helpers/database-utils.ts index 7befa2c49..240b18033 100644 --- a/server/helpers/database-utils.ts +++ b/server/helpers/database-utils.ts @@ -58,7 +58,7 @@ function transactionRetryer (func: (err: any, data: T) => any) { errorFilter: err => { const willRetry = (err.name === 'SequelizeDatabaseError') - logger.debug('Maybe retrying the transaction function.', { willRetry, err }) + logger.debug('Maybe retrying the transaction function.', { willRetry, err, tags: [ 'sql', 'retry' ] }) return willRetry } }, @@ -68,6 +68,8 @@ function transactionRetryer (func: (err: any, data: T) => any) { }) } +// --------------------------------------------------------------------------- + function updateInstanceWithAnother > (instanceToUpdate: T, baseInstance: U) { const obj = baseInstance.toJSON() @@ -82,12 +84,6 @@ function resetSequelizeInstance (instance: Model, savedFields: object) { }) } -function afterCommitIfTransaction (t: Transaction, fn: Function) { - if (t) return t.afterCommit(() => fn()) - - return fn() -} - function deleteNonExistingModels > ( fromDatabase: T[], newModels: T[], @@ -111,6 +107,18 @@ function setAsUpdated (table: string, id: number, transaction?: Transaction) { // --------------------------------------------------------------------------- +function runInReadCommittedTransaction (fn: (t: Transaction) => Promise) { + return sequelizeTypescript.transaction(t => fn(t)) +} + +function afterCommitIfTransaction (t: Transaction, fn: Function) { + if (t) return t.afterCommit(() => fn()) + + return fn() +} + +// --------------------------------------------------------------------------- + export { resetSequelizeInstance, retryTransactionWrapper, @@ -118,5 +126,6 @@ export { updateInstanceWithAnother, afterCommitIfTransaction, deleteNonExistingModels, - setAsUpdated + setAsUpdated, + runInReadCommittedTransaction } diff --git a/server/lib/activitypub/actors/get.ts b/server/lib/activitypub/actors/get.ts index c7b49d6e4..de93aa964 100644 --- a/server/lib/activitypub/actors/get.ts +++ b/server/lib/activitypub/actors/get.ts @@ -56,7 +56,7 @@ async function getOrCreateAPActor ( if (actor.Account) (actor as MActorAccountChannelIdActor).Account.Actor = actor if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor - const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType) + const { actor: actorRefreshed, refreshed } = await refreshActorIfNeeded(actor, fetchType) if (!actorRefreshed) throw new Error('Actor ' + actor.url + ' does not exist anymore.') await scheduleOutboxFetchIfNeeded(actor, created, refreshed, updateCollections) diff --git a/server/lib/activitypub/actors/updater.ts b/server/lib/activitypub/actors/updater.ts index 471688f11..fb880a767 100644 --- a/server/lib/activitypub/actors/updater.ts +++ b/server/lib/activitypub/actors/updater.ts @@ -1,6 +1,5 @@ -import { resetSequelizeInstance } from '@server/helpers/database-utils' +import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils' import { logger } from '@server/helpers/logger' -import { sequelizeTypescript } from '@server/initializers/database' import { VideoChannelModel } from '@server/models/video/video-channel' import { MAccount, MActor, MActorFull, MChannel } from '@server/types/models' import { ActivityPubActor, ActorImageType } from '@shared/models' @@ -32,20 +31,21 @@ export class APActorUpdater { const bannerInfo = getImageInfoFromObject(this.actorObject, ActorImageType.BANNER) try { - await sequelizeTypescript.transaction(async t => { - await this.updateActorInstance(this.actor, this.actorObject) + await this.updateActorInstance(this.actor, this.actorObject) + this.accountOrChannel.name = this.actorObject.name || this.actorObject.preferredUsername + this.accountOrChannel.description = this.actorObject.summary + + if (this.accountOrChannel instanceof VideoChannelModel) this.accountOrChannel.support = this.actorObject.support + + await runInReadCommittedTransaction(async t => { + await this.actor.save({ transaction: t }) + await this.accountOrChannel.save({ transaction: t }) + }) + + await runInReadCommittedTransaction(async t => { await updateActorImageInstance(this.actor, ActorImageType.AVATAR, avatarInfo, t) await updateActorImageInstance(this.actor, ActorImageType.BANNER, bannerInfo, t) - - await this.actor.save({ transaction: t }) - - this.accountOrChannel.name = this.actorObject.name || this.actorObject.preferredUsername - this.accountOrChannel.description = this.actorObject.summary - - if (this.accountOrChannel instanceof VideoChannelModel) this.accountOrChannel.support = this.actorObject.support - - await this.accountOrChannel.save({ transaction: t }) }) logger.info('Remote account %s updated', this.actorObject.url) diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts index c2d41dd28..0a0231a3a 100644 --- a/server/lib/activitypub/process/process-view.ts +++ b/server/lib/activitypub/process/process-view.ts @@ -24,12 +24,11 @@ async function processCreateView (activity: ActivityView | ActivityCreate, byAct ? activity.object : (activity.object as ViewObject).object - const options = { + const { video } = await getOrCreateAPVideo({ videoObject, - fetchType: 'only-video' as 'only-video', - allowRefresh: false as false - } - const { video } = await getOrCreateAPVideo(options) + fetchType: 'only-video', + allowRefresh: false + }) if (!video.isLive) { await Redis.Instance.addVideoView(video.id) diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts index 0b58ddb33..22280fce1 100644 --- a/server/lib/activitypub/videos/shared/abstract-builder.ts +++ b/server/lib/activitypub/videos/shared/abstract-builder.ts @@ -49,7 +49,7 @@ export abstract class APVideoAbstractBuilder { }) } - protected async setPreview (video: MVideoFullLight, t: Transaction) { + protected async setPreview (video: MVideoFullLight, t?: Transaction) { // Don't fetch the preview that could be big, create a placeholder instead const previewIcon = getPreviewFromIcons(this.videoObject) if (!previewIcon) return diff --git a/server/lib/activitypub/videos/shared/trackers.ts b/server/lib/activitypub/videos/shared/trackers.ts index fcb2a5091..1c5fc4f84 100644 --- a/server/lib/activitypub/videos/shared/trackers.ts +++ b/server/lib/activitypub/videos/shared/trackers.ts @@ -28,7 +28,7 @@ function getTrackerUrls (object: VideoObject, video: MVideoWithHost) { async function setVideoTrackers (options: { video: MVideo trackers: string[] - transaction?: Transaction + transaction: Transaction }) { const { video, trackers, transaction } = options diff --git a/server/lib/activitypub/videos/updater.ts b/server/lib/activitypub/videos/updater.ts index 3339611fc..e17e5fdc2 100644 --- a/server/lib/activitypub/videos/updater.ts +++ b/server/lib/activitypub/videos/updater.ts @@ -1,7 +1,6 @@ import { Transaction } from 'sequelize/types' -import { resetSequelizeInstance } from '@server/helpers/database-utils' +import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils' import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' -import { sequelizeTypescript } from '@server/initializers/database' import { Notifier } from '@server/lib/notifier' import { PeerTubeSocket } from '@server/lib/peertube-socket' import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' @@ -48,24 +47,26 @@ export class APVideoUpdater extends APVideoAbstractBuilder { const thumbnailModel = await this.tryToGenerateThumbnail(this.video) - const videoUpdated = await sequelizeTypescript.transaction(async t => { - this.checkChannelUpdateOrThrow(channelActor) + this.checkChannelUpdateOrThrow(channelActor) - const videoUpdated = await this.updateVideo(channelActor.VideoChannel, t, overrideTo) + const videoUpdated = await this.updateVideo(channelActor.VideoChannel, undefined, overrideTo) - if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t) + if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel) - await this.setPreview(videoUpdated, t) + await runInReadCommittedTransaction(async t => { await this.setWebTorrentFiles(videoUpdated, t) await this.setStreamingPlaylists(videoUpdated, t) - await this.setTags(videoUpdated, t) - await this.setTrackers(videoUpdated, t) - await this.setCaptions(videoUpdated, t) - await this.setOrDeleteLive(videoUpdated, t) - - return videoUpdated }) + await Promise.all([ + runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)), + runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)), + this.setOrDeleteLive(videoUpdated), + this.setPreview(videoUpdated) + ]) + + await runInReadCommittedTransaction(t => this.setCaptions(videoUpdated, t)) + await autoBlacklistVideoIfNeeded({ video: videoUpdated, user: undefined, @@ -103,7 +104,7 @@ export class APVideoUpdater extends APVideoAbstractBuilder { } } - private updateVideo (channel: MChannelId, transaction: Transaction, overrideTo?: string[]) { + private updateVideo (channel: MChannelId, transaction?: Transaction, overrideTo?: string[]) { const to = overrideTo || this.videoObject.to const videoData = getVideoAttributesFromObject(channel, this.videoObject, to) this.video.name = videoData.name @@ -140,7 +141,9 @@ export class APVideoUpdater extends APVideoAbstractBuilder { await this.insertOrReplaceCaptions(videoUpdated, t) } - private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction: Transaction) { + private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction?: Transaction) { + if (!this.video.isLive) return + if (this.video.isLive) return this.insertOrReplaceLive(videoUpdated, transaction) // Delete existing live if it exists diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 8f561116b..44aaa24ef 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -1886,7 +1886,7 @@ export class VideoModel extends Model>> { return Array.isArray(this.VideoFiles) === true && this.VideoFiles.length !== 0 } - async addAndSaveThumbnail (thumbnail: MThumbnail, transaction: Transaction) { + async addAndSaveThumbnail (thumbnail: MThumbnail, transaction?: Transaction) { thumbnail.videoId = this.id const savedThumbnail = await thumbnail.save({ transaction })