Don't refresh videos when processing views

It allows us to use a cache
pull/2456/head
Chocobozzz 2020-02-04 15:45:41 +01:00
parent 7eba5e1fa8
commit 943e519390
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
9 changed files with 95 additions and 34 deletions

View File

@ -38,14 +38,23 @@ function fetchVideo (
if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id)
}
type VideoFetchByUrlType = 'all' | 'only-video'
type VideoFetchByUrlType = 'all' | 'only-video' | 'only-immutable-attributes'
function fetchVideoByUrl (url: string, fetchType: 'all'): Bluebird<MVideoAccountLightBlacklistAllFiles>
function fetchVideoByUrl (url: string, fetchType: 'only-immutable-attributes'): Bluebird<MVideoImmutable>
function fetchVideoByUrl (url: string, fetchType: 'only-video'): Bluebird<MVideoThumbnail>
function fetchVideoByUrl (url: string, fetchType: VideoFetchByUrlType): Bluebird<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail>
function fetchVideoByUrl (url: string, fetchType: VideoFetchByUrlType): Bluebird<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail> {
function fetchVideoByUrl (
url: string,
fetchType: VideoFetchByUrlType
): Bluebird<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable>
function fetchVideoByUrl (
url: string,
fetchType: VideoFetchByUrlType
): Bluebird<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url)
if (fetchType === 'only-immutable-attributes') return VideoModel.loadByUrlImmutableAttributes(url)
if (fetchType === 'only-video') return VideoModel.loadByUrl(url)
}

View File

@ -4,7 +4,15 @@ import { ACTIVITY_PUB } from '../../initializers/constants'
import { ActorModel } from '../../models/activitypub/actor'
import { VideoModel } from '../../models/video/video'
import { VideoShareModel } from '../../models/video/video-share'
import { MActorFollowersUrl, MActorLight, MCommentOwner, MCommentOwnerVideo, MVideo, MVideoAccountLight } from '../../typings/models'
import {
MActorFollowersUrl,
MActorLight,
MCommentOwner,
MCommentOwnerVideo,
MVideo,
MVideoAccountLight,
MVideoId
} from '../../typings/models'
function getRemoteVideoAudience (video: MVideoAccountLight, actorsInvolvedInVideo: MActorFollowersUrl[]): ActivityAudience {
return {
@ -48,7 +56,7 @@ function getAudienceFromFollowersOf (actorsInvolvedInObject: MActorFollowersUrl[
}
}
async function getActorsInvolvedInVideo (video: MVideo, t: Transaction) {
async function getActorsInvolvedInVideo (video: MVideoId, t: Transaction) {
const actors: MActorLight[] = await VideoShareModel.loadActorsByShare(video.id, t)
const videoAll = video as VideoModel

View File

@ -23,7 +23,8 @@ async function processCreateView (activity: ActivityView | ActivityCreate, byAct
const options = {
videoObject,
fetchType: 'only-video' as 'only-video'
fetchType: 'only-immutable-attributes' as 'only-immutable-attributes',
allowRefresh: false as false
}
const { video } = await getOrCreateVideoAndAccountAndChannel(options)

View File

@ -7,7 +7,7 @@ import { JobQueue } from '../../job-queue'
import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience'
import { getServerActor } from '../../../helpers/utils'
import { afterCommitIfTransaction } from '../../../helpers/database-utils'
import { MActorWithInboxes, MActor, MActorId, MActorLight, MVideo, MVideoAccountLight } from '../../../typings/models'
import { MActorWithInboxes, MActor, MActorId, MActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../../typings/models'
import { ContextType } from '@server/helpers/activitypub'
async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: {
@ -43,7 +43,7 @@ async function forwardVideoRelatedActivity (
activity: Activity,
t: Transaction,
followersException: MActorWithInboxes[] = [],
video: MVideo
video: MVideoId
) {
// Mastodon does not add our announces in audience, so we forward to them manually
const additionalActors = await getActorsInvolvedInVideo(video, t)

View File

@ -68,7 +68,7 @@ import {
MVideoAPWithoutCaption,
MVideoFile,
MVideoFullLight,
MVideoId,
MVideoId, MVideoImmutable,
MVideoThumbnail
} from '../../typings/models'
import { MThumbnail } from '../../typings/models/video/thumbnail'
@ -200,24 +200,41 @@ async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoTo
await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }))
}
function getOrCreateVideoAndAccountAndChannel (options: {
type GetVideoResult <T> = Promise<{
video: T
created: boolean
autoBlacklisted?: boolean
}>
type GetVideoParamAll = {
videoObject: { id: string } | string
syncParam?: SyncParam
fetchType?: 'all'
allowRefresh?: boolean
}): Promise<{ video: MVideoAccountLightBlacklistAllFiles, created: boolean, autoBlacklisted?: boolean }>
function getOrCreateVideoAndAccountAndChannel (options: {
}
type GetVideoParamImmutable = {
videoObject: { id: string } | string
syncParam?: SyncParam
fetchType?: VideoFetchByUrlType
fetchType: 'only-immutable-attributes'
allowRefresh: false
}
type GetVideoParamOther = {
videoObject: { id: string } | string
syncParam?: SyncParam
fetchType?: 'all' | 'only-video'
allowRefresh?: boolean
}): Promise<{ video: MVideoAccountLightBlacklistAllFiles | MVideoThumbnail, created: boolean, autoBlacklisted?: boolean }>
async function getOrCreateVideoAndAccountAndChannel (options: {
videoObject: { id: string } | string
syncParam?: SyncParam
fetchType?: VideoFetchByUrlType
allowRefresh?: boolean // true by default
}): Promise<{ video: MVideoAccountLightBlacklistAllFiles | MVideoThumbnail, created: boolean, autoBlacklisted?: boolean }> {
}
function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles>
function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable>
function getOrCreateVideoAndAccountAndChannel (
options: GetVideoParamOther
): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail>
async function getOrCreateVideoAndAccountAndChannel (
options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther
): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
// Default params
const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
const fetchType = options.fetchType || 'all'
@ -225,12 +242,13 @@ async function getOrCreateVideoAndAccountAndChannel (options: {
// Get video url
const videoUrl = getAPId(options.videoObject)
let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
if (videoFromDatabase) {
if (videoFromDatabase.isOutdated() && allowRefresh === true) {
// If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type
if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) {
const refreshOptions = {
video: videoFromDatabase,
video: videoFromDatabase as MVideoThumbnail,
fetchedType: fetchType,
syncParam
}

View File

@ -160,6 +160,9 @@ const videosCustomGetValidator = (
if (areValidationErrors(req, res)) return
if (!await doesVideoExist(req.params.id, res, fetchType)) return
// Controllers does not need to check video rights
if (fetchType === 'only-immutable-attributes') return next()
const video = getVideoWithAttributes(res)
const videoAll = video as MVideoFullLight

View File

@ -6,7 +6,8 @@ type ModelCacheType =
'local-account-name'
| 'local-actor-name'
| 'local-actor-url'
| 'video-immutable'
| 'load-video-immutable-id'
| 'load-video-immutable-url'
type DeleteKey =
'video'
@ -19,7 +20,8 @@ class ModelCache {
'local-account-name': new Map(),
'local-actor-name': new Map(),
'local-actor-url': new Map(),
'video-immutable': new Map()
'load-video-immutable-id': new Map(),
'load-video-immutable-url': new Map()
}
private readonly deleteIds: {

View File

@ -145,6 +145,7 @@ export enum ScopeNames {
WITH_USER_HISTORY = 'WITH_USER_HISTORY',
WITH_STREAMING_PLAYLISTS = 'WITH_STREAMING_PLAYLISTS',
WITH_USER_ID = 'WITH_USER_ID',
WITH_IMMUTABLE_ATTRIBUTES = 'WITH_IMMUTABLE_ATTRIBUTES',
WITH_THUMBNAILS = 'WITH_THUMBNAILS'
}
@ -188,6 +189,9 @@ export type AvailableForListIDsOptions = {
}
@Scopes(() => ({
[ScopeNames.WITH_IMMUTABLE_ATTRIBUTES]: {
attributes: [ 'id', 'url', 'uuid', 'remote' ]
},
[ScopeNames.FOR_API]: (options: ForAPIOptions) => {
const query: FindOptions = {
include: [
@ -1476,20 +1480,16 @@ export class VideoModel extends Model<VideoModel> {
static loadImmutableAttributes (id: number | string, t?: Transaction): Bluebird<MVideoImmutable> {
const fun = () => {
const where = buildWhereIdOrUUID(id)
const options = {
attributes: [
'id', 'url', 'uuid'
],
where,
const query = {
where: buildWhereIdOrUUID(id),
transaction: t
}
return VideoModel.unscoped().findOne(options)
return VideoModel.scope(ScopeNames.WITH_IMMUTABLE_ATTRIBUTES).findOne(query)
}
return ModelCache.Instance.doCache({
cacheType: 'video-immutable',
cacheType: 'load-video-immutable-id',
key: '' + id,
deleteKey: 'video',
fun
@ -1559,6 +1559,26 @@ export class VideoModel extends Model<VideoModel> {
return VideoModel.scope(ScopeNames.WITH_THUMBNAILS).findOne(query)
}
static loadByUrlImmutableAttributes (url: string, transaction?: Transaction): Bluebird<MVideoImmutable> {
const fun = () => {
const query: FindOptions = {
where: {
url
},
transaction
}
return VideoModel.scope(ScopeNames.WITH_IMMUTABLE_ATTRIBUTES).findOne(query)
}
return ModelCache.Instance.doCache({
cacheType: 'load-video-immutable-url',
key: url,
deleteKey: 'video',
fun
})
}
static loadByUrlAndPopulateAccount (url: string, transaction?: Transaction): Bluebird<MVideoAccountLightBlacklistAllFiles> {
const query: FindOptions = {
where: {

View File

@ -37,7 +37,7 @@ export type MVideoId = Pick<MVideo, 'id'>
export type MVideoUrl = Pick<MVideo, 'url'>
export type MVideoUUID = Pick<MVideo, 'uuid'>
export type MVideoImmutable = Pick<MVideo, 'id' | 'url' | 'uuid'>
export type MVideoImmutable = Pick<MVideo, 'id' | 'url' | 'uuid' | 'remote' | 'isOwned'>
export type MVideoIdUrl = MVideoId & MVideoUrl
export type MVideoFeed = Pick<MVideo, 'name' | 'uuid'>