diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index 4068e3d7b..0191b55ef 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts @@ -21,9 +21,9 @@ import { WEBSERVER } from '../initializers/constants' import { AccountModel } from '../models/account/account' +import { getActivityStreamDuration } from '../models/video/formatter/video-format-utils' import { VideoModel } from '../models/video/video' import { VideoChannelModel } from '../models/video/video-channel' -import { getActivityStreamDuration } from '../models/video/video-format-utils' import { VideoPlaylistModel } from '../models/video/video-playlist' import { MAccountActor, MChannelActor } from '../types/models' import { ServerConfigManager } from './server-config-manager' diff --git a/server/models/video/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts similarity index 95% rename from server/models/video/video-format-utils.ts rename to server/models/video/formatter/video-format-utils.ts index 551cb2842..5ddbf74da 100644 --- a/server/models/video/video-format-utils.ts +++ b/server/models/video/formatter/video-format-utils.ts @@ -1,17 +1,17 @@ import { generateMagnetUri } from '@server/helpers/webtorrent' import { getLocalVideoFileMetadataUrl } from '@server/lib/video-paths' import { VideoFile } from '@shared/models/videos/video-file.model' -import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../shared/models/activitypub/objects' -import { Video, VideoDetails } from '../../../shared/models/videos' -import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model' -import { isArray } from '../../helpers/custom-validators/misc' -import { MIMETYPES, WEBSERVER } from '../../initializers/constants' +import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../../shared/models/activitypub/objects' +import { Video, VideoDetails } from '../../../../shared/models/videos' +import { VideoStreamingPlaylist } from '../../../../shared/models/videos/video-streaming-playlist.model' +import { isArray } from '../../../helpers/custom-validators/misc' +import { MIMETYPES, WEBSERVER } from '../../../initializers/constants' import { getLocalVideoCommentsActivityPubUrl, getLocalVideoDislikesActivityPubUrl, getLocalVideoLikesActivityPubUrl, getLocalVideoSharesActivityPubUrl -} from '../../lib/activitypub/url' +} from '../../../lib/activitypub/url' import { MStreamingPlaylistRedundanciesOpt, MVideo, @@ -19,10 +19,10 @@ import { MVideoFile, MVideoFormattable, MVideoFormattableDetails -} from '../../types/models' -import { MVideoFileRedundanciesOpt } from '../../types/models/video/video-file' -import { VideoModel } from './video' -import { VideoCaptionModel } from './video-caption' +} from '../../../types/models' +import { MVideoFileRedundanciesOpt } from '../../../types/models/video/video-file' +import { VideoModel } from '../video' +import { VideoCaptionModel } from '../video-caption' export type VideoFormattingJSONOptions = { completeDescription?: boolean diff --git a/server/models/video/sql/abstract-videos-query-builder.ts b/server/models/video/sql/abstract-videos-query-builder.ts new file mode 100644 index 000000000..597a02af7 --- /dev/null +++ b/server/models/video/sql/abstract-videos-query-builder.ts @@ -0,0 +1,15 @@ +import { logger } from '@server/helpers/logger' +import { Sequelize, QueryTypes } from 'sequelize' + +export class AbstractVideosQueryBuilder { + protected sequelize: Sequelize + + protected query: string + protected replacements: any = {} + + protected runQuery (nest?: boolean) { + logger.info('Running video query.', { query: this.query, replacements: this.replacements }) + + return this.sequelize.query(this.query, { replacements: this.replacements, type: QueryTypes.SELECT, nest }) + } +} diff --git a/server/models/video/sql/video-model-builder.ts b/server/models/video/sql/video-model-builder.ts new file mode 100644 index 000000000..c428312fe --- /dev/null +++ b/server/models/video/sql/video-model-builder.ts @@ -0,0 +1,162 @@ +import { pick } from 'lodash' +import { AccountModel } from '@server/models/account/account' +import { ActorModel } from '@server/models/actor/actor' +import { ActorImageModel } from '@server/models/actor/actor-image' +import { ServerModel } from '@server/models/server/server' +import { UserVideoHistoryModel } from '@server/models/user/user-video-history' +import { ThumbnailModel } from '../thumbnail' +import { VideoModel } from '../video' +import { VideoChannelModel } from '../video-channel' +import { VideoFileModel } from '../video-file' +import { VideoStreamingPlaylistModel } from '../video-streaming-playlist' + +function buildVideosFromRows (rows: any[]) { + const videosMemo: { [ id: number ]: VideoModel } = {} + const videoStreamingPlaylistMemo: { [ id: number ]: VideoStreamingPlaylistModel } = {} + + const thumbnailsDone = new Set() + const historyDone = new Set() + const videoFilesDone = new Set() + + const videos: VideoModel[] = [] + + const avatarKeys = [ 'id', 'filename', 'fileUrl', 'onDisk', 'createdAt', 'updatedAt' ] + const actorKeys = [ 'id', 'preferredUsername', 'url', 'serverId', 'avatarId' ] + const serverKeys = [ 'id', 'host' ] + const videoFileKeys = [ + 'id', + 'createdAt', + 'updatedAt', + 'resolution', + 'size', + 'extname', + 'filename', + 'fileUrl', + 'torrentFilename', + 'torrentUrl', + 'infoHash', + 'fps', + 'videoId', + 'videoStreamingPlaylistId' + ] + const videoStreamingPlaylistKeys = [ 'id', 'type', 'playlistUrl' ] + const videoKeys = [ + 'id', + 'uuid', + 'name', + 'category', + 'licence', + 'language', + 'privacy', + 'nsfw', + 'description', + 'support', + 'duration', + 'views', + 'likes', + 'dislikes', + 'remote', + 'isLive', + 'url', + 'commentsEnabled', + 'downloadEnabled', + 'waitTranscoding', + 'state', + 'publishedAt', + 'originallyPublishedAt', + 'channelId', + 'createdAt', + 'updatedAt' + ] + const buildOpts = { raw: true } + + function buildActor (rowActor: any) { + const avatarModel = rowActor.Avatar.id !== null + ? new ActorImageModel(pick(rowActor.Avatar, avatarKeys), buildOpts) + : null + + const serverModel = rowActor.Server.id !== null + ? new ServerModel(pick(rowActor.Server, serverKeys), buildOpts) + : null + + const actorModel = new ActorModel(pick(rowActor, actorKeys), buildOpts) + actorModel.Avatar = avatarModel + actorModel.Server = serverModel + + return actorModel + } + + for (const row of rows) { + if (!videosMemo[row.id]) { + // Build Channel + const channel = row.VideoChannel + const channelModel = new VideoChannelModel(pick(channel, [ 'id', 'name', 'description', 'actorId' ]), buildOpts) + channelModel.Actor = buildActor(channel.Actor) + + const account = row.VideoChannel.Account + const accountModel = new AccountModel(pick(account, [ 'id', 'name' ]), buildOpts) + accountModel.Actor = buildActor(account.Actor) + + channelModel.Account = accountModel + + const videoModel = new VideoModel(pick(row, videoKeys), buildOpts) + videoModel.VideoChannel = channelModel + + videoModel.UserVideoHistories = [] + videoModel.Thumbnails = [] + videoModel.VideoFiles = [] + videoModel.VideoStreamingPlaylists = [] + + videosMemo[row.id] = videoModel + // Don't take object value to have a sorted array + videos.push(videoModel) + } + + const videoModel = videosMemo[row.id] + + if (row.userVideoHistory?.id && !historyDone.has(row.userVideoHistory.id)) { + const historyModel = new UserVideoHistoryModel(pick(row.userVideoHistory, [ 'id', 'currentTime' ]), buildOpts) + videoModel.UserVideoHistories.push(historyModel) + + historyDone.add(row.userVideoHistory.id) + } + + if (row.Thumbnails?.id && !thumbnailsDone.has(row.Thumbnails.id)) { + const thumbnailModel = new ThumbnailModel(pick(row.Thumbnails, [ 'id', 'type', 'filename' ]), buildOpts) + videoModel.Thumbnails.push(thumbnailModel) + + thumbnailsDone.add(row.Thumbnails.id) + } + + if (row.VideoFiles?.id && !videoFilesDone.has(row.VideoFiles.id)) { + const videoFileModel = new VideoFileModel(pick(row.VideoFiles, videoFileKeys), buildOpts) + videoModel.VideoFiles.push(videoFileModel) + + videoFilesDone.add(row.VideoFiles.id) + } + + if (row.VideoStreamingPlaylists?.id && !videoStreamingPlaylistMemo[row.VideoStreamingPlaylists.id]) { + const streamingPlaylist = new VideoStreamingPlaylistModel(pick(row.VideoStreamingPlaylists, videoStreamingPlaylistKeys), buildOpts) + streamingPlaylist.VideoFiles = [] + + videoModel.VideoStreamingPlaylists.push(streamingPlaylist) + + videoStreamingPlaylistMemo[streamingPlaylist.id] = streamingPlaylist + } + + if (row.VideoStreamingPlaylists?.VideoFiles?.id && !videoFilesDone.has(row.VideoStreamingPlaylists.VideoFiles.id)) { + const streamingPlaylist = videoStreamingPlaylistMemo[row.VideoStreamingPlaylists.id] + + const videoFileModel = new VideoFileModel(pick(row.VideoStreamingPlaylists.VideoFiles, videoFileKeys), buildOpts) + streamingPlaylist.VideoFiles.push(videoFileModel) + + videoFilesDone.add(row.VideoStreamingPlaylists.VideoFiles.id) + } + } + + return videos +} + +export { + buildVideosFromRows +} diff --git a/server/models/video/sql/videos-id-list-query-builder.ts b/server/models/video/sql/videos-id-list-query-builder.ts new file mode 100644 index 000000000..7bb942ea4 --- /dev/null +++ b/server/models/video/sql/videos-id-list-query-builder.ts @@ -0,0 +1,609 @@ +import { Sequelize } from 'sequelize' +import validator from 'validator' +import { exists } from '@server/helpers/custom-validators/misc' +import { buildDirectionAndField, createSafeIn } from '@server/models/utils' +import { MUserAccountId, MUserId } from '@server/types/models' +import { VideoFilter, VideoPrivacy, VideoState } from '@shared/models' +import { AbstractVideosQueryBuilder } from './abstract-videos-query-builder' + +export type BuildVideosListQueryOptions = { + attributes?: string[] + + serverAccountId: number + followerActorId: number + includeLocalVideos: boolean + + count: number + start: number + sort: string + + nsfw?: boolean + filter?: VideoFilter + isLive?: boolean + + categoryOneOf?: number[] + licenceOneOf?: number[] + languageOneOf?: string[] + tagsOneOf?: string[] + tagsAllOf?: string[] + + withFiles?: boolean + + accountId?: number + videoChannelId?: number + + videoPlaylistId?: number + + trendingAlgorithm?: string // best, hot, or any other algorithm implemented + trendingDays?: number + + user?: MUserAccountId + historyOfUser?: MUserId + + startDate?: string // ISO 8601 + endDate?: string // ISO 8601 + originallyPublishedStartDate?: string + originallyPublishedEndDate?: string + + durationMin?: number // seconds + durationMax?: number // seconds + + search?: string + + isCount?: boolean + + group?: string + having?: string +} + +export class VideosIdListQueryBuilder extends AbstractVideosQueryBuilder { + private attributes: string[] + + protected replacements: any = {} + private readonly and: string[] = [] + private joins: string[] = [] + + private readonly cte: string[] = [] + + private group = '' + private having = '' + + private sort = '' + private limit = '' + private offset = '' + + constructor (protected readonly sequelize: Sequelize) { + super() + } + + queryVideoIds (options: BuildVideosListQueryOptions) { + this.buildIdsListQuery(options) + + return this.runQuery() + } + + countVideoIds (countOptions: BuildVideosListQueryOptions): Promise { + this.buildIdsListQuery(countOptions) + + return this.runQuery().then(rows => rows.length !== 0 ? rows[0].total : 0) + } + + getIdsListQueryAndSort (options: BuildVideosListQueryOptions) { + this.buildIdsListQuery(options) + return { query: this.query, sort: this.sort, replacements: this.replacements } + } + + private buildIdsListQuery (options: BuildVideosListQueryOptions) { + this.attributes = options.attributes || [ '"video"."id"' ] + + if (options.group) this.group = options.group + if (options.having) this.having = options.having + + this.joins = this.joins.concat([ + 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId"', + 'INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId"', + 'INNER JOIN "actor" "accountActor" ON "account"."actorId" = "accountActor"."id"' + ]) + + this.whereNotBlacklisted() + + if (options.serverAccountId) { + this.whereNotBlocked(options.serverAccountId, options.user) + } + + // Only list public/published videos + if (!options.filter || (options.filter !== 'all-local' && options.filter !== 'all')) { + this.whereStateAndPrivacyAvailable(options.user) + } + + if (options.videoPlaylistId) { + this.joinPlaylist(options.videoPlaylistId) + } + + if (options.filter && (options.filter === 'local' || options.filter === 'all-local')) { + this.whereOnlyLocal() + } + + if (options.accountId) { + this.whereAccountId(options.accountId) + } + + if (options.videoChannelId) { + this.whereChannelId(options.videoChannelId) + } + + if (options.followerActorId) { + this.whereFollowerActorId(options.followerActorId, options.includeLocalVideos) + } + + if (options.withFiles === true) { + this.whereFileExists() + } + + if (options.tagsOneOf) { + this.whereTagsOneOf(options.tagsOneOf) + } + + if (options.tagsAllOf) { + this.whereTagsAllOf(options.tagsAllOf) + } + + if (options.nsfw === true) { + this.whereNSFW() + } else if (options.nsfw === false) { + this.whereSFW() + } + + if (options.isLive === true) { + this.whereLive() + } else if (options.isLive === false) { + this.whereVOD() + } + + if (options.categoryOneOf) { + this.whereCategoryOneOf(options.categoryOneOf) + } + + if (options.licenceOneOf) { + this.whereLicenceOneOf(options.licenceOneOf) + } + + if (options.languageOneOf) { + this.whereLanguageOneOf(options.languageOneOf) + } + + // We don't exclude results in this so if we do a count we don't need to add this complex clause + if (options.isCount !== true) { + if (options.trendingDays) { + this.groupForTrending(options.trendingDays) + } else if ([ 'best', 'hot' ].includes(options.trendingAlgorithm)) { + this.groupForHotOrBest(options.trendingAlgorithm, options.user) + } + } + + if (options.historyOfUser) { + this.joinHistory(options.historyOfUser.id) + } + + if (options.startDate) { + this.whereStartDate(options.startDate) + } + + if (options.endDate) { + this.whereEndDate(options.endDate) + } + + if (options.originallyPublishedStartDate) { + this.whereOriginallyPublishedStartDate(options.originallyPublishedStartDate) + } + + if (options.originallyPublishedEndDate) { + this.whereOriginallyPublishedEndDate(options.originallyPublishedEndDate) + } + + if (options.durationMin) { + this.whereDurationMin(options.durationMin) + } + + if (options.durationMax) { + this.whereDurationMax(options.durationMax) + } + + this.whereSearch(options.search) + + if (options.isCount === true) { + this.setCountAttribute() + } else { + if (exists(options.sort)) { + this.setSort(options.sort) + } + + if (exists(options.count)) { + this.setLimit(options.count) + } + + if (exists(options.start)) { + this.setOffset(options.start) + } + } + + const cteString = this.cte.length !== 0 + ? `WITH ${this.cte.join(', ')} ` + : '' + + this.query = cteString + + 'SELECT ' + this.attributes.join(', ') + ' ' + + 'FROM "video" ' + this.joins.join(' ') + ' ' + + 'WHERE ' + this.and.join(' AND ') + ' ' + + this.group + ' ' + + this.having + ' ' + + this.sort + ' ' + + this.limit + ' ' + + this.offset + } + + private setCountAttribute () { + this.attributes = [ 'COUNT(*) as "total"' ] + } + + private joinHistory (userId: number) { + this.joins.push('INNER JOIN "userVideoHistory" ON "video"."id" = "userVideoHistory"."videoId"') + + this.and.push('"userVideoHistory"."userId" = :historyOfUser') + + this.replacements.historyOfUser = userId + } + + private joinPlaylist (playlistId: number) { + this.joins.push( + 'INNER JOIN "videoPlaylistElement" "video"."id" = "videoPlaylistElement"."videoId" ' + + 'AND "videoPlaylistElement"."videoPlaylistId" = :videoPlaylistId' + ) + + this.replacements.videoPlaylistId = playlistId + } + + private whereStateAndPrivacyAvailable (user?: MUserAccountId) { + this.and.push( + `("video"."state" = ${VideoState.PUBLISHED} OR ` + + `("video"."state" = ${VideoState.TO_TRANSCODE} AND "video"."waitTranscoding" IS false))` + ) + + if (user) { + this.and.push( + `("video"."privacy" = ${VideoPrivacy.PUBLIC} OR "video"."privacy" = ${VideoPrivacy.INTERNAL})` + ) + } else { // Or only public videos + this.and.push( + `"video"."privacy" = ${VideoPrivacy.PUBLIC}` + ) + } + } + + private whereOnlyLocal () { + this.and.push('"video"."remote" IS FALSE') + } + + private whereAccountId (accountId: number) { + this.and.push('"account"."id" = :accountId') + this.replacements.accountId = accountId + } + + private whereChannelId (channelId: number) { + this.and.push('"videoChannel"."id" = :videoChannelId') + this.replacements.videoChannelId = channelId + } + + private whereFollowerActorId (followerActorId: number, includeLocalVideos: boolean) { + let query = + '(' + + ' EXISTS (' + + ' SELECT 1 FROM "videoShare" ' + + ' INNER JOIN "actorFollow" "actorFollowShare" ON "actorFollowShare"."targetActorId" = "videoShare"."actorId" ' + + ' AND "actorFollowShare"."actorId" = :followerActorId AND "actorFollowShare"."state" = \'accepted\' ' + + ' WHERE "videoShare"."videoId" = "video"."id"' + + ' )' + + ' OR' + + ' EXISTS (' + + ' SELECT 1 from "actorFollow" ' + + ' WHERE "actorFollow"."targetActorId" = "videoChannel"."actorId" AND "actorFollow"."actorId" = :followerActorId ' + + ' AND "actorFollow"."state" = \'accepted\'' + + ' )' + + if (includeLocalVideos) { + query += ' OR "video"."remote" IS FALSE' + } + + query += ')' + + this.and.push(query) + this.replacements.followerActorId = followerActorId + } + + private whereFileExists () { + this.and.push( + '(' + + ' EXISTS (SELECT 1 FROM "videoFile" WHERE "videoFile"."videoId" = "video"."id") ' + + ' OR EXISTS (' + + ' SELECT 1 FROM "videoStreamingPlaylist" ' + + ' INNER JOIN "videoFile" ON "videoFile"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ' + + ' WHERE "videoStreamingPlaylist"."videoId" = "video"."id"' + + ' )' + + ')' + ) + } + + private whereTagsOneOf (tagsOneOf: string[]) { + const tagsOneOfLower = tagsOneOf.map(t => t.toLowerCase()) + + this.and.push( + 'EXISTS (' + + ' SELECT 1 FROM "videoTag" ' + + ' INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' + + ' WHERE lower("tag"."name") IN (' + createSafeIn(this.sequelize, tagsOneOfLower) + ') ' + + ' AND "video"."id" = "videoTag"."videoId"' + + ')' + ) + } + + private whereTagsAllOf (tagsAllOf: string[]) { + const tagsAllOfLower = tagsAllOf.map(t => t.toLowerCase()) + + this.and.push( + 'EXISTS (' + + ' SELECT 1 FROM "videoTag" ' + + ' INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' + + ' WHERE lower("tag"."name") IN (' + createSafeIn(this.sequelize, tagsAllOfLower) + ') ' + + ' AND "video"."id" = "videoTag"."videoId" ' + + ' GROUP BY "videoTag"."videoId" HAVING COUNT(*) = ' + tagsAllOfLower.length + + ')' + ) + } + + private whereCategoryOneOf (categoryOneOf: number[]) { + this.and.push('"video"."category" IN (:categoryOneOf)') + this.replacements.categoryOneOf = categoryOneOf + } + + private whereLicenceOneOf (licenceOneOf: number[]) { + this.and.push('"video"."licence" IN (:licenceOneOf)') + this.replacements.licenceOneOf = licenceOneOf + } + + private whereLanguageOneOf (languageOneOf: string[]) { + const languages = languageOneOf.filter(l => l && l !== '_unknown') + const languagesQueryParts: string[] = [] + + if (languages.length !== 0) { + languagesQueryParts.push('"video"."language" IN (:languageOneOf)') + this.replacements.languageOneOf = languages + + languagesQueryParts.push( + 'EXISTS (' + + ' SELECT 1 FROM "videoCaption" WHERE "videoCaption"."language" ' + + ' IN (' + createSafeIn(this.sequelize, languages) + ') AND ' + + ' "videoCaption"."videoId" = "video"."id"' + + ')' + ) + } + + if (languageOneOf.includes('_unknown')) { + languagesQueryParts.push('"video"."language" IS NULL') + } + + if (languagesQueryParts.length !== 0) { + this.and.push('(' + languagesQueryParts.join(' OR ') + ')') + } + } + + private whereNSFW () { + this.and.push('"video"."nsfw" IS TRUE') + } + + private whereSFW () { + this.and.push('"video"."nsfw" IS FALSE') + } + + private whereLive () { + this.and.push('"video"."isLive" IS TRUE') + } + + private whereVOD () { + this.and.push('"video"."isLive" IS FALSE') + } + + private whereNotBlocked (serverAccountId: number, user?: MUserAccountId) { + const blockerIds = [ serverAccountId ] + if (user) blockerIds.push(user.Account.id) + + const inClause = createSafeIn(this.sequelize, blockerIds) + + this.and.push( + 'NOT EXISTS (' + + ' SELECT 1 FROM "accountBlocklist" ' + + ' WHERE "accountBlocklist"."accountId" IN (' + inClause + ') ' + + ' AND "accountBlocklist"."targetAccountId" = "account"."id" ' + + ')' + + 'AND NOT EXISTS (' + + ' SELECT 1 FROM "serverBlocklist" WHERE "serverBlocklist"."accountId" IN (' + inClause + ') ' + + ' AND "serverBlocklist"."targetServerId" = "accountActor"."serverId"' + + ')' + ) + } + + private whereSearch (search?: string) { + if (!search) { + this.attributes.push('0 as similarity') + return + } + + const escapedSearch = this.sequelize.escape(search) + const escapedLikeSearch = this.sequelize.escape('%' + search + '%') + + this.cte.push( + '"trigramSearch" AS (' + + ' SELECT "video"."id", ' + + ` similarity(lower(immutable_unaccent("video"."name")), lower(immutable_unaccent(${escapedSearch}))) as similarity ` + + ' FROM "video" ' + + ' WHERE lower(immutable_unaccent("video"."name")) % lower(immutable_unaccent(' + escapedSearch + ')) OR ' + + ' lower(immutable_unaccent("video"."name")) LIKE lower(immutable_unaccent(' + escapedLikeSearch + '))' + + ')' + ) + + this.joins.push('LEFT JOIN "trigramSearch" ON "video"."id" = "trigramSearch"."id"') + + let base = '(' + + ' "trigramSearch"."id" IS NOT NULL OR ' + + ' EXISTS (' + + ' SELECT 1 FROM "videoTag" ' + + ' INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' + + ` WHERE lower("tag"."name") = ${escapedSearch} ` + + ' AND "video"."id" = "videoTag"."videoId"' + + ' )' + + if (validator.isUUID(search)) { + base += ` OR "video"."uuid" = ${escapedSearch}` + } + + base += ')' + + this.and.push(base) + this.attributes.push(`COALESCE("trigramSearch"."similarity", 0) as similarity`) + } + + private whereNotBlacklisted () { + this.and.push('"video"."id" NOT IN (SELECT "videoBlacklist"."videoId" FROM "videoBlacklist")') + } + + private whereStartDate (startDate: string) { + this.and.push('"video"."publishedAt" >= :startDate') + this.replacements.startDate = startDate + } + + private whereEndDate (endDate: string) { + this.and.push('"video"."publishedAt" <= :endDate') + this.replacements.endDate = endDate + } + + private whereOriginallyPublishedStartDate (startDate: string) { + this.and.push('"video"."originallyPublishedAt" >= :originallyPublishedStartDate') + this.replacements.originallyPublishedStartDate = startDate + } + + private whereOriginallyPublishedEndDate (endDate: string) { + this.and.push('"video"."originallyPublishedAt" <= :originallyPublishedEndDate') + this.replacements.originallyPublishedEndDate = endDate + } + + private whereDurationMin (durationMin: number) { + this.and.push('"video"."duration" >= :durationMin') + this.replacements.durationMin = durationMin + } + + private whereDurationMax (durationMax: number) { + this.and.push('"video"."duration" <= :durationMax') + this.replacements.durationMax = durationMax + } + + private groupForTrending (trendingDays: number) { + const viewsGteDate = new Date(new Date().getTime() - (24 * 3600 * 1000) * trendingDays) + + this.joins.push('LEFT JOIN "videoView" ON "video"."id" = "videoView"."videoId" AND "videoView"."startDate" >= :viewsGteDate') + this.replacements.viewsGteDate = viewsGteDate + + this.attributes.push('COALESCE(SUM("videoView"."views"), 0) AS "score"') + + this.group = 'GROUP BY "video"."id"' + } + + private groupForHotOrBest (trendingAlgorithm: string, user?: MUserAccountId) { + /** + * "Hotness" is a measure based on absolute view/comment/like/dislike numbers, + * with fixed weights only applied to their log values. + * + * This algorithm gives little chance for an old video to have a good score, + * for which recent spikes in interactions could be a sign of "hotness" and + * justify a better score. However there are multiple ways to achieve that + * goal, which is left for later. Yes, this is a TODO :) + * + * notes: + * - weights and base score are in number of half-days. + * - all comments are counted, regardless of being written by the video author or not + * see https://github.com/reddit-archive/reddit/blob/master/r2/r2/lib/db/_sorts.pyx#L47-L58 + * - we have less interactions than on reddit, so multiply weights by an arbitrary factor + */ + const weights = { + like: 3 * 50, + dislike: -3 * 50, + view: Math.floor((1 / 3) * 50), + comment: 2 * 50, // a comment takes more time than a like to do, but can be done multiple times + history: -2 * 50 + } + + this.joins.push('LEFT JOIN "videoComment" ON "video"."id" = "videoComment"."videoId"') + + let attribute = + `LOG(GREATEST(1, "video"."likes" - 1)) * ${weights.like} ` + // likes (+) + `+ LOG(GREATEST(1, "video"."dislikes" - 1)) * ${weights.dislike} ` + // dislikes (-) + `+ LOG("video"."views" + 1) * ${weights.view} ` + // views (+) + `+ LOG(GREATEST(1, COUNT(DISTINCT "videoComment"."id"))) * ${weights.comment} ` + // comments (+) + '+ (SELECT (EXTRACT(epoch FROM "video"."publishedAt") - 1446156582) / 47000) ' // base score (in number of half-days) + + if (trendingAlgorithm === 'best' && user) { + this.joins.push( + 'LEFT JOIN "userVideoHistory" ON "video"."id" = "userVideoHistory"."videoId" AND "userVideoHistory"."userId" = :bestUser' + ) + this.replacements.bestUser = user.id + + attribute += `+ POWER(COUNT(DISTINCT "userVideoHistory"."id"), 2.0) * ${weights.history} ` + } + + attribute += 'AS "score"' + this.attributes.push(attribute) + + this.group = 'GROUP BY "video"."id"' + } + + private setSort (sort: string) { + if (sort === '-originallyPublishedAt' || sort === 'originallyPublishedAt') { + this.attributes.push('COALESCE("video"."originallyPublishedAt", "video"."publishedAt") AS "publishedAtForOrder"') + } + + this.sort = this.buildOrder(sort) + } + + private buildOrder (value: string) { + const { direction, field } = buildDirectionAndField(value) + if (field.match(/^[a-zA-Z."]+$/) === null) throw new Error('Invalid sort column ' + field) + + if (field.toLowerCase() === 'random') return 'ORDER BY RANDOM()' + + if ([ 'trending', 'hot', 'best' ].includes(field.toLowerCase())) { // Sort by aggregation + return `ORDER BY "score" ${direction}, "video"."views" ${direction}` + } + + let firstSort: string + + if (field.toLowerCase() === 'match') { // Search + firstSort = '"similarity"' + } else if (field === 'originallyPublishedAt') { + firstSort = '"publishedAtForOrder"' + } else if (field.includes('.')) { + firstSort = field + } else { + firstSort = `"video"."${field}"` + } + + return `ORDER BY ${firstSort} ${direction}, "video"."id" ASC` + } + + private setLimit (countArg: number) { + const count = parseInt(countArg + '', 10) + this.limit = `LIMIT ${count}` + } + + private setOffset (startArg: number) { + const start = parseInt(startArg + '', 10) + this.offset = `OFFSET ${start}` + } +} diff --git a/server/models/video/sql/videos-model-list-query-builder.ts b/server/models/video/sql/videos-model-list-query-builder.ts new file mode 100644 index 000000000..4ba9dd878 --- /dev/null +++ b/server/models/video/sql/videos-model-list-query-builder.ts @@ -0,0 +1,234 @@ + +import { MUserId } from '@server/types/models' +import { Sequelize } from 'sequelize' +import { AbstractVideosQueryBuilder } from './abstract-videos-query-builder' +import { buildVideosFromRows } from './video-model-builder' +import { BuildVideosListQueryOptions, VideosIdListQueryBuilder } from './videos-id-list-query-builder' + +export class VideosModelListQueryBuilder extends AbstractVideosQueryBuilder { + private attributes: { [key: string]: string } + + private joins: string[] = [] + + private innerQuery: string + private innerSort: string + + constructor (protected readonly sequelize: Sequelize) { + super() + } + + queryVideos (options: BuildVideosListQueryOptions) { + this.buildInnerQuery(options) + this.buildListQueryFromIdsQuery(options) + + return this.runQuery(true).then(rows => buildVideosFromRows(rows)) + } + + private buildInnerQuery (options: BuildVideosListQueryOptions) { + const idsQueryBuilder = new VideosIdListQueryBuilder(this.sequelize) + const { query, sort, replacements } = idsQueryBuilder.getIdsListQueryAndSort(options) + + this.replacements = replacements + this.innerQuery = query + this.innerSort = sort + } + + private buildListQueryFromIdsQuery (options: BuildVideosListQueryOptions) { + this.attributes = { + '"video".*': '' + } + + this.joins = [ 'INNER JOIN "video" ON "tmp"."id" = "video"."id"' ] + + this.includeChannels() + this.includeAccounts() + this.includeThumbnails() + + if (options.withFiles) { + this.includeFiles() + } + + if (options.user) { + this.includeUserHistory(options.user) + } + + if (options.videoPlaylistId) { + this.includePlaylist(options.videoPlaylistId) + } + + const select = this.buildSelect() + + this.query = `${select} FROM (${this.innerQuery}) AS "tmp" ${this.joins.join(' ')} ${this.innerSort}` + } + + private includeChannels () { + this.attributes = { + ...this.attributes, + + '"VideoChannel"."id"': '"VideoChannel.id"', + '"VideoChannel"."name"': '"VideoChannel.name"', + '"VideoChannel"."description"': '"VideoChannel.description"', + '"VideoChannel"."actorId"': '"VideoChannel.actorId"', + '"VideoChannel->Actor"."id"': '"VideoChannel.Actor.id"', + '"VideoChannel->Actor"."preferredUsername"': '"VideoChannel.Actor.preferredUsername"', + '"VideoChannel->Actor"."url"': '"VideoChannel.Actor.url"', + '"VideoChannel->Actor"."serverId"': '"VideoChannel.Actor.serverId"', + '"VideoChannel->Actor"."avatarId"': '"VideoChannel.Actor.avatarId"', + '"VideoChannel->Actor->Server"."id"': '"VideoChannel.Actor.Server.id"', + '"VideoChannel->Actor->Server"."host"': '"VideoChannel.Actor.Server.host"', + '"VideoChannel->Actor->Avatar"."id"': '"VideoChannel.Actor.Avatar.id"', + '"VideoChannel->Actor->Avatar"."filename"': '"VideoChannel.Actor.Avatar.filename"', + '"VideoChannel->Actor->Avatar"."fileUrl"': '"VideoChannel.Actor.Avatar.fileUrl"', + '"VideoChannel->Actor->Avatar"."onDisk"': '"VideoChannel.Actor.Avatar.onDisk"', + '"VideoChannel->Actor->Avatar"."createdAt"': '"VideoChannel.Actor.Avatar.createdAt"', + '"VideoChannel->Actor->Avatar"."updatedAt"': '"VideoChannel.Actor.Avatar.updatedAt"' + } + + this.joins = this.joins.concat([ + 'INNER JOIN "videoChannel" AS "VideoChannel" ON "video"."channelId" = "VideoChannel"."id"', + 'INNER JOIN "actor" AS "VideoChannel->Actor" ON "VideoChannel"."actorId" = "VideoChannel->Actor"."id"', + + 'LEFT OUTER JOIN "server" AS "VideoChannel->Actor->Server" ON "VideoChannel->Actor"."serverId" = "VideoChannel->Actor->Server"."id"', + 'LEFT OUTER JOIN "actorImage" AS "VideoChannel->Actor->Avatar" ' + + 'ON "VideoChannel->Actor"."avatarId" = "VideoChannel->Actor->Avatar"."id"' + ]) + } + + private includeAccounts () { + this.attributes = { + ...this.attributes, + + '"VideoChannel->Account"."id"': '"VideoChannel.Account.id"', + '"VideoChannel->Account"."name"': '"VideoChannel.Account.name"', + '"VideoChannel->Account->Actor"."id"': '"VideoChannel.Account.Actor.id"', + '"VideoChannel->Account->Actor"."preferredUsername"': '"VideoChannel.Account.Actor.preferredUsername"', + '"VideoChannel->Account->Actor"."url"': '"VideoChannel.Account.Actor.url"', + '"VideoChannel->Account->Actor"."serverId"': '"VideoChannel.Account.Actor.serverId"', + '"VideoChannel->Account->Actor"."avatarId"': '"VideoChannel.Account.Actor.avatarId"', + '"VideoChannel->Account->Actor->Server"."id"': '"VideoChannel.Account.Actor.Server.id"', + '"VideoChannel->Account->Actor->Server"."host"': '"VideoChannel.Account.Actor.Server.host"', + '"VideoChannel->Account->Actor->Avatar"."id"': '"VideoChannel.Account.Actor.Avatar.id"', + '"VideoChannel->Account->Actor->Avatar"."filename"': '"VideoChannel.Account.Actor.Avatar.filename"', + '"VideoChannel->Account->Actor->Avatar"."fileUrl"': '"VideoChannel.Account.Actor.Avatar.fileUrl"', + '"VideoChannel->Account->Actor->Avatar"."onDisk"': '"VideoChannel.Account.Actor.Avatar.onDisk"', + '"VideoChannel->Account->Actor->Avatar"."createdAt"': '"VideoChannel.Account.Actor.Avatar.createdAt"', + '"VideoChannel->Account->Actor->Avatar"."updatedAt"': '"VideoChannel.Account.Actor.Avatar.updatedAt"' + } + + this.joins = this.joins.concat([ + 'INNER JOIN "account" AS "VideoChannel->Account" ON "VideoChannel"."accountId" = "VideoChannel->Account"."id"', + 'INNER JOIN "actor" AS "VideoChannel->Account->Actor" ON "VideoChannel->Account"."actorId" = "VideoChannel->Account->Actor"."id"', + + 'LEFT OUTER JOIN "server" AS "VideoChannel->Account->Actor->Server" ' + + 'ON "VideoChannel->Account->Actor"."serverId" = "VideoChannel->Account->Actor->Server"."id"', + + 'LEFT OUTER JOIN "actorImage" AS "VideoChannel->Account->Actor->Avatar" ' + + 'ON "VideoChannel->Account->Actor"."avatarId" = "VideoChannel->Account->Actor->Avatar"."id"' + ]) + } + + private includeThumbnails () { + this.attributes = { + ...this.attributes, + + '"Thumbnails"."id"': '"Thumbnails.id"', + '"Thumbnails"."type"': '"Thumbnails.type"', + '"Thumbnails"."filename"': '"Thumbnails.filename"' + } + + this.joins.push('LEFT OUTER JOIN "thumbnail" AS "Thumbnails" ON "video"."id" = "Thumbnails"."videoId"') + } + + private includeFiles () { + this.attributes = { + ...this.attributes, + + '"VideoFiles"."id"': '"VideoFiles.id"', + '"VideoFiles"."createdAt"': '"VideoFiles.createdAt"', + '"VideoFiles"."updatedAt"': '"VideoFiles.updatedAt"', + '"VideoFiles"."resolution"': '"VideoFiles.resolution"', + '"VideoFiles"."size"': '"VideoFiles.size"', + '"VideoFiles"."extname"': '"VideoFiles.extname"', + '"VideoFiles"."filename"': '"VideoFiles.filename"', + '"VideoFiles"."fileUrl"': '"VideoFiles.fileUrl"', + '"VideoFiles"."torrentFilename"': '"VideoFiles.torrentFilename"', + '"VideoFiles"."torrentUrl"': '"VideoFiles.torrentUrl"', + '"VideoFiles"."infoHash"': '"VideoFiles.infoHash"', + '"VideoFiles"."fps"': '"VideoFiles.fps"', + '"VideoFiles"."videoId"': '"VideoFiles.videoId"', + + '"VideoStreamingPlaylists"."id"': '"VideoStreamingPlaylists.id"', + '"VideoStreamingPlaylists"."playlistUrl"': '"VideoStreamingPlaylists.playlistUrl"', + '"VideoStreamingPlaylists"."type"': '"VideoStreamingPlaylists.type"', + '"VideoStreamingPlaylists->VideoFiles"."id"': '"VideoStreamingPlaylists.VideoFiles.id"', + '"VideoStreamingPlaylists->VideoFiles"."createdAt"': '"VideoStreamingPlaylists.VideoFiles.createdAt"', + '"VideoStreamingPlaylists->VideoFiles"."updatedAt"': '"VideoStreamingPlaylists.VideoFiles.updatedAt"', + '"VideoStreamingPlaylists->VideoFiles"."resolution"': '"VideoStreamingPlaylists.VideoFiles.resolution"', + '"VideoStreamingPlaylists->VideoFiles"."size"': '"VideoStreamingPlaylists.VideoFiles.size"', + '"VideoStreamingPlaylists->VideoFiles"."extname"': '"VideoStreamingPlaylists.VideoFiles.extname"', + '"VideoStreamingPlaylists->VideoFiles"."filename"': '"VideoStreamingPlaylists.VideoFiles.filename"', + '"VideoStreamingPlaylists->VideoFiles"."fileUrl"': '"VideoStreamingPlaylists.VideoFiles.fileUrl"', + '"VideoStreamingPlaylists->VideoFiles"."torrentFilename"': '"VideoStreamingPlaylists.VideoFiles.torrentFilename"', + '"VideoStreamingPlaylists->VideoFiles"."torrentUrl"': '"VideoStreamingPlaylists.VideoFiles.torrentUrl"', + '"VideoStreamingPlaylists->VideoFiles"."infoHash"': '"VideoStreamingPlaylists.VideoFiles.infoHash"', + '"VideoStreamingPlaylists->VideoFiles"."fps"': '"VideoStreamingPlaylists.VideoFiles.fps"', + '"VideoStreamingPlaylists->VideoFiles"."videoStreamingPlaylistId"': '"VideoStreamingPlaylists.VideoFiles.videoStreamingPlaylistId"', + '"VideoStreamingPlaylists->VideoFiles"."videoId"': '"VideoStreamingPlaylists.VideoFiles.videoId"' + } + + this.joins = this.joins.concat([ + 'LEFT JOIN "videoFile" AS "VideoFiles" ON "VideoFiles"."videoId" = "video"."id"', + + 'LEFT JOIN "videoStreamingPlaylist" AS "VideoStreamingPlaylists" ON "VideoStreamingPlaylists"."videoId" = "video"."id"', + + 'LEFT JOIN "videoFile" AS "VideoStreamingPlaylists->VideoFiles" ' + + 'ON "VideoStreamingPlaylists->VideoFiles"."videoStreamingPlaylistId" = "VideoStreamingPlaylists"."id"' + ]) + } + + private includeUserHistory (user: MUserId) { + this.attributes = { + ...this.attributes, + + '"userVideoHistory"."id"': '"userVideoHistory.id"', + '"userVideoHistory"."currentTime"': '"userVideoHistory.currentTime"' + } + + this.joins.push( + 'LEFT OUTER JOIN "userVideoHistory" ' + + 'ON "video"."id" = "userVideoHistory"."videoId" AND "userVideoHistory"."userId" = :userVideoHistoryId' + ) + + this.replacements.userVideoHistoryId = user.id + } + + private includePlaylist (playlistId: number) { + this.attributes = { + ...this.attributes, + + '"VideoPlaylistElement"."createdAt"': '"VideoPlaylistElement.createdAt"', + '"VideoPlaylistElement"."updatedAt"': '"VideoPlaylistElement.updatedAt"', + '"VideoPlaylistElement"."url"': '"VideoPlaylistElement.url"', + '"VideoPlaylistElement"."position"': '"VideoPlaylistElement.position"', + '"VideoPlaylistElement"."startTimestamp"': '"VideoPlaylistElement.startTimestamp"', + '"VideoPlaylistElement"."stopTimestamp"': '"VideoPlaylistElement.stopTimestamp"', + '"VideoPlaylistElement"."videoPlaylistId"': '"VideoPlaylistElement.videoPlaylistId"' + } + + this.joins.push( + 'INNER JOIN "videoPlaylistElement" as "VideoPlaylistElement" ON "videoPlaylistElement"."videoId" = "video"."id" ' + + 'AND "VideoPlaylistElement"."videoPlaylistId" = :videoPlaylistId' + ) + + this.replacements.videoPlaylistId = playlistId + } + + private buildSelect () { + return 'SELECT ' + Object.keys(this.attributes).map(key => { + const value = this.attributes[key] + if (value) return `${key} AS ${value}` + + return key + }).join(', ') + } +} diff --git a/server/models/video/video-query-builder.ts b/server/models/video/video-query-builder.ts deleted file mode 100644 index 2aa5e65c8..000000000 --- a/server/models/video/video-query-builder.ts +++ /dev/null @@ -1,599 +0,0 @@ -import { Sequelize } from 'sequelize/types' -import validator from 'validator' -import { exists } from '@server/helpers/custom-validators/misc' -import { buildDirectionAndField, createSafeIn } from '@server/models/utils' -import { MUserAccountId, MUserId } from '@server/types/models' -import { VideoFilter, VideoPrivacy, VideoState } from '@shared/models' - -export type BuildVideosQueryOptions = { - attributes?: string[] - - serverAccountId: number - followerActorId: number - includeLocalVideos: boolean - - count: number - start: number - sort: string - - nsfw?: boolean - filter?: VideoFilter - isLive?: boolean - - categoryOneOf?: number[] - licenceOneOf?: number[] - languageOneOf?: string[] - tagsOneOf?: string[] - tagsAllOf?: string[] - - withFiles?: boolean - - accountId?: number - videoChannelId?: number - - videoPlaylistId?: number - - trendingAlgorithm?: string // best, hot, or any other algorithm implemented - trendingDays?: number - - user?: MUserAccountId - historyOfUser?: MUserId - - startDate?: string // ISO 8601 - endDate?: string // ISO 8601 - originallyPublishedStartDate?: string - originallyPublishedEndDate?: string - - durationMin?: number // seconds - durationMax?: number // seconds - - search?: string - - isCount?: boolean - - group?: string - having?: string -} - -function buildListQuery (sequelize: Sequelize, options: BuildVideosQueryOptions) { - const and: string[] = [] - const joins: string[] = [] - const replacements: any = {} - const cte: string[] = [] - - let attributes: string[] = options.attributes || [ '"video"."id"' ] - let group = options.group || '' - const having = options.having || '' - - joins.push( - 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId"' + - 'INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId"' + - 'INNER JOIN "actor" "accountActor" ON "account"."actorId" = "accountActor"."id"' - ) - - and.push('"video"."id" NOT IN (SELECT "videoBlacklist"."videoId" FROM "videoBlacklist")') - - if (options.serverAccountId) { - const blockerIds = [ options.serverAccountId ] - if (options.user) blockerIds.push(options.user.Account.id) - - const inClause = createSafeIn(sequelize, blockerIds) - - and.push( - 'NOT EXISTS (' + - ' SELECT 1 FROM "accountBlocklist" ' + - ' WHERE "accountBlocklist"."accountId" IN (' + inClause + ') ' + - ' AND "accountBlocklist"."targetAccountId" = "account"."id" ' + - ')' + - 'AND NOT EXISTS (' + - ' SELECT 1 FROM "serverBlocklist" WHERE "serverBlocklist"."accountId" IN (' + inClause + ') ' + - ' AND "serverBlocklist"."targetServerId" = "accountActor"."serverId"' + - ')' - ) - } - - // Only list public/published videos - if (!options.filter || (options.filter !== 'all-local' && options.filter !== 'all')) { - and.push( - `("video"."state" = ${VideoState.PUBLISHED} OR ` + - `("video"."state" = ${VideoState.TO_TRANSCODE} AND "video"."waitTranscoding" IS false))` - ) - - if (options.user) { - and.push( - `("video"."privacy" = ${VideoPrivacy.PUBLIC} OR "video"."privacy" = ${VideoPrivacy.INTERNAL})` - ) - } else { // Or only public videos - and.push( - `"video"."privacy" = ${VideoPrivacy.PUBLIC}` - ) - } - } - - if (options.videoPlaylistId) { - joins.push( - 'INNER JOIN "videoPlaylistElement" "video"."id" = "videoPlaylistElement"."videoId" ' + - 'AND "videoPlaylistElement"."videoPlaylistId" = :videoPlaylistId' - ) - - replacements.videoPlaylistId = options.videoPlaylistId - } - - if (options.filter && (options.filter === 'local' || options.filter === 'all-local')) { - and.push('"video"."remote" IS FALSE') - } - - if (options.accountId) { - and.push('"account"."id" = :accountId') - replacements.accountId = options.accountId - } - - if (options.videoChannelId) { - and.push('"videoChannel"."id" = :videoChannelId') - replacements.videoChannelId = options.videoChannelId - } - - if (options.followerActorId) { - let query = - '(' + - ' EXISTS (' + - ' SELECT 1 FROM "videoShare" ' + - ' INNER JOIN "actorFollow" "actorFollowShare" ON "actorFollowShare"."targetActorId" = "videoShare"."actorId" ' + - ' AND "actorFollowShare"."actorId" = :followerActorId AND "actorFollowShare"."state" = \'accepted\' ' + - ' WHERE "videoShare"."videoId" = "video"."id"' + - ' )' + - ' OR' + - ' EXISTS (' + - ' SELECT 1 from "actorFollow" ' + - ' WHERE "actorFollow"."targetActorId" = "videoChannel"."actorId" AND "actorFollow"."actorId" = :followerActorId ' + - ' AND "actorFollow"."state" = \'accepted\'' + - ' )' - - if (options.includeLocalVideos) { - query += ' OR "video"."remote" IS FALSE' - } - - query += ')' - - and.push(query) - replacements.followerActorId = options.followerActorId - } - - if (options.withFiles === true) { - and.push( - '(' + - ' EXISTS (SELECT 1 FROM "videoFile" WHERE "videoFile"."videoId" = "video"."id") ' + - ' OR EXISTS (' + - ' SELECT 1 FROM "videoStreamingPlaylist" ' + - ' INNER JOIN "videoFile" ON "videoFile"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ' + - ' WHERE "videoStreamingPlaylist"."videoId" = "video"."id"' + - ' )' + - ')' - ) - } - - if (options.tagsOneOf) { - const tagsOneOfLower = options.tagsOneOf.map(t => t.toLowerCase()) - - and.push( - 'EXISTS (' + - ' SELECT 1 FROM "videoTag" ' + - ' INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' + - ' WHERE lower("tag"."name") IN (' + createSafeIn(sequelize, tagsOneOfLower) + ') ' + - ' AND "video"."id" = "videoTag"."videoId"' + - ')' - ) - } - - if (options.tagsAllOf) { - const tagsAllOfLower = options.tagsAllOf.map(t => t.toLowerCase()) - - and.push( - 'EXISTS (' + - ' SELECT 1 FROM "videoTag" ' + - ' INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' + - ' WHERE lower("tag"."name") IN (' + createSafeIn(sequelize, tagsAllOfLower) + ') ' + - ' AND "video"."id" = "videoTag"."videoId" ' + - ' GROUP BY "videoTag"."videoId" HAVING COUNT(*) = ' + tagsAllOfLower.length + - ')' - ) - } - - if (options.nsfw === true) { - and.push('"video"."nsfw" IS TRUE') - } else if (options.nsfw === false) { - and.push('"video"."nsfw" IS FALSE') - } - - if (options.isLive === true) { - and.push('"video"."isLive" IS TRUE') - } else if (options.isLive === false) { - and.push('"video"."isLive" IS FALSE') - } - - if (options.categoryOneOf) { - and.push('"video"."category" IN (:categoryOneOf)') - replacements.categoryOneOf = options.categoryOneOf - } - - if (options.licenceOneOf) { - and.push('"video"."licence" IN (:licenceOneOf)') - replacements.licenceOneOf = options.licenceOneOf - } - - if (options.languageOneOf) { - const languages = options.languageOneOf.filter(l => l && l !== '_unknown') - const languagesQueryParts: string[] = [] - - if (languages.length !== 0) { - languagesQueryParts.push('"video"."language" IN (:languageOneOf)') - replacements.languageOneOf = languages - - languagesQueryParts.push( - 'EXISTS (' + - ' SELECT 1 FROM "videoCaption" WHERE "videoCaption"."language" ' + - ' IN (' + createSafeIn(sequelize, languages) + ') AND ' + - ' "videoCaption"."videoId" = "video"."id"' + - ')' - ) - } - - if (options.languageOneOf.includes('_unknown')) { - languagesQueryParts.push('"video"."language" IS NULL') - } - - if (languagesQueryParts.length !== 0) { - and.push('(' + languagesQueryParts.join(' OR ') + ')') - } - } - - // We don't exclude results in this so if we do a count we don't need to add this complex clause - if (options.isCount !== true) { - if (options.trendingDays) { - const viewsGteDate = new Date(new Date().getTime() - (24 * 3600 * 1000) * options.trendingDays) - - joins.push('LEFT JOIN "videoView" ON "video"."id" = "videoView"."videoId" AND "videoView"."startDate" >= :viewsGteDate') - replacements.viewsGteDate = viewsGteDate - - attributes.push('COALESCE(SUM("videoView"."views"), 0) AS "score"') - - group = 'GROUP BY "video"."id"' - } else if ([ 'best', 'hot' ].includes(options.trendingAlgorithm)) { - /** - * "Hotness" is a measure based on absolute view/comment/like/dislike numbers, - * with fixed weights only applied to their log values. - * - * This algorithm gives little chance for an old video to have a good score, - * for which recent spikes in interactions could be a sign of "hotness" and - * justify a better score. However there are multiple ways to achieve that - * goal, which is left for later. Yes, this is a TODO :) - * - * notes: - * - weights and base score are in number of half-days. - * - all comments are counted, regardless of being written by the video author or not - * see https://github.com/reddit-archive/reddit/blob/master/r2/r2/lib/db/_sorts.pyx#L47-L58 - * - we have less interactions than on reddit, so multiply weights by an arbitrary factor - */ - const weights = { - like: 3 * 50, - dislike: -3 * 50, - view: Math.floor((1 / 3) * 50), - comment: 2 * 50, // a comment takes more time than a like to do, but can be done multiple times - history: -2 * 50 - } - - joins.push('LEFT JOIN "videoComment" ON "video"."id" = "videoComment"."videoId"') - - let attribute = - `LOG(GREATEST(1, "video"."likes" - 1)) * ${weights.like} ` + // likes (+) - `+ LOG(GREATEST(1, "video"."dislikes" - 1)) * ${weights.dislike} ` + // dislikes (-) - `+ LOG("video"."views" + 1) * ${weights.view} ` + // views (+) - `+ LOG(GREATEST(1, COUNT(DISTINCT "videoComment"."id"))) * ${weights.comment} ` + // comments (+) - '+ (SELECT (EXTRACT(epoch FROM "video"."publishedAt") - 1446156582) / 47000) ' // base score (in number of half-days) - - if (options.trendingAlgorithm === 'best' && options.user) { - joins.push( - 'LEFT JOIN "userVideoHistory" ON "video"."id" = "userVideoHistory"."videoId" AND "userVideoHistory"."userId" = :bestUser' - ) - replacements.bestUser = options.user.id - - attribute += `+ POWER(COUNT(DISTINCT "userVideoHistory"."id"), 2.0) * ${weights.history} ` - } - - attribute += 'AS "score"' - attributes.push(attribute) - - group = 'GROUP BY "video"."id"' - } - } - - if (options.historyOfUser) { - joins.push('INNER JOIN "userVideoHistory" ON "video"."id" = "userVideoHistory"."videoId"') - - and.push('"userVideoHistory"."userId" = :historyOfUser') - replacements.historyOfUser = options.historyOfUser.id - } - - if (options.startDate) { - and.push('"video"."publishedAt" >= :startDate') - replacements.startDate = options.startDate - } - - if (options.endDate) { - and.push('"video"."publishedAt" <= :endDate') - replacements.endDate = options.endDate - } - - if (options.originallyPublishedStartDate) { - and.push('"video"."originallyPublishedAt" >= :originallyPublishedStartDate') - replacements.originallyPublishedStartDate = options.originallyPublishedStartDate - } - - if (options.originallyPublishedEndDate) { - and.push('"video"."originallyPublishedAt" <= :originallyPublishedEndDate') - replacements.originallyPublishedEndDate = options.originallyPublishedEndDate - } - - if (options.durationMin) { - and.push('"video"."duration" >= :durationMin') - replacements.durationMin = options.durationMin - } - - if (options.durationMax) { - and.push('"video"."duration" <= :durationMax') - replacements.durationMax = options.durationMax - } - - if (options.search) { - const escapedSearch = sequelize.escape(options.search) - const escapedLikeSearch = sequelize.escape('%' + options.search + '%') - - cte.push( - '"trigramSearch" AS (' + - ' SELECT "video"."id", ' + - ` similarity(lower(immutable_unaccent("video"."name")), lower(immutable_unaccent(${escapedSearch}))) as similarity ` + - ' FROM "video" ' + - ' WHERE lower(immutable_unaccent("video"."name")) % lower(immutable_unaccent(' + escapedSearch + ')) OR ' + - ' lower(immutable_unaccent("video"."name")) LIKE lower(immutable_unaccent(' + escapedLikeSearch + '))' + - ')' - ) - - joins.push('LEFT JOIN "trigramSearch" ON "video"."id" = "trigramSearch"."id"') - - let base = '(' + - ' "trigramSearch"."id" IS NOT NULL OR ' + - ' EXISTS (' + - ' SELECT 1 FROM "videoTag" ' + - ' INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' + - ` WHERE lower("tag"."name") = ${escapedSearch} ` + - ' AND "video"."id" = "videoTag"."videoId"' + - ' )' - - if (validator.isUUID(options.search)) { - base += ` OR "video"."uuid" = ${escapedSearch}` - } - - base += ')' - and.push(base) - - attributes.push(`COALESCE("trigramSearch"."similarity", 0) as similarity`) - } else { - attributes.push('0 as similarity') - } - - if (options.isCount === true) attributes = [ 'COUNT(*) as "total"' ] - - let suffix = '' - let order = '' - if (options.isCount !== true) { - - if (exists(options.sort)) { - if (options.sort === '-originallyPublishedAt' || options.sort === 'originallyPublishedAt') { - attributes.push('COALESCE("video"."originallyPublishedAt", "video"."publishedAt") AS "publishedAtForOrder"') - } - - order = buildOrder(options.sort) - suffix += `${order} ` - } - - if (exists(options.count)) { - const count = parseInt(options.count + '', 10) - suffix += `LIMIT ${count} ` - } - - if (exists(options.start)) { - const start = parseInt(options.start + '', 10) - suffix += `OFFSET ${start} ` - } - } - - const cteString = cte.length !== 0 - ? `WITH ${cte.join(', ')} ` - : '' - - const query = cteString + - 'SELECT ' + attributes.join(', ') + ' ' + - 'FROM "video" ' + joins.join(' ') + ' ' + - 'WHERE ' + and.join(' AND ') + ' ' + - group + ' ' + - having + ' ' + - suffix - - return { query, replacements, order } -} - -function buildOrder (value: string) { - const { direction, field } = buildDirectionAndField(value) - if (field.match(/^[a-zA-Z."]+$/) === null) throw new Error('Invalid sort column ' + field) - - if (field.toLowerCase() === 'random') return 'ORDER BY RANDOM()' - - if ([ 'trending', 'hot', 'best' ].includes(field.toLowerCase())) { // Sort by aggregation - return `ORDER BY "score" ${direction}, "video"."views" ${direction}` - } - - let firstSort: string - - if (field.toLowerCase() === 'match') { // Search - firstSort = '"similarity"' - } else if (field === 'originallyPublishedAt') { - firstSort = '"publishedAtForOrder"' - } else if (field.includes('.')) { - firstSort = field - } else { - firstSort = `"video"."${field}"` - } - - return `ORDER BY ${firstSort} ${direction}, "video"."id" ASC` -} - -function wrapForAPIResults (baseQuery: string, replacements: any, options: BuildVideosQueryOptions, order: string) { - const attributes = { - '"video".*': '', - '"VideoChannel"."id"': '"VideoChannel.id"', - '"VideoChannel"."name"': '"VideoChannel.name"', - '"VideoChannel"."description"': '"VideoChannel.description"', - '"VideoChannel"."actorId"': '"VideoChannel.actorId"', - '"VideoChannel->Actor"."id"': '"VideoChannel.Actor.id"', - '"VideoChannel->Actor"."preferredUsername"': '"VideoChannel.Actor.preferredUsername"', - '"VideoChannel->Actor"."url"': '"VideoChannel.Actor.url"', - '"VideoChannel->Actor"."serverId"': '"VideoChannel.Actor.serverId"', - '"VideoChannel->Actor"."avatarId"': '"VideoChannel.Actor.avatarId"', - '"VideoChannel->Account"."id"': '"VideoChannel.Account.id"', - '"VideoChannel->Account"."name"': '"VideoChannel.Account.name"', - '"VideoChannel->Account->Actor"."id"': '"VideoChannel.Account.Actor.id"', - '"VideoChannel->Account->Actor"."preferredUsername"': '"VideoChannel.Account.Actor.preferredUsername"', - '"VideoChannel->Account->Actor"."url"': '"VideoChannel.Account.Actor.url"', - '"VideoChannel->Account->Actor"."serverId"': '"VideoChannel.Account.Actor.serverId"', - '"VideoChannel->Account->Actor"."avatarId"': '"VideoChannel.Account.Actor.avatarId"', - '"VideoChannel->Actor->Server"."id"': '"VideoChannel.Actor.Server.id"', - '"VideoChannel->Actor->Server"."host"': '"VideoChannel.Actor.Server.host"', - '"VideoChannel->Actor->Avatar"."id"': '"VideoChannel.Actor.Avatar.id"', - '"VideoChannel->Actor->Avatar"."filename"': '"VideoChannel.Actor.Avatar.filename"', - '"VideoChannel->Actor->Avatar"."fileUrl"': '"VideoChannel.Actor.Avatar.fileUrl"', - '"VideoChannel->Actor->Avatar"."onDisk"': '"VideoChannel.Actor.Avatar.onDisk"', - '"VideoChannel->Actor->Avatar"."createdAt"': '"VideoChannel.Actor.Avatar.createdAt"', - '"VideoChannel->Actor->Avatar"."updatedAt"': '"VideoChannel.Actor.Avatar.updatedAt"', - '"VideoChannel->Account->Actor->Server"."id"': '"VideoChannel.Account.Actor.Server.id"', - '"VideoChannel->Account->Actor->Server"."host"': '"VideoChannel.Account.Actor.Server.host"', - '"VideoChannel->Account->Actor->Avatar"."id"': '"VideoChannel.Account.Actor.Avatar.id"', - '"VideoChannel->Account->Actor->Avatar"."filename"': '"VideoChannel.Account.Actor.Avatar.filename"', - '"VideoChannel->Account->Actor->Avatar"."fileUrl"': '"VideoChannel.Account.Actor.Avatar.fileUrl"', - '"VideoChannel->Account->Actor->Avatar"."onDisk"': '"VideoChannel.Account.Actor.Avatar.onDisk"', - '"VideoChannel->Account->Actor->Avatar"."createdAt"': '"VideoChannel.Account.Actor.Avatar.createdAt"', - '"VideoChannel->Account->Actor->Avatar"."updatedAt"': '"VideoChannel.Account.Actor.Avatar.updatedAt"', - '"Thumbnails"."id"': '"Thumbnails.id"', - '"Thumbnails"."type"': '"Thumbnails.type"', - '"Thumbnails"."filename"': '"Thumbnails.filename"' - } - - const joins = [ - 'INNER JOIN "video" ON "tmp"."id" = "video"."id"', - - 'INNER JOIN "videoChannel" AS "VideoChannel" ON "video"."channelId" = "VideoChannel"."id"', - 'INNER JOIN "actor" AS "VideoChannel->Actor" ON "VideoChannel"."actorId" = "VideoChannel->Actor"."id"', - 'INNER JOIN "account" AS "VideoChannel->Account" ON "VideoChannel"."accountId" = "VideoChannel->Account"."id"', - 'INNER JOIN "actor" AS "VideoChannel->Account->Actor" ON "VideoChannel->Account"."actorId" = "VideoChannel->Account->Actor"."id"', - - 'LEFT OUTER JOIN "server" AS "VideoChannel->Actor->Server" ON "VideoChannel->Actor"."serverId" = "VideoChannel->Actor->Server"."id"', - 'LEFT OUTER JOIN "actorImage" AS "VideoChannel->Actor->Avatar" ' + - 'ON "VideoChannel->Actor"."avatarId" = "VideoChannel->Actor->Avatar"."id"', - - 'LEFT OUTER JOIN "server" AS "VideoChannel->Account->Actor->Server" ' + - 'ON "VideoChannel->Account->Actor"."serverId" = "VideoChannel->Account->Actor->Server"."id"', - - 'LEFT OUTER JOIN "actorImage" AS "VideoChannel->Account->Actor->Avatar" ' + - 'ON "VideoChannel->Account->Actor"."avatarId" = "VideoChannel->Account->Actor->Avatar"."id"', - - 'LEFT OUTER JOIN "thumbnail" AS "Thumbnails" ON "video"."id" = "Thumbnails"."videoId"' - ] - - if (options.withFiles) { - joins.push('LEFT JOIN "videoFile" AS "VideoFiles" ON "VideoFiles"."videoId" = "video"."id"') - - joins.push('LEFT JOIN "videoStreamingPlaylist" AS "VideoStreamingPlaylists" ON "VideoStreamingPlaylists"."videoId" = "video"."id"') - joins.push( - 'LEFT JOIN "videoFile" AS "VideoStreamingPlaylists->VideoFiles" ' + - 'ON "VideoStreamingPlaylists->VideoFiles"."videoStreamingPlaylistId" = "VideoStreamingPlaylists"."id"' - ) - - Object.assign(attributes, { - '"VideoFiles"."id"': '"VideoFiles.id"', - '"VideoFiles"."createdAt"': '"VideoFiles.createdAt"', - '"VideoFiles"."updatedAt"': '"VideoFiles.updatedAt"', - '"VideoFiles"."resolution"': '"VideoFiles.resolution"', - '"VideoFiles"."size"': '"VideoFiles.size"', - '"VideoFiles"."extname"': '"VideoFiles.extname"', - '"VideoFiles"."filename"': '"VideoFiles.filename"', - '"VideoFiles"."fileUrl"': '"VideoFiles.fileUrl"', - '"VideoFiles"."torrentFilename"': '"VideoFiles.torrentFilename"', - '"VideoFiles"."torrentUrl"': '"VideoFiles.torrentUrl"', - '"VideoFiles"."infoHash"': '"VideoFiles.infoHash"', - '"VideoFiles"."fps"': '"VideoFiles.fps"', - '"VideoFiles"."videoId"': '"VideoFiles.videoId"', - - '"VideoStreamingPlaylists"."id"': '"VideoStreamingPlaylists.id"', - '"VideoStreamingPlaylists"."playlistUrl"': '"VideoStreamingPlaylists.playlistUrl"', - '"VideoStreamingPlaylists"."type"': '"VideoStreamingPlaylists.type"', - '"VideoStreamingPlaylists->VideoFiles"."id"': '"VideoStreamingPlaylists.VideoFiles.id"', - '"VideoStreamingPlaylists->VideoFiles"."createdAt"': '"VideoStreamingPlaylists.VideoFiles.createdAt"', - '"VideoStreamingPlaylists->VideoFiles"."updatedAt"': '"VideoStreamingPlaylists.VideoFiles.updatedAt"', - '"VideoStreamingPlaylists->VideoFiles"."resolution"': '"VideoStreamingPlaylists.VideoFiles.resolution"', - '"VideoStreamingPlaylists->VideoFiles"."size"': '"VideoStreamingPlaylists.VideoFiles.size"', - '"VideoStreamingPlaylists->VideoFiles"."extname"': '"VideoStreamingPlaylists.VideoFiles.extname"', - '"VideoStreamingPlaylists->VideoFiles"."filename"': '"VideoStreamingPlaylists.VideoFiles.filename"', - '"VideoStreamingPlaylists->VideoFiles"."fileUrl"': '"VideoStreamingPlaylists.VideoFiles.fileUrl"', - '"VideoStreamingPlaylists->VideoFiles"."torrentFilename"': '"VideoStreamingPlaylists.VideoFiles.torrentFilename"', - '"VideoStreamingPlaylists->VideoFiles"."torrentUrl"': '"VideoStreamingPlaylists.VideoFiles.torrentUrl"', - '"VideoStreamingPlaylists->VideoFiles"."infoHash"': '"VideoStreamingPlaylists.VideoFiles.infoHash"', - '"VideoStreamingPlaylists->VideoFiles"."fps"': '"VideoStreamingPlaylists.VideoFiles.fps"', - '"VideoStreamingPlaylists->VideoFiles"."videoStreamingPlaylistId"': '"VideoStreamingPlaylists.VideoFiles.videoStreamingPlaylistId"', - '"VideoStreamingPlaylists->VideoFiles"."videoId"': '"VideoStreamingPlaylists.VideoFiles.videoId"' - }) - } - - if (options.user) { - joins.push( - 'LEFT OUTER JOIN "userVideoHistory" ' + - 'ON "video"."id" = "userVideoHistory"."videoId" AND "userVideoHistory"."userId" = :userVideoHistoryId' - ) - replacements.userVideoHistoryId = options.user.id - - Object.assign(attributes, { - '"userVideoHistory"."id"': '"userVideoHistory.id"', - '"userVideoHistory"."currentTime"': '"userVideoHistory.currentTime"' - }) - } - - if (options.videoPlaylistId) { - joins.push( - 'INNER JOIN "videoPlaylistElement" as "VideoPlaylistElement" ON "videoPlaylistElement"."videoId" = "video"."id" ' + - 'AND "VideoPlaylistElement"."videoPlaylistId" = :videoPlaylistId' - ) - replacements.videoPlaylistId = options.videoPlaylistId - - Object.assign(attributes, { - '"VideoPlaylistElement"."createdAt"': '"VideoPlaylistElement.createdAt"', - '"VideoPlaylistElement"."updatedAt"': '"VideoPlaylistElement.updatedAt"', - '"VideoPlaylistElement"."url"': '"VideoPlaylistElement.url"', - '"VideoPlaylistElement"."position"': '"VideoPlaylistElement.position"', - '"VideoPlaylistElement"."startTimestamp"': '"VideoPlaylistElement.startTimestamp"', - '"VideoPlaylistElement"."stopTimestamp"': '"VideoPlaylistElement.stopTimestamp"', - '"VideoPlaylistElement"."videoPlaylistId"': '"VideoPlaylistElement.videoPlaylistId"' - }) - } - - const select = 'SELECT ' + Object.keys(attributes).map(key => { - const value = attributes[key] - if (value) return `${key} AS ${value}` - - return key - }).join(', ') - - return `${select} FROM (${baseQuery}) AS "tmp" ${joins.join(' ')} ${order}` -} - -export { - buildListQuery, - wrapForAPIResults -} diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 44aaa24ef..4979cee50 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -1,6 +1,6 @@ import * as Bluebird from 'bluebird' import { remove } from 'fs-extra' -import { maxBy, minBy, pick } from 'lodash' +import { maxBy, minBy } from 'lodash' import { join } from 'path' import { FindOptions, Includeable, IncludeOptions, Op, QueryTypes, ScopeOptions, Sequelize, Transaction, WhereOptions } from 'sequelize' import { @@ -110,7 +110,16 @@ import { VideoTrackerModel } from '../server/video-tracker' import { UserModel } from '../user/user' import { UserVideoHistoryModel } from '../user/user-video-history' import { buildTrigramSearchIndex, buildWhereIdOrUUID, getVideoSort, isOutdated, throwIfNotValid } from '../utils' +import { + videoFilesModelToFormattedJSON, + VideoFormattingJSONOptions, + videoModelToActivityPubObject, + videoModelToFormattedDetailsJSON, + videoModelToFormattedJSON +} from './formatter/video-format-utils' import { ScheduleVideoUpdateModel } from './schedule-video-update' +import { BuildVideosListQueryOptions, VideosIdListQueryBuilder } from './sql/videos-id-list-query-builder' +import { VideosModelListQueryBuilder } from './sql/videos-model-list-query-builder' import { TagModel } from './tag' import { ThumbnailModel } from './thumbnail' import { VideoBlacklistModel } from './video-blacklist' @@ -118,17 +127,9 @@ import { VideoCaptionModel } from './video-caption' import { ScopeNames as VideoChannelScopeNames, SummaryOptions, VideoChannelModel } from './video-channel' import { VideoCommentModel } from './video-comment' import { VideoFileModel } from './video-file' -import { - videoFilesModelToFormattedJSON, - VideoFormattingJSONOptions, - videoModelToActivityPubObject, - videoModelToFormattedDetailsJSON, - videoModelToFormattedJSON -} from './video-format-utils' import { VideoImportModel } from './video-import' import { VideoLiveModel } from './video-live' import { VideoPlaylistElementModel } from './video-playlist-element' -import { buildListQuery, BuildVideosQueryOptions, wrapForAPIResults } from './video-query-builder' import { VideoShareModel } from './video-share' import { VideoStreamingPlaylistModel } from './video-streaming-playlist' import { VideoTagModel } from './video-tag' @@ -1607,7 +1608,7 @@ export class VideoModel extends Model>> { const serverActor = await getServerActor() const followerActorId = serverActor.id - const queryOptions: BuildVideosQueryOptions = { + const queryOptions: BuildVideosListQueryOptions = { attributes: [ `"${field}"` ], group: `GROUP BY "${field}"`, having: `HAVING COUNT("${field}") >= ${threshold}`, @@ -1619,10 +1620,10 @@ export class VideoModel extends Model>> { includeLocalVideos: true } - const { query, replacements } = buildListQuery(VideoModel.sequelize, queryOptions) + const queryBuilder = new VideosIdListQueryBuilder(VideoModel.sequelize) - return this.sequelize.query(query, { replacements, type: QueryTypes.SELECT }) - .then(rows => rows.map(r => r[field])) + return queryBuilder.queryVideoIds(queryOptions) + .then(rows => rows.map(r => r[field])) } static buildTrendingQuery (trendingDays: number) { @@ -1640,27 +1641,24 @@ export class VideoModel extends Model>> { } private static async getAvailableForApi ( - options: BuildVideosQueryOptions, + options: BuildVideosListQueryOptions, countVideos = true ): Promise> { function getCount () { if (countVideos !== true) return Promise.resolve(undefined) const countOptions = Object.assign({}, options, { isCount: true }) - const { query: queryCount, replacements: replacementsCount } = buildListQuery(VideoModel.sequelize, countOptions) + const queryBuilder = new VideosIdListQueryBuilder(VideoModel.sequelize) - return VideoModel.sequelize.query(queryCount, { replacements: replacementsCount, type: QueryTypes.SELECT }) - .then(rows => rows.length !== 0 ? rows[0].total : 0) + return queryBuilder.countVideoIds(countOptions) } function getModels () { if (options.count === 0) return Promise.resolve([]) - const { query, replacements, order } = buildListQuery(VideoModel.sequelize, options) - const queryModels = wrapForAPIResults(query, replacements, options, order) + const queryBuilder = new VideosModelListQueryBuilder(VideoModel.sequelize) - return VideoModel.sequelize.query(queryModels, { replacements, type: QueryTypes.SELECT, nest: true }) - .then(rows => VideoModel.buildAPIResult(rows)) + return queryBuilder.queryVideos(options) } const [ count, rows ] = await Promise.all([ getCount(), getModels() ]) @@ -1671,153 +1669,6 @@ export class VideoModel extends Model>> { } } - private static buildAPIResult (rows: any[]) { - const videosMemo: { [ id: number ]: VideoModel } = {} - const videoStreamingPlaylistMemo: { [ id: number ]: VideoStreamingPlaylistModel } = {} - - const thumbnailsDone = new Set() - const historyDone = new Set() - const videoFilesDone = new Set() - - const videos: VideoModel[] = [] - - const avatarKeys = [ 'id', 'filename', 'fileUrl', 'onDisk', 'createdAt', 'updatedAt' ] - const actorKeys = [ 'id', 'preferredUsername', 'url', 'serverId', 'avatarId' ] - const serverKeys = [ 'id', 'host' ] - const videoFileKeys = [ - 'id', - 'createdAt', - 'updatedAt', - 'resolution', - 'size', - 'extname', - 'filename', - 'fileUrl', - 'torrentFilename', - 'torrentUrl', - 'infoHash', - 'fps', - 'videoId', - 'videoStreamingPlaylistId' - ] - const videoStreamingPlaylistKeys = [ 'id', 'type', 'playlistUrl' ] - const videoKeys = [ - 'id', - 'uuid', - 'name', - 'category', - 'licence', - 'language', - 'privacy', - 'nsfw', - 'description', - 'support', - 'duration', - 'views', - 'likes', - 'dislikes', - 'remote', - 'isLive', - 'url', - 'commentsEnabled', - 'downloadEnabled', - 'waitTranscoding', - 'state', - 'publishedAt', - 'originallyPublishedAt', - 'channelId', - 'createdAt', - 'updatedAt' - ] - const buildOpts = { raw: true } - - function buildActor (rowActor: any) { - const avatarModel = rowActor.Avatar.id !== null - ? new ActorImageModel(pick(rowActor.Avatar, avatarKeys), buildOpts) - : null - - const serverModel = rowActor.Server.id !== null - ? new ServerModel(pick(rowActor.Server, serverKeys), buildOpts) - : null - - const actorModel = new ActorModel(pick(rowActor, actorKeys), buildOpts) - actorModel.Avatar = avatarModel - actorModel.Server = serverModel - - return actorModel - } - - for (const row of rows) { - if (!videosMemo[row.id]) { - // Build Channel - const channel = row.VideoChannel - const channelModel = new VideoChannelModel(pick(channel, [ 'id', 'name', 'description', 'actorId' ]), buildOpts) - channelModel.Actor = buildActor(channel.Actor) - - const account = row.VideoChannel.Account - const accountModel = new AccountModel(pick(account, [ 'id', 'name' ]), buildOpts) - accountModel.Actor = buildActor(account.Actor) - - channelModel.Account = accountModel - - const videoModel = new VideoModel(pick(row, videoKeys), buildOpts) - videoModel.VideoChannel = channelModel - - videoModel.UserVideoHistories = [] - videoModel.Thumbnails = [] - videoModel.VideoFiles = [] - videoModel.VideoStreamingPlaylists = [] - - videosMemo[row.id] = videoModel - // Don't take object value to have a sorted array - videos.push(videoModel) - } - - const videoModel = videosMemo[row.id] - - if (row.userVideoHistory?.id && !historyDone.has(row.userVideoHistory.id)) { - const historyModel = new UserVideoHistoryModel(pick(row.userVideoHistory, [ 'id', 'currentTime' ]), buildOpts) - videoModel.UserVideoHistories.push(historyModel) - - historyDone.add(row.userVideoHistory.id) - } - - if (row.Thumbnails?.id && !thumbnailsDone.has(row.Thumbnails.id)) { - const thumbnailModel = new ThumbnailModel(pick(row.Thumbnails, [ 'id', 'type', 'filename' ]), buildOpts) - videoModel.Thumbnails.push(thumbnailModel) - - thumbnailsDone.add(row.Thumbnails.id) - } - - if (row.VideoFiles?.id && !videoFilesDone.has(row.VideoFiles.id)) { - const videoFileModel = new VideoFileModel(pick(row.VideoFiles, videoFileKeys), buildOpts) - videoModel.VideoFiles.push(videoFileModel) - - videoFilesDone.add(row.VideoFiles.id) - } - - if (row.VideoStreamingPlaylists?.id && !videoStreamingPlaylistMemo[row.VideoStreamingPlaylists.id]) { - const streamingPlaylist = new VideoStreamingPlaylistModel(pick(row.VideoStreamingPlaylists, videoStreamingPlaylistKeys), buildOpts) - streamingPlaylist.VideoFiles = [] - - videoModel.VideoStreamingPlaylists.push(streamingPlaylist) - - videoStreamingPlaylistMemo[streamingPlaylist.id] = streamingPlaylist - } - - if (row.VideoStreamingPlaylists?.VideoFiles?.id && !videoFilesDone.has(row.VideoStreamingPlaylists.VideoFiles.id)) { - const streamingPlaylist = videoStreamingPlaylistMemo[row.VideoStreamingPlaylists.id] - - const videoFileModel = new VideoFileModel(pick(row.VideoStreamingPlaylists.VideoFiles, videoFileKeys), buildOpts) - streamingPlaylist.VideoFiles.push(videoFileModel) - - videoFilesDone.add(row.VideoStreamingPlaylists.VideoFiles.id) - } - } - - return videos - } - static getCategoryLabel (id: number) { return VIDEO_CATEGORIES[id] || 'Misc' }