PeerTube/server/models/redundancy/video-redundancy.ts

571 lines
14 KiB
TypeScript
Raw Normal View History

2018-09-11 16:27:07 +02:00
import {
AllowNull,
2018-10-03 16:43:57 +02:00
BeforeDestroy,
2018-09-11 16:27:07 +02:00
BelongsTo,
Column,
CreatedAt,
DataType,
ForeignKey,
Is,
Model,
Scopes,
Table,
UpdatedAt
} from 'sequelize-typescript'
import { ActorModel } from '../activitypub/actor'
2019-04-23 09:50:57 +02:00
import { getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
2018-09-11 16:27:07 +02:00
import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
2018-09-11 16:27:07 +02:00
import { VideoFileModel } from '../video/video-file'
import { getServerActor } from '../../helpers/utils'
import { VideoModel } from '../video/video'
import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
import { logger } from '../../helpers/logger'
2018-09-25 18:05:54 +02:00
import { CacheFileObject, VideoPrivacy } from '../../../shared'
2018-09-11 16:27:07 +02:00
import { VideoChannelModel } from '../video/video-channel'
import { ServerModel } from '../server/server'
import { sample } from 'lodash'
import { isTestInstance } from '../../helpers/core-utils'
2018-09-14 11:05:38 +02:00
import * as Bluebird from 'bluebird'
2019-04-23 09:50:57 +02:00
import { col, FindOptions, fn, literal, Op, Transaction } from 'sequelize'
2019-01-29 08:37:25 +01:00
import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
2019-04-11 11:33:44 +02:00
import { CONFIG } from '../../initializers/config'
2019-08-21 14:31:57 +02:00
import { MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/typings/models'
2018-09-11 16:27:07 +02:00
export enum ScopeNames {
WITH_VIDEO = 'WITH_VIDEO'
}
2019-04-23 09:50:57 +02:00
@Scopes(() => ({
2018-09-11 16:27:07 +02:00
[ ScopeNames.WITH_VIDEO ]: {
include: [
{
2019-04-23 09:50:57 +02:00
model: VideoFileModel,
2019-01-29 08:37:25 +01:00
required: false,
include: [
{
2019-04-23 09:50:57 +02:00
model: VideoModel,
2019-01-29 08:37:25 +01:00
required: true
}
]
},
{
2019-04-23 09:50:57 +02:00
model: VideoStreamingPlaylistModel,
2019-01-29 08:37:25 +01:00
required: false,
2018-09-11 16:27:07 +02:00
include: [
{
2019-04-23 09:50:57 +02:00
model: VideoModel,
2018-09-11 16:27:07 +02:00
required: true
}
]
}
2019-04-23 09:50:57 +02:00
]
2018-09-11 16:27:07 +02:00
}
2019-04-23 09:50:57 +02:00
}))
2018-09-11 16:27:07 +02:00
@Table({
tableName: 'videoRedundancy',
indexes: [
{
fields: [ 'videoFileId' ]
},
{
fields: [ 'actorId' ]
},
{
fields: [ 'url' ],
unique: true
}
]
})
export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
@CreatedAt
createdAt: Date
@UpdatedAt
updatedAt: Date
@AllowNull(false)
@Column
expiresOn: Date
@AllowNull(false)
@Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
@Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
fileUrl: string
@AllowNull(false)
@Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
@Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
url: string
@AllowNull(true)
@Column
strategy: string // Only used by us
@ForeignKey(() => VideoFileModel)
@Column
videoFileId: number
@BelongsTo(() => VideoFileModel, {
foreignKey: {
2019-01-29 08:37:25 +01:00
allowNull: true
2018-09-11 16:27:07 +02:00
},
onDelete: 'cascade'
})
VideoFile: VideoFileModel
2019-01-29 08:37:25 +01:00
@ForeignKey(() => VideoStreamingPlaylistModel)
@Column
videoStreamingPlaylistId: number
@BelongsTo(() => VideoStreamingPlaylistModel, {
foreignKey: {
allowNull: true
},
onDelete: 'cascade'
})
VideoStreamingPlaylist: VideoStreamingPlaylistModel
2018-09-11 16:27:07 +02:00
@ForeignKey(() => ActorModel)
@Column
actorId: number
@BelongsTo(() => ActorModel, {
foreignKey: {
allowNull: false
},
onDelete: 'cascade'
})
Actor: ActorModel
2018-10-03 16:43:57 +02:00
@BeforeDestroy
static async removeFile (instance: VideoRedundancyModel) {
2018-11-16 11:18:13 +01:00
if (!instance.isOwned()) return
2018-09-11 16:27:07 +02:00
2019-01-29 08:37:25 +01:00
if (instance.videoFileId) {
const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
2018-09-11 16:27:07 +02:00
2019-01-29 08:37:25 +01:00
const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
logger.info('Removing duplicated video file %s.', logIdentifier)
2018-10-03 16:43:57 +02:00
2019-01-29 08:37:25 +01:00
videoFile.Video.removeFile(videoFile, true)
.catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
}
if (instance.videoStreamingPlaylistId) {
const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
const videoUUID = videoStreamingPlaylist.Video.uuid
logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
videoStreamingPlaylist.Video.removeStreamingPlaylist(true)
.catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
}
return undefined
2018-09-11 16:27:07 +02:00
}
2019-08-15 11:53:26 +02:00
static async loadLocalByFileId (videoFileId: number): Promise<MVideoRedundancyVideo> {
const actor = await getServerActor()
2018-09-11 16:27:07 +02:00
const query = {
where: {
actorId: actor.id,
2018-09-11 16:27:07 +02:00
videoFileId
}
}
return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
}
2019-08-15 11:53:26 +02:00
static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise<MVideoRedundancyVideo> {
2019-01-29 08:37:25 +01:00
const actor = await getServerActor()
const query = {
where: {
actorId: actor.id,
videoStreamingPlaylistId
}
}
return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
}
2019-08-15 11:53:26 +02:00
static loadByUrl (url: string, transaction?: Transaction): Bluebird<MVideoRedundancy> {
2018-09-11 16:27:07 +02:00
const query = {
where: {
url
},
transaction
2018-09-11 16:27:07 +02:00
}
return VideoRedundancyModel.findOne(query)
}
2018-09-28 10:56:13 +02:00
static async isLocalByVideoUUIDExists (uuid: string) {
const actor = await getServerActor()
const query = {
raw: true,
attributes: [ 'id' ],
where: {
actorId: actor.id
},
include: [
{
attributes: [ ],
model: VideoFileModel,
required: true,
include: [
{
attributes: [ ],
model: VideoModel,
required: true,
where: {
uuid
}
}
]
}
]
}
return VideoRedundancyModel.findOne(query)
.then(r => !!r)
}
2018-09-14 11:05:38 +02:00
static async getVideoSample (p: Bluebird<VideoModel[]>) {
const rows = await p
2019-08-13 10:22:54 +02:00
if (rows.length === 0) return undefined
2018-09-14 09:57:21 +02:00
const ids = rows.map(r => r.id)
const id = sample(ids)
2019-01-29 08:37:25 +01:00
return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
2018-09-14 09:57:21 +02:00
}
2018-09-11 16:27:07 +02:00
static async findMostViewToDuplicate (randomizedFactor: number) {
// On VideoModel!
const query = {
2018-09-14 09:57:21 +02:00
attributes: [ 'id', 'views' ],
2018-09-11 16:27:07 +02:00
limit: randomizedFactor,
2018-09-14 09:57:21 +02:00
order: getVideoSort('-views'),
2018-09-25 18:05:54 +02:00
where: {
privacy: VideoPrivacy.PUBLIC
},
2018-09-11 16:27:07 +02:00
include: [
2018-09-14 09:57:21 +02:00
await VideoRedundancyModel.buildVideoFileForDuplication(),
VideoRedundancyModel.buildServerRedundancyInclude()
]
}
2018-09-14 11:05:38 +02:00
return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
2018-09-14 09:57:21 +02:00
}
static async findTrendingToDuplicate (randomizedFactor: number) {
// On VideoModel!
const query = {
attributes: [ 'id', 'views' ],
subQuery: false,
group: 'VideoModel.id',
limit: randomizedFactor,
order: getVideoSort('-trending'),
2018-09-25 18:05:54 +02:00
where: {
privacy: VideoPrivacy.PUBLIC
},
2018-09-14 09:57:21 +02:00
include: [
await VideoRedundancyModel.buildVideoFileForDuplication(),
VideoRedundancyModel.buildServerRedundancyInclude(),
VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
2018-09-11 16:27:07 +02:00
]
}
2018-09-14 11:05:38 +02:00
return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
}
static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
// On VideoModel!
const query = {
attributes: [ 'id', 'publishedAt' ],
limit: randomizedFactor,
order: getVideoSort('-publishedAt'),
where: {
2018-09-25 18:05:54 +02:00
privacy: VideoPrivacy.PUBLIC,
2018-09-14 11:05:38 +02:00
views: {
2019-04-23 09:50:57 +02:00
[ Op.gte ]: minViews
2018-09-14 11:05:38 +02:00
}
},
include: [
await VideoRedundancyModel.buildVideoFileForDuplication(),
VideoRedundancyModel.buildServerRedundancyInclude()
]
}
2018-09-11 16:27:07 +02:00
2018-09-14 11:05:38 +02:00
return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
2018-09-11 16:27:07 +02:00
}
2019-08-15 11:53:26 +02:00
static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise<MVideoRedundancyVideo> {
const expiredDate = new Date()
expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
const actor = await getServerActor()
const query = {
where: {
actorId: actor.id,
strategy,
createdAt: {
2019-04-23 09:50:57 +02:00
[ Op.lt ]: expiredDate
}
}
}
return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
}
2018-09-14 11:05:38 +02:00
static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
2018-09-11 16:27:07 +02:00
const actor = await getServerActor()
2019-08-12 08:46:46 +02:00
const redundancyInclude = {
attributes: [],
model: VideoRedundancyModel,
required: true,
where: {
actorId: actor.id,
strategy
}
}
const queryFiles: FindOptions = {
include: [ redundancyInclude ]
}
2018-09-11 16:27:07 +02:00
2019-08-12 08:46:46 +02:00
const queryStreamingPlaylists: FindOptions = {
2018-09-14 11:05:38 +02:00
include: [
{
attributes: [],
2019-08-12 08:46:46 +02:00
model: VideoModel.unscoped(),
2018-09-14 11:05:38 +02:00
required: true,
2019-08-12 08:46:46 +02:00
include: [
{
2019-08-13 10:22:54 +02:00
required: true,
2019-08-12 08:46:46 +02:00
attributes: [],
model: VideoStreamingPlaylistModel.unscoped(),
include: [
redundancyInclude
]
}
]
2018-09-14 11:05:38 +02:00
}
]
2018-09-11 16:27:07 +02:00
}
2019-08-12 08:46:46 +02:00
return Promise.all([
VideoFileModel.aggregate('size', 'SUM', queryFiles),
VideoFileModel.aggregate('size', 'SUM', queryStreamingPlaylists)
]).then(([ r1, r2 ]) => {
return parseAggregateResult(r1) + parseAggregateResult(r2)
})
2018-09-11 16:27:07 +02:00
}
static async listLocalExpired () {
const actor = await getServerActor()
const query = {
where: {
actorId: actor.id,
expiresOn: {
2019-04-23 09:50:57 +02:00
[ Op.lt ]: new Date()
}
}
}
return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
}
static async listRemoteExpired () {
const actor = await getServerActor()
2018-09-11 16:27:07 +02:00
const query = {
where: {
actorId: {
2019-04-23 09:50:57 +02:00
[Op.ne]: actor.id
},
2018-09-11 16:27:07 +02:00
expiresOn: {
2019-04-23 09:50:57 +02:00
[ Op.lt ]: new Date()
2018-09-11 16:27:07 +02:00
}
}
}
return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
2018-09-11 16:27:07 +02:00
}
static async listLocalOfServer (serverId: number) {
const actor = await getServerActor()
2019-01-29 08:37:25 +01:00
const buildVideoInclude = () => ({
model: VideoModel,
required: true,
include: [
{
2019-01-29 08:37:25 +01:00
attributes: [],
model: VideoChannelModel.unscoped(),
required: true,
include: [
{
2019-01-29 08:37:25 +01:00
attributes: [],
model: ActorModel.unscoped(),
required: true,
2019-01-29 08:37:25 +01:00
where: {
serverId
}
}
]
}
]
2019-01-29 08:37:25 +01:00
})
const query = {
where: {
actorId: actor.id
},
include: [
{
model: VideoFileModel,
required: false,
include: [ buildVideoInclude() ]
},
{
model: VideoStreamingPlaylistModel,
required: false,
include: [ buildVideoInclude() ]
}
]
}
return VideoRedundancyModel.findAll(query)
}
2018-09-14 14:57:59 +02:00
static async getStats (strategy: VideoRedundancyStrategy) {
const actor = await getServerActor()
2019-04-23 09:50:57 +02:00
const query: FindOptions = {
2018-09-14 14:57:59 +02:00
raw: true,
attributes: [
2019-04-23 09:50:57 +02:00
[ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ],
[ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ],
[ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ]
2018-09-14 14:57:59 +02:00
],
where: {
strategy,
actorId: actor.id
},
include: [
{
attributes: [],
model: VideoFileModel,
required: true
}
]
}
2019-04-23 09:50:57 +02:00
return VideoRedundancyModel.findOne(query)
2018-09-14 14:57:59 +02:00
.then((r: any) => ({
2019-04-23 09:50:57 +02:00
totalUsed: parseAggregateResult(r.totalUsed),
2018-09-14 14:57:59 +02:00
totalVideos: r.totalVideos,
totalVideoFiles: r.totalVideoFiles
}))
}
2019-01-29 08:37:25 +01:00
getVideo () {
if (this.VideoFile) return this.VideoFile.Video
return this.VideoStreamingPlaylist.Video
}
2018-11-16 11:18:13 +01:00
isOwned () {
return !!this.strategy
}
2019-08-21 14:31:57 +02:00
toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject {
2019-01-29 08:37:25 +01:00
if (this.VideoStreamingPlaylist) {
return {
id: this.url,
type: 'CacheFile' as 'CacheFile',
object: this.VideoStreamingPlaylist.Video.url,
expires: this.expiresOn.toISOString(),
url: {
type: 'Link',
mimeType: 'application/x-mpegURL',
mediaType: 'application/x-mpegURL',
href: this.fileUrl
}
}
}
2018-09-11 16:27:07 +02:00
return {
id: this.url,
type: 'CacheFile' as 'CacheFile',
object: this.VideoFile.Video.url,
expires: this.expiresOn.toISOString(),
url: {
type: 'Link',
2018-12-11 14:52:50 +01:00
mimeType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
2018-09-11 16:27:07 +02:00
href: this.fileUrl,
height: this.VideoFile.resolution,
size: this.VideoFile.size,
fps: this.VideoFile.fps
}
}
}
2018-09-14 09:57:21 +02:00
// Don't include video files we already duplicated
private static async buildVideoFileForDuplication () {
2018-09-11 16:27:07 +02:00
const actor = await getServerActor()
2019-04-23 09:50:57 +02:00
const notIn = literal(
2018-09-11 16:27:07 +02:00
'(' +
2019-01-29 08:37:25 +01:00
`SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
2018-09-11 16:27:07 +02:00
')'
)
2018-09-14 09:57:21 +02:00
return {
attributes: [],
model: VideoFileModel.unscoped(),
required: true,
where: {
id: {
2019-04-23 09:50:57 +02:00
[ Op.notIn ]: notIn
2018-09-14 09:57:21 +02:00
}
}
}
}
private static buildServerRedundancyInclude () {
return {
attributes: [],
model: VideoChannelModel.unscoped(),
required: true,
include: [
{
attributes: [],
model: ActorModel.unscoped(),
required: true,
include: [
{
attributes: [],
model: ServerModel.unscoped(),
required: true,
where: {
redundancyAllowed: true
}
}
]
}
]
}
2018-09-11 16:27:07 +02:00
}
}