OpenCloud
/
PeerTube
mirror of https://github.com/Chocobozzz/PeerTube
1
0
Fork 0
Code Issues Releases Wiki Activity

Don't guess remote tracker URL

pull/3756/head
Chocobozzz 2021-02-18 10:15:11 +01:00 committed by Chocobozzz
parent 2451916e45
commit d9a2a03196
21 changed files with 458 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
server/helpers/activitypub.ts
View File

@ -201,10 +201,12 @@ function checkUrlsSameHost (url1: string, url2: string) {
return idHost && actorHost && idHost.toLowerCase() === actorHost.toLowerCase()
}
function buildRemoteVideoBaseUrl (video: MVideoWithHost, path: string) {
function buildRemoteVideoBaseUrl (video: MVideoWithHost, path: string, scheme?: string) {
if (!scheme) scheme = REMOTE_SCHEME.HTTP
const host = video.VideoChannel.Actor.Server.host
return REMOTE_SCHEME.HTTP + '://' + host + path
return scheme + '://' + host + path
}
// ---------------------------------------------------------------------------

24
server/helpers/custom-validators/activitypub/videos.ts
View File

@ -1,4 +1,7 @@
import validator from 'validator'
import { logger } from '@server/helpers/logger'
import { ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject } from '@shared/models'
import { VideoState } from '../../../../shared/models/videos'
import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers/constants'
import { peertubeTruncate } from '../../core-utils'
import { exists, isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc'
@ -11,9 +14,6 @@ import {
isVideoViewsValid
} from '../videos'
import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc'
import { VideoState } from '../../../../shared/models/videos'
import { logger } from '@server/helpers/logger'
import { ActivityVideoFileMetadataObject } from '@shared/models'
function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) {
return isBaseActivityValid(activity, 'Update') &&
@ -84,6 +84,7 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
function isRemoteVideoUrlValid (url: any) {
return url.type === 'Link' &&
// Video file link
(
ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.includes(url.mediaType) &&
isActivityPubUrlValid(url.href) &&
@ -91,31 +92,41 @@ function isRemoteVideoUrlValid (url: any) {
validator.isInt(url.size + '', { min: 0 }) &&
(!url.fps || validator.isInt(url.fps + '', { min: -1 }))
) ||
// Torrent link
(
ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.includes(url.mediaType) &&
isActivityPubUrlValid(url.href) &&
validator.isInt(url.height + '', { min: 0 })
) ||
// Magnet link
(
ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.includes(url.mediaType) &&
validator.isLength(url.href, { min: 5 }) &&
validator.isInt(url.height + '', { min: 0 })
) ||
// HLS playlist link
(
(url.mediaType || url.mimeType) === 'application/x-mpegURL' &&
isActivityPubUrlValid(url.href) &&
isArray(url.tag)
) ||
isAPVideoFileMetadataObject(url)
isAPVideoTrackerUrlObject(url) ||
isAPVideoFileUrlMetadataObject(url)
}
function isAPVideoFileMetadataObject (url: any): url is ActivityVideoFileMetadataObject {
function isAPVideoFileUrlMetadataObject (url: any): url is ActivityVideoFileMetadataUrlObject {
return url &&
url.type === 'Link' &&
url.mediaType === 'application/json' &&
isArray(url.rel) && url.rel.includes('metadata')
}
function isAPVideoTrackerUrlObject (url: any): url is ActivityTrackerUrlObject {
return isArray(url.rel) &&
url.rel.includes('tracker') &&
isActivityPubUrlValid(url.href)
}
// ---------------------------------------------------------------------------
export {
@ -123,7 +134,8 @@ export {
isRemoteStringIdentifierValid,
sanitizeAndCheckVideoTorrentObject,
isRemoteVideoUrlValid,
isAPVideoFileMetadataObject
isAPVideoFileUrlMetadataObject,
isAPVideoTrackerUrlObject
}
// ---------------------------------------------------------------------------

7
server/helpers/webtorrent.ts
View File

@ -107,16 +107,13 @@ async function createTorrentAndSetInfoHash (
videoFile.torrentFilename = torrentFilename
}
// FIXME: merge/refactor videoOrPlaylist and video arguments
function generateMagnetUri (
videoOrPlaylist: MVideo | MStreamingPlaylistVideo,
video: MVideoWithHost,
videoFile: MVideoFileRedundanciesOpt,
baseUrlHttp: string,
baseUrlWs: string
trackerUrls: string[]
) {
const xs = videoFile.getTorrentUrl()
const announce = videoOrPlaylist.getTrackerUrls(baseUrlHttp, baseUrlWs)
const announce = trackerUrls
let urlList = [ videoFile.getFileUrl(video) ]
const redundancies = videoFile.RedundancyVideos

2
server/initializers/constants.ts
View File

@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
// ---------------------------------------------------------------------------
const LAST_MIGRATION_VERSION = 585
const LAST_MIGRATION_VERSION = 595
// ---------------------------------------------------------------------------

4
server/initializers/database.ts
View File

@ -1,3 +1,5 @@
import { TrackerModel } from '@server/models/server/tracker'
import { VideoTrackerModel } from '@server/models/server/video-tracker'
import { QueryTypes, Transaction } from 'sequelize'
import { Sequelize as SequelizeTypescript } from 'sequelize-typescript'
import { isTestInstance } from '../helpers/core-utils'
@ -128,6 +130,8 @@ async function initDatabaseModels (silent: boolean) {
VideoPlaylistModel,
VideoPlaylistElementModel,
ThumbnailModel,
TrackerModel,
VideoTrackerModel,
PluginModel
])

44
server/initializers/migrations/0590-trackers.ts Normal file
View File

@ -0,0 +1,44 @@
import * as Sequelize from 'sequelize'
async function up (utils: {
transaction: Sequelize.Transaction
queryInterface: Sequelize.QueryInterface
sequelize: Sequelize.Sequelize
db: any
}): Promise<void> {
{
const query = `CREATE TABLE IF NOT EXISTS "tracker" (
"id" serial,
"url" varchar(255) NOT NULL,
"createdAt" timestamp WITH time zone NOT NULL,
"updatedAt" timestamp WITH time zone NOT NULL,
PRIMARY KEY ("id")
);`
await utils.sequelize.query(query)
}
{
const query = `CREATE TABLE IF NOT EXISTS "videoTracker" (
"videoId" integer REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
"trackerId" integer REFERENCES "tracker" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
"createdAt" timestamp WITH time zone NOT NULL,
"updatedAt" timestamp WITH time zone NOT NULL,
UNIQUE ("videoId", "trackerId"),
PRIMARY KEY ("videoId", "trackerId")
);`
await utils.sequelize.query(query)
}
await utils.sequelize.query(`CREATE UNIQUE INDEX "tracker_url" ON "tracker" ("url")`)
}
function down (options) {
throw new Error('Not implemented.')
}
export {
up,
down
}

130
server/initializers/migrations/0595-remote-url.ts Normal file
View File

@ -0,0 +1,130 @@
import * as Sequelize from 'sequelize'
async function up (utils: {
transaction: Sequelize.Transaction
queryInterface: Sequelize.QueryInterface
sequelize: Sequelize.Sequelize
db: any
}): Promise<void> {
// Torrent and file URLs
{
const fromQueryWebtorrent = `SELECT 'https://' || server.host AS "serverUrl", '/static/webseed/' AS "filePath", "videoFile".id ` +
`FROM video ` +
`INNER JOIN "videoChannel" ON "videoChannel".id = video."channelId" ` +
`INNER JOIN actor ON actor.id = "videoChannel"."actorId" ` +
`INNER JOIN server ON server.id = actor."serverId" ` +
`INNER JOIN "videoFile" ON "videoFile"."videoId" = video.id ` +
`WHERE video.remote IS TRUE`
const fromQueryHLS = `SELECT 'https://' || server.host AS "serverUrl", ` +
`'/static/streaming-playlists/hls/' || video.uuid || '/' AS "filePath", "videoFile".id ` +
`FROM video ` +
`INNER JOIN "videoChannel" ON "videoChannel".id = video."channelId" ` +
`INNER JOIN actor ON actor.id = "videoChannel"."actorId" ` +
`INNER JOIN server ON server.id = actor."serverId" ` +
`INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."videoId" = video.id ` +
`INNER JOIN "videoFile" ON "videoFile"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id ` +
`WHERE video.remote IS TRUE`
for (const fromQuery of [ fromQueryWebtorrent, fromQueryHLS ]) {
const query = `UPDATE "videoFile" ` +
`SET "torrentUrl" = t."serverUrl" || '/static/torrents/' || "videoFile"."torrentFilename", ` +
`"fileUrl" = t."serverUrl" || t."filePath" || "videoFile"."filename" ` +
`FROM (${fromQuery}) AS t WHERE t.id = "videoFile"."id" AND "videoFile"."fileUrl" IS NULL`
await utils.sequelize.query(query)
}
}
// Caption URLs
{
const fromQuery = `SELECT 'https://' || server.host AS "serverUrl", "video".uuid, "videoCaption".id ` +
`FROM video ` +
`INNER JOIN "videoChannel" ON "videoChannel".id = video."channelId" ` +
`INNER JOIN actor ON actor.id = "videoChannel"."actorId" ` +
`INNER JOIN server ON server.id = actor."serverId" ` +
`INNER JOIN "videoCaption" ON "videoCaption"."videoId" = video.id ` +
`WHERE video.remote IS TRUE`
const query = `UPDATE "videoCaption" ` +
`SET "fileUrl" = t."serverUrl" || '/lazy-static/video-captions/' || t.uuid || '-' || "videoCaption"."language" || '.vtt' ` +
`FROM (${fromQuery}) AS t WHERE t.id = "videoCaption"."id" AND "videoCaption"."fileUrl" IS NULL`
await utils.sequelize.query(query)
}
// Thumbnail URLs
{
const fromQuery = `SELECT 'https://' || server.host AS "serverUrl", "video".uuid, "thumbnail".id ` +
`FROM video ` +
`INNER JOIN "videoChannel" ON "videoChannel".id = video."channelId" ` +
`INNER JOIN actor ON actor.id = "videoChannel"."actorId" ` +
`INNER JOIN server ON server.id = actor."serverId" ` +
`INNER JOIN "thumbnail" ON "thumbnail"."videoId" = video.id ` +
`WHERE video.remote IS TRUE`
// Thumbnails
{
const query = `UPDATE "thumbnail" ` +
`SET "fileUrl" = t."serverUrl" || '/static/thumbnails/' || t.uuid || '.jpg' ` +
`FROM (${fromQuery}) AS t WHERE t.id = "thumbnail"."id" AND "thumbnail"."fileUrl" IS NULL AND thumbnail.type = 1`
await utils.sequelize.query(query)
}
{
// Previews
const query = `UPDATE "thumbnail" ` +
`SET "fileUrl" = t."serverUrl" || '/lazy-static/previews/' || t.uuid || '.jpg' ` +
`FROM (${fromQuery}) AS t WHERE t.id = "thumbnail"."id" AND "thumbnail"."fileUrl" IS NULL AND thumbnail.type = 2`
await utils.sequelize.query(query)
}
}
// Trackers
{
const trackerUrls = [
`'https://' || server.host || '/tracker/announce'`,
`'wss://' || server.host || '/tracker/socket'`
]
for (const trackerUrl of trackerUrls) {
{
const query = `INSERT INTO "tracker" ("url", "createdAt", "updatedAt") ` +
`SELECT ${trackerUrl} AS "url", NOW(), NOW() ` +
`FROM video ` +
`INNER JOIN "videoChannel" ON "videoChannel".id = video."channelId" ` +
`INNER JOIN actor ON actor.id = "videoChannel"."actorId" ` +
`INNER JOIN server ON server.id = actor."serverId" ` +
`WHERE video.remote IS TRUE ` +
`ON CONFLICT DO NOTHING`
await utils.sequelize.query(query)
}
{
const query = `INSERT INTO "videoTracker" ("videoId", "trackerId", "createdAt", "updatedAt") ` +
`SELECT video.id, (SELECT tracker.id FROM tracker WHERE url = ${trackerUrl}) AS "trackerId", NOW(), NOW()` +
`FROM video ` +
`INNER JOIN "videoChannel" ON "videoChannel".id = video."channelId" ` +
`INNER JOIN actor ON actor.id = "videoChannel"."actorId" ` +
`INNER JOIN server ON server.id = actor."serverId" ` +
`WHERE video.remote IS TRUE`
await utils.sequelize.query(query)
}
}
}
}
function down (options) {
throw new Error('Not implemented.')
}
export {
up,
down
}

59
server/lib/activitypub/videos.ts
View File

@ -3,7 +3,8 @@ import { maxBy, minBy } from 'lodash'
import * as magnetUtil from 'magnet-uri'
import { basename, join } from 'path'
import * as request from 'request'
import * as sequelize from 'sequelize'
import { Transaction } from 'sequelize/types'
import { TrackerModel } from '@server/models/server/tracker'
import { VideoLiveModel } from '@server/models/video/video-live'
import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
import {
@ -16,12 +17,16 @@ import {
ActivityUrlObject,
ActivityVideoUrlObject
} from '../../../shared/index'
import { ActivityIconObject, VideoObject } from '../../../shared/models/activitypub/objects'
import { ActivityIconObject, ActivityTrackerUrlObject, VideoObject } from '../../../shared/models/activitypub/objects'
import { VideoPrivacy } from '../../../shared/models/videos'
import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
import { buildRemoteVideoBaseUrl, checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
import { isAPVideoFileMetadataObject, sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
import {
isAPVideoFileUrlMetadataObject,
isAPVideoTrackerUrlObject,
sanitizeAndCheckVideoTorrentObject
} from '../../helpers/custom-validators/activitypub/videos'
import { isArray } from '../../helpers/custom-validators/misc'
import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
@ -83,7 +88,7 @@ import { addVideoShares, shareVideoByServerAndChannel } from './share'
import { addVideoComments } from './video-comments'
import { createRates } from './video-rates'
async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: sequelize.Transaction) {
async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: Transaction) {
const video = videoArg as MVideoAP
if (
@ -433,6 +438,12 @@ async function updateVideoFromAP (options: {
await setVideoTags({ video: videoUpdated, tags, transaction: t, defaultValue: videoUpdated.Tags })
}
// Update trackers
{
const trackers = getTrackerUrls(videoObject, videoUpdated)
await setVideoTrackers({ video: videoUpdated, trackers, transaction: t })
}
{
// Update captions
await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoUpdated.id, t)
@ -577,7 +588,7 @@ function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject {
return MIMETYPES.VIDEO.MIMETYPE_EXT[urlMediaType] && urlMediaType.startsWith('video/')
}
function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject {
function isAPStreamingPlaylistUrlObject (url: any): url is ActivityPlaylistUrlObject {
return url && url.mediaType === 'application/x-mpegURL'
}
@ -671,6 +682,12 @@ async function createVideo (videoObject: VideoObject, channel: MChannelAccountLi
})
await Promise.all(videoCaptionsPromises)
// Process trackers
{
const trackers = getTrackerUrls(videoObject, videoCreated)
await setVideoTrackers({ video: videoCreated, trackers, transaction: t })
}
videoCreated.VideoFiles = videoFiles
if (videoCreated.isLive) {
@ -797,7 +814,7 @@ function videoFileActivityUrlToDBAttributes (
: parsed.xs
// Fetch associated metadata url, if any
const metadata = urls.filter(isAPVideoFileMetadataObject)
const metadata = urls.filter(isAPVideoFileUrlMetadataObject)
.find(u => {
return u.height === fileUrl.height &&
u.fps === fileUrl.fps &&
@ -889,3 +906,33 @@ function getPreviewUrl (previewIcon: ActivityIconObject, video: MVideoWithHost)
? previewIcon.url
: buildRemoteVideoBaseUrl(video, join(LAZY_STATIC_PATHS.PREVIEWS, video.generatePreviewName()))
}
function getTrackerUrls (object: VideoObject, video: MVideoWithHost) {
let wsFound = false
const trackers = object.url.filter(u => isAPVideoTrackerUrlObject(u))
.map((u: ActivityTrackerUrlObject) => {
if (u.rel.includes('websocket')) wsFound = true
return u.href
})
if (wsFound) return trackers
return [
buildRemoteVideoBaseUrl(video, '/tracker/socket', REMOTE_SCHEME.WS),
buildRemoteVideoBaseUrl(video, '/tracker/announce')
]
}
async function setVideoTrackers (options: {
video: MVideo
trackers: string[]
transaction?: Transaction
}) {
const { video, trackers, transaction } = options
const trackerInstances = await TrackerModel.findOrCreateTrackers(trackers, transaction)
await video.$set('Trackers', trackerInstances, { transaction })
}

5
server/lib/schedulers/videos-redundancy-scheduler.ts
View File

@ -1,6 +1,7 @@
import { move } from 'fs-extra'
import { join } from 'path'
import { getServerActor } from '@server/models/application/application'
import { TrackerModel } from '@server/models/server/tracker'
import { VideoModel } from '@server/models/video/video'
import {
MStreamingPlaylist,
@ -221,8 +222,8 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy)
const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
const magnetUri = generateMagnetUri(video, video, file, baseUrlHttp, baseUrlWs)
const trackerUrls = await TrackerModel.listUrlsByVideoId(video.id)
const magnetUri = generateMagnetUri(video, file, trackerUrls)
const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT)

16
server/lib/thumbnail.ts
View File

@ -1,5 +1,6 @@
import { copy } from 'fs-extra'
import { join } from 'path'
import { logger } from '@server/helpers/logger'
import { ThumbnailType } from '../../shared/models/videos/thumbnail.type'
import { generateImageFromVideoFile } from '../helpers/ffmpeg-utils'
import { processImage } from '../helpers/image-utils'
@ -62,7 +63,7 @@ function createVideoMiniatureFromUrl (options: {
size?: ImageSize
}) {
const { downloadUrl, video, type, size } = options
const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size)
const { filename: updatedFilename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size)
// Only save the file URL if it is a remote video
const fileUrl = video.isOwned()
@ -76,10 +77,16 @@ function createVideoMiniatureFromUrl (options: {
// If the thumbnail URL did not change and has a unique filename (introduced in 3.2), avoid thumbnail processing
const thumbnailUrlChanged = !existingUrl || existingUrl !== downloadUrl || downloadUrl.endsWith(`${video.uuid}.jpg`)
// Do not change the thumbnail filename if the file did not change
const filename = thumbnailUrlChanged
? updatedFilename
: existingThumbnail.filename
const thumbnailCreator = () => {
if (thumbnailUrlChanged) return downloadImage(downloadUrl, basePath, filename, { width, height })
return copy(existingThumbnail.getPath(), ThumbnailModel.buildPath(type, filename))
return Promise.resolve()
}
return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl })
@ -236,7 +243,7 @@ async function createThumbnailFromFunction (parameters: {
fileUrl = null
} = parameters
const oldFilename = existingThumbnail
const oldFilename = existingThumbnail && existingThumbnail.filename !== filename
? existingThumbnail.filename
: undefined
@ -248,7 +255,8 @@ async function createThumbnailFromFunction (parameters: {
thumbnail.type = type
thumbnail.fileUrl = fileUrl
thumbnail.automaticallyGenerated = automaticallyGenerated
thumbnail.previousThumbnailFilename = oldFilename
if (oldFilename) thumbnail.previousThumbnailFilename = oldFilename
await thumbnailCreator()

73
server/models/server/tracker.ts Normal file
View File

@ -0,0 +1,73 @@
import { AllowNull, BelongsToMany, Column, CreatedAt, Model, Table, UpdatedAt } from 'sequelize-typescript'
import { Transaction } from 'sequelize/types'
import { MTracker } from '@server/types/models/server/tracker'
import { VideoModel } from '../video/video'
import { VideoTrackerModel } from './video-tracker'
@Table({
tableName: 'tracker',
indexes: [
{
fields: [ 'url' ],
unique: true
}
]
})
export class TrackerModel extends Model {
@AllowNull(false)
@Column
url: string
@CreatedAt
createdAt: Date
@UpdatedAt
updatedAt: Date
@BelongsToMany(() => VideoModel, {
foreignKey: 'trackerId',
through: () => VideoTrackerModel,
onDelete: 'CASCADE'
})
Videos: VideoModel[]
static listUrlsByVideoId (videoId: number) {
const query = {
include: [
{
attributes: [ 'id', 'trackerId' ],
model: VideoModel.unscoped(),
required: true,
where: { id: videoId }
}
]
}
return TrackerModel.findAll(query)
.then(rows => rows.map(rows => rows.url))
}
static findOrCreateTrackers (trackers: string[], transaction: Transaction): Promise<MTracker[]> {
if (trackers === null) return Promise.resolve([])
const tasks: Promise<MTracker>[] = []
trackers.forEach(tracker => {
const query = {
where: {
url: tracker
},
defaults: {
url: tracker
},
transaction
}
const promise = TrackerModel.findOrCreate<MTracker>(query)
.then(([ trackerInstance ]) => trackerInstance)
tasks.push(promise)
})
return Promise.all(tasks)
}
}

30
server/models/server/video-tracker.ts Normal file
View File

@ -0,0 +1,30 @@
import { Column, CreatedAt, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript'
import { VideoModel } from '../video/video'
import { TrackerModel } from './tracker'
@Table({
tableName: 'videoTracker',
indexes: [
{
fields: [ 'videoId' ]
},
{
fields: [ 'trackerId' ]
}
]
})
export class VideoTrackerModel extends Model {
@CreatedAt
createdAt: Date
@UpdatedAt
updatedAt: Date
@ForeignKey(() => VideoModel)
@Column
videoId: number
@ForeignKey(() => TrackerModel)
@Column
trackerId: number
}

5
server/models/video/thumbnail.ts
View File

@ -15,7 +15,6 @@ import {
Table,
UpdatedAt
} from 'sequelize-typescript'
import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
import { afterCommitIfTransaction } from '@server/helpers/database-utils'
import { MThumbnail, MThumbnailVideo, MVideoWithHost } from '@server/types/models'
import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
@ -168,10 +167,8 @@ export class ThumbnailModel extends Model {
const staticPath = ThumbnailModel.types[this.type].staticPath + this.filename
if (video.isOwned()) return WEBSERVER.URL + staticPath
if (this.fileUrl) return this.fileUrl
// Fallback if we don't have a file URL
return buildRemoteVideoBaseUrl(video, staticPath)
return this.fileUrl
}
getPath () {

5
server/models/video/video-caption.ts
View File

@ -16,7 +16,6 @@ import {
UpdatedAt
} from 'sequelize-typescript'
import { v4 as uuidv4 } from 'uuid'
import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
import { MVideoCaption, MVideoCaptionFormattable, MVideoCaptionVideo, MVideoWithHost } from '@server/types/models'
import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model'
import { isVideoCaptionLanguageValid } from '../../helpers/custom-validators/video-captions'
@ -208,9 +207,7 @@ export class VideoCaptionModel extends Model {
if (!this.Video) this.Video = video as VideoModel
if (video.isOwned()) return WEBSERVER.URL + this.getCaptionStaticPath()
if (this.fileUrl) return this.fileUrl
// Fallback if we don't have a file URL
return buildRemoteVideoBaseUrl(video, this.getCaptionStaticPath())
return this.fileUrl
}
}

9
server/models/video/video-file.ts
View File

@ -427,10 +427,8 @@ export class VideoFileModel extends Model {
if (!this.Video) this.Video = video as VideoModel
if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video)
if (this.fileUrl) return this.fileUrl
// Fallback if we don't have a file URL
return buildRemoteVideoBaseUrl(video, this.getFileStaticPath(video))
return this.fileUrl
}
getFileStaticPath (video: MVideo) {
@ -454,10 +452,7 @@ export class VideoFileModel extends Model {
getRemoteTorrentUrl (video: MVideoWithHost) {
if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`)
if (this.torrentUrl) return this.torrentUrl
// Fallback if we don't have a torrent URL
return buildRemoteVideoBaseUrl(video, this.getTorrentStaticPath())
return this.torrentUrl
}
// We proxify torrent requests so use a local URL

49
server/models/video/video-format-utils.ts
View File

@ -14,8 +14,6 @@ import {
} from '../../lib/activitypub/url'
import {
MStreamingPlaylistRedundanciesOpt,
MStreamingPlaylistVideo,
MVideo,
MVideoAP,
MVideoFile,
MVideoFormattable,
@ -127,8 +125,6 @@ function videoModelToFormattedDetailsJSON (video: MVideoFormattableDetails): Vid
}
})
const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
const tags = video.Tags ? video.Tags.map(t => t.name) : []
const streamingPlaylists = streamingPlaylistsModelToFormattedJSON(video, video.VideoStreamingPlaylists)
@ -147,14 +143,14 @@ function videoModelToFormattedDetailsJSON (video: MVideoFormattableDetails): Vid
label: VideoModel.getStateLabel(video.state)
},
trackerUrls: video.getTrackerUrls(baseUrlHttp, baseUrlWs),
trackerUrls: video.getTrackerUrls(),
files: [],
streamingPlaylists
}
// Format and sort video files
detailsJson.files = videoFilesModelToFormattedJSON(video, video, baseUrlHttp, baseUrlWs, video.VideoFiles)
detailsJson.files = videoFilesModelToFormattedJSON(video, video.VideoFiles)
return Object.assign(formattedJson, detailsJson)
}
@ -165,17 +161,13 @@ function streamingPlaylistsModelToFormattedJSON (
): VideoStreamingPlaylist[] {
if (isArray(playlists) === false) return []
const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
return playlists
.map(playlist => {
const playlistWithVideo = Object.assign(playlist, { Video: video })
const redundancies = isArray(playlist.RedundancyVideos)
? playlist.RedundancyVideos.map(r => ({ baseUrl: r.fileUrl }))
: []
const files = videoFilesModelToFormattedJSON(playlistWithVideo, video, baseUrlHttp, baseUrlWs, playlist.VideoFiles)
const files = videoFilesModelToFormattedJSON(video, playlist.VideoFiles)
return {
id: playlist.id,
@ -194,14 +186,12 @@ function sortByResolutionDesc (fileA: MVideoFile, fileB: MVideoFile) {
return -1
}
// FIXME: refactor/merge model and video arguments
function videoFilesModelToFormattedJSON (
model: MVideo | MStreamingPlaylistVideo,
video: MVideoFormattableDetails,
baseUrlHttp: string,
baseUrlWs: string,
videoFiles: MVideoFileRedundanciesOpt[]
): VideoFile[] {
const trackerUrls = video.getTrackerUrls()
return [ ...videoFiles ]
.filter(f => !f.isLive())
.sort(sortByResolutionDesc)
@ -213,7 +203,7 @@ function videoFilesModelToFormattedJSON (
},
// FIXME: deprecated in 3.2
magnetUri: generateMagnetUri(model, video, videoFile, baseUrlHttp, baseUrlWs),
magnetUri: generateMagnetUri(video, videoFile, trackerUrls),
size: videoFile.size,
fps: videoFile.fps,
@ -229,15 +219,13 @@ function videoFilesModelToFormattedJSON (
})
}
// FIXME: refactor/merge model and video arguments
function addVideoFilesInAPAcc (
acc: ActivityUrlObject[] | ActivityTagObject[],
model: MVideoAP | MStreamingPlaylistVideo,
video: MVideoWithHost,
baseUrlHttp: string,
baseUrlWs: string,
files: MVideoFile[]
) {
const trackerUrls = video.getTrackerUrls()
const sortedFiles = [ ...files ]
.filter(f => !f.isLive())
.sort(sortByResolutionDesc)
@ -271,14 +259,13 @@ function addVideoFilesInAPAcc (
acc.push({
type: 'Link',
mediaType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet',
href: generateMagnetUri(model, video, file, baseUrlHttp, baseUrlWs),
href: generateMagnetUri(video, file, trackerUrls),
height: file.resolution
})
}
}
function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
if (!video.Tags) video.Tags = []
const tag = video.Tags.map(t => ({
@ -319,7 +306,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
}
]
addVideoFilesInAPAcc(url, video, video, baseUrlHttp, baseUrlWs, video.VideoFiles || [])
addVideoFilesInAPAcc(url, video, video.VideoFiles || [])
for (const playlist of (video.VideoStreamingPlaylists || [])) {
const tag = playlist.p2pMediaLoaderInfohashes
@ -331,8 +318,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
href: playlist.segmentsSha256Url
})
const playlistWithVideo = Object.assign(playlist, { Video: video })
addVideoFilesInAPAcc(tag, playlistWithVideo, video, baseUrlHttp, baseUrlWs, playlist.VideoFiles || [])
addVideoFilesInAPAcc(tag, video, playlist.VideoFiles || [])
url.push({
type: 'Link',
@ -342,6 +328,19 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
})
}
for (const trackerUrl of video.getTrackerUrls()) {
const rel2 = trackerUrl.startsWith('http')
? 'http'
: 'websocket'
url.push({
type: 'Link',
name: `tracker-${rel2}`,
rel: [ 'tracker', rel2 ],
href: trackerUrl
})
}
const subtitleLanguage = []
for (const caption of video.VideoCaptions) {
subtitleLanguage.push({

56
server/models/video/video.ts
View File

@ -60,7 +60,6 @@ import {
API_VERSION,
CONSTRAINTS_FIELDS,
LAZY_STATIC_PATHS,
REMOTE_SCHEME,
STATIC_PATHS,
VIDEO_CATEGORIES,
VIDEO_LANGUAGES,
@ -107,6 +106,8 @@ import { ActorModel } from '../activitypub/actor'
import { AvatarModel } from '../avatar/avatar'
import { VideoRedundancyModel } from '../redundancy/video-redundancy'
import { ServerModel } from '../server/server'
import { TrackerModel } from '../server/tracker'
import { VideoTrackerModel } from '../server/video-tracker'
import { buildTrigramSearchIndex, buildWhereIdOrUUID, getVideoSort, isOutdated, throwIfNotValid } from '../utils'
import { ScheduleVideoUpdateModel } from './schedule-video-update'
import { TagModel } from './tag'
@ -137,6 +138,7 @@ export enum ScopeNames {
FOR_API = 'FOR_API',
WITH_ACCOUNT_DETAILS = 'WITH_ACCOUNT_DETAILS',
WITH_TAGS = 'WITH_TAGS',
WITH_TRACKERS = 'WITH_TRACKERS',
WITH_WEBTORRENT_FILES = 'WITH_WEBTORRENT_FILES',
WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE',
WITH_BLACKLISTED = 'WITH_BLACKLISTED',
@ -320,6 +322,14 @@ export type AvailableForListIDsOptions = {
[ScopeNames.WITH_TAGS]: {
include: [ TagModel ]
},
[ScopeNames.WITH_TRACKERS]: {
include: [
{
attributes: [ 'id', 'url' ],
model: TrackerModel
}
]
},
[ScopeNames.WITH_BLACKLISTED]: {
include: [
{
@ -616,6 +626,13 @@ export class VideoModel extends Model {
})
Tags: TagModel[]
@BelongsToMany(() => TrackerModel, {
foreignKey: 'videoId',
through: () => VideoTrackerModel,
onDelete: 'CASCADE'
})
Trackers: TrackerModel[]
@HasMany(() => ThumbnailModel, {
foreignKey: {
name: 'videoId',
@ -1436,6 +1453,7 @@ export class VideoModel extends Model {
ScopeNames.WITH_SCHEDULED_UPDATE,
ScopeNames.WITH_THUMBNAILS,
ScopeNames.WITH_LIVE,
ScopeNames.WITH_TRACKERS,
{ method: [ ScopeNames.WITH_WEBTORRENT_FILES, true ] },
{ method: [ ScopeNames.WITH_STREAMING_PLAYLISTS, true ] }
]
@ -1887,18 +1905,15 @@ export class VideoModel extends Model {
}
getFormattedVideoFilesJSON (): VideoFile[] {
const { baseUrlHttp, baseUrlWs } = this.getBaseUrls()
let files: VideoFile[] = []
if (Array.isArray(this.VideoFiles)) {
const result = videoFilesModelToFormattedJSON(this, this, baseUrlHttp, baseUrlWs, this.VideoFiles)
const result = videoFilesModelToFormattedJSON(this, this.VideoFiles)
files = files.concat(result)
}
for (const p of (this.VideoStreamingPlaylists || [])) {
p.Video = this
const result = videoFilesModelToFormattedJSON(p, this, baseUrlHttp, baseUrlWs, p.VideoFiles)
const result = videoFilesModelToFormattedJSON(this, p.VideoFiles)
files = files.concat(result)
}
@ -2030,25 +2045,18 @@ export class VideoModel extends Model {
return false
}
getBaseUrls () {
if (this.isOwned()) {
return {
baseUrlHttp: WEBSERVER.URL,
baseUrlWs: WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT
}
}
return {
baseUrlHttp: REMOTE_SCHEME.HTTP + '://' + this.VideoChannel.Account.Actor.Server.host,
baseUrlWs: REMOTE_SCHEME.WS + '://' + this.VideoChannel.Account.Actor.Server.host
}
}
getTrackerUrls (baseUrlHttp: string, baseUrlWs: string) {
return [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ]
}
getBandwidthBits (videoFile: MVideoFile) {
return Math.ceil((videoFile.size * 8) / this.duration)
}
getTrackerUrls () {
if (this.isOwned()) {
return [
WEBSERVER.URL + '/tracker/announce',
WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket'
]
}
return this.Trackers.map(t => t.url)
}
}

7
server/types/models/server/tracker.ts Normal file
View File

@ -0,0 +1,7 @@
import { TrackerModel } from '../../../models/server/tracker'
export type MTracker = Omit<TrackerModel, 'Videos'>
// ############################################################################
export type MTrackerUrl = Pick<MTracker, 'url'>

4
server/types/models/video/video.ts
View File

@ -1,5 +1,6 @@
import { PickWith, PickWithOpt } from '@shared/core-utils'
import { VideoModel } from '../../../models/video/video'
import { MTrackerUrl } from '../server/tracker'
import { MUserVideoHistoryTime } from '../user/user-video-history'
import { MScheduleVideoUpdate } from './schedule-video-update'
import { MTag } from './tag'
@ -216,4 +217,5 @@ export type MVideoFormattableDetails =
Use<'VideoChannel', MChannelFormattable> &
Use<'Tags', MTag[]> &
Use<'VideoStreamingPlaylists', MStreamingPlaylistRedundanciesOpt[]> &
Use<'VideoFiles', MVideoFileRedundanciesOpt[]>
Use<'VideoFiles', MVideoFileRedundanciesOpt[]> &
Use<'Trackers', MTrackerUrl[]>

14
shared/models/activitypub/objects/common-objects.ts
View File

@ -30,7 +30,7 @@ export type ActivityPlaylistSegmentHashesObject = {
href: string
}
export type ActivityVideoFileMetadataObject = {
export type ActivityVideoFileMetadataUrlObject = {
type: 'Link'
rel: [ 'metadata', any ]
mediaType: 'application/json'
@ -39,6 +39,13 @@ export type ActivityVideoFileMetadataObject = {
fps: number
}
export type ActivityTrackerUrlObject = {
type: 'Link'
rel: [ 'tracker', 'websocket' | 'http' ]
name: string
href: string
}
export type ActivityPlaylistInfohashesObject = {
type: 'Infohash'
name: string
@ -96,7 +103,7 @@ export type ActivityTagObject =
| ActivityMentionObject
| ActivityBitTorrentUrlObject
| ActivityMagnetUrlObject
| ActivityVideoFileMetadataObject
| ActivityVideoFileMetadataUrlObject
export type ActivityUrlObject =
ActivityVideoUrlObject
@ -104,7 +111,8 @@ export type ActivityUrlObject =
| ActivityBitTorrentUrlObject
| ActivityMagnetUrlObject
| ActivityHtmlUrlObject
| ActivityVideoFileMetadataObject
| ActivityVideoFileMetadataUrlObject
| ActivityTrackerUrlObject
export interface ActivityPubAttributedTo {
type: 'Group' | 'Person'

3
shared/models/activitypub/objects/video-torrent-object.ts
View File

@ -40,11 +40,14 @@ export interface VideoObject {
icon: ActivityIconObject[]
url: ActivityUrlObject[]
likes: string
dislikes: string
shares: string
comments: string
attributedTo: ActivityPubAttributedTo[]
to?: string[]
cc?: string[]
}