mirror of https://github.com/Chocobozzz/PeerTube
Refactor AP playlists
Chocobozzz
parent
9777fe9eeb
commit
49af5ac8c2
|
|
@ -1,8 +1,51 @@
|
|||
import { CacheFileObject } from '../../../shared/index'
|
||||
import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
|
||||
import { Transaction } from 'sequelize'
|
||||
import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
|
||||
import { MActorId, MVideoRedundancy, MVideoWithAllFiles } from '@server/types/models'
|
||||
import { CacheFileObject } from '../../../shared/index'
|
||||
import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
|
||||
import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
|
||||
|
||||
async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) {
|
||||
const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t)
|
||||
|
||||
if (redundancyModel) {
|
||||
return updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t)
|
||||
}
|
||||
|
||||
return createCacheFile(cacheFileObject, video, byActor, t)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
createOrUpdateCacheFile
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function createCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) {
|
||||
const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
|
||||
|
||||
return VideoRedundancyModel.create(attributes, { transaction: t })
|
||||
}
|
||||
|
||||
function updateCacheFile (
|
||||
cacheFileObject: CacheFileObject,
|
||||
redundancyModel: MVideoRedundancy,
|
||||
video: MVideoWithAllFiles,
|
||||
byActor: MActorId,
|
||||
t: Transaction
|
||||
) {
|
||||
if (redundancyModel.actorId !== byActor.id) {
|
||||
throw new Error('Cannot update redundancy ' + redundancyModel.url + ' of another actor.')
|
||||
}
|
||||
|
||||
const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
|
||||
|
||||
redundancyModel.expiresOn = attributes.expiresOn
|
||||
redundancyModel.fileUrl = attributes.fileUrl
|
||||
|
||||
return redundancyModel.save({ transaction: t })
|
||||
}
|
||||
|
||||
function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId) {
|
||||
|
||||
|
|
@ -38,45 +81,3 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
|
|||
actorId: byActor.id
|
||||
}
|
||||
}
|
||||
|
||||
async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) {
|
||||
const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t)
|
||||
|
||||
if (!redundancyModel) {
|
||||
await createCacheFile(cacheFileObject, video, byActor, t)
|
||||
} else {
|
||||
await updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t)
|
||||
}
|
||||
}
|
||||
|
||||
function createCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) {
|
||||
const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
|
||||
|
||||
return VideoRedundancyModel.create(attributes, { transaction: t })
|
||||
}
|
||||
|
||||
function updateCacheFile (
|
||||
cacheFileObject: CacheFileObject,
|
||||
redundancyModel: MVideoRedundancy,
|
||||
video: MVideoWithAllFiles,
|
||||
byActor: MActorId,
|
||||
t: Transaction
|
||||
) {
|
||||
if (redundancyModel.actorId !== byActor.id) {
|
||||
throw new Error('Cannot update redundancy ' + redundancyModel.url + ' of another actor.')
|
||||
}
|
||||
|
||||
const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
|
||||
|
||||
redundancyModel.expiresOn = attributes.expiresOn
|
||||
redundancyModel.fileUrl = attributes.fileUrl
|
||||
|
||||
return redundancyModel.save({ transaction: t })
|
||||
}
|
||||
|
||||
export {
|
||||
createOrUpdateCacheFile,
|
||||
createCacheFile,
|
||||
updateCacheFile,
|
||||
cacheFileActivityObjectToDBAttributes
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,204 +0,0 @@
|
|||
import * as Bluebird from 'bluebird'
|
||||
import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
|
||||
import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
|
||||
import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
|
||||
import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
|
||||
import { checkUrlsSameHost } from '../../helpers/activitypub'
|
||||
import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
|
||||
import { isArray } from '../../helpers/custom-validators/misc'
|
||||
import { logger } from '../../helpers/logger'
|
||||
import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests'
|
||||
import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
|
||||
import { sequelizeTypescript } from '../../initializers/database'
|
||||
import { VideoPlaylistModel } from '../../models/video/video-playlist'
|
||||
import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
|
||||
import { MAccountDefault, MAccountId, MVideoId } from '../../types/models'
|
||||
import { MVideoPlaylist, MVideoPlaylistId, MVideoPlaylistOwner } from '../../types/models/video/video-playlist'
|
||||
import { FilteredModelAttributes } from '../../types/sequelize'
|
||||
import { createPlaylistMiniatureFromUrl } from '../thumbnail'
|
||||
import { getOrCreateActorAndServerAndModel } from './actor'
|
||||
import { crawlCollectionPage } from './crawl'
|
||||
import { getOrCreateAPVideo } from './videos'
|
||||
|
||||
function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
|
||||
const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
|
||||
? VideoPlaylistPrivacy.PUBLIC
|
||||
: VideoPlaylistPrivacy.UNLISTED
|
||||
|
||||
return {
|
||||
name: playlistObject.name,
|
||||
description: playlistObject.content,
|
||||
privacy,
|
||||
url: playlistObject.id,
|
||||
uuid: playlistObject.uuid,
|
||||
ownerAccountId: byAccount.id,
|
||||
videoChannelId: null,
|
||||
createdAt: new Date(playlistObject.published),
|
||||
updatedAt: new Date(playlistObject.updated)
|
||||
}
|
||||
}
|
||||
|
||||
function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) {
|
||||
return {
|
||||
position: elementObject.position,
|
||||
url: elementObject.id,
|
||||
startTimestamp: elementObject.startTimestamp || null,
|
||||
stopTimestamp: elementObject.stopTimestamp || null,
|
||||
videoPlaylistId: videoPlaylist.id,
|
||||
videoId: video.id
|
||||
}
|
||||
}
|
||||
|
||||
async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) {
|
||||
await Bluebird.map(playlistUrls, async playlistUrl => {
|
||||
try {
|
||||
const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
|
||||
if (exists === true) return
|
||||
|
||||
// Fetch url
|
||||
const { body } = await doJSONRequest<PlaylistObject>(playlistUrl, { activityPub: true })
|
||||
|
||||
if (!isPlaylistObjectValid(body)) {
|
||||
throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`)
|
||||
}
|
||||
|
||||
if (!isArray(body.to)) {
|
||||
throw new Error('Playlist does not have an audience.')
|
||||
}
|
||||
|
||||
return createOrUpdateVideoPlaylist(body, account, body.to)
|
||||
} catch (err) {
|
||||
logger.warn('Cannot add playlist element %s.', playlistUrl, { err })
|
||||
}
|
||||
}, { concurrency: CRAWL_REQUEST_CONCURRENCY })
|
||||
}
|
||||
|
||||
async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
|
||||
const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
|
||||
|
||||
if (isArray(playlistObject.attributedTo) && playlistObject.attributedTo.length === 1) {
|
||||
const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0])
|
||||
|
||||
if (actor.VideoChannel) {
|
||||
playlistAttributes.videoChannelId = actor.VideoChannel.id
|
||||
} else {
|
||||
logger.warn('Attributed to of video playlist %s is not a video channel.', playlistObject.id, { playlistObject })
|
||||
}
|
||||
}
|
||||
|
||||
const [ playlist ] = await VideoPlaylistModel.upsert<MVideoPlaylist>(playlistAttributes, { returning: true })
|
||||
|
||||
let accItems: string[] = []
|
||||
await crawlCollectionPage<string>(playlistObject.id, items => {
|
||||
accItems = accItems.concat(items)
|
||||
|
||||
return Promise.resolve()
|
||||
})
|
||||
|
||||
const refreshedPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(playlist.id, null)
|
||||
|
||||
if (playlistObject.icon) {
|
||||
try {
|
||||
const thumbnailModel = await createPlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist: refreshedPlaylist })
|
||||
await refreshedPlaylist.setAndSaveThumbnail(thumbnailModel, undefined)
|
||||
} catch (err) {
|
||||
logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err })
|
||||
}
|
||||
} else if (refreshedPlaylist.hasThumbnail()) {
|
||||
await refreshedPlaylist.Thumbnail.destroy()
|
||||
refreshedPlaylist.Thumbnail = null
|
||||
}
|
||||
|
||||
return resetVideoPlaylistElements(accItems, refreshedPlaylist)
|
||||
}
|
||||
|
||||
async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> {
|
||||
if (!videoPlaylist.isOutdated()) return videoPlaylist
|
||||
|
||||
try {
|
||||
const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
|
||||
|
||||
if (playlistObject === undefined) {
|
||||
logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url)
|
||||
|
||||
await videoPlaylist.setAsRefreshed()
|
||||
return videoPlaylist
|
||||
}
|
||||
|
||||
const byAccount = videoPlaylist.OwnerAccount
|
||||
await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
|
||||
|
||||
return videoPlaylist
|
||||
} catch (err) {
|
||||
if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
|
||||
logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
|
||||
|
||||
await videoPlaylist.destroy()
|
||||
return undefined
|
||||
}
|
||||
|
||||
logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err })
|
||||
|
||||
await videoPlaylist.setAsRefreshed()
|
||||
return videoPlaylist
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
createAccountPlaylists,
|
||||
playlistObjectToDBAttributes,
|
||||
playlistElementObjectToDBAttributes,
|
||||
createOrUpdateVideoPlaylist,
|
||||
refreshVideoPlaylistIfNeeded
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
|
||||
const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
|
||||
|
||||
await Bluebird.map(elementUrls, async elementUrl => {
|
||||
try {
|
||||
const { body } = await doJSONRequest<PlaylistElementObject>(elementUrl, { activityPub: true })
|
||||
|
||||
if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`)
|
||||
|
||||
if (checkUrlsSameHost(body.id, elementUrl) !== true) {
|
||||
throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
|
||||
}
|
||||
|
||||
const { video } = await getOrCreateAPVideo({ videoObject: { id: body.url }, fetchType: 'only-video' })
|
||||
|
||||
elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video))
|
||||
} catch (err) {
|
||||
logger.warn('Cannot add playlist element %s.', elementUrl, { err })
|
||||
}
|
||||
}, { concurrency: CRAWL_REQUEST_CONCURRENCY })
|
||||
|
||||
await sequelizeTypescript.transaction(async t => {
|
||||
await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
|
||||
|
||||
for (const element of elementsToCreate) {
|
||||
await VideoPlaylistElementModel.create(element, { transaction: t })
|
||||
}
|
||||
})
|
||||
|
||||
logger.info('Reset playlist %s with %s elements.', playlist.url, elementsToCreate.length)
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
|
||||
logger.info('Fetching remote playlist %s.', playlistUrl)
|
||||
|
||||
const { body, statusCode } = await doJSONRequest<any>(playlistUrl, { activityPub: true })
|
||||
|
||||
if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
|
||||
logger.debug('Remote video playlist JSON is not valid.', { body })
|
||||
return { statusCode, playlistObject: undefined }
|
||||
}
|
||||
|
||||
return { statusCode, playlistObject: body }
|
||||
}
|
||||
|
|
@ -0,0 +1,146 @@
|
|||
import { isArray } from '@server/helpers/custom-validators/misc'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger'
|
||||
import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants'
|
||||
import { sequelizeTypescript } from '@server/initializers/database'
|
||||
import { createPlaylistMiniatureFromUrl } from '@server/lib/thumbnail'
|
||||
import { VideoPlaylistModel } from '@server/models/video/video-playlist'
|
||||
import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
|
||||
import { FilteredModelAttributes } from '@server/types'
|
||||
import { MAccountDefault, MAccountId, MVideoPlaylist, MVideoPlaylistFull } from '@server/types/models'
|
||||
import { AttributesOnly } from '@shared/core-utils'
|
||||
import { PlaylistObject } from '@shared/models'
|
||||
import { getOrCreateActorAndServerAndModel } from '../actor'
|
||||
import { crawlCollectionPage } from '../crawl'
|
||||
import { getOrCreateAPVideo } from '../videos'
|
||||
import {
|
||||
fetchRemotePlaylistElement,
|
||||
fetchRemoteVideoPlaylist,
|
||||
playlistElementObjectToDBAttributes,
|
||||
playlistObjectToDBAttributes
|
||||
} from './shared'
|
||||
|
||||
import Bluebird = require('bluebird')
|
||||
|
||||
const lTags = loggerTagsFactory('ap', 'video-playlist')
|
||||
|
||||
async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) {
|
||||
await Bluebird.map(playlistUrls, async playlistUrl => {
|
||||
try {
|
||||
const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
|
||||
if (exists === true) return
|
||||
|
||||
const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl)
|
||||
|
||||
if (playlistObject === undefined) {
|
||||
throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`)
|
||||
}
|
||||
|
||||
return createOrUpdateVideoPlaylist(playlistObject, account, playlistObject.to)
|
||||
} catch (err) {
|
||||
logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) })
|
||||
}
|
||||
}, { concurrency: CRAWL_REQUEST_CONCURRENCY })
|
||||
}
|
||||
|
||||
async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
|
||||
const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
|
||||
|
||||
await setVideoChannelIfNeeded(playlistObject, playlistAttributes)
|
||||
|
||||
const [ upsertPlaylist ] = await VideoPlaylistModel.upsert<MVideoPlaylist>(playlistAttributes, { returning: true })
|
||||
|
||||
const playlistElementUrls = await fetchElementUrls(playlistObject)
|
||||
|
||||
// Refetch playlist from DB since elements fetching could be long in time
|
||||
const playlist = await VideoPlaylistModel.loadWithAccountAndChannel(upsertPlaylist.id, null)
|
||||
|
||||
try {
|
||||
await updatePlaylistThumbnail(playlistObject, playlist)
|
||||
} catch (err) {
|
||||
logger.warn('Cannot update thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) })
|
||||
}
|
||||
|
||||
return rebuildVideoPlaylistElements(playlistElementUrls, playlist)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
createAccountPlaylists,
|
||||
createOrUpdateVideoPlaylist
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function setVideoChannelIfNeeded (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly<VideoPlaylistModel>) {
|
||||
if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) return
|
||||
|
||||
const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0])
|
||||
|
||||
if (!actor.VideoChannel) {
|
||||
logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) })
|
||||
return
|
||||
}
|
||||
|
||||
playlistAttributes.videoChannelId = actor.VideoChannel.id
|
||||
}
|
||||
|
||||
async function fetchElementUrls (playlistObject: PlaylistObject) {
|
||||
let accItems: string[] = []
|
||||
await crawlCollectionPage<string>(playlistObject.id, items => {
|
||||
accItems = accItems.concat(items)
|
||||
|
||||
return Promise.resolve()
|
||||
})
|
||||
|
||||
return accItems
|
||||
}
|
||||
|
||||
async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) {
|
||||
if (playlistObject.icon) {
|
||||
const thumbnailModel = await createPlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist })
|
||||
await playlist.setAndSaveThumbnail(thumbnailModel, undefined)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Playlist does not have an icon, destroy existing one
|
||||
if (playlist.hasThumbnail()) {
|
||||
await playlist.Thumbnail.destroy()
|
||||
playlist.Thumbnail = null
|
||||
}
|
||||
}
|
||||
|
||||
async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
|
||||
const elementsToCreate = await buildElementsDBAttributes(elementUrls, playlist)
|
||||
|
||||
await sequelizeTypescript.transaction(async t => {
|
||||
await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
|
||||
|
||||
for (const element of elementsToCreate) {
|
||||
await VideoPlaylistElementModel.create(element, { transaction: t })
|
||||
}
|
||||
})
|
||||
|
||||
logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url))
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) {
|
||||
const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
|
||||
|
||||
await Bluebird.map(elementUrls, async elementUrl => {
|
||||
try {
|
||||
const { elementObject } = await fetchRemotePlaylistElement(elementUrl)
|
||||
|
||||
const { video } = await getOrCreateAPVideo({ videoObject: { id: elementObject.url }, fetchType: 'only-video' })
|
||||
|
||||
elementsToCreate.push(playlistElementObjectToDBAttributes(elementObject, playlist, video))
|
||||
} catch (err) {
|
||||
logger.warn('Cannot add playlist element %s.', elementUrl, { err, ...lTags(playlist.uuid, playlist.url) })
|
||||
}
|
||||
}, { concurrency: CRAWL_REQUEST_CONCURRENCY })
|
||||
|
||||
return elementsToCreate
|
||||
}
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
export * from './create-update'
|
||||
export * from './refresh'
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
import { logger, loggerTagsFactory } from '@server/helpers/logger'
|
||||
import { PeerTubeRequestError } from '@server/helpers/requests'
|
||||
import { MVideoPlaylistOwner } from '@server/types/models'
|
||||
import { HttpStatusCode } from '@shared/core-utils'
|
||||
import { createOrUpdateVideoPlaylist } from './create-update'
|
||||
import { fetchRemoteVideoPlaylist } from './shared'
|
||||
|
||||
async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> {
|
||||
if (!videoPlaylist.isOutdated()) return videoPlaylist
|
||||
|
||||
const lTags = loggerTagsFactory('ap', 'video-playlist', 'refresh', videoPlaylist.uuid, videoPlaylist.url)
|
||||
|
||||
try {
|
||||
const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
|
||||
|
||||
if (playlistObject === undefined) {
|
||||
logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url, lTags())
|
||||
|
||||
await videoPlaylist.setAsRefreshed()
|
||||
return videoPlaylist
|
||||
}
|
||||
|
||||
const byAccount = videoPlaylist.OwnerAccount
|
||||
await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
|
||||
|
||||
return videoPlaylist
|
||||
} catch (err) {
|
||||
if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
|
||||
logger.info('Cannot refresh not existing playlist %s. Deleting it.', videoPlaylist.url, lTags())
|
||||
|
||||
await videoPlaylist.destroy()
|
||||
return undefined
|
||||
}
|
||||
|
||||
logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err, ...lTags() })
|
||||
|
||||
await videoPlaylist.setAsRefreshed()
|
||||
return videoPlaylist
|
||||
}
|
||||
}
|
||||
|
||||
export {
|
||||
refreshVideoPlaylistIfNeeded
|
||||
}
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
export * from './object-to-model-attributes'
|
||||
export * from './url-to-object'
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
import { ACTIVITY_PUB } from '@server/initializers/constants'
|
||||
import { VideoPlaylistModel } from '@server/models/video/video-playlist'
|
||||
import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
|
||||
import { MAccountId, MVideoId, MVideoPlaylistId } from '@server/types/models'
|
||||
import { AttributesOnly } from '@shared/core-utils'
|
||||
import { PlaylistElementObject, PlaylistObject, VideoPlaylistPrivacy } from '@shared/models'
|
||||
|
||||
function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
|
||||
const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
|
||||
? VideoPlaylistPrivacy.PUBLIC
|
||||
: VideoPlaylistPrivacy.UNLISTED
|
||||
|
||||
return {
|
||||
name: playlistObject.name,
|
||||
description: playlistObject.content,
|
||||
privacy,
|
||||
url: playlistObject.id,
|
||||
uuid: playlistObject.uuid,
|
||||
ownerAccountId: byAccount.id,
|
||||
videoChannelId: null,
|
||||
createdAt: new Date(playlistObject.published),
|
||||
updatedAt: new Date(playlistObject.updated)
|
||||
} as AttributesOnly<VideoPlaylistModel>
|
||||
}
|
||||
|
||||
function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) {
|
||||
return {
|
||||
position: elementObject.position,
|
||||
url: elementObject.id,
|
||||
startTimestamp: elementObject.startTimestamp || null,
|
||||
stopTimestamp: elementObject.stopTimestamp || null,
|
||||
videoPlaylistId: videoPlaylist.id,
|
||||
videoId: video.id
|
||||
} as AttributesOnly<VideoPlaylistElementModel>
|
||||
}
|
||||
|
||||
export {
|
||||
playlistObjectToDBAttributes,
|
||||
playlistElementObjectToDBAttributes
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
import { isArray } from 'lodash'
|
||||
import { checkUrlsSameHost } from '@server/helpers/activitypub'
|
||||
import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '@server/helpers/custom-validators/activitypub/playlist'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger'
|
||||
import { doJSONRequest } from '@server/helpers/requests'
|
||||
import { PlaylistElementObject, PlaylistObject } from '@shared/models'
|
||||
|
||||
async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
|
||||
const lTags = loggerTagsFactory('ap', 'video-playlist', playlistUrl)
|
||||
|
||||
logger.info('Fetching remote playlist %s.', playlistUrl, lTags())
|
||||
|
||||
const { body, statusCode } = await doJSONRequest<any>(playlistUrl, { activityPub: true })
|
||||
|
||||
if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
|
||||
logger.debug('Remote video playlist JSON is not valid.', { body, ...lTags() })
|
||||
return { statusCode, playlistObject: undefined }
|
||||
}
|
||||
|
||||
if (!isArray(body.to)) {
|
||||
logger.debug('Remote video playlist JSON does not have a valid audience.', { body, ...lTags() })
|
||||
return { statusCode, playlistObject: undefined }
|
||||
}
|
||||
|
||||
return { statusCode, playlistObject: body }
|
||||
}
|
||||
|
||||
async function fetchRemotePlaylistElement (elementUrl: string): Promise<{ statusCode: number, elementObject: PlaylistElementObject }> {
|
||||
const lTags = loggerTagsFactory('ap', 'video-playlist', 'element', elementUrl)
|
||||
|
||||
logger.debug('Fetching remote playlist element %s.', elementUrl, lTags())
|
||||
|
||||
const { body, statusCode } = await doJSONRequest<PlaylistElementObject>(elementUrl, { activityPub: true })
|
||||
|
||||
if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in fetch playlist element ${elementUrl}`)
|
||||
|
||||
if (checkUrlsSameHost(body.id, elementUrl) !== true) {
|
||||
throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
|
||||
}
|
||||
|
||||
return { statusCode, elementObject: body }
|
||||
}
|
||||
|
||||
export {
|
||||
fetchRemoteVideoPlaylist,
|
||||
fetchRemotePlaylistElement
|
||||
}
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import { isBlockedByServerOrAccount } from '@server/lib/blocklist'
|
||||
import { isRedundancyAccepted } from '@server/lib/redundancy'
|
||||
import { ActivityCreate, CacheFileObject, VideoObject } from '../../../../shared'
|
||||
import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object'
|
||||
|
|
@ -9,11 +10,10 @@ import { APProcessorOptions } from '../../../types/activitypub-processor.model'
|
|||
import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models'
|
||||
import { Notifier } from '../../notifier'
|
||||
import { createOrUpdateCacheFile } from '../cache-file'
|
||||
import { createOrUpdateVideoPlaylist } from '../playlist'
|
||||
import { createOrUpdateVideoPlaylist } from '../playlists'
|
||||
import { forwardVideoRelatedActivity } from '../send/utils'
|
||||
import { resolveThread } from '../video-comments'
|
||||
import { getOrCreateAPVideo } from '../videos'
|
||||
import { isBlockedByServerOrAccount } from '@server/lib/blocklist'
|
||||
|
||||
async function processCreateActivity (options: APProcessorOptions<ActivityCreate>) {
|
||||
const { activity, byActor } = options
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import { APProcessorOptions } from '../../../types/activitypub-processor.model'
|
|||
import { MActorSignature } from '../../../types/models'
|
||||
import { getImageInfoIfExists, updateActorImageInstance, updateActorInstance } from '../actor'
|
||||
import { createOrUpdateCacheFile } from '../cache-file'
|
||||
import { createOrUpdateVideoPlaylist } from '../playlist'
|
||||
import { createOrUpdateVideoPlaylist } from '../playlists'
|
||||
import { forwardVideoRelatedActivity } from '../send/utils'
|
||||
import { APVideoUpdater, getOrCreateAPVideo } from '../videos'
|
||||
|
||||
|
|
|
|||
|
|
@ -40,23 +40,7 @@ async function changeVideoChannelShare (
|
|||
async function addVideoShares (shareUrls: string[], video: MVideoId) {
|
||||
await Bluebird.map(shareUrls, async shareUrl => {
|
||||
try {
|
||||
const { body } = await doJSONRequest<any>(shareUrl, { activityPub: true })
|
||||
if (!body || !body.actor) throw new Error('Body or body actor is invalid')
|
||||
|
||||
const actorUrl = getAPId(body.actor)
|
||||
if (checkUrlsSameHost(shareUrl, actorUrl) !== true) {
|
||||
throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`)
|
||||
}
|
||||
|
||||
const actor = await getOrCreateActorAndServerAndModel(actorUrl)
|
||||
|
||||
const entry = {
|
||||
actorId: actor.id,
|
||||
videoId: video.id,
|
||||
url: shareUrl
|
||||
}
|
||||
|
||||
await VideoShareModel.upsert(entry)
|
||||
await addVideoShare(shareUrl, video)
|
||||
} catch (err) {
|
||||
logger.warn('Cannot add share %s.', shareUrl, { err })
|
||||
}
|
||||
|
|
@ -71,6 +55,26 @@ export {
|
|||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function addVideoShare (shareUrl: string, video: MVideoId) {
|
||||
const { body } = await doJSONRequest<any>(shareUrl, { activityPub: true })
|
||||
if (!body || !body.actor) throw new Error('Body or body actor is invalid')
|
||||
|
||||
const actorUrl = getAPId(body.actor)
|
||||
if (checkUrlsSameHost(shareUrl, actorUrl) !== true) {
|
||||
throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`)
|
||||
}
|
||||
|
||||
const actor = await getOrCreateActorAndServerAndModel(actorUrl)
|
||||
|
||||
const entry = {
|
||||
actorId: actor.id,
|
||||
videoId: video.id,
|
||||
url: shareUrl
|
||||
}
|
||||
|
||||
await VideoShareModel.upsert(entry)
|
||||
}
|
||||
|
||||
async function shareByServer (video: MVideo, t: Transaction) {
|
||||
const serverActor = await getServerActor()
|
||||
|
||||
|
|
|
|||
|
|
@ -29,10 +29,11 @@ async function addVideoComments (commentUrls: string[]) {
|
|||
|
||||
async function resolveThread (params: ResolveThreadParams): ResolveThreadResult {
|
||||
const { url, isVideo } = params
|
||||
|
||||
if (params.commentCreated === undefined) params.commentCreated = false
|
||||
if (params.comments === undefined) params.comments = []
|
||||
|
||||
// If it is not a video, or if we don't know if it's a video
|
||||
// If it is not a video, or if we don't know if it's a video, try to get the thread from DB
|
||||
if (isVideo === false || isVideo === undefined) {
|
||||
const result = await resolveCommentFromDB(params)
|
||||
if (result) return result
|
||||
|
|
@ -42,7 +43,7 @@ async function resolveThread (params: ResolveThreadParams): ResolveThreadResult
|
|||
// If it is a video, or if we don't know if it's a video
|
||||
if (isVideo === true || isVideo === undefined) {
|
||||
// Keep await so we catch the exception
|
||||
return await tryResolveThreadFromVideo(params)
|
||||
return await tryToResolveThreadFromVideo(params)
|
||||
}
|
||||
} catch (err) {
|
||||
logger.debug('Cannot resolve thread from video %s, maybe because it was not a video', url, { err })
|
||||
|
|
@ -62,28 +63,26 @@ async function resolveCommentFromDB (params: ResolveThreadParams) {
|
|||
const { url, comments, commentCreated } = params
|
||||
|
||||
const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideoUrlAndAccount(url)
|
||||
if (commentFromDatabase) {
|
||||
let parentComments = comments.concat([ commentFromDatabase ])
|
||||
if (!commentFromDatabase) return undefined
|
||||
|
||||
// Speed up things and resolve directly the thread
|
||||
if (commentFromDatabase.InReplyToVideoComment) {
|
||||
const data = await VideoCommentModel.listThreadParentComments(commentFromDatabase, undefined, 'DESC')
|
||||
let parentComments = comments.concat([ commentFromDatabase ])
|
||||
|
||||
parentComments = parentComments.concat(data)
|
||||
}
|
||||
// Speed up things and resolve directly the thread
|
||||
if (commentFromDatabase.InReplyToVideoComment) {
|
||||
const data = await VideoCommentModel.listThreadParentComments(commentFromDatabase, undefined, 'DESC')
|
||||
|
||||
return resolveThread({
|
||||
url: commentFromDatabase.Video.url,
|
||||
comments: parentComments,
|
||||
isVideo: true,
|
||||
commentCreated
|
||||
})
|
||||
parentComments = parentComments.concat(data)
|
||||
}
|
||||
|
||||
return undefined
|
||||
return resolveThread({
|
||||
url: commentFromDatabase.Video.url,
|
||||
comments: parentComments,
|
||||
isVideo: true,
|
||||
commentCreated
|
||||
})
|
||||
}
|
||||
|
||||
async function tryResolveThreadFromVideo (params: ResolveThreadParams) {
|
||||
async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {
|
||||
const { url, comments, commentCreated } = params
|
||||
|
||||
// Maybe it's a reply to a video?
|
||||
|
|
|
|||
|
|
@ -15,30 +15,7 @@ import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlBy
|
|||
async function createRates (ratesUrl: string[], video: MVideo, rate: VideoRateType) {
|
||||
await Bluebird.map(ratesUrl, async rateUrl => {
|
||||
try {
|
||||
// Fetch url
|
||||
const { body } = await doJSONRequest<any>(rateUrl, { activityPub: true })
|
||||
if (!body || !body.actor) throw new Error('Body or body actor is invalid')
|
||||
|
||||
const actorUrl = getAPId(body.actor)
|
||||
if (checkUrlsSameHost(actorUrl, rateUrl) !== true) {
|
||||
throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`)
|
||||
}
|
||||
|
||||
if (checkUrlsSameHost(body.id, rateUrl) !== true) {
|
||||
throw new Error(`Rate url ${rateUrl} host is different from the AP object id ${body.id}`)
|
||||
}
|
||||
|
||||
const actor = await getOrCreateActorAndServerAndModel(actorUrl)
|
||||
|
||||
const entry = {
|
||||
videoId: video.id,
|
||||
accountId: actor.Account.id,
|
||||
type: rate,
|
||||
url: body.id
|
||||
}
|
||||
|
||||
// Video "likes"/"dislikes" will be updated by the caller
|
||||
await AccountVideoRateModel.upsert(entry)
|
||||
await createRate(rateUrl, video, rate)
|
||||
} catch (err) {
|
||||
logger.warn('Cannot add rate %s.', rateUrl, { err })
|
||||
}
|
||||
|
|
@ -73,8 +50,39 @@ function getLocalRateUrl (rateType: VideoRateType, actor: MActorUrl, video: MVid
|
|||
: getVideoDislikeActivityPubUrlByLocalActor(actor, video)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
getLocalRateUrl,
|
||||
createRates,
|
||||
sendVideoRateChange
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function createRate (rateUrl: string, video: MVideo, rate: VideoRateType) {
|
||||
// Fetch url
|
||||
const { body } = await doJSONRequest<any>(rateUrl, { activityPub: true })
|
||||
if (!body || !body.actor) throw new Error('Body or body actor is invalid')
|
||||
|
||||
const actorUrl = getAPId(body.actor)
|
||||
if (checkUrlsSameHost(actorUrl, rateUrl) !== true) {
|
||||
throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`)
|
||||
}
|
||||
|
||||
if (checkUrlsSameHost(body.id, rateUrl) !== true) {
|
||||
throw new Error(`Rate url ${rateUrl} host is different from the AP object id ${body.id}`)
|
||||
}
|
||||
|
||||
const actor = await getOrCreateActorAndServerAndModel(actorUrl)
|
||||
|
||||
const entry = {
|
||||
videoId: video.id,
|
||||
accountId: actor.Account.id,
|
||||
type: rate,
|
||||
url: body.id
|
||||
}
|
||||
|
||||
// Video "likes"/"dislikes" will be updated by the caller
|
||||
await AccountVideoRateModel.upsert(entry)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import { VideoCommentModel } from '../../../models/video/video-comment'
|
|||
import { VideoShareModel } from '../../../models/video/video-share'
|
||||
import { MAccountDefault, MVideoFullLight } from '../../../types/models'
|
||||
import { crawlCollectionPage } from '../../activitypub/crawl'
|
||||
import { createAccountPlaylists } from '../../activitypub/playlist'
|
||||
import { createAccountPlaylists } from '../../activitypub/playlists'
|
||||
import { processActivities } from '../../activitypub/process'
|
||||
import { addVideoShares } from '../../activitypub/share'
|
||||
import { addVideoComments } from '../../activitypub/video-comments'
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import * as Bull from 'bull'
|
||||
import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlist'
|
||||
import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlists'
|
||||
import { refreshVideoIfNeeded } from '@server/lib/activitypub/videos'
|
||||
import { RefreshPayload } from '@shared/models'
|
||||
import { logger } from '../../../helpers/logger'
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import {
|
|||
UpdatedAt
|
||||
} from 'sequelize-typescript'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import { setAsUpdated } from '@server/helpers/database-utils'
|
||||
import { MAccountId, MChannelId } from '@server/types/models'
|
||||
import { AttributesOnly } from '@shared/core-utils'
|
||||
import { ActivityIconObject } from '../../../shared/models/activitypub/objects'
|
||||
|
|
@ -531,9 +532,7 @@ export class VideoPlaylistModel extends Model<Partial<AttributesOnly<VideoPlayli
|
|||
}
|
||||
|
||||
setAsRefreshed () {
|
||||
this.changed('updatedAt', true)
|
||||
|
||||
return this.save()
|
||||
return setAsUpdated('videoPlaylist', this.id)
|
||||
}
|
||||
|
||||
isOwned () {
|
||||
|
|
|
|||
|
|
@ -2,5 +2,9 @@ export * from './abuse-object'
|
|||
export * from './cache-file-object'
|
||||
export * from './common-objects'
|
||||
export * from './dislike-object'
|
||||
export * from './object.model'
|
||||
export * from './playlist-element-object'
|
||||
export * from './playlist-object'
|
||||
export * from './video-comment-object'
|
||||
export * from './video-torrent-object'
|
||||
export * from './view-object'
|
||||
|
|
|
|||
Loading…
Reference in New Issue