Support chapter import/export

pull/6266/head
Chocobozzz 2024-02-13 14:23:32 +01:00 committed by Chocobozzz
parent 967702d6c7
commit 7986ab8452
14 changed files with 651 additions and 446 deletions

View File

@ -6,6 +6,7 @@ import {
ActivityTagObject,
ActivityUrlObject
} from './common-objects.js'
import { VideoChapterObject } from './video-chapters-object.js'
export interface VideoObject {
type: 'Video'
@ -51,7 +52,7 @@ export interface VideoObject {
dislikes: string
shares: string
comments: string
hasParts: string
hasParts: string | VideoChapterObject[]
attributedTo: ActivityPubAttributedTo[]

View File

@ -70,6 +70,11 @@ export interface VideoExportJSON {
fileUrl: string
}[]
chapters: {
timecode: number
title: string
}[]
files: VideoFileExportJSON[]
streamingPlaylists: {

View File

@ -20,9 +20,11 @@ import {
FollowingExportJSON,
HttpStatusCode,
LikesExportJSON,
LiveVideoLatencyMode,
UserExportState,
UserNotificationSettingValue,
UserSettingsExportJSON,
VideoChapterObject,
VideoCommentObject,
VideoCreateResult,
VideoExportJSON, VideoPlaylistCreateResult,
@ -59,6 +61,7 @@ function runTest (withObjectStorage: boolean) {
let externalVideo: VideoCreateResult
let noahPrivateVideo: VideoCreateResult
let noahVideo: VideoCreateResult
let noahLive: VideoCreateResult
let mouskaVideo: VideoCreateResult
let noahPlaylist: VideoPlaylistCreateResult
@ -81,6 +84,7 @@ function runTest (withObjectStorage: boolean) {
noahPrivateVideo,
mouskaVideo,
noahVideo,
noahLive,
noahToken,
server,
remoteServer
@ -249,29 +253,58 @@ function runTest (withObjectStorage: boolean) {
expect(outbox.type).to.equal('OrderedCollection')
// 3 videos and 2 comments
expect(outbox.totalItems).to.equal(5)
expect(outbox.orderedItems).to.have.lengthOf(5)
expect(outbox.totalItems).to.equal(6)
expect(outbox.orderedItems).to.have.lengthOf(6)
expect(outbox.orderedItems.filter(i => i.object.type === 'Video')).to.have.lengthOf(3)
expect(outbox.orderedItems.filter(i => i.object.type === 'Video')).to.have.lengthOf(4)
expect(outbox.orderedItems.filter(i => i.object.type === 'Note')).to.have.lengthOf(2)
const { object: video } = findVideoObjectInOutbox(outbox, 'noah public video')
{
const { object: video } = findVideoObjectInOutbox(outbox, 'noah public video')
// Thumbnail
expect(video.icon).to.have.lengthOf(1)
expect(video.icon[0].url).to.equal('../files/videos/thumbnails/' + noahVideo.uuid + '.jpg')
// Thumbnail
expect(video.icon).to.have.lengthOf(1)
expect(video.icon[0].url).to.equal('../files/videos/thumbnails/' + noahVideo.uuid + '.jpg')
await checkFileExistsInZIP(zip, video.icon[0].url, '/activity-pub')
await checkFileExistsInZIP(zip, video.icon[0].url, '/activity-pub')
// Subtitles
expect(video.subtitleLanguage).to.have.lengthOf(2)
for (const subtitle of video.subtitleLanguage) {
await checkFileExistsInZIP(zip, subtitle.url, '/activity-pub')
// Subtitles
expect(video.subtitleLanguage).to.have.lengthOf(2)
for (const subtitle of video.subtitleLanguage) {
await checkFileExistsInZIP(zip, subtitle.url, '/activity-pub')
}
// Chapters
expect(video.hasParts).to.have.lengthOf(2)
const chapters = video.hasParts as VideoChapterObject[]
expect(chapters[0].name).to.equal('chapter 1')
expect(chapters[0].startOffset).to.equal(1)
expect(chapters[0].endOffset).to.equal(3)
expect(chapters[1].name).to.equal('chapter 2')
expect(chapters[1].startOffset).to.equal(3)
expect(chapters[1].endOffset).to.equal(5)
// Video file
expect(video.attachment).to.have.lengthOf(1)
expect(video.attachment[0].url).to.equal('../files/videos/video-files/' + noahVideo.uuid + '.webm')
await checkFileExistsInZIP(zip, video.attachment[0].url, '/activity-pub')
}
expect(video.attachment).to.have.lengthOf(1)
expect(video.attachment[0].url).to.equal('../files/videos/video-files/' + noahVideo.uuid + '.webm')
await checkFileExistsInZIP(zip, video.attachment[0].url, '/activity-pub')
{
const { object: live } = findVideoObjectInOutbox(outbox, 'noah live video')
expect(live.isLiveBroadcast).to.be.true
// Thumbnail
expect(live.icon).to.have.lengthOf(1)
expect(live.icon[0].url).to.equal('../files/videos/thumbnails/' + noahLive.uuid + '.jpg')
await checkFileExistsInZIP(zip, live.icon[0].url, '/activity-pub')
expect(live.subtitleLanguage).to.have.lengthOf(0)
expect(live.attachment).to.not.exist
}
}
})
@ -438,7 +471,7 @@ function runTest (withObjectStorage: boolean) {
{
const json = await parseZIPJSONFile<VideoExportJSON>(zip, 'peertube/videos.json')
expect(json.videos).to.have.lengthOf(3)
expect(json.videos).to.have.lengthOf(4)
{
const privateVideo = json.videos.find(v => v.name === 'noah private video')
@ -460,6 +493,8 @@ function runTest (withObjectStorage: boolean) {
expect(publicVideo.files).to.have.lengthOf(1)
expect(publicVideo.streamingPlaylists).to.have.lengthOf(0)
expect(publicVideo.chapters).to.have.lengthOf(2)
expect(publicVideo.captions).to.have.lengthOf(2)
expect(publicVideo.captions.find(c => c.language === 'ar')).to.exist
@ -476,6 +511,32 @@ function runTest (withObjectStorage: boolean) {
}
}
{
const liveVideo = json.videos.find(v => v.name === 'noah live video')
expect(liveVideo).to.exist
expect(liveVideo.isLive).to.be.true
expect(liveVideo.live.latencyMode).to.equal(LiveVideoLatencyMode.SMALL_LATENCY)
expect(liveVideo.live.saveReplay).to.be.true
expect(liveVideo.live.permanentLive).to.be.true
expect(liveVideo.live.streamKey).to.exist
expect(liveVideo.live.replaySettings.privacy).to.equal(VideoPrivacy.PUBLIC)
expect(liveVideo.channel.name).to.equal('noah_second_channel')
expect(liveVideo.privacy).to.equal(VideoPrivacy.PASSWORD_PROTECTED)
expect(liveVideo.passwords).to.deep.equal([ 'password1' ])
expect(liveVideo.duration).to.equal(0)
expect(liveVideo.captions).to.have.lengthOf(0)
expect(liveVideo.files).to.have.lengthOf(0)
expect(liveVideo.streamingPlaylists).to.have.lengthOf(0)
expect(liveVideo.source).to.not.exist
expect(liveVideo.archiveFiles.captions).to.deep.equal({})
expect(liveVideo.archiveFiles.thumbnail).to.exist
expect(liveVideo.archiveFiles.videoFile).to.not.exist
}
{
const secondaryChannelVideo = json.videos.find(v => v.name === 'noah public video second channel')
expect(secondaryChannelVideo.channel.name).to.equal('noah_second_channel')
@ -513,7 +574,7 @@ function runTest (withObjectStorage: boolean) {
{
const videoThumbnails = files.filter(f => f.startsWith('files/videos/thumbnails/'))
expect(videoThumbnails).to.have.lengthOf(3)
expect(videoThumbnails).to.have.lengthOf(4)
const videoFiles = files.filter(f => f.startsWith('files/videos/video-files/'))
expect(videoFiles).to.have.lengthOf(3)
@ -620,9 +681,9 @@ function runTest (withObjectStorage: boolean) {
expect(json.videos).to.have.lengthOf(1)
const video = json.videos[0]
expect(video.files).to.have.lengthOf(4)
expect(video.files).to.have.lengthOf(2)
expect(video.streamingPlaylists).to.have.lengthOf(1)
expect(video.streamingPlaylists[0].files).to.have.lengthOf(4)
expect(video.streamingPlaylists[0].files).to.have.lengthOf(2)
}
{

View File

@ -8,6 +8,7 @@ import {
} from '@peertube/peertube-server-commands'
import {
HttpStatusCode,
LiveVideoLatencyMode,
UserImportState,
UserNotificationSettingValue,
VideoCreateResult,
@ -327,7 +328,7 @@ function runTest (withObjectStorage: boolean) {
it('Should have correctly imported user videos', async function () {
const { data } = await remoteServer.videos.listMyVideos({ token: remoteNoahToken })
expect(data).to.have.lengthOf(4)
expect(data).to.have.lengthOf(5)
{
const privateVideo = data.find(v => v.name === 'noah private video')
@ -425,6 +426,29 @@ function runTest (withObjectStorage: boolean) {
const source = await remoteServer.videos.getSource({ id: otherVideo.uuid })
expect(source.filename).to.equal('video_short.webm')
}
{
const liveVideo = data.find(v => v.name === 'noah live video')
expect(liveVideo).to.exist
await remoteServer.videos.get({ id: liveVideo.uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
const video = await remoteServer.videos.getWithPassword({ id: liveVideo.uuid, password: 'password1' })
const live = await remoteServer.live.get({ videoId: liveVideo.uuid, token: remoteNoahToken })
expect(video.isLive).to.be.true
expect(live.latencyMode).to.equal(LiveVideoLatencyMode.SMALL_LATENCY)
expect(live.saveReplay).to.be.true
expect(live.permanentLive).to.be.true
expect(live.streamKey).to.exist
expect(live.replaySettings.privacy).to.equal(VideoPrivacy.PUBLIC)
expect(video.channel.name).to.equal('noah_second_channel')
expect(video.privacy.id).to.equal(VideoPrivacy.PASSWORD_PROTECTED)
expect(video.duration).to.equal(0)
expect(video.files).to.have.lengthOf(0)
expect(video.streamingPlaylists).to.have.lengthOf(0)
}
})
it('Should re-import the same file', async function () {
@ -494,7 +518,7 @@ function runTest (withObjectStorage: boolean) {
// Videos
{
const { data } = await remoteServer.videos.listMyVideos({ token: remoteNoahToken })
expect(data).to.have.lengthOf(4)
expect(data).to.have.lengthOf(5)
}
})
@ -505,7 +529,7 @@ function runTest (withObjectStorage: boolean) {
})
expect(email).to.exist
expect(email['text']).to.contain('as considered duplicate: 4') // 4 videos are considered as duplicates
expect(email['text']).to.contain('as considered duplicate: 5') // 5 videos are considered as duplicates
})
it('Should auto blacklist imported videos if enabled by the administrator', async function () {
@ -519,7 +543,7 @@ function runTest (withObjectStorage: boolean) {
{
const { data } = await blockedServer.videos.listMyVideos({ token })
expect(data).to.have.lengthOf(4)
expect(data).to.have.lengthOf(5)
for (const video of data) {
expect(video.blacklisted).to.be.true

View File

@ -3,6 +3,7 @@ import {
ActivityCreate,
ActivityPubOrderedCollection,
HttpStatusCode,
LiveVideoLatencyMode,
UserExport,
UserNotificationSettingValue,
VideoCommentObject,
@ -218,6 +219,15 @@ export async function prepareImportExportTests (options: {
await server.captions.add({ language: 'ar', videoId: noahVideo.uuid, fixture: 'subtitle-good1.vtt' })
await server.captions.add({ language: 'fr', videoId: noahVideo.uuid, fixture: 'subtitle-good1.vtt' })
// Chapters
await server.chapters.update({
videoId: noahVideo.uuid,
chapters: [
{ timecode: 1, title: 'chapter 1' },
{ timecode: 3, title: 'chapter 2' }
]
})
// My settings
await server.users.updateMe({ token: noahToken, description: 'super noah description', p2pEnabled: false })
@ -275,6 +285,26 @@ export async function prepareImportExportTests (options: {
const remoteRootId = (await remoteServer.users.getMyInfo()).id
const remoteNoahId = (await remoteServer.users.getMyInfo({ token: remoteNoahToken })).id
// Lives
await server.config.enableMinimumTranscoding()
await server.config.enableLive({ allowReplay: true })
const noahLive = await server.live.create({
fields: {
permanentLive: true,
saveReplay: true,
latencyMode: LiveVideoLatencyMode.SMALL_LATENCY,
replaySettings: {
privacy: VideoPrivacy.PUBLIC
},
videoPasswords: [ 'password1' ],
channelId: noahSecondChannelId,
name: 'noah live video',
privacy: VideoPrivacy.PASSWORD_PROTECTED
},
token: noahToken
})
return {
rootId,
@ -292,6 +322,7 @@ export async function prepareImportExportTests (options: {
noahPlaylist,
noahPrivateVideo,
noahVideo,
noahLive,
server,
remoteServer,

View File

@ -1,7 +1,6 @@
import cors from 'cors'
import express from 'express'
import {
VideoChapterObject,
VideoChaptersObject,
VideoCommentObject,
VideoPlaylistPrivacy,
@ -57,6 +56,7 @@ import { VideoShareModel } from '../../models/video/video-share.js'
import { activityPubResponse } from './utils.js'
import { VideoChapterModel } from '@server/models/video/video-chapter.js'
import { InternalEventEmitter } from '@server/lib/internal-event-emitter.js'
import { buildChaptersAPHasPart } from '@server/lib/activitypub/video-chapters.js'
const activityPubClientRouter = express.Router()
activityPubClientRouter.use(cors())
@ -433,19 +433,9 @@ async function videoChaptersController (req: express.Request, res: express.Respo
const chapters = await VideoChapterModel.listChaptersOfVideo(video.id)
const hasPart: VideoChapterObject[] = []
if (chapters.length !== 0) {
for (let i = 0; i < chapters.length - 1; i++) {
hasPart.push(chapters[i].toActivityPubJSON({ video, nextChapter: chapters[i + 1] }))
}
hasPart.push(chapters[chapters.length - 1].toActivityPubJSON({ video: res.locals.onlyVideo, nextChapter: null }))
}
const chaptersObject: VideoChaptersObject = {
id: getLocalVideoChaptersActivityPubUrl(video),
hasPart
hasPart: buildChaptersAPHasPart(video, chapters)
}
return activityPubResponse(activityPubContextify(chaptersObject, 'Chapters', getContextFilter()), res)

View File

@ -2,20 +2,17 @@ import express from 'express'
import {
HttpStatusCode,
LiveVideoCreate,
LiveVideoLatencyMode,
LiveVideoUpdate,
ThumbnailType,
UserRight,
VideoPrivacy,
VideoState
} from '@peertube/peertube-models'
import { exists } from '@server/helpers/custom-validators/misc.js'
import { createReqFiles } from '@server/helpers/express-utils.js'
import { getFormattedObjects } from '@server/helpers/utils.js'
import { ASSETS_PATH, MIMETYPES } from '@server/initializers/constants.js'
import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url.js'
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
import { Hooks } from '@server/lib/plugins/hooks.js'
import { buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video.js'
import {
videoLiveAddValidator,
videoLiveFindReplaySessionValidator,
@ -25,15 +22,14 @@ import {
} from '@server/middlewares/validators/videos/video-live.js'
import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting.js'
import { VideoLiveSessionModel } from '@server/models/video/video-live-session.js'
import { VideoLiveModel } from '@server/models/video/video-live.js'
import { VideoPasswordModel } from '@server/models/video/video-password.js'
import { MVideoDetails, MVideoFullLight, MVideoLive } from '@server/types/models/index.js'
import { buildUUID, uuidToShort } from '@peertube/peertube-node-utils'
import { logger } from '../../../helpers/logger.js'
import { sequelizeTypescript } from '../../../initializers/database.js'
import { updateLocalVideoMiniatureFromExisting } from '../../../lib/thumbnail.js'
import { MVideoLive } from '@server/types/models/index.js'
import { uuidToShort } from '@peertube/peertube-node-utils'
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, optionalAuthenticate } from '../../../middlewares/index.js'
import { VideoModel } from '../../../models/video/video.js'
import { LocalVideoCreator } from '@server/lib/local-video-creator.js'
import { pick } from '@peertube/peertube-core-utils'
const lTags = loggerTagsFactory('api', 'live')
const liveRouter = express.Router()
@ -153,80 +149,59 @@ async function updateReplaySettings (videoLive: MVideoLive, body: LiveVideoUpdat
async function addLiveVideo (req: express.Request, res: express.Response) {
const videoInfo: LiveVideoCreate = req.body
// Prepare data so we don't block the transaction
let videoData = buildLocalVideoFromReq(videoInfo, res.locals.videoChannel.id)
videoData = await Hooks.wrapObject(videoData, 'filter:api.video.live.video-attribute.result')
const thumbnails = [ { type: ThumbnailType.MINIATURE, field: 'thumbnailfile' }, { type: ThumbnailType.PREVIEW, field: 'previewfile' } ]
.map(({ type, field }) => {
if (req.files?.[field]?.[0]) {
return {
path: req.files[field][0].path,
type,
automaticallyGenerated: false,
keepOriginal: false
}
}
videoData.isLive = true
videoData.state = VideoState.WAITING_FOR_LIVE
videoData.duration = 0
const video = new VideoModel(videoData) as MVideoDetails
video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object
const videoLive = new VideoLiveModel()
videoLive.saveReplay = videoInfo.saveReplay || false
videoLive.permanentLive = videoInfo.permanentLive || false
videoLive.latencyMode = videoInfo.latencyMode || LiveVideoLatencyMode.DEFAULT
videoLive.streamKey = buildUUID()
const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
video,
files: req.files,
fallback: type => {
return updateLocalVideoMiniatureFromExisting({
inputPath: ASSETS_PATH.DEFAULT_LIVE_BACKGROUND,
video,
return {
path: ASSETS_PATH.DEFAULT_LIVE_BACKGROUND,
type,
automaticallyGenerated: true,
keepOriginal: true
})
}
}
})
const localVideoCreator = new LocalVideoCreator({
channel: res.locals.videoChannel,
chapters: undefined,
fallbackChapters: {
fromDescription: false,
finalFallback: undefined
},
liveAttributes: pick(videoInfo, [ 'saveReplay', 'permanentLive', 'latencyMode', 'replaySettings' ]),
videoAttributeResultHook: 'filter:api.video.live.video-attribute.result',
lTags,
videoAttributes: {
...videoInfo,
duration: 0,
state: VideoState.WAITING_FOR_LIVE,
isLive: true,
filename: null
},
videoFilePath: undefined,
user: res.locals.oauth.token.User,
thumbnails
})
const { videoCreated } = await sequelizeTypescript.transaction(async t => {
const sequelizeOptions = { transaction: t }
const { video } = await localVideoCreator.create()
const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight
logger.info('Video live %s with uuid %s created.', videoInfo.name, video.uuid, lTags())
if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
// Do not forget to add video channel information to the created video
videoCreated.VideoChannel = res.locals.videoChannel
if (videoLive.saveReplay) {
const replaySettings = new VideoLiveReplaySettingModel({
privacy: videoInfo.replaySettings?.privacy ?? videoCreated.privacy
})
await replaySettings.save(sequelizeOptions)
videoLive.replaySettingId = replaySettings.id
}
videoLive.videoId = videoCreated.id
videoCreated.VideoLive = await videoLive.save(sequelizeOptions)
await setVideoTags({ video, tags: videoInfo.tags, transaction: t })
await federateVideoIfNeeded(videoCreated, true, t)
if (videoInfo.privacy === VideoPrivacy.PASSWORD_PROTECTED) {
await VideoPasswordModel.addPasswords(videoInfo.videoPasswords, video.id, t)
}
logger.info('Video live %s with uuid %s created.', videoInfo.name, videoCreated.uuid)
return { videoCreated }
})
Hooks.runAction('action:api.live-video.created', { video: videoCreated, req, res })
Hooks.runAction('action:api.live-video.created', { video, req, res })
return res.json({
video: {
id: videoCreated.id,
shortUUID: uuidToShort(videoCreated.uuid),
uuid: videoCreated.uuid
id: video.id,
shortUUID: uuidToShort(video.uuid),
uuid: video.uuid
}
})
}

View File

@ -1,16 +1,16 @@
import express from 'express'
import express, { UploadFiles } from 'express'
import { Transaction } from 'sequelize'
import { forceNumber } from '@peertube/peertube-core-utils'
import { HttpStatusCode, VideoPrivacy, VideoPrivacyType, VideoUpdate } from '@peertube/peertube-models'
import { HttpStatusCode, ThumbnailType, VideoPrivacy, VideoPrivacyType, VideoUpdate } from '@peertube/peertube-models'
import { exists } from '@server/helpers/custom-validators/misc.js'
import { changeVideoChannelShare } from '@server/lib/activitypub/share.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { setVideoPrivacy } from '@server/lib/video-privacy.js'
import { buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video.js'
import { setVideoTags } from '@server/lib/video.js'
import { openapiOperationDoc } from '@server/middlewares/doc.js'
import { VideoPasswordModel } from '@server/models/video/video-password.js'
import { FilteredModelAttributes } from '@server/types/index.js'
import { MVideoFullLight } from '@server/types/models/index.js'
import { MVideoFullLight, MVideoThumbnail } from '@server/types/models/index.js'
import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger.js'
import { resetSequelizeInstance } from '../../../helpers/database-utils.js'
import { createReqFiles } from '../../../helpers/express-utils.js'
@ -24,6 +24,7 @@ import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-u
import { VideoModel } from '../../../models/video/video.js'
import { replaceChaptersFromDescriptionIfNeeded } from '@server/lib/video-chapters.js'
import { addVideoJobsAfterUpdate } from '@server/lib/video-jobs.js'
import { updateLocalVideoMiniatureFromExisting } from '@server/lib/thumbnail.js'
const lTags = loggerTagsFactory('api', 'video')
const auditLogger = auditLoggerFactory('videos')
@ -55,13 +56,7 @@ async function updateVideo (req: express.Request, res: express.Response) {
const hadPrivacyForFederation = videoFromReq.hasPrivacyForFederation()
const oldPrivacy = videoFromReq.privacy
const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
video: videoFromReq,
files: req.files,
fallback: () => Promise.resolve(undefined),
automaticallyGenerated: false
})
const thumbnails = await buildVideoThumbnailsFromReq(videoFromReq, req.files)
const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(videoFromReq.uuid)
try {
@ -115,8 +110,9 @@ async function updateVideo (req: express.Request, res: express.Response) {
const videoInstanceUpdated = await video.save({ transaction: t }) as MVideoFullLight
// Thumbnail & preview updates?
if (thumbnailModel) await videoInstanceUpdated.addAndSaveThumbnail(thumbnailModel, t)
if (previewModel) await videoInstanceUpdated.addAndSaveThumbnail(previewModel, t)
for (const thumbnail of thumbnails) {
await videoInstanceUpdated.addAndSaveThumbnail(thumbnail, t)
}
// Video tags update?
if (videoInfoToUpdate.tags !== undefined) {
@ -229,3 +225,30 @@ function updateSchedule (videoInstance: MVideoFullLight, videoInfoToUpdate: Vide
return ScheduleVideoUpdateModel.deleteByVideoId(videoInstance.id, transaction)
}
}
async function buildVideoThumbnailsFromReq (video: MVideoThumbnail, files: UploadFiles) {
const promises = [
{
type: ThumbnailType.MINIATURE,
fieldName: 'thumbnailfile'
},
{
type: ThumbnailType.PREVIEW,
fieldName: 'previewfile'
}
].map(p => {
const fields = files?.[p.fieldName]
if (!fields) return undefined
return updateLocalVideoMiniatureFromExisting({
inputPath: fields[0].path,
video,
type: p.type,
automaticallyGenerated: false
})
})
const thumbnailsOrUndefined = await Promise.all(promises)
return thumbnailsOrUndefined.filter(t => !!t)
}

View File

@ -1,31 +1,16 @@
import express, { UploadFiles } from 'express'
import { move } from 'fs-extra/esm'
import { basename } from 'path'
import express from 'express'
import { getResumableUploadPath } from '@server/helpers/upload.js'
import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url.js'
import { Redis } from '@server/lib/redis.js'
import { uploadx } from '@server/lib/uploadx.js'
import {
buildLocalVideoFromReq, buildVideoThumbnailsFromReq,
setVideoTags
} from '@server/lib/video.js'
import { buildNewFile } from '@server/lib/video-file.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { buildNextVideoState } from '@server/lib/video-state.js'
import { openapiOperationDoc } from '@server/middlewares/doc.js'
import { VideoPasswordModel } from '@server/models/video/video-password.js'
import { VideoSourceModel } from '@server/models/video/video-source.js'
import { MVideoFile, MVideoFullLight, MVideoThumbnail } from '@server/types/models/index.js'
import { uuidToShort } from '@peertube/peertube-node-utils'
import { HttpStatusCode, ThumbnailType, VideoCreate, VideoPrivacy } from '@peertube/peertube-models'
import { HttpStatusCode, ThumbnailType, VideoCreate } from '@peertube/peertube-models'
import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger.js'
import { createReqFiles } from '../../../helpers/express-utils.js'
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../../initializers/constants.js'
import { sequelizeTypescript } from '../../../initializers/database.js'
import { Hooks } from '../../../lib/plugins/hooks.js'
import { generateLocalVideoMiniature } from '../../../lib/thumbnail.js'
import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist.js'
import {
asyncMiddleware,
asyncRetryTransactionMiddleware,
@ -34,12 +19,8 @@ import {
videosAddResumableInitValidator,
videosAddResumableValidator
} from '../../../middlewares/index.js'
import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update.js'
import { VideoModel } from '../../../models/video/video.js'
import { ffprobePromise, getChaptersFromContainer } from '@peertube/peertube-ffmpeg'
import { replaceChapters, replaceChaptersFromDescriptionIfNeeded } from '@server/lib/video-chapters.js'
import { FfprobeData } from 'fluent-ffmpeg'
import { addVideoJobsAfterCreation } from '@server/lib/video-jobs.js'
import { LocalVideoCreator } from '@server/lib/local-video-creator.js'
const lTags = loggerTagsFactory('api', 'video')
const auditLogger = auditLoggerFactory('videos')
@ -134,109 +115,65 @@ async function addVideo (options: {
files: express.UploadFiles
}) {
const { req, res, videoPhysicalFile, videoInfo, files } = options
const videoChannel = res.locals.videoChannel
const user = res.locals.oauth.token.User
let videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id)
videoData = await Hooks.wrapObject(videoData, 'filter:api.video.upload.video-attribute.result')
videoData.state = buildNextVideoState()
videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware
const video = new VideoModel(videoData) as MVideoFullLight
video.VideoChannel = videoChannel
video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object
const ffprobe = await ffprobePromise(videoPhysicalFile.path)
const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video', ffprobe })
const originalFilename = videoPhysicalFile.originalname
const containerChapters = await getChaptersFromContainer({
path: videoPhysicalFile.path,
maxTitleLength: CONSTRAINTS_FIELDS.VIDEO_CHAPTERS.TITLE.max,
ffprobe
})
logger.debug(`Got ${containerChapters.length} chapters from video "${video.name}" container`, { containerChapters, ...lTags(video.uuid) })
logger.debug(`Got ${containerChapters.length} chapters from video "${videoInfo.name}" container`, { containerChapters, ...lTags() })
// Move physical file
const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile)
await move(videoPhysicalFile.path, destination)
// This is important in case if there is another attempt in the retry process
videoPhysicalFile.filename = basename(destination)
videoPhysicalFile.path = destination
const thumbnails = [ { type: ThumbnailType.MINIATURE, field: 'thumbnailfile' }, { type: ThumbnailType.PREVIEW, field: 'previewfile' } ]
.filter(({ field }) => !!files?.[field]?.[0])
.map(({ type, field }) => ({
path: files[field][0].path,
type,
automaticallyGenerated: false,
keepOriginal: false
}))
const thumbnails = await createThumbnailFiles({ video, files, videoFile, ffprobe })
const localVideoCreator = new LocalVideoCreator({
lTags,
videoFilePath: videoPhysicalFile.path,
user: res.locals.oauth.token.User,
channel: res.locals.videoChannel,
const { videoCreated } = await sequelizeTypescript.transaction(async t => {
const sequelizeOptions = { transaction: t }
chapters: undefined,
fallbackChapters: {
fromDescription: true,
finalFallback: containerChapters
},
const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight
videoAttributes: {
...videoInfo,
for (const thumbnail of thumbnails) {
await videoCreated.addAndSaveThumbnail(thumbnail, t)
}
duration: videoPhysicalFile.duration,
filename: videoPhysicalFile.originalname,
state: buildNextVideoState(),
isLive: false
},
// Do not forget to add video channel information to the created video
videoCreated.VideoChannel = res.locals.videoChannel
liveAttributes: undefined,
videoFile.videoId = video.id
await videoFile.save(sequelizeOptions)
videoAttributeResultHook: 'filter:api.video.upload.video-attribute.result',
video.VideoFiles = [ videoFile ]
await VideoSourceModel.create({
filename: originalFilename,
videoId: video.id
}, { transaction: t })
await setVideoTags({ video, tags: videoInfo.tags, transaction: t })
// Schedule an update in the future?
if (videoInfo.scheduleUpdate) {
await ScheduleVideoUpdateModel.create({
videoId: video.id,
updateAt: new Date(videoInfo.scheduleUpdate.updateAt),
privacy: videoInfo.scheduleUpdate.privacy || null
}, sequelizeOptions)
}
if (!await replaceChaptersFromDescriptionIfNeeded({ newDescription: video.description, video, transaction: t })) {
await replaceChapters({ video, chapters: containerChapters, transaction: t })
}
await autoBlacklistVideoIfNeeded({
video,
user,
isRemote: false,
isNew: true,
isNewFile: true,
transaction: t
})
if (videoInfo.privacy === VideoPrivacy.PASSWORD_PROTECTED) {
await VideoPasswordModel.addPasswords(videoInfo.videoPasswords, video.id, t)
}
auditLogger.create(getAuditIdFromRes(res), new VideoAuditView(videoCreated.toFormattedDetailsJSON()))
logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid, lTags(videoCreated.uuid))
return { videoCreated }
thumbnails
})
// Channel has a new content, set as updated
await videoCreated.VideoChannel.setAsUpdated()
const { video } = await localVideoCreator.create()
addVideoJobsAfterCreation({ video: videoCreated, videoFile })
.catch(err => logger.error('Cannot build new video jobs of %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) }))
auditLogger.create(getAuditIdFromRes(res), new VideoAuditView(video.toFormattedDetailsJSON()))
logger.info('Video with name %s and uuid %s created.', videoInfo.name, video.uuid, lTags(video.uuid))
Hooks.runAction('action:api.video.uploaded', { video: videoCreated, req, res })
Hooks.runAction('action:api.video.uploaded', { video, req, res })
return {
video: {
id: videoCreated.id,
shortUUID: uuidToShort(videoCreated.uuid),
uuid: videoCreated.uuid
id: video.id,
shortUUID: uuidToShort(video.uuid),
uuid: video.uuid
}
}
}
@ -246,27 +183,3 @@ async function deleteUploadResumableCache (req: express.Request, res: express.Re
return next()
}
async function createThumbnailFiles (options: {
video: MVideoThumbnail
files: UploadFiles
videoFile: MVideoFile
ffprobe?: FfprobeData
}) {
const { video, videoFile, files, ffprobe } = options
const models = await buildVideoThumbnailsFromReq({
video,
files,
fallback: () => Promise.resolve(undefined)
})
const filteredModels = models.filter(m => !!m)
const thumbnailsToGenerate = [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ].filter(type => {
// Generate missing thumbnail types
return !filteredModels.some(m => m.type === type)
})
return [ ...filteredModels, ...await generateLocalVideoMiniature({ video, videoFile, types: thumbnailsToGenerate, ffprobe }) ]
}

View File

@ -0,0 +1,16 @@
import { VideoChapterObject } from '@peertube/peertube-models'
import { MVideo, MVideoChapter } from '@server/types/models/index.js'
export function buildChaptersAPHasPart (video: MVideo, chapters: MVideoChapter[]) {
const hasPart: VideoChapterObject[] = []
if (chapters.length !== 0) {
for (let i = 0; i < chapters.length - 1; i++) {
hasPart.push(chapters[i].toActivityPubJSON({ video, nextChapter: chapters[i + 1] }))
}
hasPart.push(chapters[chapters.length - 1].toActivityPubJSON({ video, nextChapter: null }))
}
return hasPart
}

View File

@ -0,0 +1,268 @@
import { ffprobePromise } from '@peertube/peertube-ffmpeg'
import {
LiveVideoCreate,
LiveVideoLatencyMode,
ThumbnailType,
ThumbnailType_Type,
VideoCreate,
VideoPrivacy,
VideoStateType
} from '@peertube/peertube-models'
import { buildUUID } from '@peertube/peertube-node-utils'
import { sequelizeTypescript } from '@server/initializers/database.js'
import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting.js'
import { VideoLiveModel } from '@server/models/video/video-live.js'
import { VideoPasswordModel } from '@server/models/video/video-password.js'
import { VideoSourceModel } from '@server/models/video/video-source.js'
import { VideoModel } from '@server/models/video/video.js'
import { MVideoFullLight, MThumbnail, MChannel, MChannelAccountLight, MVideoFile, MUser } from '@server/types/models/index.js'
import { move } from 'fs-extra/esm'
import { getLocalVideoActivityPubUrl } from './activitypub/url.js'
import { generateLocalVideoMiniature, updateLocalVideoMiniatureFromExisting } from './thumbnail.js'
import { autoBlacklistVideoIfNeeded } from './video-blacklist.js'
import { buildNewFile } from './video-file.js'
import { addVideoJobsAfterCreation } from './video-jobs.js'
import { VideoPathManager } from './video-path-manager.js'
import { setVideoTags } from './video.js'
import { FilteredModelAttributes } from '@server/types/sequelize.js'
import { CONFIG } from '@server/initializers/config.js'
import { Hooks } from './plugins/hooks.js'
import Ffmpeg from 'fluent-ffmpeg'
import { ScheduleVideoUpdateModel } from '@server/models/video/schedule-video-update.js'
import { replaceChapters, replaceChaptersFromDescriptionIfNeeded } from './video-chapters.js'
import { LoggerTagsFn, logger } from '@server/helpers/logger.js'
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
import { federateVideoIfNeeded } from './activitypub/videos/federate.js'
type VideoAttributes = Omit<VideoCreate, 'channelId'> & {
duration: number
isLive: boolean
state: VideoStateType
filename: string
}
type LiveAttributes = Pick<LiveVideoCreate, 'permanentLive' | 'latencyMode' | 'saveReplay' | 'replaySettings'> & {
streamKey?: string
}
export type ThumbnailOptions = {
path: string
type: ThumbnailType_Type
automaticallyGenerated: boolean
keepOriginal: boolean
}[]
type ChaptersOption = { timecode: number, title: string }[]
type VideoAttributeHookFilter =
'filter:api.video.user-import.video-attribute.result' |
'filter:api.video.upload.video-attribute.result' |
'filter:api.video.live.video-attribute.result'
export class LocalVideoCreator {
private readonly lTags: LoggerTagsFn
private readonly videoFilePath: string | undefined
private readonly videoAttributes: VideoAttributes
private readonly liveAttributes: LiveAttributes | undefined
private readonly channel: MChannelAccountLight
private readonly videoAttributeResultHook: VideoAttributeHookFilter
private video: MVideoFullLight
private videoFile: MVideoFile
private ffprobe: Ffmpeg.FfprobeData
constructor (private readonly options: {
lTags: LoggerTagsFn
videoFilePath: string
videoAttributes: VideoAttributes
liveAttributes: LiveAttributes
channel: MChannelAccountLight
user: MUser
videoAttributeResultHook: VideoAttributeHookFilter
thumbnails: ThumbnailOptions
chapters: ChaptersOption | undefined
fallbackChapters: {
fromDescription: boolean
finalFallback: ChaptersOption | undefined
}
}) {
this.videoFilePath = options.videoFilePath
this.videoAttributes = options.videoAttributes
this.liveAttributes = options.liveAttributes
this.channel = options.channel
this.videoAttributeResultHook = options.videoAttributeResultHook
}
async create () {
this.video = new VideoModel(
await Hooks.wrapObject(this.buildVideo(this.videoAttributes, this.channel), this.videoAttributeResultHook)
) as MVideoFullLight
this.video.VideoChannel = this.channel
this.video.url = getLocalVideoActivityPubUrl(this.video)
if (this.videoFilePath) {
this.ffprobe = await ffprobePromise(this.videoFilePath)
this.videoFile = await buildNewFile({ path: this.videoFilePath, mode: 'web-video', ffprobe: this.ffprobe })
const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(this.video, this.videoFile)
await move(this.videoFilePath, destination)
}
const thumbnails = await this.createThumbnails()
await retryTransactionWrapper(() => {
return sequelizeTypescript.transaction(async transaction => {
await this.video.save({ transaction })
for (const thumbnail of thumbnails) {
await this.video.addAndSaveThumbnail(thumbnail, transaction)
}
if (this.videoFile) {
this.videoFile.videoId = this.video.id
await this.videoFile.save({ transaction })
this.video.VideoFiles = [ this.videoFile ]
}
await setVideoTags({ video: this.video, tags: this.videoAttributes.tags, transaction })
// Schedule an update in the future?
if (this.videoAttributes.scheduleUpdate) {
await ScheduleVideoUpdateModel.create({
videoId: this.video.id,
updateAt: new Date(this.videoAttributes.scheduleUpdate.updateAt),
privacy: this.videoAttributes.scheduleUpdate.privacy || null
}, { transaction })
}
if (this.options.chapters) {
await replaceChapters({ video: this.video, chapters: this.options.chapters, transaction })
} else if (this.options.fallbackChapters.fromDescription) {
if (!await replaceChaptersFromDescriptionIfNeeded({ newDescription: this.video.description, video: this.video, transaction })) {
await replaceChapters({ video: this.video, chapters: this.options.fallbackChapters.finalFallback, transaction })
}
}
await autoBlacklistVideoIfNeeded({
video: this.video,
user: this.options.user,
isRemote: false,
isNew: true,
isNewFile: true,
transaction
})
if (this.videoAttributes.filename) {
await VideoSourceModel.create({
filename: this.videoAttributes.filename,
videoId: this.video.id
}, { transaction })
}
if (this.videoAttributes.privacy === VideoPrivacy.PASSWORD_PROTECTED) {
await VideoPasswordModel.addPasswords(this.videoAttributes.videoPasswords, this.video.id, transaction)
}
if (this.videoAttributes.isLive) {
const videoLive = new VideoLiveModel({
saveReplay: this.liveAttributes.saveReplay || false,
permanentLive: this.liveAttributes.permanentLive || false,
latencyMode: this.liveAttributes.latencyMode || LiveVideoLatencyMode.DEFAULT,
streamKey: this.liveAttributes.streamKey || buildUUID()
})
if (videoLive.saveReplay) {
const replaySettings = new VideoLiveReplaySettingModel({
privacy: this.liveAttributes.replaySettings?.privacy ?? this.video.privacy
})
await replaySettings.save({ transaction })
videoLive.replaySettingId = replaySettings.id
}
videoLive.videoId = this.video.id
this.video.VideoLive = await videoLive.save({ transaction })
}
if (this.videoFile) {
transaction.afterCommit(() => {
addVideoJobsAfterCreation({ video: this.video, videoFile: this.videoFile })
.catch(err => logger.error('Cannot build new video jobs of %s.', this.video.uuid, { err, ...this.lTags(this.video.uuid) }))
})
} else {
await federateVideoIfNeeded(this.video, true, transaction)
}
})
})
// Channel has a new content, set as updated
await this.channel.setAsUpdated()
return { video: this.video, videoFile: this.videoFile }
}
private async createThumbnails () {
const promises: Promise<MThumbnail>[] = []
let toGenerate = [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ]
for (const type of [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ]) {
const thumbnail = this.options.thumbnails.find(t => t.type === type)
if (!thumbnail) continue
promises.push(
updateLocalVideoMiniatureFromExisting({
inputPath: thumbnail.path,
video: this.video,
type,
automaticallyGenerated: thumbnail.automaticallyGenerated || false,
keepOriginal: thumbnail.keepOriginal
})
)
toGenerate = toGenerate.filter(t => t !== thumbnail.type)
}
return [
...await Promise.all(promises),
...await generateLocalVideoMiniature({ video: this.video, videoFile: this.videoFile, types: toGenerate, ffprobe: this.ffprobe })
]
}
private buildVideo (videoInfo: VideoAttributes, channel: MChannel): FilteredModelAttributes<VideoModel> {
return {
name: videoInfo.name,
state: videoInfo.state,
remote: false,
category: videoInfo.category,
licence: videoInfo.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
language: videoInfo.language,
commentsEnabled: videoInfo.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
downloadEnabled: videoInfo.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
waitTranscoding: videoInfo.waitTranscoding || false,
nsfw: videoInfo.nsfw || false,
description: videoInfo.description,
support: videoInfo.support,
privacy: videoInfo.privacy || VideoPrivacy.PRIVATE,
isLive: videoInfo.isLive,
channelId: channel.id,
originallyPublishedAt: videoInfo.originallyPublishedAt
? new Date(videoInfo.originallyPublishedAt)
: null,
uuid: buildUUID(),
duration: videoInfo.duration
}
}
}

View File

@ -7,6 +7,7 @@ import {
MStreamingPlaylistFiles,
MThumbnail, MVideo, MVideoAP, MVideoCaption,
MVideoCaptionLanguageUrl,
MVideoChapter,
MVideoFile,
MVideoFullLight, MVideoLiveWithSetting,
MVideoPassword
@ -25,6 +26,8 @@ import { pick } from '@peertube/peertube-core-utils'
import { VideoPasswordModel } from '@server/models/video/video-password.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
import { VideoSourceModel } from '@server/models/video/video-source.js'
import { VideoChapterModel } from '@server/models/video/video-chapter.js'
import { buildChaptersAPHasPart } from '@server/lib/activitypub/video-chapters.js'
export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
@ -65,10 +68,11 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
}
private async exportVideo (videoId: number) {
const [ video, captions, source ] = await Promise.all([
const [ video, captions, source, chapters ] = await Promise.all([
VideoModel.loadFull(videoId),
VideoCaptionModel.listVideoCaptions(videoId),
VideoSourceModel.loadLatest(videoId)
VideoSourceModel.loadLatest(videoId),
VideoChapterModel.listChaptersOfVideo(videoId)
])
const passwords = video.privacy === VideoPrivacy.PASSWORD_PROTECTED
@ -87,10 +91,10 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
const { relativePathsFromJSON, staticFiles } = this.exportVideoFiles({ video, captions })
return {
json: this.exportVideoJSON({ video, captions, live, passwords, source, archiveFiles: relativePathsFromJSON }),
json: this.exportVideoJSON({ video, captions, live, passwords, source, chapters, archiveFiles: relativePathsFromJSON }),
staticFiles,
relativePathsFromJSON,
activityPubOutbox: await this.exportVideoAP(videoAP)
activityPubOutbox: await this.exportVideoAP(videoAP, chapters)
}
}
@ -102,9 +106,10 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
live: MVideoLiveWithSetting
passwords: MVideoPassword[]
source: MVideoSource
chapters: MVideoChapter[]
archiveFiles: VideoExportJSON['videos'][0]['archiveFiles']
}): VideoExportJSON['videos'][0] {
const { video, captions, live, passwords, source, archiveFiles } = options
const { video, captions, live, passwords, source, chapters, archiveFiles } = options
return {
uuid: video.uuid,
@ -156,6 +161,7 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
},
captions: this.exportCaptionsJSON(video, captions),
chapters: this.exportChaptersJSON(chapters),
files: this.exportFilesJSON(video, video.VideoFiles),
@ -194,6 +200,13 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
}))
}
private exportChaptersJSON (chapters: MVideoChapter[]) {
return chapters.map(c => ({
timecode: c.timecode,
title: c.title
}))
}
private exportFilesJSON (video: MVideo, files: MVideoFile[]) {
return files.map(f => ({
resolution: f.resolution,
@ -216,12 +229,10 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
// ---------------------------------------------------------------------------
private async exportVideoAP (video: MVideoAP): Promise<ActivityCreate<VideoObject>> {
private async exportVideoAP (video: MVideoAP, chapters: MVideoChapter[]): Promise<ActivityCreate<VideoObject>> {
const videoFile = video.getMaxQualityFile()
const icon = video.getPreview()
const videoFileAP = videoFile.toActivityPubObject(video)
const audience = getAudience(video.VideoChannel.Account.Actor, video.privacy === VideoPrivacy.PUBLIC)
const videoObject = {
...audiencify(await video.toActivityPubObject(), audience),
@ -240,13 +251,15 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
url: join(this.options.relativeStaticDirPath, this.getArchiveCaptionFilePath(video, c))
})),
attachment: this.options.withVideoFiles
hasParts: buildChaptersAPHasPart(video, chapters),
attachment: this.options.withVideoFiles && videoFile
? [
{
type: 'Video' as 'Video',
url: join(this.options.relativeStaticDirPath, this.getArchiveVideoFilePath(video, videoFile)),
...pick(videoFileAP, [ 'mediaType', 'height', 'size', 'fps' ])
...pick(videoFile.toActivityPubObject(video), [ 'mediaType', 'height', 'size', 'fps' ])
}
]
: undefined

View File

@ -5,25 +5,14 @@ import { buildNextVideoState } from '@server/lib/video-state.js'
import { VideoModel } from '@server/models/video/video.js'
import { pick } from '@peertube/peertube-core-utils'
import { buildUUID, getFileSize } from '@peertube/peertube-node-utils'
import { MChannelId, MThumbnail, MVideoCaption, MVideoFullLight } from '@server/types/models/index.js'
import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url.js'
import { buildNewFile } from '@server/lib/video-file.js'
import { MChannelId, MVideoCaption, MVideoFullLight } from '@server/types/models/index.js'
import { ffprobePromise, getVideoStreamDuration } from '@peertube/peertube-ffmpeg'
import { updateLocalVideoMiniatureFromExisting } from '@server/lib/thumbnail.js'
import { sequelizeTypescript } from '@server/initializers/database.js'
import { setVideoTags } from '@server/lib/video.js'
import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js'
import { VideoPasswordModel } from '@server/models/video/video-password.js'
import { addVideoJobsAfterCreation } from '@server/lib/video-jobs.js'
import { VideoChannelModel } from '@server/models/video/video-channel.js'
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils.js'
import { VideoLiveModel } from '@server/models/video/video-live.js'
import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting.js'
import { AbstractUserImporter } from './abstract-user-importer.js'
import { isUserQuotaValid } from '@server/lib/user.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { move } from 'fs-extra'
import {
isPasswordValid,
isVideoCategoryValid,
@ -45,16 +34,17 @@ import { isArray, isBooleanValid, isUUIDValid } from '@server/helpers/custom-val
import { CONFIG } from '@server/initializers/config.js'
import { isVideoCaptionLanguageValid } from '@server/helpers/custom-validators/video-captions.js'
import { isLiveLatencyModeValid } from '@server/helpers/custom-validators/video-lives.js'
import { VideoSourceModel } from '@server/models/video/video-source.js'
import { parse } from 'path'
import { isLocalVideoFileAccepted } from '@server/lib/moderation.js'
import { LocalVideoCreator, ThumbnailOptions } from '@server/lib/local-video-creator.js'
import { isVideoChapterTimecodeValid, isVideoChapterTitleValid } from '@server/helpers/custom-validators/video-chapters.js'
const lTags = loggerTagsFactory('user-import')
type ImportObject = VideoExportJSON['videos'][0]
type SanitizedObject = Pick<ImportObject, 'name' | 'duration' | 'channel' | 'privacy' | 'archiveFiles' | 'captions' | 'category' |
'licence' | 'language' | 'description' | 'support' | 'nsfw' | 'isLive' | 'commentsEnabled' | 'downloadEnabled' | 'waitTranscoding' |
'originallyPublishedAt' | 'tags' | 'live' | 'passwords' | 'source'>
'originallyPublishedAt' | 'tags' | 'live' | 'passwords' | 'source' | 'chapters'>
export class VideosImporter extends AbstractUserImporter <VideoExportJSON, ImportObject, SanitizedObject> {
@ -67,7 +57,7 @@ export class VideosImporter extends AbstractUserImporter <VideoExportJSON, Impor
if (!isVideoDurationValid(o.duration + '')) return undefined
if (!isVideoChannelUsernameValid(o.channel?.name)) return undefined
if (!isVideoPrivacyValid(o.privacy)) return undefined
if (!o.archiveFiles?.videoFile) return undefined
if (o.isLive !== true && !o.archiveFiles?.videoFile) return undefined
if (!isVideoCategoryValid(o.category)) o.category = null
if (!isVideoLicenceValid(o.licence)) o.licence = CONFIG.DEFAULTS.PUBLISH.LICENCE
@ -87,9 +77,11 @@ export class VideosImporter extends AbstractUserImporter <VideoExportJSON, Impor
if (!isArray(o.tags)) o.tags = []
if (!isArray(o.captions)) o.captions = []
if (!isArray(o.chapters)) o.chapters = []
o.tags = o.tags.filter(t => isVideoTagValid(t))
o.captions = o.captions.filter(c => isVideoCaptionLanguageValid(c.language))
o.chapters = o.chapters.filter(c => isVideoChapterTimecodeValid(c.timecode) && isVideoChapterTitleValid(c.title))
if (o.isLive) {
if (!o.live) return undefined
@ -131,17 +123,15 @@ export class VideosImporter extends AbstractUserImporter <VideoExportJSON, Impor
'captions',
'live',
'passwords',
'source'
'source',
'chapters'
])
}
protected async importObject (videoImportData: SanitizedObject) {
const videoFilePath = this.getSafeArchivePathOrThrow(videoImportData.archiveFiles.videoFile)
const videoSize = await getFileSize(videoFilePath)
if (await isUserQuotaValid({ userId: this.user.id, uploadSize: videoSize, checkDaily: false }) === false) {
throw new Error(`Cannot import video ${videoImportData.name} for user ${this.user.username} because of exceeded quota`)
}
const videoFilePath = !videoImportData.isLive
? this.getSafeArchivePathOrThrow(videoImportData.archiveFiles.videoFile)
: null
const videoChannel = await VideoChannelModel.loadLocalByNameAndPopulateAccount(videoImportData.channel.name)
if (!videoChannel) throw new Error(`Channel ${videoImportData} not found`)
@ -155,124 +145,85 @@ export class VideosImporter extends AbstractUserImporter <VideoExportJSON, Impor
return { duplicate: true }
}
const ffprobe = await ffprobePromise(videoFilePath)
const duration = await getVideoStreamDuration(videoFilePath, ffprobe)
const videoFile = await buildNewFile({ path: videoFilePath, mode: 'web-video', ffprobe })
const videoSize = videoFilePath
? await getFileSize(videoFilePath)
: undefined
await this.checkVideoFileIsAcceptedOrThrow({ videoFilePath, size: videoFile.size, channel: videoChannel, videoImportData })
let duration = 0
let videoData = {
...pick(videoImportData, [
'name',
'category',
'licence',
'language',
'privacy',
'description',
'support',
'isLive',
'nsfw',
'commentsEnabled',
'downloadEnabled',
'waitTranscoding'
]),
if (videoFilePath) {
if (await isUserQuotaValid({ userId: this.user.id, uploadSize: videoSize, checkDaily: false }) === false) {
throw new Error(`Cannot import video ${videoImportData.name} for user ${this.user.username} because of exceeded quota`)
}
uuid: buildUUID(),
duration,
remote: false,
state: buildNextVideoState(),
channelId: videoChannel.id,
originallyPublishedAt: videoImportData.originallyPublishedAt
? new Date(videoImportData.originallyPublishedAt)
: undefined
await this.checkVideoFileIsAcceptedOrThrow({ videoFilePath, size: videoSize, channel: videoChannel, videoImportData })
const ffprobe = await ffprobePromise(videoFilePath)
duration = await getVideoStreamDuration(videoFilePath, ffprobe)
}
videoData = await Hooks.wrapObject(videoData, 'filter:api.video.user-import.video-attribute.result')
const video = new VideoModel(videoData) as MVideoFullLight
video.VideoChannel = videoChannel
video.url = getLocalVideoActivityPubUrl(video)
const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile)
await move(videoFilePath, destination)
const thumbnailPath = this.getSafeArchivePathOrThrow(videoImportData.archiveFiles.thumbnail)
const thumbnails: MThumbnail[] = []
const thumbnails: ThumbnailOptions = []
for (const type of [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ]) {
if (!await this.isFileValidOrLog(thumbnailPath, CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max)) continue
thumbnails.push(
await updateLocalVideoMiniatureFromExisting({
inputPath: thumbnailPath,
video,
type,
automaticallyGenerated: false,
keepOriginal: true
})
)
thumbnails.push({
path: thumbnailPath,
automaticallyGenerated: false,
keepOriginal: true,
type
})
}
const { videoCreated } = await sequelizeTypescript.transaction(async t => {
const sequelizeOptions = { transaction: t }
const localVideoCreator = new LocalVideoCreator({
lTags,
videoFilePath,
user: this.user,
channel: videoChannel,
const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight
chapters: videoImportData.chapters,
fallbackChapters: {
fromDescription: false,
finalFallback: undefined
},
for (const thumbnail of thumbnails) {
await videoCreated.addAndSaveThumbnail(thumbnail, t)
}
videoAttributes: {
...pick(videoImportData, [
'name',
'category',
'licence',
'language',
'privacy',
'description',
'support',
'isLive',
'nsfw',
'tags',
'commentsEnabled',
'downloadEnabled',
'waitTranscoding',
'originallyPublishedAt'
]),
videoFile.videoId = video.id
await videoFile.save(sequelizeOptions)
videoPasswords: videoImportData.passwords,
duration,
filename: videoImportData.source?.filename,
state: buildNextVideoState()
},
video.VideoFiles = [ videoFile ]
liveAttributes: videoImportData.live,
await setVideoTags({ video, tags: videoImportData.tags, transaction: t })
videoAttributeResultHook: 'filter:api.video.user-import.video-attribute.result',
await autoBlacklistVideoIfNeeded({
video,
user: this.user,
isRemote: false,
isNew: true,
isNewFile: true,
transaction: t
})
if (videoImportData.source?.filename) {
await VideoSourceModel.create({
filename: videoImportData.source.filename,
videoId: video.id
}, { transaction: t })
}
if (videoImportData.privacy === VideoPrivacy.PASSWORD_PROTECTED) {
await VideoPasswordModel.addPasswords(videoImportData.passwords, video.id, t)
}
if (videoImportData.isLive) {
const videoLive = new VideoLiveModel(pick(videoImportData.live, [ 'saveReplay', 'permanentLive', 'latencyMode', 'streamKey' ]))
if (videoLive.saveReplay) {
const replaySettings = new VideoLiveReplaySettingModel({
privacy: videoImportData.live.replaySettings.privacy
})
await replaySettings.save(sequelizeOptions)
videoLive.replaySettingId = replaySettings.id
}
videoLive.videoId = videoCreated.id
videoCreated.VideoLive = await videoLive.save(sequelizeOptions)
}
return { videoCreated }
thumbnails
})
await this.importCaptions(videoCreated, videoImportData)
const { video } = await localVideoCreator.create()
await addVideoJobsAfterCreation({ video: videoCreated, videoFile })
await this.importCaptions(video, videoImportData)
logger.info('Video %s imported.', video.name, lTags(videoCreated.uuid))
logger.info('Video %s imported.', video.name, lTags(video.uuid))
return { duplicate: false }
}

View File

@ -1,75 +1,9 @@
import { UploadFiles } from 'express'
import memoizee from 'memoizee'
import { Transaction } from 'sequelize'
import {
ThumbnailType,
ThumbnailType_Type,
VideoCreate,
VideoPrivacy
} from '@peertube/peertube-models'
import { CONFIG } from '@server/initializers/config.js'
import { MEMOIZE_LENGTH, MEMOIZE_TTL } from '@server/initializers/constants.js'
import { TagModel } from '@server/models/video/tag.js'
import { VideoModel } from '@server/models/video/video.js'
import { FilteredModelAttributes } from '@server/types/index.js'
import { MThumbnail, MVideoTag, MVideoThumbnail } from '@server/types/models/index.js'
import { updateLocalVideoMiniatureFromExisting } from './thumbnail.js'
export function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
return {
name: videoInfo.name,
remote: false,
category: videoInfo.category,
licence: videoInfo.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
language: videoInfo.language,
commentsEnabled: videoInfo.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
downloadEnabled: videoInfo.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
waitTranscoding: videoInfo.waitTranscoding || false,
nsfw: videoInfo.nsfw || false,
description: videoInfo.description,
support: videoInfo.support,
privacy: videoInfo.privacy || VideoPrivacy.PRIVATE,
channelId,
originallyPublishedAt: videoInfo.originallyPublishedAt
? new Date(videoInfo.originallyPublishedAt)
: null
}
}
export async function buildVideoThumbnailsFromReq (options: {
video: MVideoThumbnail
files: UploadFiles
fallback: (type: ThumbnailType_Type) => Promise<MThumbnail>
automaticallyGenerated?: boolean
}) {
const { video, files, fallback, automaticallyGenerated } = options
const promises = [
{
type: ThumbnailType.MINIATURE,
fieldName: 'thumbnailfile'
},
{
type: ThumbnailType.PREVIEW,
fieldName: 'previewfile'
}
].map(p => {
const fields = files?.[p.fieldName]
if (fields) {
return updateLocalVideoMiniatureFromExisting({
inputPath: fields[0].path,
video,
type: p.type,
automaticallyGenerated: automaticallyGenerated || false
})
}
return fallback(p.type)
})
return Promise.all(promises)
}
import { MVideoTag } from '@server/types/models/index.js'
// ---------------------------------------------------------------------------
@ -89,7 +23,7 @@ export async function setVideoTags (options: {
// ---------------------------------------------------------------------------
export async function getVideoDuration (videoId: number | string) {
async function getVideoDuration (videoId: number | string) {
const video = await VideoModel.load(videoId)
const duration = video.isLive