Add server plugin filter hooks for import with torrent and url (#2621)

* Add server plugin filter hooks for import with torrent and url

* WIP: pre and post-import filter hooks

* Rebased

* Cleanup filters to accept imports

Co-authored-by: Chocobozzz <me@florianbigard.com>
pull/2780/head
Rigel Kent 2020-05-14 11:10:26 +02:00 committed by GitHub
parent 7405b6ba89
commit 2158ac9034
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 261 additions and 43 deletions

View File

@ -372,7 +372,8 @@ const VIDEO_STATES = {
const VIDEO_IMPORT_STATES = { const VIDEO_IMPORT_STATES = {
[VideoImportState.FAILED]: 'Failed', [VideoImportState.FAILED]: 'Failed',
[VideoImportState.PENDING]: 'Pending', [VideoImportState.PENDING]: 'Pending',
[VideoImportState.SUCCESS]: 'Success' [VideoImportState.SUCCESS]: 'Success',
[VideoImportState.REJECTED]: 'Rejected'
} }
const VIDEO_ABUSE_STATES = { const VIDEO_ABUSE_STATES = {

View File

@ -1,27 +1,36 @@
import * as Bull from 'bull' import * as Bull from 'bull'
import { logger } from '../../../helpers/logger'
import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl'
import { VideoImportModel } from '../../../models/video/video-import'
import { VideoImportState } from '../../../../shared/models/videos'
import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
import { extname } from 'path'
import { VideoFileModel } from '../../../models/video/video-file'
import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
import { VideoImportPayload, VideoImportTorrentPayload, VideoImportYoutubeDLPayload, VideoState } from '../../../../shared'
import { federateVideoIfNeeded } from '../../activitypub/videos'
import { VideoModel } from '../../../models/video/video'
import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
import { getSecureTorrentName } from '../../../helpers/utils'
import { move, remove, stat } from 'fs-extra' import { move, remove, stat } from 'fs-extra'
import { Notifier } from '../../notifier' import { extname } from 'path'
import { CONFIG } from '../../../initializers/config'
import { sequelizeTypescript } from '../../../initializers/database'
import { generateVideoMiniature } from '../../thumbnail'
import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
import { MThumbnail } from '../../../typings/models/video/thumbnail'
import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import'
import { getVideoFilePath } from '@server/lib/video-paths'
import { addOptimizeOrMergeAudioJob } from '@server/helpers/video' import { addOptimizeOrMergeAudioJob } from '@server/helpers/video'
import { isPostImportVideoAccepted } from '@server/lib/moderation'
import { Hooks } from '@server/lib/plugins/hooks'
import { getVideoFilePath } from '@server/lib/video-paths'
import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import'
import {
VideoImportPayload,
VideoImportTorrentPayload,
VideoImportTorrentPayloadType,
VideoImportYoutubeDLPayload,
VideoImportYoutubeDLPayloadType,
VideoState
} from '../../../../shared'
import { VideoImportState } from '../../../../shared/models/videos'
import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
import { logger } from '../../../helpers/logger'
import { getSecureTorrentName } from '../../../helpers/utils'
import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl'
import { CONFIG } from '../../../initializers/config'
import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
import { sequelizeTypescript } from '../../../initializers/database'
import { VideoModel } from '../../../models/video/video'
import { VideoFileModel } from '../../../models/video/video-file'
import { VideoImportModel } from '../../../models/video/video-import'
import { MThumbnail } from '../../../typings/models/video/thumbnail'
import { federateVideoIfNeeded } from '../../activitypub/videos'
import { Notifier } from '../../notifier'
import { generateVideoMiniature } from '../../thumbnail'
async function processVideoImport (job: Bull.Job) { async function processVideoImport (job: Bull.Job) {
const payload = job.data as VideoImportPayload const payload = job.data as VideoImportPayload
@ -44,6 +53,7 @@ async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentP
const videoImport = await getVideoImportOrDie(payload.videoImportId) const videoImport = await getVideoImportOrDie(payload.videoImportId)
const options = { const options = {
type: payload.type,
videoImportId: payload.videoImportId, videoImportId: payload.videoImportId,
generateThumbnail: true, generateThumbnail: true,
@ -61,6 +71,7 @@ async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutub
const videoImport = await getVideoImportOrDie(payload.videoImportId) const videoImport = await getVideoImportOrDie(payload.videoImportId)
const options = { const options = {
type: payload.type,
videoImportId: videoImport.id, videoImportId: videoImport.id,
generateThumbnail: payload.generateThumbnail, generateThumbnail: payload.generateThumbnail,
@ -80,6 +91,7 @@ async function getVideoImportOrDie (videoImportId: number) {
} }
type ProcessFileOptions = { type ProcessFileOptions = {
type: VideoImportYoutubeDLPayloadType | VideoImportTorrentPayloadType
videoImportId: number videoImportId: number
generateThumbnail: boolean generateThumbnail: boolean
@ -105,7 +117,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
const fps = await getVideoFileFPS(tempVideoPath) const fps = await getVideoFileFPS(tempVideoPath)
const duration = await getDurationFromVideoFile(tempVideoPath) const duration = await getDurationFromVideoFile(tempVideoPath)
// Create video file object in database // Prepare video file object for creation in database
const videoFileData = { const videoFileData = {
extname: extname(tempVideoPath), extname: extname(tempVideoPath),
resolution: videoFileResolution, resolution: videoFileResolution,
@ -115,6 +127,30 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
} }
videoFile = new VideoFileModel(videoFileData) videoFile = new VideoFileModel(videoFileData)
const hookName = options.type === 'youtube-dl'
? 'filter:api.video.post-import-url.accept.result'
: 'filter:api.video.post-import-torrent.accept.result'
// Check we accept this video
const acceptParameters = {
videoImport,
video: videoImport.Video,
videoFilePath: tempVideoPath,
videoFile,
user: videoImport.User
}
const acceptedResult = await Hooks.wrapFun(isPostImportVideoAccepted, acceptParameters, hookName)
if (acceptedResult.accepted !== true) {
logger.info('Refused imported video.', { acceptedResult, acceptParameters })
videoImport.state = VideoImportState.REJECTED
await videoImport.save()
throw new Error(acceptedResult.errorMessage)
}
// Video is accepted, resuming preparation
const videoWithFiles = Object.assign(videoImport.Video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] }) const videoWithFiles = Object.assign(videoImport.Video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] })
// To clean files if the import fails // To clean files if the import fails
const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles }) const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles })
@ -194,7 +230,9 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
} }
videoImport.error = err.message videoImport.error = err.message
if (videoImport.state !== VideoImportState.REJECTED) {
videoImport.state = VideoImportState.FAILED videoImport.state = VideoImportState.FAILED
}
await videoImport.save() await videoImport.save()
Notifier.Instance.notifyOnFinishedVideoImport(videoImport, false) Notifier.Instance.notifyOnFinishedVideoImport(videoImport, false)

View File

@ -1,12 +1,15 @@
import { VideoModel } from '../models/video/video' import { VideoModel } from '../models/video/video'
import { VideoCommentModel } from '../models/video/video-comment' import { VideoCommentModel } from '../models/video/video-comment'
import { VideoCommentCreate } from '../../shared/models/videos/video-comment.model' import { VideoCommentCreate } from '../../shared/models/videos/video-comment.model'
import { VideoCreate } from '../../shared/models/videos' import { VideoCreate, VideoImportCreate } from '../../shared/models/videos'
import { UserModel } from '../models/account/user' import { UserModel } from '../models/account/user'
import { VideoTorrentObject } from '../../shared/models/activitypub/objects' import { VideoTorrentObject } from '../../shared/models/activitypub/objects'
import { ActivityCreate } from '../../shared/models/activitypub' import { ActivityCreate } from '../../shared/models/activitypub'
import { ActorModel } from '../models/activitypub/actor' import { ActorModel } from '../models/activitypub/actor'
import { VideoCommentObject } from '../../shared/models/activitypub/objects/video-comment-object' import { VideoCommentObject } from '../../shared/models/activitypub/objects/video-comment-object'
import { VideoFileModel } from '@server/models/video/video-file'
import { PathLike } from 'fs-extra'
import { MUser } from '@server/typings/models'
export type AcceptResult = { export type AcceptResult = {
accepted: boolean accepted: boolean
@ -55,10 +58,27 @@ function isRemoteVideoCommentAccepted (_object: {
return { accepted: true } return { accepted: true }
} }
function isPreImportVideoAccepted (object: {
videoImportBody: VideoImportCreate
user: MUser
}): AcceptResult {
return { accepted: true }
}
function isPostImportVideoAccepted (object: {
videoFilePath: PathLike
videoFile: VideoFileModel
user: MUser
}): AcceptResult {
return { accepted: true }
}
export { export {
isLocalVideoAccepted, isLocalVideoAccepted,
isLocalVideoThreadAccepted, isLocalVideoThreadAccepted,
isRemoteVideoAccepted, isRemoteVideoAccepted,
isRemoteVideoCommentAccepted, isRemoteVideoCommentAccepted,
isLocalVideoCommentReplyAccepted isLocalVideoCommentReplyAccepted,
isPreImportVideoAccepted,
isPostImportVideoAccepted
} }

View File

@ -1,15 +1,18 @@
import * as express from 'express' import * as express from 'express'
import { body } from 'express-validator' import { body } from 'express-validator'
import { isPreImportVideoAccepted } from '@server/lib/moderation'
import { Hooks } from '@server/lib/plugins/hooks'
import { VideoImportCreate } from '@shared/models/videos/import/video-import-create.model'
import { isIdValid, toIntOrNull } from '../../../helpers/custom-validators/misc' import { isIdValid, toIntOrNull } from '../../../helpers/custom-validators/misc'
import { logger } from '../../../helpers/logger'
import { areValidationErrors } from '../utils'
import { getCommonVideoEditAttributes } from './videos'
import { isVideoImportTargetUrlValid, isVideoImportTorrentFile } from '../../../helpers/custom-validators/video-imports' import { isVideoImportTargetUrlValid, isVideoImportTorrentFile } from '../../../helpers/custom-validators/video-imports'
import { cleanUpReqFiles } from '../../../helpers/express-utils'
import { isVideoMagnetUriValid, isVideoNameValid } from '../../../helpers/custom-validators/videos' import { isVideoMagnetUriValid, isVideoNameValid } from '../../../helpers/custom-validators/videos'
import { cleanUpReqFiles } from '../../../helpers/express-utils'
import { logger } from '../../../helpers/logger'
import { doesVideoChannelOfAccountExist } from '../../../helpers/middlewares'
import { CONFIG } from '../../../initializers/config' import { CONFIG } from '../../../initializers/config'
import { CONSTRAINTS_FIELDS } from '../../../initializers/constants' import { CONSTRAINTS_FIELDS } from '../../../initializers/constants'
import { doesVideoChannelOfAccountExist } from '../../../helpers/middlewares' import { areValidationErrors } from '../utils'
import { getCommonVideoEditAttributes } from './videos'
const videoImportAddValidator = getCommonVideoEditAttributes().concat([ const videoImportAddValidator = getCommonVideoEditAttributes().concat([
body('channelId') body('channelId')
@ -64,6 +67,8 @@ const videoImportAddValidator = getCommonVideoEditAttributes().concat([
.end() .end()
} }
if (!await isImportAccepted(req, res)) return cleanUpReqFiles(req)
return next() return next()
} }
]) ])
@ -75,3 +80,31 @@ export {
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
async function isImportAccepted (req: express.Request, res: express.Response) {
const body: VideoImportCreate = req.body
const hookName = body.targetUrl
? 'filter:api.video.pre-import-url.accept.result'
: 'filter:api.video.pre-import-torrent.accept.result'
// Check we accept this video
const acceptParameters = {
videoImportBody: body,
user: res.locals.oauth.token.User
}
const acceptedResult = await Hooks.wrapFun(
isPreImportVideoAccepted,
acceptParameters,
hookName
)
if (!acceptedResult || acceptedResult.accepted !== true) {
logger.info('Refused to import video.', { acceptedResult, acceptParameters })
res.status(403)
.json({ error: acceptedResult.errorMessage || 'Refused to import video' })
return false
}
return true
}

View File

@ -50,7 +50,47 @@ async function register ({ registerHook, registerSetting, settingsManager, stora
target: 'filter:api.video.upload.accept.result', target: 'filter:api.video.upload.accept.result',
handler: ({ accepted }, { videoBody }) => { handler: ({ accepted }, { videoBody }) => {
if (!accepted) return { accepted: false } if (!accepted) return { accepted: false }
if (videoBody.name.indexOf('bad word') !== -1) return { accepted: false, errorMessage: 'bad word '} if (videoBody.name.indexOf('bad word') !== -1) return { accepted: false, errorMessage: 'bad word' }
return { accepted: true }
}
})
registerHook({
target: 'filter:api.video.pre-import-url.accept.result',
handler: ({ accepted }, { videoImportBody }) => {
if (!accepted) return { accepted: false }
if (videoImportBody.targetUrl.includes('bad')) return { accepted: false, errorMessage: 'bad target url' }
return { accepted: true }
}
})
registerHook({
target: 'filter:api.video.pre-import-torrent.accept.result',
handler: ({ accepted }, { videoImportBody }) => {
if (!accepted) return { accepted: false }
if (videoImportBody.name.includes('bad torrent')) return { accepted: false, errorMessage: 'bad torrent' }
return { accepted: true }
}
})
registerHook({
target: 'filter:api.video.post-import-url.accept.result',
handler: ({ accepted }, { video }) => {
if (!accepted) return { accepted: false }
if (video.name.includes('bad word')) return { accepted: false, errorMessage: 'bad word' }
return { accepted: true }
}
})
registerHook({
target: 'filter:api.video.post-import-torrent.accept.result',
handler: ({ accepted }, { video }) => {
if (!accepted) return { accepted: false }
if (video.name.includes('bad word')) return { accepted: false, errorMessage: 'bad word' }
return { accepted: true } return { accepted: true }
} }

View File

@ -1,8 +1,8 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import * as chai from 'chai'
import 'mocha' import 'mocha'
import { cleanupTests, flushAndRunMultipleServers, ServerInfo } from '../../../shared/extra-utils/server/servers' import * as chai from 'chai'
import { ServerConfig } from '@shared/models'
import { import {
addVideoCommentReply, addVideoCommentReply,
addVideoCommentThread, addVideoCommentThread,
@ -23,10 +23,10 @@ import {
uploadVideo, uploadVideo,
waitJobs waitJobs
} from '../../../shared/extra-utils' } from '../../../shared/extra-utils'
import { cleanupTests, flushAndRunMultipleServers, ServerInfo } from '../../../shared/extra-utils/server/servers'
import { getMyVideoImports, getYoutubeVideoUrl, importVideo } from '../../../shared/extra-utils/videos/video-imports'
import { VideoDetails, VideoImport, VideoImportState, VideoPrivacy } from '../../../shared/models/videos'
import { VideoCommentThreadTree } from '../../../shared/models/videos/video-comment.model' import { VideoCommentThreadTree } from '../../../shared/models/videos/video-comment.model'
import { VideoDetails } from '../../../shared/models/videos'
import { getYoutubeVideoUrl, importVideo } from '../../../shared/extra-utils/videos/video-imports'
import { ServerConfig } from '@shared/models'
const expect = chai.expect const expect = chai.expect
@ -87,6 +87,84 @@ describe('Test plugin filter hooks', function () {
await uploadVideo(servers[0].url, servers[0].accessToken, { name: 'video with bad word' }, 403) await uploadVideo(servers[0].url, servers[0].accessToken, { name: 'video with bad word' }, 403)
}) })
it('Should run filter:api.video.pre-import-url.accept.result', async function () {
const baseAttributes = {
name: 'normal title',
privacy: VideoPrivacy.PUBLIC,
channelId: servers[0].videoChannel.id,
targetUrl: getYoutubeVideoUrl() + 'bad'
}
await importVideo(servers[0].url, servers[0].accessToken, baseAttributes, 403)
})
it('Should run filter:api.video.pre-import-torrent.accept.result', async function () {
const baseAttributes = {
name: 'bad torrent',
privacy: VideoPrivacy.PUBLIC,
channelId: servers[0].videoChannel.id,
torrentfile: 'video-720p.torrent' as any
}
await importVideo(servers[0].url, servers[0].accessToken, baseAttributes, 403)
})
it('Should run filter:api.video.post-import-url.accept.result', async function () {
this.timeout(60000)
let videoImportId: number
{
const baseAttributes = {
name: 'title with bad word',
privacy: VideoPrivacy.PUBLIC,
channelId: servers[0].videoChannel.id,
targetUrl: getYoutubeVideoUrl()
}
const res = await importVideo(servers[0].url, servers[0].accessToken, baseAttributes)
videoImportId = res.body.id
}
await waitJobs(servers)
{
const res = await getMyVideoImports(servers[0].url, servers[0].accessToken)
const videoImports = res.body.data as VideoImport[]
const videoImport = videoImports.find(i => i.id === videoImportId)
expect(videoImport.state.id).to.equal(VideoImportState.REJECTED)
expect(videoImport.state.label).to.equal('Rejected')
}
})
it('Should run filter:api.video.post-import-torrent.accept.result', async function () {
this.timeout(60000)
let videoImportId: number
{
const baseAttributes = {
name: 'title with bad word',
privacy: VideoPrivacy.PUBLIC,
channelId: servers[0].videoChannel.id,
torrentfile: 'video-720p.torrent' as any
}
const res = await importVideo(servers[0].url, servers[0].accessToken, baseAttributes)
videoImportId = res.body.id
}
await waitJobs(servers)
{
const res = await getMyVideoImports(servers[0].url, servers[0].accessToken)
const videoImports = res.body.data as VideoImport[]
const videoImport = videoImports.find(i => i.id === videoImportId)
expect(videoImport.state.id).to.equal(VideoImportState.REJECTED)
expect(videoImport.state.label).to.equal('Rejected')
}
})
it('Should run filter:api.video-thread.create.accept.result', async function () { it('Should run filter:api.video-thread.create.accept.result', async function () {
await addVideoCommentThread(servers[0].url, servers[0].accessToken, videoUUID, 'comment with bad word', 403) await addVideoCommentThread(servers[0].url, servers[0].accessToken, videoUUID, 'comment with bad word', 403)
}) })

View File

@ -15,7 +15,7 @@ function getBadVideoUrl () {
return 'https://download.cpy.re/peertube/bad_video.mp4' return 'https://download.cpy.re/peertube/bad_video.mp4'
} }
function importVideo (url: string, token: string, attributes: VideoImportCreate) { function importVideo (url: string, token: string, attributes: VideoImportCreate & { torrentfile?: string }, statusCodeExpected = 200) {
const path = '/api/v1/videos/imports' const path = '/api/v1/videos/imports'
let attaches: any = {} let attaches: any = {}
@ -27,7 +27,7 @@ function importVideo (url: string, token: string, attributes: VideoImportCreate)
token, token,
attaches, attaches,
fields: attributes, fields: attributes,
statusCodeExpected: 200 statusCodeExpected
}) })
} }

View File

@ -9,9 +9,13 @@ export const serverFilterHookObject = {
// Used to get detailed video information (video watch page for example) // Used to get detailed video information (video watch page for example)
'filter:api.video.get.result': true, 'filter:api.video.get.result': true,
// Filter the result of the accept upload function // Filter the result of the accept upload, import via torrent or url functions
// If this function returns false then the upload is aborted with an error // If this function returns false then the upload is aborted with an error
'filter:api.video.upload.accept.result': true, 'filter:api.video.upload.accept.result': true,
'filter:api.video.pre-import-url.accept.result': true,
'filter:api.video.pre-import-torrent.accept.result': true,
'filter:api.video.post-import-url.accept.result': true,
'filter:api.video.post-import-torrent.accept.result': true,
// Filter the result of the accept comment (thread or reply) functions // Filter the result of the accept comment (thread or reply) functions
// If the functions return false then the user cannot post its comment // If the functions return false then the user cannot post its comment
'filter:api.video-thread.create.accept.result': true, 'filter:api.video-thread.create.accept.result': true,

View File

@ -70,8 +70,11 @@ export type VideoFileImportPayload = {
filePath: string filePath: string
} }
export type VideoImportTorrentPayloadType = 'magnet-uri' | 'torrent-file'
export type VideoImportYoutubeDLPayloadType = 'youtube-dl'
export type VideoImportYoutubeDLPayload = { export type VideoImportYoutubeDLPayload = {
type: 'youtube-dl' type: VideoImportYoutubeDLPayloadType
videoImportId: number videoImportId: number
generateThumbnail: boolean generateThumbnail: boolean
@ -80,7 +83,7 @@ export type VideoImportYoutubeDLPayload = {
fileExt?: string fileExt?: string
} }
export type VideoImportTorrentPayload = { export type VideoImportTorrentPayload = {
type: 'magnet-uri' | 'torrent-file' type: VideoImportTorrentPayloadType
videoImportId: number videoImportId: number
} }
export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload

View File

@ -1,5 +1,6 @@
export enum VideoImportState { export enum VideoImportState {
PENDING = 1, PENDING = 1,
SUCCESS = 2, SUCCESS = 2,
FAILED = 3 FAILED = 3,
REJECTED = 4
} }