diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index e5cac64d4..676d9804b 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -372,7 +372,8 @@ const VIDEO_STATES = { const VIDEO_IMPORT_STATES = { [VideoImportState.FAILED]: 'Failed', [VideoImportState.PENDING]: 'Pending', - [VideoImportState.SUCCESS]: 'Success' + [VideoImportState.SUCCESS]: 'Success', + [VideoImportState.REJECTED]: 'Rejected' } const VIDEO_ABUSE_STATES = { diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index ad549c6fc..a197ef629 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -1,27 +1,36 @@ import * as Bull from 'bull' -import { logger } from '../../../helpers/logger' -import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl' -import { VideoImportModel } from '../../../models/video/video-import' -import { VideoImportState } from '../../../../shared/models/videos' -import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' -import { extname } from 'path' -import { VideoFileModel } from '../../../models/video/video-file' -import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' -import { VideoImportPayload, VideoImportTorrentPayload, VideoImportYoutubeDLPayload, VideoState } from '../../../../shared' -import { federateVideoIfNeeded } from '../../activitypub/videos' -import { VideoModel } from '../../../models/video/video' -import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' -import { getSecureTorrentName } from '../../../helpers/utils' import { move, remove, stat } from 'fs-extra' -import { Notifier } from '../../notifier' -import { CONFIG } from '../../../initializers/config' -import { sequelizeTypescript } from '../../../initializers/database' -import { generateVideoMiniature } from '../../thumbnail' -import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type' -import { MThumbnail } from '../../../typings/models/video/thumbnail' -import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import' -import { getVideoFilePath } from '@server/lib/video-paths' +import { extname } from 'path' import { addOptimizeOrMergeAudioJob } from '@server/helpers/video' +import { isPostImportVideoAccepted } from '@server/lib/moderation' +import { Hooks } from '@server/lib/plugins/hooks' +import { getVideoFilePath } from '@server/lib/video-paths' +import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import' +import { + VideoImportPayload, + VideoImportTorrentPayload, + VideoImportTorrentPayloadType, + VideoImportYoutubeDLPayload, + VideoImportYoutubeDLPayloadType, + VideoState +} from '../../../../shared' +import { VideoImportState } from '../../../../shared/models/videos' +import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type' +import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' +import { logger } from '../../../helpers/logger' +import { getSecureTorrentName } from '../../../helpers/utils' +import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' +import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl' +import { CONFIG } from '../../../initializers/config' +import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' +import { sequelizeTypescript } from '../../../initializers/database' +import { VideoModel } from '../../../models/video/video' +import { VideoFileModel } from '../../../models/video/video-file' +import { VideoImportModel } from '../../../models/video/video-import' +import { MThumbnail } from '../../../typings/models/video/thumbnail' +import { federateVideoIfNeeded } from '../../activitypub/videos' +import { Notifier } from '../../notifier' +import { generateVideoMiniature } from '../../thumbnail' async function processVideoImport (job: Bull.Job) { const payload = job.data as VideoImportPayload @@ -44,6 +53,7 @@ async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentP const videoImport = await getVideoImportOrDie(payload.videoImportId) const options = { + type: payload.type, videoImportId: payload.videoImportId, generateThumbnail: true, @@ -61,6 +71,7 @@ async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutub const videoImport = await getVideoImportOrDie(payload.videoImportId) const options = { + type: payload.type, videoImportId: videoImport.id, generateThumbnail: payload.generateThumbnail, @@ -80,6 +91,7 @@ async function getVideoImportOrDie (videoImportId: number) { } type ProcessFileOptions = { + type: VideoImportYoutubeDLPayloadType | VideoImportTorrentPayloadType videoImportId: number generateThumbnail: boolean @@ -105,7 +117,7 @@ async function processFile (downloader: () => Promise, videoImport: MVid const fps = await getVideoFileFPS(tempVideoPath) const duration = await getDurationFromVideoFile(tempVideoPath) - // Create video file object in database + // Prepare video file object for creation in database const videoFileData = { extname: extname(tempVideoPath), resolution: videoFileResolution, @@ -115,6 +127,30 @@ async function processFile (downloader: () => Promise, videoImport: MVid } videoFile = new VideoFileModel(videoFileData) + const hookName = options.type === 'youtube-dl' + ? 'filter:api.video.post-import-url.accept.result' + : 'filter:api.video.post-import-torrent.accept.result' + + // Check we accept this video + const acceptParameters = { + videoImport, + video: videoImport.Video, + videoFilePath: tempVideoPath, + videoFile, + user: videoImport.User + } + const acceptedResult = await Hooks.wrapFun(isPostImportVideoAccepted, acceptParameters, hookName) + + if (acceptedResult.accepted !== true) { + logger.info('Refused imported video.', { acceptedResult, acceptParameters }) + + videoImport.state = VideoImportState.REJECTED + await videoImport.save() + + throw new Error(acceptedResult.errorMessage) + } + + // Video is accepted, resuming preparation const videoWithFiles = Object.assign(videoImport.Video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] }) // To clean files if the import fails const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles }) @@ -194,7 +230,9 @@ async function processFile (downloader: () => Promise, videoImport: MVid } videoImport.error = err.message - videoImport.state = VideoImportState.FAILED + if (videoImport.state !== VideoImportState.REJECTED) { + videoImport.state = VideoImportState.FAILED + } await videoImport.save() Notifier.Instance.notifyOnFinishedVideoImport(videoImport, false) diff --git a/server/lib/moderation.ts b/server/lib/moderation.ts index 55f7a985d..4afebb32a 100644 --- a/server/lib/moderation.ts +++ b/server/lib/moderation.ts @@ -1,12 +1,15 @@ import { VideoModel } from '../models/video/video' import { VideoCommentModel } from '../models/video/video-comment' import { VideoCommentCreate } from '../../shared/models/videos/video-comment.model' -import { VideoCreate } from '../../shared/models/videos' +import { VideoCreate, VideoImportCreate } from '../../shared/models/videos' import { UserModel } from '../models/account/user' import { VideoTorrentObject } from '../../shared/models/activitypub/objects' import { ActivityCreate } from '../../shared/models/activitypub' import { ActorModel } from '../models/activitypub/actor' import { VideoCommentObject } from '../../shared/models/activitypub/objects/video-comment-object' +import { VideoFileModel } from '@server/models/video/video-file' +import { PathLike } from 'fs-extra' +import { MUser } from '@server/typings/models' export type AcceptResult = { accepted: boolean @@ -55,10 +58,27 @@ function isRemoteVideoCommentAccepted (_object: { return { accepted: true } } +function isPreImportVideoAccepted (object: { + videoImportBody: VideoImportCreate + user: MUser +}): AcceptResult { + return { accepted: true } +} + +function isPostImportVideoAccepted (object: { + videoFilePath: PathLike + videoFile: VideoFileModel + user: MUser +}): AcceptResult { + return { accepted: true } +} + export { isLocalVideoAccepted, isLocalVideoThreadAccepted, isRemoteVideoAccepted, isRemoteVideoCommentAccepted, - isLocalVideoCommentReplyAccepted + isLocalVideoCommentReplyAccepted, + isPreImportVideoAccepted, + isPostImportVideoAccepted } diff --git a/server/middlewares/validators/videos/video-imports.ts b/server/middlewares/validators/videos/video-imports.ts index 5dc5db533..e3d900a9e 100644 --- a/server/middlewares/validators/videos/video-imports.ts +++ b/server/middlewares/validators/videos/video-imports.ts @@ -1,15 +1,18 @@ import * as express from 'express' import { body } from 'express-validator' +import { isPreImportVideoAccepted } from '@server/lib/moderation' +import { Hooks } from '@server/lib/plugins/hooks' +import { VideoImportCreate } from '@shared/models/videos/import/video-import-create.model' import { isIdValid, toIntOrNull } from '../../../helpers/custom-validators/misc' -import { logger } from '../../../helpers/logger' -import { areValidationErrors } from '../utils' -import { getCommonVideoEditAttributes } from './videos' import { isVideoImportTargetUrlValid, isVideoImportTorrentFile } from '../../../helpers/custom-validators/video-imports' -import { cleanUpReqFiles } from '../../../helpers/express-utils' import { isVideoMagnetUriValid, isVideoNameValid } from '../../../helpers/custom-validators/videos' +import { cleanUpReqFiles } from '../../../helpers/express-utils' +import { logger } from '../../../helpers/logger' +import { doesVideoChannelOfAccountExist } from '../../../helpers/middlewares' import { CONFIG } from '../../../initializers/config' import { CONSTRAINTS_FIELDS } from '../../../initializers/constants' -import { doesVideoChannelOfAccountExist } from '../../../helpers/middlewares' +import { areValidationErrors } from '../utils' +import { getCommonVideoEditAttributes } from './videos' const videoImportAddValidator = getCommonVideoEditAttributes().concat([ body('channelId') @@ -64,6 +67,8 @@ const videoImportAddValidator = getCommonVideoEditAttributes().concat([ .end() } + if (!await isImportAccepted(req, res)) return cleanUpReqFiles(req) + return next() } ]) @@ -75,3 +80,31 @@ export { } // --------------------------------------------------------------------------- + +async function isImportAccepted (req: express.Request, res: express.Response) { + const body: VideoImportCreate = req.body + const hookName = body.targetUrl + ? 'filter:api.video.pre-import-url.accept.result' + : 'filter:api.video.pre-import-torrent.accept.result' + + // Check we accept this video + const acceptParameters = { + videoImportBody: body, + user: res.locals.oauth.token.User + } + const acceptedResult = await Hooks.wrapFun( + isPreImportVideoAccepted, + acceptParameters, + hookName + ) + + if (!acceptedResult || acceptedResult.accepted !== true) { + logger.info('Refused to import video.', { acceptedResult, acceptParameters }) + res.status(403) + .json({ error: acceptedResult.errorMessage || 'Refused to import video' }) + + return false + } + + return true +} diff --git a/server/tests/fixtures/peertube-plugin-test/main.js b/server/tests/fixtures/peertube-plugin-test/main.js index 69796ab07..a45e98fb5 100644 --- a/server/tests/fixtures/peertube-plugin-test/main.js +++ b/server/tests/fixtures/peertube-plugin-test/main.js @@ -50,7 +50,47 @@ async function register ({ registerHook, registerSetting, settingsManager, stora target: 'filter:api.video.upload.accept.result', handler: ({ accepted }, { videoBody }) => { if (!accepted) return { accepted: false } - if (videoBody.name.indexOf('bad word') !== -1) return { accepted: false, errorMessage: 'bad word '} + if (videoBody.name.indexOf('bad word') !== -1) return { accepted: false, errorMessage: 'bad word' } + + return { accepted: true } + } + }) + + registerHook({ + target: 'filter:api.video.pre-import-url.accept.result', + handler: ({ accepted }, { videoImportBody }) => { + if (!accepted) return { accepted: false } + if (videoImportBody.targetUrl.includes('bad')) return { accepted: false, errorMessage: 'bad target url' } + + return { accepted: true } + } + }) + + registerHook({ + target: 'filter:api.video.pre-import-torrent.accept.result', + handler: ({ accepted }, { videoImportBody }) => { + if (!accepted) return { accepted: false } + if (videoImportBody.name.includes('bad torrent')) return { accepted: false, errorMessage: 'bad torrent' } + + return { accepted: true } + } + }) + + registerHook({ + target: 'filter:api.video.post-import-url.accept.result', + handler: ({ accepted }, { video }) => { + if (!accepted) return { accepted: false } + if (video.name.includes('bad word')) return { accepted: false, errorMessage: 'bad word' } + + return { accepted: true } + } + }) + + registerHook({ + target: 'filter:api.video.post-import-torrent.accept.result', + handler: ({ accepted }, { video }) => { + if (!accepted) return { accepted: false } + if (video.name.includes('bad word')) return { accepted: false, errorMessage: 'bad word' } return { accepted: true } } diff --git a/server/tests/plugins/filter-hooks.ts b/server/tests/plugins/filter-hooks.ts index 6c1fd40ba..41242318e 100644 --- a/server/tests/plugins/filter-hooks.ts +++ b/server/tests/plugins/filter-hooks.ts @@ -1,8 +1,8 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import * as chai from 'chai' import 'mocha' -import { cleanupTests, flushAndRunMultipleServers, ServerInfo } from '../../../shared/extra-utils/server/servers' +import * as chai from 'chai' +import { ServerConfig } from '@shared/models' import { addVideoCommentReply, addVideoCommentThread, @@ -23,10 +23,10 @@ import { uploadVideo, waitJobs } from '../../../shared/extra-utils' +import { cleanupTests, flushAndRunMultipleServers, ServerInfo } from '../../../shared/extra-utils/server/servers' +import { getMyVideoImports, getYoutubeVideoUrl, importVideo } from '../../../shared/extra-utils/videos/video-imports' +import { VideoDetails, VideoImport, VideoImportState, VideoPrivacy } from '../../../shared/models/videos' import { VideoCommentThreadTree } from '../../../shared/models/videos/video-comment.model' -import { VideoDetails } from '../../../shared/models/videos' -import { getYoutubeVideoUrl, importVideo } from '../../../shared/extra-utils/videos/video-imports' -import { ServerConfig } from '@shared/models' const expect = chai.expect @@ -87,6 +87,84 @@ describe('Test plugin filter hooks', function () { await uploadVideo(servers[0].url, servers[0].accessToken, { name: 'video with bad word' }, 403) }) + it('Should run filter:api.video.pre-import-url.accept.result', async function () { + const baseAttributes = { + name: 'normal title', + privacy: VideoPrivacy.PUBLIC, + channelId: servers[0].videoChannel.id, + targetUrl: getYoutubeVideoUrl() + 'bad' + } + await importVideo(servers[0].url, servers[0].accessToken, baseAttributes, 403) + }) + + it('Should run filter:api.video.pre-import-torrent.accept.result', async function () { + const baseAttributes = { + name: 'bad torrent', + privacy: VideoPrivacy.PUBLIC, + channelId: servers[0].videoChannel.id, + torrentfile: 'video-720p.torrent' as any + } + await importVideo(servers[0].url, servers[0].accessToken, baseAttributes, 403) + }) + + it('Should run filter:api.video.post-import-url.accept.result', async function () { + this.timeout(60000) + + let videoImportId: number + + { + const baseAttributes = { + name: 'title with bad word', + privacy: VideoPrivacy.PUBLIC, + channelId: servers[0].videoChannel.id, + targetUrl: getYoutubeVideoUrl() + } + const res = await importVideo(servers[0].url, servers[0].accessToken, baseAttributes) + videoImportId = res.body.id + } + + await waitJobs(servers) + + { + const res = await getMyVideoImports(servers[0].url, servers[0].accessToken) + const videoImports = res.body.data as VideoImport[] + + const videoImport = videoImports.find(i => i.id === videoImportId) + + expect(videoImport.state.id).to.equal(VideoImportState.REJECTED) + expect(videoImport.state.label).to.equal('Rejected') + } + }) + + it('Should run filter:api.video.post-import-torrent.accept.result', async function () { + this.timeout(60000) + + let videoImportId: number + + { + const baseAttributes = { + name: 'title with bad word', + privacy: VideoPrivacy.PUBLIC, + channelId: servers[0].videoChannel.id, + torrentfile: 'video-720p.torrent' as any + } + const res = await importVideo(servers[0].url, servers[0].accessToken, baseAttributes) + videoImportId = res.body.id + } + + await waitJobs(servers) + + { + const res = await getMyVideoImports(servers[0].url, servers[0].accessToken) + const videoImports = res.body.data as VideoImport[] + + const videoImport = videoImports.find(i => i.id === videoImportId) + + expect(videoImport.state.id).to.equal(VideoImportState.REJECTED) + expect(videoImport.state.label).to.equal('Rejected') + } + }) + it('Should run filter:api.video-thread.create.accept.result', async function () { await addVideoCommentThread(servers[0].url, servers[0].accessToken, videoUUID, 'comment with bad word', 403) }) diff --git a/shared/extra-utils/videos/video-imports.ts b/shared/extra-utils/videos/video-imports.ts index 8e5abd2f5..d235181b0 100644 --- a/shared/extra-utils/videos/video-imports.ts +++ b/shared/extra-utils/videos/video-imports.ts @@ -15,7 +15,7 @@ function getBadVideoUrl () { return 'https://download.cpy.re/peertube/bad_video.mp4' } -function importVideo (url: string, token: string, attributes: VideoImportCreate) { +function importVideo (url: string, token: string, attributes: VideoImportCreate & { torrentfile?: string }, statusCodeExpected = 200) { const path = '/api/v1/videos/imports' let attaches: any = {} @@ -27,7 +27,7 @@ function importVideo (url: string, token: string, attributes: VideoImportCreate) token, attaches, fields: attributes, - statusCodeExpected: 200 + statusCodeExpected }) } diff --git a/shared/models/plugins/server-hook.model.ts b/shared/models/plugins/server-hook.model.ts index 20f89b86d..5f812904f 100644 --- a/shared/models/plugins/server-hook.model.ts +++ b/shared/models/plugins/server-hook.model.ts @@ -9,9 +9,13 @@ export const serverFilterHookObject = { // Used to get detailed video information (video watch page for example) 'filter:api.video.get.result': true, - // Filter the result of the accept upload function + // Filter the result of the accept upload, import via torrent or url functions // If this function returns false then the upload is aborted with an error 'filter:api.video.upload.accept.result': true, + 'filter:api.video.pre-import-url.accept.result': true, + 'filter:api.video.pre-import-torrent.accept.result': true, + 'filter:api.video.post-import-url.accept.result': true, + 'filter:api.video.post-import-torrent.accept.result': true, // Filter the result of the accept comment (thread or reply) functions // If the functions return false then the user cannot post its comment 'filter:api.video-thread.create.accept.result': true, diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index 57d61c480..61010e5a8 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts @@ -70,8 +70,11 @@ export type VideoFileImportPayload = { filePath: string } +export type VideoImportTorrentPayloadType = 'magnet-uri' | 'torrent-file' +export type VideoImportYoutubeDLPayloadType = 'youtube-dl' + export type VideoImportYoutubeDLPayload = { - type: 'youtube-dl' + type: VideoImportYoutubeDLPayloadType videoImportId: number generateThumbnail: boolean @@ -80,7 +83,7 @@ export type VideoImportYoutubeDLPayload = { fileExt?: string } export type VideoImportTorrentPayload = { - type: 'magnet-uri' | 'torrent-file' + type: VideoImportTorrentPayloadType videoImportId: number } export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload diff --git a/shared/models/videos/import/video-import-state.enum.ts b/shared/models/videos/import/video-import-state.enum.ts index b178fbf3a..8421b8ca7 100644 --- a/shared/models/videos/import/video-import-state.enum.ts +++ b/shared/models/videos/import/video-import-state.enum.ts @@ -1,5 +1,6 @@ export enum VideoImportState { PENDING = 1, SUCCESS = 2, - FAILED = 3 + FAILED = 3, + REJECTED = 4 }