From 990b6a0b0c4fbebc165e5cf7cec8fbc1cbaa6c66 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 7 Aug 2018 09:54:36 +0200 Subject: [PATCH] Import torrents with webtorrent --- .../my-account-video-imports.component.html | 7 ++- .../video-import-torrent.component.html | 12 +++- .../video-import-torrent.component.scss | 20 +++++++ .../video-import-torrent.component.ts | 16 ++++-- .../video-import-url.component.ts | 2 +- .../+video-edit/video-add.component.scss | 2 +- server/controllers/api/videos/import.ts | 57 +++++++++++++++---- server/helpers/core-utils.ts | 6 ++ .../custom-validators/video-imports.ts | 14 +++-- server/helpers/utils.ts | 15 +++-- server/helpers/webtorrent.ts | 15 +++-- server/initializers/constants.ts | 11 ++++ server/lib/job-queue/handlers/video-import.ts | 12 +++- .../middlewares/validators/video-imports.ts | 13 ++++- server/models/video/video-import.ts | 4 ++ shared/models/videos/video-import.model.ts | 4 ++ 16 files changed, 169 insertions(+), 41 deletions(-) diff --git a/client/src/app/+my-account/my-account-video-imports/my-account-video-imports.component.html b/client/src/app/+my-account/my-account-video-imports/my-account-video-imports.component.html index 00b2d7cb0..b2b6c3d60 100644 --- a/client/src/app/+my-account/my-account-video-imports/my-account-video-imports.component.html +++ b/client/src/app/+my-account/my-account-video-imports/my-account-video-imports.component.html @@ -5,7 +5,7 @@ - URL + Target Video State Created @@ -22,7 +22,10 @@ - {{ videoImport.targetUrl }} + {{ videoImport.targetUrl }} + + {{ videoImport.torrentName || videoImport.magnetUri }} + diff --git a/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.html b/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.html index 409e4de5e..2f0c9abb5 100644 --- a/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.html +++ b/client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.html @@ -2,8 +2,16 @@
-
- +
+ Select the torrent to import + +
+ (.torrent) + +
Or
+ +
+ () + @ViewChild('torrentfileInput') torrentfileInput videoFileName: string magnetUri = '' @@ -33,7 +34,7 @@ export class VideoImportTorrentComponent extends VideoSend implements OnInit, Ca video: VideoEdit - protected readonly DEFAULT_VIDEO_PRIVACY = VideoPrivacy.PRIVATE + protected readonly DEFAULT_VIDEO_PRIVACY = VideoPrivacy.PUBLIC constructor ( protected formValidatorService: FormValidatorService, @@ -62,7 +63,14 @@ export class VideoImportTorrentComponent extends VideoSend implements OnInit, Ca return !!this.magnetUri } - importVideo () { + fileChange () { + const torrentfile = this.torrentfileInput.nativeElement.files[0] as File + if (!torrentfile) return + + this.importVideo(torrentfile) + } + + importVideo (torrentfile?: Blob) { this.isImportingVideo = true const videoUpdate: VideoUpdate = { @@ -74,7 +82,7 @@ export class VideoImportTorrentComponent extends VideoSend implements OnInit, Ca this.loadingBar.start() - this.videoImportService.importVideoTorrent(this.magnetUri, videoUpdate).subscribe( + this.videoImportService.importVideoTorrent(torrentfile || this.magnetUri, videoUpdate).subscribe( res => { this.loadingBar.complete() this.firstStepDone.emit(res.video.name) diff --git a/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts b/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts index 842ede732..97b402bfe 100644 --- a/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts +++ b/client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts @@ -33,7 +33,7 @@ export class VideoImportUrlComponent extends VideoSend implements OnInit, CanCom video: VideoEdit - protected readonly DEFAULT_VIDEO_PRIVACY = VideoPrivacy.PRIVATE + protected readonly DEFAULT_VIDEO_PRIVACY = VideoPrivacy.PUBLIC constructor ( protected formValidatorService: FormValidatorService, diff --git a/client/src/app/videos/+video-edit/video-add.component.scss b/client/src/app/videos/+video-edit/video-add.component.scss index 02ee295f9..443361f50 100644 --- a/client/src/app/videos/+video-edit/video-add.component.scss +++ b/client/src/app/videos/+video-edit/video-add.component.scss @@ -49,7 +49,7 @@ $background-color: #F7F7F7; background-color: $background-color; border-radius: 3px; width: 100%; - height: 440px; + min-height: 440px; display: flex; justify-content: center; align-items: center; diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index c16a254d2..df151e79d 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts @@ -1,8 +1,16 @@ -import * as magnetUtil from 'magnet-uri' import * as express from 'express' +import * as magnetUtil from 'magnet-uri' +import 'multer' import { auditLoggerFactory, VideoImportAuditView } from '../../../helpers/audit-logger' import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares' -import { CONFIG, IMAGE_MIMETYPE_EXT, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE } from '../../../initializers' +import { + CONFIG, + IMAGE_MIMETYPE_EXT, + PREVIEWS_SIZE, + sequelizeTypescript, + THUMBNAILS_SIZE, + TORRENT_MIMETYPE_EXT +} from '../../../initializers' import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl' import { createReqFiles } from '../../../helpers/express-utils' import { logger } from '../../../helpers/logger' @@ -18,16 +26,20 @@ import { isArray } from '../../../helpers/custom-validators/misc' import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' import { VideoChannelModel } from '../../../models/video/video-channel' import * as Bluebird from 'bluebird' +import * as parseTorrent from 'parse-torrent' +import { readFileBufferPromise, renamePromise } from '../../../helpers/core-utils' +import { getSecureTorrentName } from '../../../helpers/utils' const auditLogger = auditLoggerFactory('video-imports') const videoImportsRouter = express.Router() const reqVideoFileImport = createReqFiles( - [ 'thumbnailfile', 'previewfile' ], - IMAGE_MIMETYPE_EXT, + [ 'thumbnailfile', 'previewfile', 'torrentfile' ], + Object.assign({}, TORRENT_MIMETYPE_EXT, IMAGE_MIMETYPE_EXT), { thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR, - previewfile: CONFIG.STORAGE.PREVIEWS_DIR + previewfile: CONFIG.STORAGE.PREVIEWS_DIR, + torrentfile: CONFIG.STORAGE.TORRENTS_DIR } ) @@ -49,17 +61,37 @@ export { function addVideoImport (req: express.Request, res: express.Response) { if (req.body.targetUrl) return addYoutubeDLImport(req, res) - if (req.body.magnetUri) return addTorrentImport(req, res) + const file = req.files['torrentfile'][0] + if (req.body.magnetUri || file) return addTorrentImport(req, res, file) } -async function addTorrentImport (req: express.Request, res: express.Response) { +async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { const body: VideoImportCreate = req.body - const magnetUri = body.magnetUri - const parsed = magnetUtil.decode(magnetUri) - const magnetName = isArray(parsed.name) ? parsed.name[0] : parsed.name as string + let videoName: string + let torrentName: string + let magnetUri: string - const video = buildVideo(res.locals.videoChannel.id, body, { name: magnetName }) + if (torrentfile) { + torrentName = torrentfile.originalname + + // Rename the torrent to a secured name + const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName)) + await renamePromise(torrentfile.path, newTorrentPath) + torrentfile.path = newTorrentPath + + const buf = await readFileBufferPromise(torrentfile.path) + const parsedTorrent = parseTorrent(buf) + + videoName = isArray(parsedTorrent.name) ? parsedTorrent.name[ 0 ] : parsedTorrent.name as string + } else { + magnetUri = body.magnetUri + + const parsed = magnetUtil.decode(magnetUri) + videoName = isArray(parsed.name) ? parsed.name[ 0 ] : parsed.name as string + } + + const video = buildVideo(res.locals.videoChannel.id, body, { name: videoName }) await processThumbnail(req, video) await processPreview(req, video) @@ -67,13 +99,14 @@ async function addTorrentImport (req: express.Request, res: express.Response) { const tags = null const videoImportAttributes = { magnetUri, + torrentName, state: VideoImportState.PENDING } const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes) // Create job to import the video const payload = { - type: 'magnet-uri' as 'magnet-uri', + type: torrentfile ? 'torrent-file' as 'torrent-file' : 'magnet-uri' as 'magnet-uri', videoImportId: videoImport.id, magnetUri } diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 884206aad..25eb6454a 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts @@ -13,6 +13,7 @@ import * as pem from 'pem' import * as rimraf from 'rimraf' import { URL } from 'url' import { truncate } from 'lodash' +import * as crypto from 'crypto' function sanitizeUrl (url: string) { const urlObject = new URL(url) @@ -95,6 +96,10 @@ function peertubeTruncate (str: string, maxLength: number) { return truncate(str, options) } +function sha256 (str: string) { + return crypto.createHash('sha256').update(str).digest('hex') +} + function promisify0 (func: (cb: (err: any, result: A) => void) => void): () => Promise { return function promisified (): Promise { return new Promise((resolve: (arg: A) => void, reject: (err: any) => void) => { @@ -165,6 +170,7 @@ export { sanitizeHost, buildPath, peertubeTruncate, + sha256, promisify0, promisify1, diff --git a/server/helpers/custom-validators/video-imports.ts b/server/helpers/custom-validators/video-imports.ts index d8b9bfaff..4d6ab1fa4 100644 --- a/server/helpers/custom-validators/video-imports.ts +++ b/server/helpers/custom-validators/video-imports.ts @@ -1,10 +1,9 @@ import 'express-validator' import 'multer' import * as validator from 'validator' -import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers' -import { exists } from './misc' +import { CONSTRAINTS_FIELDS, TORRENT_MIMETYPE_EXT, VIDEO_IMPORT_STATES } from '../../initializers' +import { exists, isFileValid } from './misc' import * as express from 'express' -import { VideoChannelModel } from '../../models/video/video-channel' import { VideoImportModel } from '../../models/video/video-import' function isVideoImportTargetUrlValid (url: string) { @@ -25,6 +24,12 @@ function isVideoImportStateValid (value: any) { return exists(value) && VIDEO_IMPORT_STATES[ value ] !== undefined } +const videoTorrentImportTypes = Object.keys(TORRENT_MIMETYPE_EXT).map(m => `(${m})`) +const videoTorrentImportRegex = videoTorrentImportTypes.join('|') +function isVideoImportTorrentFile (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) { + return isFileValid(files, videoTorrentImportRegex, 'torrentfile', CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_FILE.FILE_SIZE.max, true) +} + async function isVideoImportExist (id: number, res: express.Response) { const videoImport = await VideoImportModel.loadAndPopulateVideo(id) @@ -45,5 +50,6 @@ async function isVideoImportExist (id: number, res: express.Response) { export { isVideoImportStateValid, isVideoImportTargetUrlValid, - isVideoImportExist + isVideoImportExist, + isVideoImportTorrentFile } diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index f4cc5547d..2ad87951e 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts @@ -6,11 +6,12 @@ import { CONFIG } from '../initializers' import { UserModel } from '../models/account/user' import { ActorModel } from '../models/activitypub/actor' import { ApplicationModel } from '../models/application/application' -import { pseudoRandomBytesPromise, unlinkPromise } from './core-utils' +import { pseudoRandomBytesPromise, sha256, unlinkPromise } from './core-utils' import { logger } from './logger' import { isArray } from './custom-validators/misc' import * as crypto from "crypto" import { join } from "path" +import { Instance as ParseTorrent } from 'parse-torrent' const isCidr = require('is-cidr') @@ -183,13 +184,18 @@ async function getServerActor () { return Promise.resolve(serverActor) } -function generateVideoTmpPath (id: string) { - const hash = crypto.createHash('sha256').update(id).digest('hex') +function generateVideoTmpPath (target: string | ParseTorrent) { + const id = typeof target === 'string' ? target : target.infoHash + + const hash = sha256(id) return join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4') } -type SortType = { sortModel: any, sortValue: string } +function getSecureTorrentName (originalName: string) { + return sha256(originalName) + '.torrent' +} +type SortType = { sortModel: any, sortValue: string } // --------------------------------------------------------------------------- @@ -199,6 +205,7 @@ export { generateRandomString, getFormattedObjects, isSignupAllowed, + getSecureTorrentName, isSignupAllowedForCurrentIP, computeResolutionsToTranscode, resetSequelizeInstance, diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index fce88a1f6..04b3ac71b 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -2,17 +2,22 @@ import { logger } from './logger' import { generateVideoTmpPath } from './utils' import * as WebTorrent from 'webtorrent' import { createWriteStream } from 'fs' +import { Instance as ParseTorrent } from 'parse-torrent' +import { CONFIG } from '../initializers' +import { join } from 'path' -function downloadWebTorrentVideo (target: string) { - const path = generateVideoTmpPath(target) +function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: string }) { + const id = target.magnetUri || target.torrentName - logger.info('Importing torrent video %s', target) + const path = generateVideoTmpPath(id) + logger.info('Importing torrent video %s', id) return new Promise((res, rej) => { const webtorrent = new WebTorrent() - const torrent = webtorrent.add(target, torrent => { - if (torrent.files.length !== 1) throw new Error('The number of files is not equal to 1 for ' + target) + const torrentId = target.magnetUri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName) + const torrent = webtorrent.add(torrentId, torrent => { + if (torrent.files.length !== 1) return rej(new Error('The number of files is not equal to 1 for ' + torrentId)) const file = torrent.files[ 0 ] file.createReadStream().pipe(createWriteStream(path)) diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 243d544ea..cf7cd3d74 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -273,6 +273,12 @@ const CONSTRAINTS_FIELDS = { VIDEO_IMPORTS: { URL: { min: 3, max: 2000 }, // Length TORRENT_NAME: { min: 3, max: 255 }, // Length + TORRENT_FILE: { + EXTNAME: [ '.torrent' ], + FILE_SIZE: { + max: 1024 * 200 // 200 KB + } + } }, VIDEOS: { NAME: { min: 3, max: 120 }, // Length @@ -417,6 +423,10 @@ const VIDEO_CAPTIONS_MIMETYPE_EXT = { 'application/x-subrip': '.srt' } +const TORRENT_MIMETYPE_EXT = { + 'application/x-bittorrent': '.torrent' +} + // --------------------------------------------------------------------------- const SERVER_ACTOR_NAME = 'peertube' @@ -596,6 +606,7 @@ export { FEEDS, JOB_TTL, NSFW_POLICY_TYPES, + TORRENT_MIMETYPE_EXT, STATIC_MAX_AGE, STATIC_PATHS, ACTIVITY_PUB, diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index c457b71fc..fd61aecad 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -14,6 +14,7 @@ import { JobQueue } from '../index' import { federateVideoIfNeeded } from '../../activitypub' import { VideoModel } from '../../../models/video/video' import { downloadWebTorrentVideo } from '../../../helpers/webtorrent' +import { getSecureTorrentName } from '../../../helpers/utils' type VideoImportYoutubeDLPayload = { type: 'youtube-dl' @@ -25,7 +26,7 @@ type VideoImportYoutubeDLPayload = { } type VideoImportTorrentPayload = { - type: 'magnet-uri' + type: 'magnet-uri' | 'torrent-file' videoImportId: number } @@ -35,7 +36,7 @@ async function processVideoImport (job: Bull.Job) { const payload = job.data as VideoImportPayload if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, payload) - if (payload.type === 'magnet-uri') return processTorrentImport(job, payload) + if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') return processTorrentImport(job, payload) } // --------------------------------------------------------------------------- @@ -50,6 +51,7 @@ async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentP logger.info('Processing torrent video import in job %d.', job.id) const videoImport = await getVideoImportOrDie(payload.videoImportId) + const options = { videoImportId: payload.videoImportId, @@ -59,7 +61,11 @@ async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentP generateThumbnail: true, generatePreview: true } - return processFile(() => downloadWebTorrentVideo(videoImport.magnetUri), videoImport, options) + const target = { + torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined, + magnetUri: videoImport.magnetUri + } + return processFile(() => downloadWebTorrentVideo(target), videoImport, options) } async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutubeDLPayload) { diff --git a/server/middlewares/validators/video-imports.ts b/server/middlewares/validators/video-imports.ts index 8ec9373fb..c03cf2e4d 100644 --- a/server/middlewares/validators/video-imports.ts +++ b/server/middlewares/validators/video-imports.ts @@ -4,10 +4,11 @@ import { isIdValid } from '../../helpers/custom-validators/misc' import { logger } from '../../helpers/logger' import { areValidationErrors } from './utils' import { getCommonVideoAttributes } from './videos' -import { isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports' +import { isVideoImportTargetUrlValid, isVideoImportTorrentFile } from '../../helpers/custom-validators/video-imports' import { cleanUpReqFiles } from '../../helpers/utils' import { isVideoChannelOfAccountExist, isVideoMagnetUriValid, isVideoNameValid } from '../../helpers/custom-validators/videos' import { CONFIG } from '../../initializers/constants' +import { CONSTRAINTS_FIELDS } from '../../initializers' const videoImportAddValidator = getCommonVideoAttributes().concat([ body('channelId') @@ -19,6 +20,11 @@ const videoImportAddValidator = getCommonVideoAttributes().concat([ body('magnetUri') .optional() .custom(isVideoMagnetUriValid).withMessage('Should have a valid video magnet URI'), + body('torrentfile') + .custom((value, { req }) => isVideoImportTorrentFile(req.files)).withMessage( + 'This torrent file is not supported or too large. Please, make sure it is of the following type: ' + + CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_FILE.EXTNAME.join(', ') + ), body('name') .optional() .custom(isVideoNameValid).withMessage('Should have a valid name'), @@ -40,11 +46,12 @@ const videoImportAddValidator = getCommonVideoAttributes().concat([ if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) // Check we have at least 1 required param - if (!req.body.targetUrl && !req.body.magnetUri) { + const file = req.files['torrentfile'][0] + if (!req.body.targetUrl && !req.body.magnetUri && !file) { cleanUpReqFiles(req) return res.status(400) - .json({ error: 'Should have a magnetUri or a targetUrl.' }) + .json({ error: 'Should have a magnetUri or a targetUrl or a torrent file.' }) .end() } diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts index 55fca28b8..d6c02e5ac 100644 --- a/server/models/video/video-import.ts +++ b/server/models/video/video-import.ts @@ -171,7 +171,11 @@ export class VideoImportModel extends Model { return { id: this.id, + targetUrl: this.targetUrl, + magnetUri: this.magnetUri, + torrentName: this.torrentName, + state: { id: this.state, label: VideoImportModel.getStateLabel(this.state) diff --git a/shared/models/videos/video-import.model.ts b/shared/models/videos/video-import.model.ts index a5c582c67..293854006 100644 --- a/shared/models/videos/video-import.model.ts +++ b/shared/models/videos/video-import.model.ts @@ -4,7 +4,11 @@ import { VideoImportState } from './video-import-state.enum' export interface VideoImport { id: number + targetUrl: string + magnetUri: string + torrentName: string + createdAt: string updatedAt: string state: VideoConstant