From a84b8fa5cf6e4cafb841af3db9bdfcc9531c09a4 Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Tue, 7 Aug 2018 10:07:53 +0200
Subject: [PATCH] Add import.video.torrent configuration
---
.../edit-custom-config.component.html | 5 ++
.../edit-custom-config.component.ts | 7 +-
client/src/app/core/server/server.service.ts | 3 +
.../+video-edit/video-add.component.scss | 1 +
.../videos/+video-edit/video-add.component.ts | 2 +-
config/default.yaml | 2 +
config/production.yaml.example | 2 +
config/test.yaml | 2 +
server/controllers/api/config.ts | 8 +-
server/controllers/api/users.ts | 2 +-
server/controllers/api/videos/import.ts | 10 ++-
server/initializers/constants.ts | 5 +-
.../migrations/0245-import-magnet.ts | 42 ----------
server/lib/job-queue/handlers/video-import.ts | 9 +-
server/middlewares/validators/config.ts | 1 +
.../middlewares/validators/video-imports.ts | 15 +++-
server/models/account/user.ts | 2 +-
server/models/video/video-import.ts | 83 ++++++++-----------
server/tests/api/check-params/config.ts | 3 +
server/tests/api/server/config.ts | 5 ++
server/tests/utils/server/config.ts | 3 +
shared/models/server/custom-config.model.ts | 3 +
shared/models/server/server-config.model.ts | 3 +
23 files changed, 112 insertions(+), 106 deletions(-)
delete mode 100644 server/initializers/migrations/0245-import-magnet.ts
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html
index 13b43306b..0a032df12 100644
--- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html
+++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html
@@ -108,6 +108,11 @@
i18n-labelText labelText="Video import with HTTP enabled"
></my-peertube-checkbox>
+ <my-peertube-checkbox
+ inputName="importVideosTorrentEnabled" formControlName="importVideosTorrentEnabled"
+ i18n-labelText labelText="Video import with a torrent file or a magnet URI enabled"
+ ></my-peertube-checkbox>
+
<div i18n class="inner-form-title">Administrator</div>
<div class="form-group">
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts
index bc5ce6e5d..fd6784415 100644
--- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts
+++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts
@@ -72,6 +72,7 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit {
signupEnabled: null,
signupLimit: this.customConfigValidatorsService.SIGNUP_LIMIT,
importVideosHttpEnabled: null,
+ importVideosTorrentEnabled: null,
adminEmail: this.customConfigValidatorsService.ADMIN_EMAIL,
userVideoQuota: this.userValidatorsService.USER_VIDEO_QUOTA,
transcodingThreads: this.customConfigValidatorsService.TRANSCODING_THREADS,
@@ -189,6 +190,9 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit {
videos: {
http: {
enabled: this.form.value['importVideosHttpEnabled']
+ },
+ torrent: {
+ enabled: this.form.value['importVideosTorrentEnabled']
}
}
}
@@ -231,7 +235,8 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit {
transcodingEnabled: this.customConfig.transcoding.enabled,
customizationJavascript: this.customConfig.instance.customizations.javascript,
customizationCSS: this.customConfig.instance.customizations.css,
- importVideosHttpEnabled: this.customConfig.import.videos.http.enabled
+ importVideosHttpEnabled: this.customConfig.import.videos.http.enabled,
+ importVideosTorrentEnabled: this.customConfig.import.videos.torrent.enabled
}
for (const resolution of this.resolutions) {
diff --git a/client/src/app/core/server/server.service.ts b/client/src/app/core/server/server.service.ts
index ab317f0aa..52b50cbe8 100644
--- a/client/src/app/core/server/server.service.ts
+++ b/client/src/app/core/server/server.service.ts
@@ -73,6 +73,9 @@ export class ServerService {
videos: {
http: {
enabled: false
+ },
+ torrent: {
+ enabled: false
}
}
}
diff --git a/client/src/app/videos/+video-edit/video-add.component.scss b/client/src/app/videos/+video-edit/video-add.component.scss
index 443361f50..c1f96cc37 100644
--- a/client/src/app/videos/+video-edit/video-add.component.scss
+++ b/client/src/app/videos/+video-edit/video-add.component.scss
@@ -50,6 +50,7 @@ $background-color: #F7F7F7;
border-radius: 3px;
width: 100%;
min-height: 440px;
+ padding-bottom: 20px;
display: flex;
justify-content: center;
align-items: center;
diff --git a/client/src/app/videos/+video-edit/video-add.component.ts b/client/src/app/videos/+video-edit/video-add.component.ts
index 7d360598d..1a9247dbe 100644
--- a/client/src/app/videos/+video-edit/video-add.component.ts
+++ b/client/src/app/videos/+video-edit/video-add.component.ts
@@ -40,6 +40,6 @@ export class VideoAddComponent implements CanComponentDeactivate {
}
isVideoImportTorrentEnabled () {
- return this.serverService.getConfig().import.videos.http.enabled
+ return this.serverService.getConfig().import.videos.torrent.enabled
}
}
diff --git a/config/default.yaml b/config/default.yaml
index 5fa7e5945..60da192b4 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -97,6 +97,8 @@ import:
videos:
http: # Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html
enabled: false
+ torrent: # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file)
+ enabled: false
instance:
name: 'PeerTube'
diff --git a/config/production.yaml.example b/config/production.yaml.example
index 635a41e9e..9e8b57829 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -111,6 +111,8 @@ import:
videos:
http: # Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html
enabled: false
+ torrent: # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file)
+ enabled: false
# Instance settings
instance:
diff --git a/config/test.yaml b/config/test.yaml
index 3c51785f2..879b6bdd4 100644
--- a/config/test.yaml
+++ b/config/test.yaml
@@ -44,6 +44,8 @@ import:
videos:
http:
enabled: true
+ torrent:
+ enabled: true
instance:
default_nsfw_policy: 'display'
\ No newline at end of file
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index 950a1498e..6f05c33db 100644
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -9,7 +9,7 @@ import { CONFIG, CONSTRAINTS_FIELDS, reloadConfig } from '../../initializers'
import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../middlewares'
import { customConfigUpdateValidator } from '../../middlewares/validators/config'
import { ClientHtml } from '../../lib/client-html'
-import { CustomConfigAuditView, auditLoggerFactory } from '../../helpers/audit-logger'
+import { auditLoggerFactory, CustomConfigAuditView } from '../../helpers/audit-logger'
const packageJSON = require('../../../../package.json')
const configRouter = express.Router()
@@ -69,6 +69,9 @@ async function getConfig (req: express.Request, res: express.Response, next: exp
videos: {
http: {
enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED
+ },
+ torrent: {
+ enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
}
}
},
@@ -237,6 +240,9 @@ function customConfig (): CustomConfig {
videos: {
http: {
enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED
+ },
+ torrent: {
+ enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
}
}
}
diff --git a/server/controllers/api/users.ts b/server/controllers/api/users.ts
index 879ba3f91..36bf0e0fe 100644
--- a/server/controllers/api/users.ts
+++ b/server/controllers/api/users.ts
@@ -196,7 +196,7 @@ async function getUserVideos (req: express.Request, res: express.Response, next:
async function getUserVideoImports (req: express.Request, res: express.Response, next: express.NextFunction) {
const user = res.locals.oauth.token.User as UserModel
const resultList = await VideoImportModel.listUserVideoImportsForApi(
- user.Account.id,
+ user.id,
req.query.start as number,
req.query.count as number,
req.query.sort
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index df151e79d..94dafcdbd 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -61,12 +61,13 @@ export {
function addVideoImport (req: express.Request, res: express.Response) {
if (req.body.targetUrl) return addYoutubeDLImport(req, res)
- const file = req.files['torrentfile'][0]
+ const file = req.files && req.files['torrentfile'] ? req.files['torrentfile'][0] : undefined
if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
}
async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
const body: VideoImportCreate = req.body
+ const user = res.locals.oauth.token.User
let videoName: string
let torrentName: string
@@ -100,7 +101,8 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
const videoImportAttributes = {
magnetUri,
torrentName,
- state: VideoImportState.PENDING
+ state: VideoImportState.PENDING,
+ userId: user.id
}
const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
@@ -120,6 +122,7 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
async function addYoutubeDLImport (req: express.Request, res: express.Response) {
const body: VideoImportCreate = req.body
const targetUrl = body.targetUrl
+ const user = res.locals.oauth.token.User
let youtubeDLInfo: YoutubeDLInfo
try {
@@ -140,7 +143,8 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
const tags = body.tags || youtubeDLInfo.tags
const videoImportAttributes = {
targetUrl,
- state: VideoImportState.PENDING
+ state: VideoImportState.PENDING,
+ userId: user.id
}
const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index cf7cd3d74..80eb3f1e7 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -15,7 +15,7 @@ let config: IConfig = require('config')
// ---------------------------------------------------------------------------
-const LAST_MIGRATION_VERSION = 245
+const LAST_MIGRATION_VERSION = 240
// ---------------------------------------------------------------------------
@@ -211,6 +211,9 @@ const CONFIG = {
VIDEOS: {
HTTP: {
get ENABLED () { return config.get<boolean>('import.videos.http.enabled') }
+ },
+ TORRENT: {
+ get ENABLED () { return config.get<boolean>('import.videos.torrent.enabled') }
}
}
},
diff --git a/server/initializers/migrations/0245-import-magnet.ts b/server/initializers/migrations/0245-import-magnet.ts
deleted file mode 100644
index 87603b006..000000000
--- a/server/initializers/migrations/0245-import-magnet.ts
+++ /dev/null
@@ -1,42 +0,0 @@
-import * as Sequelize from 'sequelize'
-import { Migration } from '../../models/migrations'
-import { CONSTRAINTS_FIELDS } from '../index'
-
-async function up (utils: {
- transaction: Sequelize.Transaction
- queryInterface: Sequelize.QueryInterface
- sequelize: Sequelize.Sequelize
-}): Promise<any> {
- {
- const data = {
- type: Sequelize.STRING,
- allowNull: true,
- defaultValue: null
- } as Migration.String
- await utils.queryInterface.changeColumn('videoImport', 'targetUrl', data)
- }
-
- {
- const data = {
- type: Sequelize.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max),
- allowNull: true,
- defaultValue: null
- }
- await utils.queryInterface.addColumn('videoImport', 'magnetUri', data)
- }
-
- {
- const data = {
- type: Sequelize.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_NAME.max),
- allowNull: true,
- defaultValue: null
- }
- await utils.queryInterface.addColumn('videoImport', 'torrentName', data)
- }
-}
-
-function down (options) {
- throw new Error('Not implemented.')
-}
-
-export { up, down }
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index fd61aecad..28a03d19e 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -114,16 +114,21 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
tempVideoPath = await downloader()
// Get information about this video
+ const { size } = await statPromise(tempVideoPath)
+ const isAble = await videoImport.User.isAbleToUploadVideo({ size })
+ if (isAble === false) {
+ throw new Error('The user video quota is exceeded with this video to import.')
+ }
+
const { videoFileResolution } = await getVideoFileResolution(tempVideoPath)
const fps = await getVideoFileFPS(tempVideoPath)
- const stats = await statPromise(tempVideoPath)
const duration = await getDurationFromVideoFile(tempVideoPath)
// Create video file object in database
const videoFileData = {
extname: extname(tempVideoPath),
resolution: videoFileResolution,
- size: stats.size,
+ size,
fps,
videoId: videoImport.videoId
}
diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts
index 9d303eee2..f3f257d57 100644
--- a/server/middlewares/validators/config.ts
+++ b/server/middlewares/validators/config.ts
@@ -25,6 +25,7 @@ const customConfigUpdateValidator = [
body('transcoding.resolutions.720p').isBoolean().withMessage('Should have a valid transcoding 720p resolution enabled boolean'),
body('transcoding.resolutions.1080p').isBoolean().withMessage('Should have a valid transcoding 1080p resolution enabled boolean'),
body('import.videos.http.enabled').isBoolean().withMessage('Should have a valid import video http enabled boolean'),
+ body('import.videos.torrent.enabled').isBoolean().withMessage('Should have a valid import video torrent enabled boolean'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking customConfigUpdateValidator parameters', { parameters: req.body })
diff --git a/server/middlewares/validators/video-imports.ts b/server/middlewares/validators/video-imports.ts
index c03cf2e4d..9ac739101 100644
--- a/server/middlewares/validators/video-imports.ts
+++ b/server/middlewares/validators/video-imports.ts
@@ -33,21 +33,28 @@ const videoImportAddValidator = getCommonVideoAttributes().concat([
logger.debug('Checking videoImportAddValidator parameters', { parameters: req.body })
const user = res.locals.oauth.token.User
+ const torrentFile = req.files && req.files['torrentfile'] ? req.files['torrentfile'][0] : undefined
if (areValidationErrors(req, res)) return cleanUpReqFiles(req)
- if (CONFIG.IMPORT.VIDEOS.HTTP.ENABLED !== true) {
+ if (req.body.targetUrl && CONFIG.IMPORT.VIDEOS.HTTP.ENABLED !== true) {
cleanUpReqFiles(req)
return res.status(409)
- .json({ error: 'Import is not enabled on this instance.' })
+ .json({ error: 'HTTP import is not enabled on this instance.' })
.end()
}
+ if (CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED !== true && (req.body.magnetUri || torrentFile)) {
+ cleanUpReqFiles(req)
+ return res.status(409)
+ .json({ error: 'Torrent/magnet URI import is not enabled on this instance.' })
+ .end()
+ }
+
if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req)
// Check we have at least 1 required param
- const file = req.files['torrentfile'][0]
- if (!req.body.targetUrl && !req.body.magnetUri && !file) {
+ if (!req.body.targetUrl && !req.body.magnetUri && !torrentFile) {
cleanUpReqFiles(req)
return res.status(400)
diff --git a/server/models/account/user.ts b/server/models/account/user.ts
index 1165285ea..1b1fc5ee8 100644
--- a/server/models/account/user.ts
+++ b/server/models/account/user.ts
@@ -295,7 +295,7 @@ export class UserModel extends Model<UserModel> {
return json
}
- isAbleToUploadVideo (videoFile: Express.Multer.File) {
+ isAbleToUploadVideo (videoFile: { size: number }) {
if (this.videoQuota === -1) return Promise.resolve(true)
return UserModel.getOriginalVideoFileTotalFromUser(this)
diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts
index d6c02e5ac..b794d8324 100644
--- a/server/models/video/video-import.ts
+++ b/server/models/video/video-import.ts
@@ -15,34 +15,21 @@ import {
} from 'sequelize-typescript'
import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers'
import { getSort, throwIfNotValid } from '../utils'
-import { VideoModel } from './video'
+import { ScopeNames as VideoModelScopeNames, VideoModel } from './video'
import { isVideoImportStateValid, isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports'
import { VideoImport, VideoImportState } from '../../../shared'
-import { VideoChannelModel } from './video-channel'
-import { AccountModel } from '../account/account'
-import { TagModel } from './tag'
import { isVideoMagnetUriValid } from '../../helpers/custom-validators/videos'
+import { UserModel } from '../account/user'
@DefaultScope({
include: [
{
- model: () => VideoModel,
- required: false,
- include: [
- {
- model: () => VideoChannelModel,
- required: true,
- include: [
- {
- model: () => AccountModel,
- required: true
- }
- ]
- },
- {
- model: () => TagModel
- }
- ]
+ model: () => UserModel.unscoped(),
+ required: true
+ },
+ {
+ model: () => VideoModel.scope([ VideoModelScopeNames.WITH_ACCOUNT_DETAILS, VideoModelScopeNames.WITH_TAGS]),
+ required: false
}
]
})
@@ -53,6 +40,9 @@ import { isVideoMagnetUriValid } from '../../helpers/custom-validators/videos'
{
fields: [ 'videoId' ],
unique: true
+ },
+ {
+ fields: [ 'userId' ]
}
]
})
@@ -91,6 +81,18 @@ export class VideoImportModel extends Model<VideoImportModel> {
@Column(DataType.TEXT)
error: string
+ @ForeignKey(() => UserModel)
+ @Column
+ userId: number
+
+ @BelongsTo(() => UserModel, {
+ foreignKey: {
+ allowNull: false
+ },
+ onDelete: 'cascade'
+ })
+ User: UserModel
+
@ForeignKey(() => VideoModel)
@Column
videoId: number
@@ -116,41 +118,24 @@ export class VideoImportModel extends Model<VideoImportModel> {
return VideoImportModel.findById(id)
}
- static listUserVideoImportsForApi (accountId: number, start: number, count: number, sort: string) {
+ static listUserVideoImportsForApi (userId: number, start: number, count: number, sort: string) {
const query = {
distinct: true,
+ include: [
+ {
+ model: UserModel.unscoped(), // FIXME: Without this, sequelize try to COUNT(DISTINCT(*)) which is an invalid SQL query
+ required: true
+ }
+ ],
offset: start,
limit: count,
order: getSort(sort),
- include: [
- {
- model: VideoModel,
- required: false,
- include: [
- {
- model: VideoChannelModel,
- required: true,
- include: [
- {
- model: AccountModel,
- required: true,
- where: {
- id: accountId
- }
- }
- ]
- },
- {
- model: TagModel,
- required: false
- }
- ]
- }
- ]
+ where: {
+ userId
+ }
}
- return VideoImportModel.unscoped()
- .findAndCountAll(query)
+ return VideoImportModel.findAndCountAll(query)
.then(({ rows, count }) => {
return {
data: rows,
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts
index 2742e26de..b26dfa252 100644
--- a/server/tests/api/check-params/config.ts
+++ b/server/tests/api/check-params/config.ts
@@ -65,6 +65,9 @@ describe('Test config API validators', function () {
videos: {
http: {
enabled: false
+ },
+ torrent: {
+ enabled: false
}
}
}
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts
index b65061a5d..f9805b6ea 100644
--- a/server/tests/api/server/config.ts
+++ b/server/tests/api/server/config.ts
@@ -45,6 +45,7 @@ function checkInitialConfig (data: CustomConfig) {
expect(data.transcoding.resolutions['720p']).to.be.true
expect(data.transcoding.resolutions['1080p']).to.be.true
expect(data.import.videos.http.enabled).to.be.true
+ expect(data.import.videos.torrent.enabled).to.be.true
}
function checkUpdatedConfig (data: CustomConfig) {
@@ -72,6 +73,7 @@ function checkUpdatedConfig (data: CustomConfig) {
expect(data.transcoding.resolutions['720p']).to.be.false
expect(data.transcoding.resolutions['1080p']).to.be.false
expect(data.import.videos.http.enabled).to.be.false
+ expect(data.import.videos.torrent.enabled).to.be.false
}
describe('Test config', function () {
@@ -167,6 +169,9 @@ describe('Test config', function () {
videos: {
http: {
enabled: false
+ },
+ torrent: {
+ enabled: false
}
}
}
diff --git a/server/tests/utils/server/config.ts b/server/tests/utils/server/config.ts
index e21614282..d6ac3ef8a 100644
--- a/server/tests/utils/server/config.ts
+++ b/server/tests/utils/server/config.ts
@@ -97,6 +97,9 @@ function updateCustomSubConfig (url: string, token: string, newConfig: any) {
videos: {
http: {
enabled: false
+ },
+ torrent: {
+ enabled: false
}
}
}
diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts
index 46320435d..d70c757b6 100644
--- a/shared/models/server/custom-config.model.ts
+++ b/shared/models/server/custom-config.model.ts
@@ -60,6 +60,9 @@ export interface CustomConfig {
videos: {
http: {
enabled: boolean
+ },
+ torrent: {
+ enabled: boolean
}
}
}
diff --git a/shared/models/server/server-config.model.ts b/shared/models/server/server-config.model.ts
index 2cafedbbc..8cb087234 100644
--- a/shared/models/server/server-config.model.ts
+++ b/shared/models/server/server-config.model.ts
@@ -28,6 +28,9 @@ export interface ServerConfig {
http: {
enabled: boolean
}
+ torrent: {
+ enabled: boolean
+ }
}
}