Add additional checks when importing a video

pull/4795/head
Chocobozzz 2022-02-09 08:58:40 +01:00
parent ba8a8367e7
commit 474542d7ac
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
8 changed files with 123 additions and 9 deletions

View File

@ -1,6 +1,6 @@
# Changelog
## v4.1.0-rc.1
## v4.1.0-rc.1 (unreleased)
### IMPORTANT NOTES

View File

@ -267,14 +267,22 @@
<my-peertube-checkbox
inputName="importVideosHttpEnabled" formControlName="enabled"
i18n-labelText labelText="Allow import with HTTP URL (e.g. YouTube)"
></my-peertube-checkbox>
>
<ng-container ngProjectAs="description">
<span i18n>⚠️ If enabled, we recommend to use <a href="https://docs.joinpeertube.org/maintain-configuration?id=security">a HTTP proxy</a> to prevent private URL access from your PeerTube server</span>
</ng-container>
</my-peertube-checkbox>
</div>
<div class="form-group" formGroupName="torrent">
<my-peertube-checkbox
inputName="importVideosTorrentEnabled" formControlName="enabled"
i18n-labelText labelText="Allow import with a torrent file or a magnet URI"
></my-peertube-checkbox>
>
<ng-container ngProjectAs="description">
<span i18n>⚠️ We don't recommend to enable this feature if you don't trust your users</span>
</ng-container>
</my-peertube-checkbox>
</div>
</ng-container>

View File

@ -42,7 +42,12 @@ export const FollowsRoutes: Routes = [
},
{
path: 'video-redundancies-list',
component: VideoRedundanciesListComponent
component: VideoRedundanciesListComponent,
data: {
meta: {
title: $localize`Redundancy`
}
}
}
]
}

View File

@ -431,7 +431,10 @@ import:
# Amount of import jobs to execute in parallel
concurrency: 1
http: # Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html
# Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html
http:
# We recommend to use a HTTP proxy if you enable HTTP import to prevent private URL access from this server
# See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
enabled: false
youtube_dl_release:
@ -452,7 +455,10 @@ import:
# IPv6 is very strongly rate-limited on most sites supported by youtube-dl
force_ipv4: false
torrent: # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file)
# Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file)
torrent:
# We recommend to only enable magnet URI/torrent import if you trust your users
# See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
enabled: false
auto_blacklist:

View File

@ -439,7 +439,10 @@ import:
# Amount of import jobs to execute in parallel
concurrency: 1
http: # Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html
# Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html
http:
# We recommend to use a HTTP proxy if you enable HTTP import to prevent private URL access from this server
# See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
enabled: false
youtube_dl_release:
@ -460,7 +463,10 @@ import:
# IPv6 is very strongly rate-limited on most sites supported by youtube-dl
force_ipv4: false
torrent: # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file)
# Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file)
torrent:
# We recommend to only enable magnet URI/torrent import if you trust your users
# See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
enabled: false
auto_blacklist:

View File

@ -1,9 +1,11 @@
import express from 'express'
import { move, readFile } from 'fs-extra'
import { move, readFile, remove } from 'fs-extra'
import { decode } from 'magnet-uri'
import parseTorrent, { Instance } from 'parse-torrent'
import { join } from 'path'
import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
import { isResolvingToUnicastOnly } from '@server/helpers/dns'
import { Hooks } from '@server/lib/plugins/hooks'
import { ServerConfigManager } from '@server/lib/server-config-manager'
import { setVideoTags } from '@server/lib/video'
@ -195,6 +197,13 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
})
}
if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
return res.fail({
status: HttpStatusCode.FORBIDDEN_403,
message: 'Cannot use non unicast IP as targetUrl.'
})
}
const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
// Process video thumbnail from request.files
@ -432,6 +441,11 @@ async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl:
logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
for (const subtitle of subtitles) {
if (!await isVTTFileValid(subtitle.path)) {
await remove(subtitle.path)
continue
}
const videoCaption = new VideoCaptionModel({
videoId,
language: subtitle.language,
@ -449,3 +463,16 @@ async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl:
logger.warn('Cannot get video subtitles.', { err })
}
}
async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
const uniqHosts = new Set(hosts)
for (const h of uniqHosts) {
if (await isResolvingToUnicastOnly(h) !== true) {
return false
}
}
return true
}

View File

@ -1,3 +1,5 @@
import { getFileSize } from '@shared/extra-utils'
import { readFile } from 'fs-extra'
import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_LANGUAGES } from '../../initializers/constants'
import { exists, isFileValid } from './misc'
@ -13,9 +15,20 @@ function isVideoCaptionFile (files: { [ fieldname: string ]: Express.Multer.File
return isFileValid(files, videoCaptionTypesRegex, field, CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max)
}
async function isVTTFileValid (filePath: string) {
const size = await getFileSize(filePath)
if (size > CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max) return false
const content = await readFile(filePath, 'utf8')
return content?.startsWith('WEBVTT\n')
}
// ---------------------------------------------------------------------------
export {
isVideoCaptionFile,
isVTTFileValid,
isVideoCaptionLanguageValid
}

View File

@ -1,5 +1,6 @@
import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../../initializers/constants'
import { peertubeTruncate } from '../core-utils'
import { isUrlValid } from '../custom-validators/activitypub/misc'
type YoutubeDLInfo = {
name?: string
@ -12,6 +13,8 @@ type YoutubeDLInfo = {
thumbnailUrl?: string
ext?: string
originallyPublishedAt?: Date
urls?: string[]
}
class YoutubeDLInfoBuilder {
@ -76,11 +79,57 @@ class YoutubeDLInfoBuilder {
nsfw: this.isNSFW(obj),
tags: this.getTags(obj.tags),
thumbnailUrl: obj.thumbnail || undefined,
urls: this.buildAvailableUrl(obj),
originallyPublishedAt: this.buildOriginallyPublishedAt(obj),
ext: obj.ext
}
}
private buildAvailableUrl (obj: any) {
const urls: string[] = []
if (obj.url) urls.push(obj.url)
if (obj.urls) {
if (Array.isArray(obj.urls)) urls.push(...obj.urls)
else urls.push(obj.urls)
}
const formats = Array.isArray(obj.formats)
? obj.formats
: []
for (const format of formats) {
if (!format.url) continue
urls.push(format.url)
}
const thumbnails = Array.isArray(obj.thumbnails)
? obj.thumbnails
: []
for (const thumbnail of thumbnails) {
if (!thumbnail.url) continue
urls.push(thumbnail.url)
}
if (obj.thumbnail) urls.push(obj.thumbnail)
for (const subtitleKey of Object.keys(obj.subtitles || {})) {
const subtitles = obj.subtitles[subtitleKey]
if (!Array.isArray(subtitles)) continue
for (const subtitle of subtitles) {
if (!subtitle.url) continue
urls.push(subtitle.url)
}
}
return urls.filter(u => u && isUrlValid(u))
}
private titleTruncation (title: string) {
return peertubeTruncate(title, {
length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max,