From d7a25329f9e607894d29ab342b9cb66638b56dc0 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 15 Nov 2019 15:06:03 +0100 Subject: [PATCH] Add ability to disable webtorrent In favour of HLS --- .../assets/player/peertube-videojs-typings.ts | 2 +- .../player/webtorrent/webtorrent-plugin.ts | 2 +- config/default.yaml | 8 +- config/production.yaml.example | 8 +- package.json | 2 +- scripts/optimize-old-videos.ts | 17 +- scripts/prune-storage.ts | 4 +- scripts/update-host.ts | 7 +- server/controllers/api/config.ts | 6 + server/controllers/api/videos/index.ts | 12 +- server/controllers/static.ts | 69 ++++-- .../custom-validators/activitypub/videos.ts | 36 ++-- server/helpers/database-utils.ts | 12 +- server/helpers/ffmpeg-utils.ts | 6 +- server/helpers/video.ts | 5 - server/helpers/webtorrent.ts | 69 +++++- server/initializers/checker-after-init.ts | 7 + server/initializers/config.ts | 3 + server/initializers/constants.ts | 5 +- .../migrations/0065-video-file-size.ts | 3 +- .../0450-streaming-playlist-files.ts | 40 ++++ server/lib/activitypub/videos.ts | 141 +++++++----- server/lib/hls.ts | 13 +- .../job-queue/handlers/video-file-import.ts | 6 +- server/lib/job-queue/handlers/video-import.ts | 12 +- .../job-queue/handlers/video-transcoding.ts | 119 +++++----- .../lib/schedulers/update-videos-scheduler.ts | 8 +- .../schedulers/videos-redundancy-scheduler.ts | 7 +- server/lib/thumbnail.ts | 3 +- server/lib/video-paths.ts | 64 ++++++ server/lib/video-transcoding.ts | 91 +++++--- server/lib/videos.ts | 11 + server/middlewares/validators/config.ts | 17 ++ .../middlewares/validators/videos/videos.ts | 2 +- server/models/redundancy/video-redundancy.ts | 2 - server/models/utils.ts | 2 +- server/models/video/schedule-video-update.ts | 10 +- server/models/video/video-change-ownership.ts | 6 +- server/models/video/video-file.ts | 87 +++++++- server/models/video/video-format-utils.ts | 127 ++++++----- .../models/video/video-streaming-playlist.ts | 40 +++- server/models/video/video.ts | 204 +++++++++--------- server/tests/api/check-params/config.ts | 24 +++ server/tests/api/server/config.ts | 5 + server/tests/api/videos/video-hls.ts | 191 ++++++++++------ .../tests/cli/create-import-video-file-job.ts | 5 +- server/typings/models/account/account.ts | 2 +- server/typings/models/account/actor-follow.ts | 5 +- .../models/account/{index.d.ts => index.ts} | 0 .../typings/models/{index.d.ts => index.ts} | 0 .../models/oauth/{index.d.ts => index.ts} | 0 server/typings/models/oauth/oauth-token.ts | 2 +- .../models/server/{index.d.ts => index.ts} | 0 .../typings/models/server/server-blocklist.ts | 3 +- .../models/user/{index.d.ts => index.ts} | 0 server/typings/models/user/user.ts | 2 +- .../models/video/{index.d.ts => index.ts} | 0 .../models/video/schedule-video-update.ts | 9 + .../typings/models/video/video-blacklist.ts | 2 +- server/typings/models/video/video-caption.ts | 2 +- .../models/video/video-change-ownership.ts | 5 +- server/typings/models/video/video-comment.ts | 2 +- server/typings/models/video/video-file.ts | 17 +- server/typings/models/video/video-import.ts | 3 +- .../models/video/video-playlist-element.ts | 3 +- server/typings/models/video/video-rate.ts | 3 +- .../typings/models/video/video-redundancy.ts | 6 +- .../models/video/video-streaming-playlist.ts | 18 +- server/typings/models/video/video.ts | 18 +- shared/extra-utils/server/config.ts | 3 + shared/extra-utils/videos/videos.ts | 3 +- .../activitypub/objects/common-objects.ts | 51 +++-- shared/models/server/custom-config.model.ts | 7 + shared/models/server/server-config.model.ts | 4 + shared/models/videos/index.ts | 1 + shared/models/videos/video-file.model.ts | 12 ++ .../videos/video-streaming-playlist.model.ts | 3 + shared/models/videos/video.model.ts | 12 +- tsconfig.json | 3 +- yarn.lock | 8 +- 80 files changed, 1189 insertions(+), 540 deletions(-) create mode 100644 server/initializers/migrations/0450-streaming-playlist-files.ts create mode 100644 server/lib/video-paths.ts create mode 100644 server/lib/videos.ts rename server/typings/models/account/{index.d.ts => index.ts} (100%) rename server/typings/models/{index.d.ts => index.ts} (100%) rename server/typings/models/oauth/{index.d.ts => index.ts} (100%) rename server/typings/models/server/{index.d.ts => index.ts} (100%) rename server/typings/models/user/{index.d.ts => index.ts} (100%) rename server/typings/models/video/{index.d.ts => index.ts} (100%) create mode 100644 shared/models/videos/video-file.model.ts diff --git a/client/src/assets/player/peertube-videojs-typings.ts b/client/src/assets/player/peertube-videojs-typings.ts index b7f2eec94..aad4dbb4f 100644 --- a/client/src/assets/player/peertube-videojs-typings.ts +++ b/client/src/assets/player/peertube-videojs-typings.ts @@ -2,12 +2,12 @@ // @ts-ignore import * as videojs from 'video.js' -import { VideoFile } from '../../../../shared/models/videos/video.model' import { PeerTubePlugin } from './peertube-plugin' import { WebTorrentPlugin } from './webtorrent/webtorrent-plugin' import { P2pMediaLoaderPlugin } from './p2p-media-loader/p2p-media-loader-plugin' import { PlayerMode } from './peertube-player-manager' import { RedundancyUrlManager } from './p2p-media-loader/redundancy-url-manager' +import { VideoFile } from '@shared/models' declare namespace videojs { interface Player { diff --git a/client/src/assets/player/webtorrent/webtorrent-plugin.ts b/client/src/assets/player/webtorrent/webtorrent-plugin.ts index 95f52dfe1..4a0b38703 100644 --- a/client/src/assets/player/webtorrent/webtorrent-plugin.ts +++ b/client/src/assets/player/webtorrent/webtorrent-plugin.ts @@ -3,7 +3,6 @@ import * as videojs from 'video.js' import * as WebTorrent from 'webtorrent' -import { VideoFile } from '../../../../../shared/models/videos/video.model' import { renderVideo } from './video-renderer' import { LoadedQualityData, PlayerNetworkInfo, VideoJSComponentInterface, WebtorrentPluginOptions } from '../peertube-videojs-typings' import { getRtcConfig, timeToInt, videoFileMaxByResolution, videoFileMinByResolution } from '../utils' @@ -15,6 +14,7 @@ import { getStoredWebTorrentEnabled, saveAverageBandwidth } from '../peertube-player-local-storage' +import { VideoFile } from '@shared/models' const CacheChunkStore = require('cache-chunk-store') diff --git a/config/default.yaml b/config/default.yaml index b16ebe934..9d102f760 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -209,12 +209,18 @@ transcoding: 720p: false 1080p: false 2160p: false + + # Generate videos in a WebTorrent format (what we do since the first PeerTube release) + # If you also enabled the hls format, it will multiply videos storage by 2 + webtorrent: + enabled: true + # /!\ Requires ffmpeg >= 4.1 # Generate HLS playlists and fragmented MP4 files. Better playback than with WebTorrent: # * Resolution change is smoother # * Faster playback in particular with long videos # * More stable playback (less bugs/infinite loading) - # /!\ Multiplies videos storage by 2 /!\ + # If you also enabled the webtorrent format, it will multiply videos storage by 2 hls: enabled: false diff --git a/config/production.yaml.example b/config/production.yaml.example index d563c7cf2..68ae22944 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -223,12 +223,18 @@ transcoding: 720p: false 1080p: false 2160p: false + + # Generate videos in a WebTorrent format (what we do since the first PeerTube release) + # If you also enabled the hls format, it will multiply videos storage by 2 + webtorrent: + enabled: true + # /!\ Requires ffmpeg >= 4.1 # Generate HLS playlists and fragmented MP4 files. Better playback than with WebTorrent: # * Resolution change is smoother # * Faster playback in particular with long videos # * More stable playback (less bugs/infinite loading) - # /!\ Multiplies videos storage by 2 /!\ + # If you also enabled the webtorrent format, it will multiply videos storage by 2 hls: enabled: false diff --git a/package.json b/package.json index de406d883..535c87cfc 100644 --- a/package.json +++ b/package.json @@ -219,7 +219,7 @@ "ts-node": "8.4.1", "tslint": "^5.7.0", "tslint-config-standard": "^8.0.1", - "typescript": "^3.4.3", + "typescript": "^3.7.2", "xliff": "^4.0.0" }, "scripty": { diff --git a/scripts/optimize-old-videos.ts b/scripts/optimize-old-videos.ts index eb61ec43c..107483c50 100644 --- a/scripts/optimize-old-videos.ts +++ b/scripts/optimize-old-videos.ts @@ -1,15 +1,16 @@ import { registerTSPaths } from '../server/helpers/register-ts-paths' -registerTSPaths() - import { VIDEO_TRANSCODING_FPS } from '../server/initializers/constants' import { getDurationFromVideoFile, getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../server/helpers/ffmpeg-utils' import { getMaxBitrate } from '../shared/models/videos' import { VideoModel } from '../server/models/video/video' -import { optimizeVideofile } from '../server/lib/video-transcoding' +import { optimizeOriginalVideofile } from '../server/lib/video-transcoding' import { initDatabaseModels } from '../server/initializers' -import { basename, dirname, join } from 'path' +import { basename, dirname } from 'path' import { copy, move, remove } from 'fs-extra' -import { CONFIG } from '../server/initializers/config' +import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' +import { getVideoFilePath } from '@server/lib/video-paths' + +registerTSPaths() run() .then(() => process.exit(0)) @@ -37,7 +38,7 @@ async function run () { currentVideoId = video.id for (const file of video.VideoFiles) { - currentFile = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file)) + currentFile = getVideoFilePath(video, file) const [ videoBitrate, fps, resolution ] = await Promise.all([ getVideoFileBitrate(currentFile), @@ -56,7 +57,7 @@ async function run () { const backupFile = `${currentFile}_backup` await copy(currentFile, backupFile) - await optimizeVideofile(video, file) + await optimizeOriginalVideofile(video, file) const originalDuration = await getDurationFromVideoFile(backupFile) const newDuration = await getDurationFromVideoFile(currentFile) @@ -69,7 +70,7 @@ async function run () { console.log('Failed to optimize %s, restoring original', basename(currentFile)) await move(backupFile, currentFile, { overwrite: true }) - await video.createTorrentAndSetInfoHash(file) + await createTorrentAndSetInfoHash(video, file) await file.save() } } diff --git a/scripts/prune-storage.ts b/scripts/prune-storage.ts index c8968013b..fa3d81744 100755 --- a/scripts/prune-storage.ts +++ b/scripts/prune-storage.ts @@ -134,9 +134,9 @@ async function doesRedundancyExist (file: string) { return true } - const videoFile = video.getFile(resolution) + const videoFile = video.getWebTorrentFile(resolution) if (!videoFile) { - console.error('Cannot find file of video %s - %d', video.url, resolution) + console.error('Cannot find webtorrent file of video %s - %d', video.url, resolution) return true } diff --git a/scripts/update-host.ts b/scripts/update-host.ts index a946d2e42..d9192d554 100755 --- a/scripts/update-host.ts +++ b/scripts/update-host.ts @@ -1,6 +1,4 @@ import { registerTSPaths } from '../server/helpers/register-ts-paths' -registerTSPaths() - import { WEBSERVER } from '../server/initializers/constants' import { ActorFollowModel } from '../server/models/activitypub/actor-follow' import { VideoModel } from '../server/models/video/video' @@ -19,6 +17,9 @@ import { AccountModel } from '../server/models/account/account' import { VideoChannelModel } from '../server/models/video/video-channel' import { VideoStreamingPlaylistModel } from '../server/models/video/video-streaming-playlist' import { initDatabaseModels } from '../server/initializers' +import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' + +registerTSPaths() run() .then(() => process.exit(0)) @@ -124,7 +125,7 @@ async function run () { for (const file of video.VideoFiles) { console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid) - await video.createTorrentAndSetInfoHash(file) + await createTorrentAndSetInfoHash(video, file) } for (const playlist of video.VideoStreamingPlaylists) { diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index 113c1e9db..70e8aa970 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -95,6 +95,9 @@ async function getConfig (req: express.Request, res: express.Response) { hls: { enabled: CONFIG.TRANSCODING.HLS.ENABLED }, + webtorrent: { + enabled: CONFIG.TRANSCODING.WEBTORRENT.ENABLED + }, enabledResolutions: getEnabledResolutions() }, import: { @@ -304,6 +307,9 @@ function customConfig (): CustomConfig { '1080p': CONFIG.TRANSCODING.RESOLUTIONS[ '1080p' ], '2160p': CONFIG.TRANSCODING.RESOLUTIONS[ '2160p' ] }, + webtorrent: { + enabled: CONFIG.TRANSCODING.WEBTORRENT.ENABLED + }, hls: { enabled: CONFIG.TRANSCODING.HLS.ENABLED } diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 0d1fbc8f4..78948ff24 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -64,6 +64,8 @@ import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type' import { VideoTranscodingPayload } from '../../../lib/job-queue/handlers/video-transcoding' import { Hooks } from '../../../lib/plugins/hooks' import { MVideoDetails, MVideoFullLight } from '@server/typings/models' +import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' +import { getVideoFilename, getVideoFilePath } from '@server/lib/video-paths' const auditLogger = auditLoggerFactory('videos') const videosRouter = express.Router() @@ -203,7 +205,8 @@ async function addVideo (req: express.Request, res: express.Response) { const videoFile = new VideoFileModel({ extname: extname(videoPhysicalFile.filename), - size: videoPhysicalFile.size + size: videoPhysicalFile.size, + videoStreamingPlaylistId: null }) if (videoFile.isAudio()) { @@ -214,11 +217,10 @@ async function addVideo (req: express.Request, res: express.Response) { } // Move physical file - const videoDir = CONFIG.STORAGE.VIDEOS_DIR - const destination = join(videoDir, video.getVideoFilename(videoFile)) + const destination = getVideoFilePath(video, videoFile) await move(videoPhysicalFile.path, destination) // This is important in case if there is another attempt in the retry process - videoPhysicalFile.filename = video.getVideoFilename(videoFile) + videoPhysicalFile.filename = getVideoFilePath(video, videoFile) videoPhysicalFile.path = destination // Process thumbnail or create it from the video @@ -234,7 +236,7 @@ async function addVideo (req: express.Request, res: express.Response) { : await generateVideoMiniature(video, videoFile, ThumbnailType.PREVIEW) // Create the torrent file - await video.createTorrentAndSetInfoHash(videoFile) + await createTorrentAndSetInfoHash(video, videoFile) const { videoCreated } = await sequelizeTypescript.transaction(async t => { const sequelizeOptions = { transaction: t } diff --git a/server/controllers/static.ts b/server/controllers/static.ts index 0f4772310..06123518f 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts @@ -19,6 +19,9 @@ import { join } from 'path' import { root } from '../helpers/core-utils' import { CONFIG } from '../initializers/config' import { getPreview, getVideoCaption } from './lazy-static' +import { VideoStreamingPlaylistType } from '@shared/models/videos/video-streaming-playlist.type' +import { MVideoFile, MVideoFullLight } from '@server/typings/models' +import { getTorrentFilePath, getVideoFilePath } from '@server/lib/video-paths' const staticRouter = express.Router() @@ -39,6 +42,11 @@ staticRouter.use( asyncMiddleware(videosGetValidator), asyncMiddleware(downloadTorrent) ) +staticRouter.use( + STATIC_DOWNLOAD_PATHS.TORRENTS + ':id-:resolution([0-9]+)-hls.torrent', + asyncMiddleware(videosGetValidator), + asyncMiddleware(downloadHLSVideoFileTorrent) +) // Videos path for webseeding staticRouter.use( @@ -58,6 +66,12 @@ staticRouter.use( asyncMiddleware(downloadVideoFile) ) +staticRouter.use( + STATIC_DOWNLOAD_PATHS.HLS_VIDEOS + ':id-:resolution([0-9]+).:extension', + asyncMiddleware(videosGetValidator), + asyncMiddleware(downloadHLSVideoFile) +) + // HLS staticRouter.use( STATIC_PATHS.STREAMING_PLAYLISTS.HLS, @@ -227,24 +241,55 @@ async function generateNodeinfo (req: express.Request, res: express.Response) { } async function downloadTorrent (req: express.Request, res: express.Response) { - const { video, videoFile } = getVideoAndFile(req, res) + const video = res.locals.videoAll + + const videoFile = getVideoFile(req, video.VideoFiles) if (!videoFile) return res.status(404).end() - return res.download(video.getTorrentFilePath(videoFile), `${video.name}-${videoFile.resolution}p.torrent`) + return res.download(getTorrentFilePath(video, videoFile), `${video.name}-${videoFile.resolution}p.torrent`) +} + +async function downloadHLSVideoFileTorrent (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + + const playlist = getHLSPlaylist(video) + if (!playlist) return res.status(404).end + + const videoFile = getVideoFile(req, playlist.VideoFiles) + if (!videoFile) return res.status(404).end() + + return res.download(getTorrentFilePath(playlist, videoFile), `${video.name}-${videoFile.resolution}p-hls.torrent`) } async function downloadVideoFile (req: express.Request, res: express.Response) { - const { video, videoFile } = getVideoAndFile(req, res) - if (!videoFile) return res.status(404).end() - - return res.download(video.getVideoFilePath(videoFile), `${video.name}-${videoFile.resolution}p${videoFile.extname}`) -} - -function getVideoAndFile (req: express.Request, res: express.Response) { - const resolution = parseInt(req.params.resolution, 10) const video = res.locals.videoAll - const videoFile = video.VideoFiles.find(f => f.resolution === resolution) + const videoFile = getVideoFile(req, video.VideoFiles) + if (!videoFile) return res.status(404).end() - return { video, videoFile } + return res.download(getVideoFilePath(video, videoFile), `${video.name}-${videoFile.resolution}p${videoFile.extname}`) +} + +async function downloadHLSVideoFile (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + const playlist = getHLSPlaylist(video) + if (!playlist) return res.status(404).end + + const videoFile = getVideoFile(req, playlist.VideoFiles) + if (!videoFile) return res.status(404).end() + + const filename = `${video.name}-${videoFile.resolution}p-${playlist.getStringType()}${videoFile.extname}` + return res.download(getVideoFilePath(playlist, videoFile), filename) +} + +function getVideoFile (req: express.Request, files: MVideoFile[]) { + const resolution = parseInt(req.params.resolution, 10) + return files.find(f => f.resolution === resolution) +} + +function getHLSPlaylist (video: MVideoFullLight) { + const playlist = video.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) + if (!playlist) return undefined + + return Object.assign(playlist, { Video: video }) } diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index 02f914326..a28bebf2d 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -12,6 +12,7 @@ import { } from '../videos' import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc' import { VideoState } from '../../../../shared/models/videos' +import { logger } from '@server/helpers/logger' function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { return isBaseActivityValid(activity, 'Update') && @@ -30,11 +31,26 @@ function isActivityPubVideoDurationValid (value: string) { function sanitizeAndCheckVideoTorrentObject (video: any) { if (!video || video.type !== 'Video') return false - if (!setValidRemoteTags(video)) return false - if (!setValidRemoteVideoUrls(video)) return false - if (!setRemoteVideoTruncatedContent(video)) return false - if (!setValidAttributedTo(video)) return false - if (!setValidRemoteCaptions(video)) return false + if (!setValidRemoteTags(video)) { + logger.debug('Video has invalid tags', { video }) + return false + } + if (!setValidRemoteVideoUrls(video)) { + logger.debug('Video has invalid urls', { video }) + return false + } + if (!setRemoteVideoTruncatedContent(video)) { + logger.debug('Video has invalid content', { video }) + return false + } + if (!setValidAttributedTo(video)) { + logger.debug('Video has invalid attributedTo', { video }) + return false + } + if (!setValidRemoteCaptions(video)) { + logger.debug('Video has invalid captions', { video }) + return false + } // Default attributes if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED @@ -62,25 +78,21 @@ function sanitizeAndCheckVideoTorrentObject (video: any) { } function isRemoteVideoUrlValid (url: any) { - // FIXME: Old bug, we used the width to represent the resolution. Remove it in a few release (currently beta.11) - if (url.width && !url.height) url.height = url.width - return url.type === 'Link' && ( - // TODO: remove mimeType (backward compatibility, introduced in v1.1.0) - ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mediaType || url.mimeType) !== -1 && + ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mediaType) !== -1 && isActivityPubUrlValid(url.href) && validator.isInt(url.height + '', { min: 0 }) && validator.isInt(url.size + '', { min: 0 }) && (!url.fps || validator.isInt(url.fps + '', { min: -1 })) ) || ( - ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mediaType || url.mimeType) !== -1 && + ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mediaType) !== -1 && isActivityPubUrlValid(url.href) && validator.isInt(url.height + '', { min: 0 }) ) || ( - ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mediaType || url.mimeType) !== -1 && + ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mediaType) !== -1 && validator.isLength(url.href, { min: 5 }) && validator.isInt(url.height + '', { min: 0 }) ) || diff --git a/server/helpers/database-utils.ts b/server/helpers/database-utils.ts index 6c5068fb0..87f10f913 100644 --- a/server/helpers/database-utils.ts +++ b/server/helpers/database-utils.ts @@ -79,6 +79,15 @@ function afterCommitIfTransaction (t: Transaction, fn: Function) { return fn() } +function deleteNonExistingModels > ( + fromDatabase: T[], + newModels: T[], + t: Transaction +) { + return fromDatabase.filter(f => !newModels.find(newModel => newModel.hasSameUniqueKeysThan(f))) + .map(f => f.destroy({ transaction: t })) +} + // --------------------------------------------------------------------------- export { @@ -86,5 +95,6 @@ export { retryTransactionWrapper, transactionRetryer, updateInstanceWithAnother, - afterCommitIfTransaction + afterCommitIfTransaction, + deleteNonExistingModels } diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index c0e9702a8..7a4ac0970 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts @@ -130,6 +130,7 @@ interface BaseTranscodeOptions { interface HLSTranscodeOptions extends BaseTranscodeOptions { type: 'hls' + copyCodecs: boolean hlsPlaylist: { videoFilename: string } @@ -232,7 +233,7 @@ export { // --------------------------------------------------------------------------- -async function buildx264Command (command: ffmpeg.FfmpegCommand, options: VideoTranscodeOptions) { +async function buildx264Command (command: ffmpeg.FfmpegCommand, options: TranscodeOptions) { let fps = await getVideoFileFPS(options.inputPath) // On small/medium resolutions, limit FPS if ( @@ -287,7 +288,8 @@ async function buildQuickTranscodeCommand (command: ffmpeg.FfmpegCommand) { async function buildHLSCommand (command: ffmpeg.FfmpegCommand, options: HLSTranscodeOptions) { const videoPath = getHLSVideoPath(options) - command = await presetCopy(command) + if (options.copyCodecs) command = await presetCopy(command) + else command = await buildx264Command(command, options) command = command.outputOption('-hls_time 4') .outputOption('-hls_list_size 0') diff --git a/server/helpers/video.ts b/server/helpers/video.ts index d066e2b1f..5b9c026b1 100644 --- a/server/helpers/video.ts +++ b/server/helpers/video.ts @@ -45,10 +45,6 @@ function fetchVideoByUrl (url: string, fetchType: VideoFetchByUrlType): Bluebird if (fetchType === 'only-video') return VideoModel.loadByUrl(url) } -function getVideo (res: Response) { - return res.locals.videoAll || res.locals.onlyVideo || res.locals.onlyVideoWithRights || res.locals.videoId -} - function getVideoWithAttributes (res: Response) { return res.locals.videoAll || res.locals.onlyVideo || res.locals.onlyVideoWithRights } @@ -57,7 +53,6 @@ export { VideoFetchType, VideoFetchByUrlType, fetchVideo, - getVideo, getVideoWithAttributes, fetchVideoByUrl } diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index d2a22e8f0..f3e41f8d6 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -1,11 +1,22 @@ import { logger } from './logger' import { generateVideoImportTmpPath } from './utils' import * as WebTorrent from 'webtorrent' -import { createWriteStream, ensureDir, remove } from 'fs-extra' +import { createWriteStream, ensureDir, remove, writeFile } from 'fs-extra' import { CONFIG } from '../initializers/config' import { dirname, join } from 'path' import * as createTorrent from 'create-torrent' import { promisify2 } from './core-utils' +import { MVideo } from '@server/typings/models/video/video' +import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/typings/models/video/video-file' +import { isStreamingPlaylist, MStreamingPlaylistVideo } from '@server/typings/models/video/video-streaming-playlist' +import { STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' +import * as parseTorrent from 'parse-torrent' +import * as magnetUtil from 'magnet-uri' +import { isArray } from '@server/helpers/custom-validators/misc' +import { extractVideo } from '@server/lib/videos' +import { getTorrentFileName, getVideoFilename, getVideoFilePath } from '@server/lib/video-paths' + +const createTorrentPromise = promisify2(createTorrent) async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: string }, timeout: number) { const id = target.magnetUri || target.torrentName @@ -59,12 +70,64 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName }) } -const createTorrentPromise = promisify2(createTorrent) +async function createTorrentAndSetInfoHash (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { + const video = extractVideo(videoOrPlaylist) + + const options = { + // Keep the extname, it's used by the client to stream the file inside a web browser + name: `${video.name} ${videoFile.resolution}p${videoFile.extname}`, + createdBy: 'PeerTube', + announceList: [ + [ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ], + [ WEBSERVER.URL + '/tracker/announce' ] + ], + urlList: [ WEBSERVER.URL + STATIC_PATHS.WEBSEED + getVideoFilename(videoOrPlaylist, videoFile) ] + } + + const torrent = await createTorrentPromise(getVideoFilePath(videoOrPlaylist, videoFile), options) + + const filePath = join(CONFIG.STORAGE.TORRENTS_DIR, getTorrentFileName(videoOrPlaylist, videoFile)) + logger.info('Creating torrent %s.', filePath) + + await writeFile(filePath, torrent) + + const parsedTorrent = parseTorrent(torrent) + videoFile.infoHash = parsedTorrent.infoHash +} + +function generateMagnetUri ( + videoOrPlaylist: MVideo | MStreamingPlaylistVideo, + videoFile: MVideoFileRedundanciesOpt, + baseUrlHttp: string, + baseUrlWs: string +) { + const video = isStreamingPlaylist(videoOrPlaylist) + ? videoOrPlaylist.Video + : videoOrPlaylist + + const xs = videoOrPlaylist.getTorrentUrl(videoFile, baseUrlHttp) + const announce = videoOrPlaylist.getTrackerUrls(baseUrlHttp, baseUrlWs) + let urlList = [ videoOrPlaylist.getVideoFileUrl(videoFile, baseUrlHttp) ] + + const redundancies = videoFile.RedundancyVideos + if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl)) + + const magnetHash = { + xs, + announce, + urlList, + infoHash: videoFile.infoHash, + name: video.name + } + + return magnetUtil.encode(magnetHash) +} // --------------------------------------------------------------------------- export { - createTorrentPromise, + createTorrentAndSetInfoHash, + generateMagnetUri, downloadWebTorrentVideo } diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts index db3115085..9fefba769 100644 --- a/server/initializers/checker-after-init.ts +++ b/server/initializers/checker-after-init.ts @@ -101,6 +101,13 @@ function checkConfig () { } } + // Transcoding + if (CONFIG.TRANSCODING.ENABLED) { + if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false && CONFIG.TRANSCODING.HLS.ENABLED === false) { + return 'You need to enable at least WebTorrent transcoding or HLS transcoding.' + } + } + return null } diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 164d714d6..6d5d55487 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -177,6 +177,9 @@ const CONFIG = { }, HLS: { get ENABLED () { return config.get('transcoding.hls.enabled') } + }, + WEBTORRENT: { + get ENABLED () { return config.get('transcoding.webtorrent.enabled') } } }, IMPORT: { diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index fd4c0fdaa..eaad84bee 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -14,7 +14,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 445 +const LAST_MIGRATION_VERSION = 450 // --------------------------------------------------------------------------- @@ -505,7 +505,8 @@ const STATIC_PATHS = { } const STATIC_DOWNLOAD_PATHS = { TORRENTS: '/download/torrents/', - VIDEOS: '/download/videos/' + VIDEOS: '/download/videos/', + HLS_VIDEOS: '/download/streaming-playlists/hls/videos/' } const LAZY_STATIC_PATHS = { AVATARS: '/lazy-static/avatars/', diff --git a/server/initializers/migrations/0065-video-file-size.ts b/server/initializers/migrations/0065-video-file-size.ts index 66f25016a..e9ce77e50 100644 --- a/server/initializers/migrations/0065-video-file-size.ts +++ b/server/initializers/migrations/0065-video-file-size.ts @@ -2,6 +2,7 @@ import * as Sequelize from 'sequelize' import * as Promise from 'bluebird' import { stat } from 'fs-extra' import { VideoModel } from '../../models/video/video' +import { getVideoFilePath } from '@server/lib/video-paths' function up (utils: { transaction: Sequelize.Transaction, @@ -16,7 +17,7 @@ function up (utils: { videos.forEach(video => { video.VideoFiles.forEach(videoFile => { const p = new Promise((res, rej) => { - stat(video.getVideoFilePath(videoFile), (err, stats) => { + stat(getVideoFilePath(video, videoFile), (err, stats) => { if (err) return rej(err) videoFile.size = stats.size diff --git a/server/initializers/migrations/0450-streaming-playlist-files.ts b/server/initializers/migrations/0450-streaming-playlist-files.ts new file mode 100644 index 000000000..536ef00f9 --- /dev/null +++ b/server/initializers/migrations/0450-streaming-playlist-files.ts @@ -0,0 +1,40 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction, + queryInterface: Sequelize.QueryInterface, + sequelize: Sequelize.Sequelize, + db: any +}): Promise { + { + const data = { + type: Sequelize.INTEGER, + allowNull: true, + references: { + model: 'videoStreamingPlaylist', + key: 'id' + }, + onDelete: 'CASCADE' + } + + await utils.queryInterface.addColumn('videoFile', 'videoStreamingPlaylistId', data) + } + + { + const data = { + type: Sequelize.INTEGER, + allowNull: true + } + + await utils.queryInterface.changeColumn('videoFile', 'videoId', data) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index c318978fd..d80173e03 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -3,8 +3,10 @@ import * as sequelize from 'sequelize' import * as magnetUtil from 'magnet-uri' import * as request from 'request' import { + ActivityHashTagObject, + ActivityMagnetUrlObject, ActivityPlaylistSegmentHashesObject, - ActivityPlaylistUrlObject, + ActivityPlaylistUrlObject, ActivityTagObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState @@ -13,7 +15,7 @@ import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' import { VideoPrivacy } from '../../../shared/models/videos' import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' -import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' +import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' import { @@ -57,6 +59,7 @@ import { MChannelAccountLight, MChannelDefault, MChannelId, + MStreamingPlaylist, MVideo, MVideoAccountLight, MVideoAccountLightBlacklistAllFiles, @@ -330,21 +333,15 @@ async function updateVideoFromAP (options: { await videoUpdated.addAndSaveThumbnail(previewModel, t) { - const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoUpdated, videoObject) + const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoUpdated, videoObject.url) const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) // Remove video files that do not exist anymore - const destroyTasks = videoUpdated.VideoFiles - .filter(f => !newVideoFiles.find(newFile => newFile.hasSameUniqueKeysThan(f))) - .map(f => f.destroy(sequelizeOptions)) + const destroyTasks = deleteNonExistingModels(videoUpdated.VideoFiles, newVideoFiles, t) await Promise.all(destroyTasks) // Update or add other one - const upsertTasks = videoFileAttributes.map(a => { - return VideoFileModel.upsert(a, { returning: true, transaction: t }) - .then(([ file ]) => file) - }) - + const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t)) videoUpdated.VideoFiles = await Promise.all(upsertTasks) } @@ -352,24 +349,39 @@ async function updateVideoFromAP (options: { const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(videoUpdated, videoObject, videoUpdated.VideoFiles) const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a)) - // Remove video files that do not exist anymore - const destroyTasks = videoUpdated.VideoStreamingPlaylists - .filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f))) - .map(f => f.destroy(sequelizeOptions)) + // Remove video playlists that do not exist anymore + const destroyTasks = deleteNonExistingModels(videoUpdated.VideoStreamingPlaylists, newStreamingPlaylists, t) await Promise.all(destroyTasks) - // Update or add other one - const upsertTasks = streamingPlaylistAttributes.map(a => { - return VideoStreamingPlaylistModel.upsert(a, { returning: true, transaction: t }) - .then(([ streamingPlaylist ]) => streamingPlaylist) - }) + let oldStreamingPlaylistFiles: MVideoFile[] = [] + for (const videoStreamingPlaylist of videoUpdated.VideoStreamingPlaylists) { + oldStreamingPlaylistFiles = oldStreamingPlaylistFiles.concat(videoStreamingPlaylist.VideoFiles) + } - videoUpdated.VideoStreamingPlaylists = await Promise.all(upsertTasks) + videoUpdated.VideoStreamingPlaylists = [] + + for (const playlistAttributes of streamingPlaylistAttributes) { + const streamingPlaylistModel = await VideoStreamingPlaylistModel.upsert(playlistAttributes, { returning: true, transaction: t }) + .then(([ streamingPlaylist ]) => streamingPlaylist) + + const newVideoFiles: MVideoFile[] = videoFileActivityUrlToDBAttributes(streamingPlaylistModel, playlistAttributes.tagAPObject) + .map(a => new VideoFileModel(a)) + const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t) + await Promise.all(destroyTasks) + + // Update or add other one + const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t)) + streamingPlaylistModel.VideoFiles = await Promise.all(upsertTasks) + + videoUpdated.VideoStreamingPlaylists.push(streamingPlaylistModel) + } } { // Update Tags - const tags = videoObject.tag.map(tag => tag.name) + const tags = videoObject.tag + .filter(isAPHashTagObject) + .map(tag => tag.name) const tagInstances = await TagModel.findOrCreateTags(tags, t) await videoUpdated.$set('Tags', tagInstances, sequelizeOptions) } @@ -478,23 +490,27 @@ export { // --------------------------------------------------------------------------- -function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { +function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject { const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) - const urlMediaType = url.mediaType || url.mimeType + const urlMediaType = url.mediaType return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') } function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject { - const urlMediaType = url.mediaType || url.mimeType - - return urlMediaType === 'application/x-mpegURL' + return url && url.mediaType === 'application/x-mpegURL' } function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject { - const urlMediaType = tag.mediaType || tag.mimeType + return tag && tag.name === 'sha256' && tag.type === 'Link' && tag.mediaType === 'application/json' +} - return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json' +function isAPMagnetUrlObject (url: any): url is ActivityMagnetUrlObject { + return url && url.mediaType === 'application/x-bittorrent;x-scheme-handler/magnet' +} + +function isAPHashTagObject (url: any): url is ActivityHashTagObject { + return url && url.type === 'Hashtag' } async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAccountLight, waitThumbnail = false) { @@ -524,21 +540,27 @@ async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAc if (thumbnailModel) await videoCreated.addAndSaveThumbnail(previewModel, t) // Process files - const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject) - if (videoFileAttributes.length === 0) { - throw new Error('Cannot find valid files for video %s ' + videoObject.url) - } + const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject.url) const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) const videoFiles = await Promise.all(videoFilePromises) - const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles) - const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t })) - const streamingPlaylists = await Promise.all(playlistPromises) + const streamingPlaylistsAttributes = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles) + videoCreated.VideoStreamingPlaylists = [] + + for (const playlistAttributes of streamingPlaylistsAttributes) { + const playlistModel = await VideoStreamingPlaylistModel.create(playlistAttributes, { transaction: t }) + + const playlistFiles = videoFileActivityUrlToDBAttributes(playlistModel, playlistAttributes.tagAPObject) + const videoFilePromises = playlistFiles.map(f => VideoFileModel.create(f, { transaction: t })) + playlistModel.VideoFiles = await Promise.all(videoFilePromises) + + videoCreated.VideoStreamingPlaylists.push(playlistModel) + } // Process tags const tags = videoObject.tag - .filter(t => t.type === 'Hashtag') + .filter(isAPHashTagObject) .map(t => t.name) const tagInstances = await TagModel.findOrCreateTags(tags, t) await videoCreated.$set('Tags', tagInstances, sequelizeOptions) @@ -550,7 +572,6 @@ async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAc await Promise.all(videoCaptionsPromises) videoCreated.VideoFiles = videoFiles - videoCreated.VideoStreamingPlaylists = streamingPlaylists videoCreated.Tags = tagInstances const autoBlacklisted = await autoBlacklistVideoIfNeeded({ @@ -628,20 +649,19 @@ async function videoActivityObjectToDBAttributes (videoChannel: MChannelId, vide } } -function videoFileActivityUrlToDBAttributes (video: MVideo, videoObject: VideoTorrentObject) { - const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] +function videoFileActivityUrlToDBAttributes ( + videoOrPlaylist: MVideo | MStreamingPlaylist, + urls: (ActivityTagObject | ActivityUrlObject)[] +) { + const fileUrls = urls.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] - if (fileUrls.length === 0) { - throw new Error('Cannot find video files for ' + video.url) - } + if (fileUrls.length === 0) return [] const attributes: FilteredModelAttributes[] = [] for (const fileUrl of fileUrls) { // Fetch associated magnet uri - const magnet = videoObject.url.find(u => { - const mediaType = u.mediaType || u.mimeType - return mediaType === 'application/x-bittorrent;x-scheme-handler/magnet' && (u as any).height === fileUrl.height - }) + const magnet = urls.filter(isAPMagnetUrlObject) + .find(u => u.height === fileUrl.height) if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) @@ -650,14 +670,17 @@ function videoFileActivityUrlToDBAttributes (video: MVideo, videoObject: VideoTo throw new Error('Cannot parse magnet URI ' + magnet.href) } - const mediaType = fileUrl.mediaType || fileUrl.mimeType + const mediaType = fileUrl.mediaType const attribute = { extname: MIMETYPES.VIDEO.MIMETYPE_EXT[ mediaType ], infoHash: parsed.infoHash, resolution: fileUrl.height, size: fileUrl.size, - videoId: video.id, - fps: fileUrl.fps || -1 + fps: fileUrl.fps || -1, + + // This is a video file owned by a video or by a streaming playlist + videoId: (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id, + videoStreamingPlaylistId: (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null } attributes.push(attribute) @@ -670,12 +693,15 @@ function streamingPlaylistActivityUrlToDBAttributes (video: MVideoId, videoObjec const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[] if (playlistUrls.length === 0) return [] - const attributes: FilteredModelAttributes[] = [] + const attributes: (FilteredModelAttributes & { tagAPObject?: ActivityTagObject[] })[] = [] for (const playlistUrlObject of playlistUrls) { - const segmentsSha256UrlObject = playlistUrlObject.tag - .find(t => { - return isAPPlaylistSegmentHashesUrlObject(t) - }) as ActivityPlaylistSegmentHashesObject + const segmentsSha256UrlObject = playlistUrlObject.tag.find(isAPPlaylistSegmentHashesUrlObject) + + let files: unknown[] = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] + + // FIXME: backward compatibility introduced in v2.1.0 + if (files.length === 0) files = videoFiles + if (!segmentsSha256UrlObject) { logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject }) continue @@ -685,9 +711,10 @@ function streamingPlaylistActivityUrlToDBAttributes (video: MVideoId, videoObjec type: VideoStreamingPlaylistType.HLS, playlistUrl: playlistUrlObject.href, segmentsSha256Url: segmentsSha256UrlObject.href, - p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, videoFiles), + p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files), p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, - videoId: video.id + videoId: video.id, + tagAPObject: playlistUrlObject.tag } attributes.push(attribute) diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 05136c21c..943721dd7 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -12,6 +12,7 @@ import { VideoFileModel } from '../models/video/video-file' import { CONFIG } from '../initializers/config' import { sequelizeTypescript } from '../initializers/database' import { MVideoWithFile } from '@server/typings/models' +import { getVideoFilename, getVideoFilePath } from './video-paths' async function updateStreamingPlaylistsInfohashesIfNeeded () { const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() @@ -32,13 +33,14 @@ async function updateMasterHLSPlaylist (video: MVideoWithFile) { const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename()) + const streamingPlaylist = video.getHLSPlaylist() - for (const file of video.VideoFiles) { + for (const file of streamingPlaylist.VideoFiles) { // If we did not generated a playlist for this resolution, skip const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) if (await pathExists(filePlaylistPath) === false) continue - const videoFilePath = video.getVideoFilePath(file) + const videoFilePath = getVideoFilePath(streamingPlaylist, file) const size = await getVideoFileSize(videoFilePath) @@ -59,12 +61,13 @@ async function updateSha256Segments (video: MVideoWithFile) { const json: { [filename: string]: { [range: string]: string } } = {} const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) + const hlsPlaylist = video.getHLSPlaylist() // For all the resolutions available for this video - for (const file of video.VideoFiles) { + for (const file of hlsPlaylist.VideoFiles) { const rangeHashes: { [range: string]: string } = {} - const videoPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution)) + const videoPath = getVideoFilePath(hlsPlaylist, file) const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) // Maybe the playlist is not generated for this resolution yet @@ -82,7 +85,7 @@ async function updateSha256Segments (video: MVideoWithFile) { } await close(fd) - const videoFilename = VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution) + const videoFilename = getVideoFilename(hlsPlaylist, file) json[videoFilename] = rangeHashes } diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts index 5c5b7dccb..99c991e72 100644 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ b/server/lib/job-queue/handlers/video-file-import.ts @@ -7,6 +7,8 @@ import { copy, stat } from 'fs-extra' import { VideoFileModel } from '../../../models/video/video-file' import { extname } from 'path' import { MVideoFile, MVideoWithFile } from '@server/typings/models' +import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' +import { getVideoFilePath } from '@server/lib/video-paths' export type VideoFileImportPayload = { videoUUID: string, @@ -68,10 +70,10 @@ async function updateVideoFile (video: MVideoWithFile, inputFilePath: string) { updatedVideoFile = currentVideoFile } - const outputPath = video.getVideoFilePath(updatedVideoFile) + const outputPath = getVideoFilePath(video, updatedVideoFile) await copy(inputFilePath, outputPath) - await video.createTorrentAndSetInfoHash(updatedVideoFile) + await createTorrentAndSetInfoHash(video, updatedVideoFile) await updatedVideoFile.save() diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 93a3e9d90..1fca17584 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -4,14 +4,14 @@ import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl' import { VideoImportModel } from '../../../models/video/video-import' import { VideoImportState } from '../../../../shared/models/videos' import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' -import { extname, join } from 'path' +import { extname } from 'path' import { VideoFileModel } from '../../../models/video/video-file' import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' import { VideoState } from '../../../../shared' import { JobQueue } from '../index' import { federateVideoIfNeeded } from '../../activitypub' import { VideoModel } from '../../../models/video/video' -import { downloadWebTorrentVideo } from '../../../helpers/webtorrent' +import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' import { getSecureTorrentName } from '../../../helpers/utils' import { move, remove, stat } from 'fs-extra' import { Notifier } from '../../notifier' @@ -21,7 +21,7 @@ import { createVideoMiniatureFromUrl, generateVideoMiniature } from '../../thumb import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type' import { MThumbnail } from '../../../typings/models/video/thumbnail' import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import' -import { MVideoBlacklistVideo, MVideoBlacklist } from '@server/typings/models' +import { getVideoFilePath } from '@server/lib/video-paths' type VideoImportYoutubeDLPayload = { type: 'youtube-dl' @@ -142,12 +142,12 @@ async function processFile (downloader: () => Promise, videoImport: MVid } videoFile = new VideoFileModel(videoFileData) - const videoWithFiles = Object.assign(videoImport.Video, { VideoFiles: [ videoFile ] }) + const videoWithFiles = Object.assign(videoImport.Video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] }) // To clean files if the import fails const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles }) // Move file - videoDestFile = join(CONFIG.STORAGE.VIDEOS_DIR, videoImportWithFiles.Video.getVideoFilename(videoFile)) + videoDestFile = getVideoFilePath(videoImportWithFiles.Video, videoFile) await move(tempVideoPath, videoDestFile) tempVideoPath = null // This path is not used anymore @@ -168,7 +168,7 @@ async function processFile (downloader: () => Promise, videoImport: MVid } // Create torrent - await videoImportWithFiles.Video.createTorrentAndSetInfoHash(videoFile) + await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile) const { videoImportUpdated, video } = await sequelizeTypescript.transaction(async t => { const videoImportToUpdate = videoImportWithFiles as MVideoImportVideo diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index 2ebe15bcb..39b9fac98 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts @@ -1,5 +1,5 @@ import * as Bull from 'bull' -import { VideoResolution, VideoState } from '../../../../shared' +import { VideoResolution } from '../../../../shared' import { logger } from '../../../helpers/logger' import { VideoModel } from '../../../models/video/video' import { JobQueue } from '../job-queue' @@ -8,10 +8,10 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils' import { sequelizeTypescript } from '../../../initializers' import * as Bluebird from 'bluebird' import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' -import { generateHlsPlaylist, optimizeVideofile, transcodeOriginalVideofile, mergeAudioVideofile } from '../../video-transcoding' +import { generateHlsPlaylist, mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewResolution } from '../../video-transcoding' import { Notifier } from '../../notifier' import { CONFIG } from '../../../initializers/config' -import { MVideoUUID, MVideoWithFile } from '@server/typings/models' +import { MVideoFullLight, MVideoUUID, MVideoWithFile } from '@server/typings/models' interface BaseTranscodingPayload { videoUUID: string @@ -22,6 +22,7 @@ interface HLSTranscodingPayload extends BaseTranscodingPayload { type: 'hls' isPortraitMode?: boolean resolution: VideoResolution + copyCodecs: boolean } interface NewResolutionTranscodingPayload extends BaseTranscodingPayload { @@ -54,11 +55,11 @@ async function processVideoTranscoding (job: Bull.Job) { } if (payload.type === 'hls') { - await generateHlsPlaylist(video, payload.resolution, payload.isPortraitMode || false) + await generateHlsPlaylist(video, payload.resolution, payload.copyCodecs, payload.isPortraitMode || false) await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video) } else if (payload.type === 'new-resolution') { - await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) + await transcodeNewResolution(video, payload.resolution, payload.isPortraitMode || false) await retryTransactionWrapper(publishNewResolutionIfNeeded, video, payload) } else if (payload.type === 'merge-audio') { @@ -66,7 +67,7 @@ async function processVideoTranscoding (job: Bull.Job) { await retryTransactionWrapper(publishNewResolutionIfNeeded, video, payload) } else { - await optimizeVideofile(video) + await optimizeOriginalVideofile(video) await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload) } @@ -74,48 +75,24 @@ async function processVideoTranscoding (job: Bull.Job) { return video } -async function onHlsPlaylistGenerationSuccess (video: MVideoUUID) { +async function onHlsPlaylistGenerationSuccess (video: MVideoFullLight) { if (video === undefined) return undefined - await sequelizeTypescript.transaction(async t => { - // Maybe the video changed in database, refresh it - let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) - // Video does not exist anymore - if (!videoDatabase) return undefined + // We generated the HLS playlist, we don't need the webtorrent files anymore if the admin disabled it + if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { + for (const file of video.VideoFiles) { + await video.removeFile(file) + await file.destroy() + } - // If the video was not published, we consider it is a new one for other instances - await federateVideoIfNeeded(videoDatabase, false, t) - }) + video.VideoFiles = [] + } + + return publishAndFederateIfNeeded(video) } async function publishNewResolutionIfNeeded (video: MVideoUUID, payload?: NewResolutionTranscodingPayload | MergeAudioTranscodingPayload) { - const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { - // Maybe the video changed in database, refresh it - let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) - // Video does not exist anymore - if (!videoDatabase) return undefined - - let videoPublished = false - - // We transcoded the video file in another format, now we can publish it - if (videoDatabase.state !== VideoState.PUBLISHED) { - videoPublished = true - - videoDatabase.state = VideoState.PUBLISHED - videoDatabase.publishedAt = new Date() - videoDatabase = await videoDatabase.save({ transaction: t }) - } - - // If the video was not published, we consider it is a new one for other instances - await federateVideoIfNeeded(videoDatabase, videoPublished, t) - - return { videoDatabase, videoPublished } - }) - - if (videoPublished) { - Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase) - Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase) - } + await publishAndFederateIfNeeded(video) await createHlsJobIfEnabled(payload) } @@ -124,7 +101,7 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O if (videoArg === undefined) return undefined // Outside the transaction (IO on disk) - const { videoFileResolution } = await videoArg.getOriginalFileResolution() + const { videoFileResolution } = await videoArg.getMaxQualityResolution() const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { // Maybe the video changed in database, refresh it @@ -141,14 +118,29 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O let videoPublished = false + const hlsPayload = Object.assign({}, payload, { resolution: videoDatabase.getMaxQualityFile().resolution }) + await createHlsJobIfEnabled(hlsPayload) + if (resolutionsEnabled.length !== 0) { const tasks: (Bluebird> | Promise>)[] = [] for (const resolution of resolutionsEnabled) { - const dataInput = { - type: 'new-resolution' as 'new-resolution', - videoUUID: videoDatabase.uuid, - resolution + let dataInput: VideoTranscodingPayload + + if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED) { + dataInput = { + type: 'new-resolution' as 'new-resolution', + videoUUID: videoDatabase.uuid, + resolution + } + } else if (CONFIG.TRANSCODING.HLS.ENABLED) { + dataInput = { + type: 'hls', + videoUUID: videoDatabase.uuid, + resolution, + isPortraitMode: false, + copyCodecs: false + } } const p = JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput }) @@ -159,11 +151,8 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O logger.info('Transcoding jobs created for uuid %s.', videoDatabase.uuid, { resolutionsEnabled }) } else { - videoPublished = true - // No transcoding to do, it's now published - videoDatabase.state = VideoState.PUBLISHED - videoDatabase = await videoDatabase.save({ transaction: t }) + videoPublished = await videoDatabase.publishIfNeededAndSave(t) logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy }) } @@ -175,9 +164,6 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase) if (videoPublished) Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase) - - const hlsPayload = Object.assign({}, payload, { resolution: videoDatabase.getOriginalFile().resolution }) - await createHlsJobIfEnabled(hlsPayload) } // --------------------------------------------------------------------------- @@ -196,9 +182,32 @@ function createHlsJobIfEnabled (payload?: { videoUUID: string, resolution: numbe type: 'hls' as 'hls', videoUUID: payload.videoUUID, resolution: payload.resolution, - isPortraitMode: payload.isPortraitMode + isPortraitMode: payload.isPortraitMode, + copyCodecs: true } return JobQueue.Instance.createJob({ type: 'video-transcoding', payload: hlsTranscodingPayload }) } } + +async function publishAndFederateIfNeeded (video: MVideoUUID) { + const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { + // Maybe the video changed in database, refresh it + const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) + // Video does not exist anymore + if (!videoDatabase) return undefined + + // We transcoded the video file in another format, now we can publish it + const videoPublished = await videoDatabase.publishIfNeededAndSave(t) + + // If the video was not published, we consider it is a new one for other instances + await federateVideoIfNeeded(videoDatabase, videoPublished, t) + + return { videoDatabase, videoPublished } + }) + + if (videoPublished) { + Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase) + Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase) + } +} diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts index 5b673b913..293bba91f 100644 --- a/server/lib/schedulers/update-videos-scheduler.ts +++ b/server/lib/schedulers/update-videos-scheduler.ts @@ -6,8 +6,8 @@ import { federateVideoIfNeeded } from '../activitypub' import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' import { VideoPrivacy } from '../../../shared/models/videos' import { Notifier } from '../notifier' -import { VideoModel } from '../../models/video/video' import { sequelizeTypescript } from '../../initializers/database' +import { MVideoFullLight } from '@server/typings/models' export class UpdateVideosScheduler extends AbstractScheduler { @@ -28,7 +28,7 @@ export class UpdateVideosScheduler extends AbstractScheduler { const publishedVideos = await sequelizeTypescript.transaction(async t => { const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate(t) - const publishedVideos: VideoModel[] = [] + const publishedVideos: MVideoFullLight[] = [] for (const schedule of schedules) { const video = schedule.Video @@ -45,8 +45,8 @@ export class UpdateVideosScheduler extends AbstractScheduler { await federateVideoIfNeeded(video, isNewVideo, t) if (oldPrivacy === VideoPrivacy.UNLISTED || oldPrivacy === VideoPrivacy.PRIVATE) { - video.ScheduleVideoUpdate = schedule - publishedVideos.push(video) + const videoToPublish: MVideoFullLight = Object.assign(video, { ScheduleVideoUpdate: schedule, UserVideoHistories: [] }) + publishedVideos.push(videoToPublish) } } diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 1e30f6ebc..f2bd75cb4 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -3,7 +3,7 @@ import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER } import { logger } from '../../helpers/logger' import { VideosRedundancy } from '../../../shared/models/redundancy' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' -import { downloadWebTorrentVideo } from '../../helpers/webtorrent' +import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent' import { join } from 'path' import { move } from 'fs-extra' import { getServerActor } from '../../helpers/utils' @@ -24,6 +24,7 @@ import { MVideoRedundancyVideo, MVideoWithAllFiles } from '@server/typings/models' +import { getVideoFilename } from '../video-paths' type CandidateToDuplicate = { redundancy: VideosRedundancy, @@ -195,11 +196,11 @@ export class VideosRedundancyScheduler extends AbstractScheduler { logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() - const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) + const magnetUri = await generateMagnetUri(video, file, baseUrlHttp, baseUrlWs) const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) - const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) + const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, getVideoFilename(video, file)) await move(tmpPath, destPath, { overwrite: true }) const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts index 84791955e..a99f71629 100644 --- a/server/lib/thumbnail.ts +++ b/server/lib/thumbnail.ts @@ -9,6 +9,7 @@ import { downloadImage } from '../helpers/requests' import { MVideoPlaylistThumbnail } from '../typings/models/video/video-playlist' import { MVideoFile, MVideoThumbnail } from '../typings/models' import { MThumbnail } from '../typings/models/video/thumbnail' +import { getVideoFilePath } from './video-paths' type ImageSize = { height: number, width: number } @@ -55,7 +56,7 @@ function createVideoMiniatureFromExisting ( } function generateVideoMiniature (video: MVideoThumbnail, videoFile: MVideoFile, type: ThumbnailType) { - const input = video.getVideoFilePath(videoFile) + const input = getVideoFilePath(video, videoFile) const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) const thumbnailCreator = videoFile.isAudio() diff --git a/server/lib/video-paths.ts b/server/lib/video-paths.ts new file mode 100644 index 000000000..63011cdb2 --- /dev/null +++ b/server/lib/video-paths.ts @@ -0,0 +1,64 @@ +import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/typings/models' +import { extractVideo } from './videos' +import { join } from 'path' +import { CONFIG } from '@server/initializers/config' +import { HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants' + +// ################## Video file name ################## + +function getVideoFilename (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { + const video = extractVideo(videoOrPlaylist) + + if (isStreamingPlaylist(videoOrPlaylist)) { + return generateVideoStreamingPlaylistName(video.uuid, videoFile.resolution) + } + + return generateWebTorrentVideoName(video.uuid, videoFile.resolution, videoFile.extname) +} + +function generateVideoStreamingPlaylistName (uuid: string, resolution: number) { + return `${uuid}-${resolution}-fragmented.mp4` +} + +function generateWebTorrentVideoName (uuid: string, resolution: number, extname: string) { + return uuid + '-' + resolution + extname +} + +function getVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, isRedundancy = false) { + if (isStreamingPlaylist(videoOrPlaylist)) { + const video = extractVideo(videoOrPlaylist) + return join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid, getVideoFilename(videoOrPlaylist, videoFile)) + } + + const baseDir = isRedundancy ? CONFIG.STORAGE.REDUNDANCY_DIR : CONFIG.STORAGE.VIDEOS_DIR + return join(baseDir, getVideoFilename(videoOrPlaylist, videoFile)) +} + +// ################## Torrents ################## + +function getTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { + const video = extractVideo(videoOrPlaylist) + const extension = '.torrent' + + if (isStreamingPlaylist(videoOrPlaylist)) { + return `${video.uuid}-${videoFile.resolution}-${videoOrPlaylist.getStringType()}${extension}` + } + + return video.uuid + '-' + videoFile.resolution + extension +} + +function getTorrentFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { + return join(CONFIG.STORAGE.TORRENTS_DIR, getTorrentFileName(videoOrPlaylist, videoFile)) +} + +// --------------------------------------------------------------------------- + +export { + generateVideoStreamingPlaylistName, + generateWebTorrentVideoName, + getVideoFilename, + getVideoFilePath, + + getTorrentFileName, + getTorrentFilePath +} diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts index 612d388ee..9243d1742 100644 --- a/server/lib/video-transcoding.ts +++ b/server/lib/video-transcoding.ts @@ -1,5 +1,5 @@ import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, WEBSERVER } from '../initializers/constants' -import { basename, join } from 'path' +import { basename, extname as extnameUtil, join } from 'path' import { canDoQuickTranscode, getDurationFromVideoFile, @@ -16,18 +16,19 @@ import { updateMasterHLSPlaylist, updateSha256Segments } from './hls' import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type' import { CONFIG } from '../initializers/config' -import { MVideoFile, MVideoWithFile, MVideoWithFileThumbnail } from '@server/typings/models' +import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoWithAllFiles, MVideoWithFile } from '@server/typings/models' +import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' +import { generateVideoStreamingPlaylistName, getVideoFilename, getVideoFilePath } from './video-paths' /** * Optimize the original video file and replace it. The resolution is not changed. */ -async function optimizeVideofile (video: MVideoWithFile, inputVideoFileArg?: MVideoFile) { - const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR +async function optimizeOriginalVideofile (video: MVideoWithFile, inputVideoFileArg?: MVideoFile) { const transcodeDirectory = CONFIG.STORAGE.TMP_DIR const newExtname = '.mp4' - const inputVideoFile = inputVideoFileArg ? inputVideoFileArg : video.getOriginalFile() - const videoInputPath = join(videosDirectory, video.getVideoFilename(inputVideoFile)) + const inputVideoFile = inputVideoFileArg || video.getMaxQualityFile() + const videoInputPath = getVideoFilePath(video, inputVideoFile) const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath) @@ -35,7 +36,7 @@ async function optimizeVideofile (video: MVideoWithFile, inputVideoFileArg?: MVi : 'video' const transcodeOptions: TranscodeOptions = { - type: transcodeType as any, // FIXME: typing issue + type: transcodeType, inputPath: videoInputPath, outputPath: videoTranscodedPath, resolution: inputVideoFile.resolution @@ -50,7 +51,7 @@ async function optimizeVideofile (video: MVideoWithFile, inputVideoFileArg?: MVi // Important to do this before getVideoFilename() to take in account the new file extension inputVideoFile.extname = newExtname - const videoOutputPath = video.getVideoFilePath(inputVideoFile) + const videoOutputPath = getVideoFilePath(video, inputVideoFile) await onVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) } catch (err) { @@ -64,13 +65,12 @@ async function optimizeVideofile (video: MVideoWithFile, inputVideoFileArg?: MVi /** * Transcode the original video file to a lower resolution. */ -async function transcodeOriginalVideofile (video: MVideoWithFile, resolution: VideoResolution, isPortrait: boolean) { - const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR +async function transcodeNewResolution (video: MVideoWithFile, resolution: VideoResolution, isPortrait: boolean) { const transcodeDirectory = CONFIG.STORAGE.TMP_DIR const extname = '.mp4' // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed - const videoInputPath = join(videosDirectory, video.getVideoFilename(video.getOriginalFile())) + const videoInputPath = getVideoFilePath(video, video.getMaxQualityFile()) const newVideoFile = new VideoFileModel({ resolution, @@ -78,8 +78,8 @@ async function transcodeOriginalVideofile (video: MVideoWithFile, resolution: Vi size: 0, videoId: video.id }) - const videoOutputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(newVideoFile)) - const videoTranscodedPath = join(transcodeDirectory, video.getVideoFilename(newVideoFile)) + const videoOutputPath = getVideoFilePath(video, newVideoFile) + const videoTranscodedPath = join(transcodeDirectory, getVideoFilename(video, newVideoFile)) const transcodeOptions = { type: 'video' as 'video', @@ -94,14 +94,13 @@ async function transcodeOriginalVideofile (video: MVideoWithFile, resolution: Vi return onVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath) } -async function mergeAudioVideofile (video: MVideoWithFileThumbnail, resolution: VideoResolution) { - const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR +async function mergeAudioVideofile (video: MVideoWithAllFiles, resolution: VideoResolution) { const transcodeDirectory = CONFIG.STORAGE.TMP_DIR const newExtname = '.mp4' - const inputVideoFile = video.getOriginalFile() + const inputVideoFile = video.getMaxQualityFile() - const audioInputPath = join(videosDirectory, video.getVideoFilename(video.getOriginalFile())) + const audioInputPath = getVideoFilePath(video, inputVideoFile) const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) // If the user updates the video preview during transcoding @@ -130,7 +129,7 @@ async function mergeAudioVideofile (video: MVideoWithFileThumbnail, resolution: // Important to do this before getVideoFilename() to take in account the new file extension inputVideoFile.extname = newExtname - const videoOutputPath = video.getVideoFilePath(inputVideoFile) + const videoOutputPath = getVideoFilePath(video, inputVideoFile) // ffmpeg generated a new video file, so update the video duration // See https://trac.ffmpeg.org/ticket/5456 video.duration = await getDurationFromVideoFile(videoTranscodedPath) @@ -139,33 +138,40 @@ async function mergeAudioVideofile (video: MVideoWithFileThumbnail, resolution: return onVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) } -async function generateHlsPlaylist (video: MVideoWithFile, resolution: VideoResolution, isPortraitMode: boolean) { +async function generateHlsPlaylist (video: MVideoWithFile, resolution: VideoResolution, copyCodecs: boolean, isPortraitMode: boolean) { const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) await ensureDir(join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)) - const videoInputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(video.getFile(resolution))) + const videoFileInput = copyCodecs + ? video.getWebTorrentFile(resolution) + : video.getMaxQualityFile() + + const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() + const videoInputPath = getVideoFilePath(videoOrStreamingPlaylist, videoFileInput) + const outputPath = join(baseHlsDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution)) + const videoFilename = generateVideoStreamingPlaylistName(video.uuid, resolution) const transcodeOptions = { type: 'hls' as 'hls', inputPath: videoInputPath, outputPath, resolution, + copyCodecs, isPortraitMode, hlsPlaylist: { - videoFilename: VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, resolution) + videoFilename } } - await transcode(transcodeOptions) + logger.debug('Will run transcode.', { transcodeOptions }) - await updateMasterHLSPlaylist(video) - await updateSha256Segments(video) + await transcode(transcodeOptions) const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid) - await VideoStreamingPlaylistModel.upsert({ + const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({ videoId: video.id, playlistUrl, segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid), @@ -173,15 +179,44 @@ async function generateHlsPlaylist (video: MVideoWithFile, resolution: VideoReso p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, type: VideoStreamingPlaylistType.HLS + }, { returning: true }) as [ MStreamingPlaylistFilesVideo, boolean ] + videoStreamingPlaylist.Video = video + + const newVideoFile = new VideoFileModel({ + resolution, + extname: extnameUtil(videoFilename), + size: 0, + fps: -1, + videoStreamingPlaylistId: videoStreamingPlaylist.id }) + + const videoFilePath = getVideoFilePath(videoStreamingPlaylist, newVideoFile) + const stats = await stat(videoFilePath) + + newVideoFile.size = stats.size + newVideoFile.fps = await getVideoFileFPS(videoFilePath) + + await createTorrentAndSetInfoHash(videoStreamingPlaylist, newVideoFile) + + const updatedVideoFile = await newVideoFile.save() + + videoStreamingPlaylist.VideoFiles = await videoStreamingPlaylist.$get('VideoFiles') as VideoFileModel[] + videoStreamingPlaylist.VideoFiles.push(updatedVideoFile) + + video.setHLSPlaylist(videoStreamingPlaylist) + + await updateMasterHLSPlaylist(video) + await updateSha256Segments(video) + + return video } // --------------------------------------------------------------------------- export { generateHlsPlaylist, - optimizeVideofile, - transcodeOriginalVideofile, + optimizeOriginalVideofile, + transcodeNewResolution, mergeAudioVideofile } @@ -196,7 +231,7 @@ async function onVideoFileTranscoding (video: MVideoWithFile, videoFile: MVideoF videoFile.size = stats.size videoFile.fps = fps - await video.createTorrentAndSetInfoHash(videoFile) + await createTorrentAndSetInfoHash(video, videoFile) const updatedVideoFile = await videoFile.save() diff --git a/server/lib/videos.ts b/server/lib/videos.ts new file mode 100644 index 000000000..22e9afbf9 --- /dev/null +++ b/server/lib/videos.ts @@ -0,0 +1,11 @@ +import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/typings/models' + +function extractVideo (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) { + return isStreamingPlaylist(videoOrPlaylist) + ? videoOrPlaylist.Video + : videoOrPlaylist +} + +export { + extractVideo +} diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts index 5059ed0f2..1db907f91 100644 --- a/server/middlewares/validators/config.ts +++ b/server/middlewares/validators/config.ts @@ -43,6 +43,9 @@ const customConfigUpdateValidator = [ body('transcoding.resolutions.720p').isBoolean().withMessage('Should have a valid transcoding 720p resolution enabled boolean'), body('transcoding.resolutions.1080p').isBoolean().withMessage('Should have a valid transcoding 1080p resolution enabled boolean'), + body('transcoding.webtorrent.enabled').isBoolean().withMessage('Should have a valid webtorrent transcoding enabled boolean'), + body('transcoding.hls.enabled').isBoolean().withMessage('Should have a valid webtorrent transcoding enabled boolean'), + body('import.videos.http.enabled').isBoolean().withMessage('Should have a valid import video http enabled boolean'), body('import.videos.torrent.enabled').isBoolean().withMessage('Should have a valid import video torrent enabled boolean'), @@ -56,6 +59,7 @@ const customConfigUpdateValidator = [ if (areValidationErrors(req, res)) return if (!checkInvalidConfigIfEmailDisabled(req.body as CustomConfig, res)) return + if (!checkInvalidTranscodingConfig(req.body as CustomConfig, res)) return return next() } @@ -79,3 +83,16 @@ function checkInvalidConfigIfEmailDisabled (customConfig: CustomConfig, res: exp return true } + +function checkInvalidTranscodingConfig (customConfig: CustomConfig, res: express.Response) { + if (customConfig.transcoding.enabled === false) return true + + if (customConfig.transcoding.webtorrent.enabled === false && customConfig.transcoding.hls.enabled === false) { + res.status(400) + .send({ error: 'You need to enable at least webtorrent transcoding or hls transcoding' }) + .end() + return false + } + + return true +} diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts index 1449903b7..53a2f193d 100644 --- a/server/middlewares/validators/videos/videos.ts +++ b/server/middlewares/validators/videos/videos.ts @@ -270,7 +270,7 @@ const videosAcceptChangeOwnershipValidator = [ const user = res.locals.oauth.token.User const videoChangeOwnership = res.locals.videoChangeOwnership - const isAble = await user.isAbleToUploadVideo(videoChangeOwnership.Video.getOriginalFile()) + const isAble = await user.isAbleToUploadVideo(videoChangeOwnership.Video.getMaxQualityFile()) if (isAble === false) { res.status(403) .json({ error: 'The user video quota is exceeded with this video.' }) diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index 61d9a5612..77f83d8aa 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -497,7 +497,6 @@ export class VideoRedundancyModel extends Model { expires: this.expiresOn.toISOString(), url: { type: 'Link', - mimeType: 'application/x-mpegURL', mediaType: 'application/x-mpegURL', href: this.fileUrl } @@ -511,7 +510,6 @@ export class VideoRedundancyModel extends Model { expires: this.expiresOn.toISOString(), url: { type: 'Link', - mimeType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any, mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any, href: this.fileUrl, height: this.VideoFile.resolution, diff --git a/server/models/utils.ts b/server/models/utils.ts index e7e6ddde1..ccdbcd1cf 100644 --- a/server/models/utils.ts +++ b/server/models/utils.ts @@ -1,7 +1,7 @@ import { Model, Sequelize } from 'sequelize-typescript' import * as validator from 'validator' import { Col } from 'sequelize/types/lib/utils' -import { col, literal, OrderItem } from 'sequelize' +import { literal, OrderItem } from 'sequelize' type SortType = { sortModel: string, sortValue: string } diff --git a/server/models/video/schedule-video-update.ts b/server/models/video/schedule-video-update.ts index fc2a424aa..eefc10f14 100644 --- a/server/models/video/schedule-video-update.ts +++ b/server/models/video/schedule-video-update.ts @@ -2,7 +2,7 @@ import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, Model, Ta import { ScopeNames as VideoScopeNames, VideoModel } from './video' import { VideoPrivacy } from '../../../shared/models/videos' import { Op, Transaction } from 'sequelize' -import { MScheduleVideoUpdateFormattable } from '@server/typings/models' +import { MScheduleVideoUpdateFormattable, MScheduleVideoUpdateVideoAll } from '@server/typings/models' @Table({ tableName: 'scheduleVideoUpdate', @@ -72,10 +72,12 @@ export class ScheduleVideoUpdateModel extends Model { { model: VideoModel.scope( [ - VideoScopeNames.WITH_FILES, + VideoScopeNames.WITH_WEBTORRENT_FILES, + VideoScopeNames.WITH_STREAMING_PLAYLISTS, VideoScopeNames.WITH_ACCOUNT_DETAILS, VideoScopeNames.WITH_BLACKLISTED, - VideoScopeNames.WITH_THUMBNAILS + VideoScopeNames.WITH_THUMBNAILS, + VideoScopeNames.WITH_TAGS ] ) } @@ -83,7 +85,7 @@ export class ScheduleVideoUpdateModel extends Model { transaction: t } - return ScheduleVideoUpdateModel.findAll(query) + return ScheduleVideoUpdateModel.findAll(query) } static deleteByVideoId (videoId: number, t: Transaction) { diff --git a/server/models/video/video-change-ownership.ts b/server/models/video/video-change-ownership.ts index f7a351329..3259b6c02 100644 --- a/server/models/video/video-change-ownership.ts +++ b/server/models/video/video-change-ownership.ts @@ -43,7 +43,11 @@ enum ScopeNames { [ScopeNames.WITH_VIDEO]: { include: [ { - model: VideoModel.scope([ VideoScopeNames.WITH_THUMBNAILS, VideoScopeNames.WITH_FILES ]), + model: VideoModel.scope([ + VideoScopeNames.WITH_THUMBNAILS, + VideoScopeNames.WITH_WEBTORRENT_FILES, + VideoScopeNames.WITH_STREAMING_PLAYLISTS + ]), required: true } ] diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 68e2d562a..cacef0106 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -23,22 +23,52 @@ import { parseAggregateResult, throwIfNotValid } from '../utils' import { VideoModel } from './video' import { VideoRedundancyModel } from '../redundancy/video-redundancy' import { VideoStreamingPlaylistModel } from './video-streaming-playlist' -import { FindOptions, QueryTypes, Transaction } from 'sequelize' +import { FindOptions, Op, QueryTypes, Transaction } from 'sequelize' import { MIMETYPES } from '../../initializers/constants' -import { MVideoFile } from '@server/typings/models' +import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../typings/models/video/video-file' +import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/typings/models' @Table({ tableName: 'videoFile', indexes: [ { - fields: [ 'videoId' ] + fields: [ 'videoId' ], + where: { + videoId: { + [Op.ne]: null + } + } }, + { + fields: [ 'videoStreamingPlaylistId' ], + where: { + videoStreamingPlaylistId: { + [Op.ne]: null + } + } + }, + { fields: [ 'infoHash' ] }, + { fields: [ 'videoId', 'resolution', 'fps' ], - unique: true + unique: true, + where: { + videoId: { + [Op.ne]: null + } + } + }, + { + fields: [ 'videoStreamingPlaylistId', 'resolution', 'fps' ], + unique: true, + where: { + videoStreamingPlaylistId: { + [Op.ne]: null + } + } } ] }) @@ -81,12 +111,24 @@ export class VideoFileModel extends Model { @BelongsTo(() => VideoModel, { foreignKey: { - allowNull: false + allowNull: true }, onDelete: 'CASCADE' }) Video: VideoModel + @ForeignKey(() => VideoStreamingPlaylistModel) + @Column + videoStreamingPlaylistId: number + + @BelongsTo(() => VideoStreamingPlaylistModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'CASCADE' + }) + VideoStreamingPlaylist: VideoStreamingPlaylistModel + @HasMany(() => VideoRedundancyModel, { foreignKey: { allowNull: true @@ -163,6 +205,36 @@ export class VideoFileModel extends Model { })) } + // Redefine upsert because sequelize does not use an appropriate where clause in the update query with 2 unique indexes + static async customUpsert ( + videoFile: MVideoFile, + mode: 'streaming-playlist' | 'video', + transaction: Transaction + ) { + const baseWhere = { + fps: videoFile.fps, + resolution: videoFile.resolution + } + + if (mode === 'streaming-playlist') Object.assign(baseWhere, { videoStreamingPlaylistId: videoFile.videoStreamingPlaylistId }) + else Object.assign(baseWhere, { videoId: videoFile.videoId }) + + const element = await VideoFileModel.findOne({ where: baseWhere, transaction }) + if (!element) return videoFile.save({ transaction }) + + for (const k of Object.keys(videoFile.toJSON())) { + element[k] = videoFile[k] + } + + return element.save({ transaction }) + } + + getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo { + if (this.videoId) return (this as MVideoFileVideo).Video + + return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist + } + isAudio () { return !!MIMETYPES.AUDIO.EXT_MIMETYPE[this.extname] } @@ -170,6 +242,9 @@ export class VideoFileModel extends Model { hasSameUniqueKeysThan (other: MVideoFile) { return this.fps === other.fps && this.resolution === other.resolution && - this.videoId === other.videoId + ( + (this.videoId !== null && this.videoId === other.videoId) || + (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId) + ) } } diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts index 2987aa780..9fed2d49d 100644 --- a/server/models/video/video-format-utils.ts +++ b/server/models/video/video-format-utils.ts @@ -1,11 +1,6 @@ -import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' +import { Video, VideoDetails } from '../../../shared/models/videos' import { VideoModel } from './video' -import { - ActivityPlaylistInfohashesObject, - ActivityPlaylistSegmentHashesObject, - ActivityUrlObject, - VideoTorrentObject -} from '../../../shared/models/activitypub/objects' +import { ActivityTagObject, ActivityUrlObject, VideoTorrentObject } from '../../../shared/models/activitypub/objects' import { MIMETYPES, WEBSERVER } from '../../initializers/constants' import { VideoCaptionModel } from './video-caption' import { @@ -16,9 +11,18 @@ import { } from '../../lib/activitypub' import { isArray } from '../../helpers/custom-validators/misc' import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model' -import { MStreamingPlaylistRedundanciesOpt, MVideo, MVideoAP, MVideoFormattable, MVideoFormattableDetails } from '../../typings/models' -import { MStreamingPlaylistRedundancies } from '../../typings/models/video/video-streaming-playlist' +import { + MStreamingPlaylistRedundanciesOpt, + MStreamingPlaylistVideo, + MVideo, + MVideoAP, + MVideoFile, + MVideoFormattable, + MVideoFormattableDetails +} from '../../typings/models' import { MVideoFileRedundanciesOpt } from '../../typings/models/video/video-file' +import { VideoFile } from '@shared/models/videos/video-file.model' +import { generateMagnetUri } from '@server/helpers/webtorrent' export type VideoFormattingJSONOptions = { completeDescription?: boolean @@ -115,7 +119,7 @@ function videoModelToFormattedDetailsJSON (video: MVideoFormattableDetails): Vid const tags = video.Tags ? video.Tags.map(t => t.name) : [] - const streamingPlaylists = streamingPlaylistsModelToFormattedJSON(video.VideoStreamingPlaylists) + const streamingPlaylists = streamingPlaylistsModelToFormattedJSON(video, video.VideoStreamingPlaylists) const detailsJson = { support: video.support, @@ -138,33 +142,43 @@ function videoModelToFormattedDetailsJSON (video: MVideoFormattableDetails): Vid } // Format and sort video files - detailsJson.files = videoFilesModelToFormattedJSON(video, video.VideoFiles) + detailsJson.files = videoFilesModelToFormattedJSON(video, baseUrlHttp, baseUrlWs, video.VideoFiles) return Object.assign(formattedJson, detailsJson) } -function streamingPlaylistsModelToFormattedJSON (playlists: MStreamingPlaylistRedundanciesOpt[]): VideoStreamingPlaylist[] { +function streamingPlaylistsModelToFormattedJSON (video: MVideo, playlists: MStreamingPlaylistRedundanciesOpt[]): VideoStreamingPlaylist[] { if (isArray(playlists) === false) return [] + const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() + return playlists .map(playlist => { + const playlistWithVideo = Object.assign(playlist, { Video: video }) + const redundancies = isArray(playlist.RedundancyVideos) ? playlist.RedundancyVideos.map(r => ({ baseUrl: r.fileUrl })) : [] + const files = videoFilesModelToFormattedJSON(playlistWithVideo, baseUrlHttp, baseUrlWs, playlist.VideoFiles) + return { id: playlist.id, type: playlist.type, playlistUrl: playlist.playlistUrl, segmentsSha256Url: playlist.segmentsSha256Url, - redundancies - } as VideoStreamingPlaylist + redundancies, + files + } }) } -function videoFilesModelToFormattedJSON (video: MVideo, videoFiles: MVideoFileRedundanciesOpt[]): VideoFile[] { - const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() - +function videoFilesModelToFormattedJSON ( + model: MVideo | MStreamingPlaylistVideo, + baseUrlHttp: string, + baseUrlWs: string, + videoFiles: MVideoFileRedundanciesOpt[] +): VideoFile[] { return videoFiles .map(videoFile => { let resolutionLabel = videoFile.resolution + 'p' @@ -174,13 +188,13 @@ function videoFilesModelToFormattedJSON (video: MVideo, videoFiles: MVideoFileRe id: videoFile.resolution, label: resolutionLabel }, - magnetUri: video.generateMagnetUri(videoFile, baseUrlHttp, baseUrlWs), + magnetUri: generateMagnetUri(model, videoFile, baseUrlHttp, baseUrlWs), size: videoFile.size, fps: videoFile.fps, - torrentUrl: video.getTorrentUrl(videoFile, baseUrlHttp), - torrentDownloadUrl: video.getTorrentDownloadUrl(videoFile, baseUrlHttp), - fileUrl: video.getVideoFileUrl(videoFile, baseUrlHttp), - fileDownloadUrl: video.getVideoFileDownloadUrl(videoFile, baseUrlHttp) + torrentUrl: model.getTorrentUrl(videoFile, baseUrlHttp), + torrentDownloadUrl: model.getTorrentDownloadUrl(videoFile, baseUrlHttp), + fileUrl: model.getVideoFileUrl(videoFile, baseUrlHttp), + fileDownloadUrl: model.getVideoFileDownloadUrl(videoFile, baseUrlHttp) } as VideoFile }) .sort((a, b) => { @@ -190,6 +204,39 @@ function videoFilesModelToFormattedJSON (video: MVideo, videoFiles: MVideoFileRe }) } +function addVideoFilesInAPAcc ( + acc: ActivityUrlObject[] | ActivityTagObject[], + model: MVideoAP | MStreamingPlaylistVideo, + baseUrlHttp: string, + baseUrlWs: string, + files: MVideoFile[] +) { + for (const file of files) { + acc.push({ + type: 'Link', + mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ file.extname ] as any, + href: model.getVideoFileUrl(file, baseUrlHttp), + height: file.resolution, + size: file.size, + fps: file.fps + }) + + acc.push({ + type: 'Link', + mediaType: 'application/x-bittorrent' as 'application/x-bittorrent', + href: model.getTorrentUrl(file, baseUrlHttp), + height: file.resolution + }) + + acc.push({ + type: 'Link', + mediaType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet', + href: generateMagnetUri(model, file, baseUrlHttp, baseUrlWs), + height: file.resolution + }) + } +} + function videoModelToActivityPubObject (video: MVideoAP): VideoTorrentObject { const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() if (!video.Tags) video.Tags = [] @@ -224,50 +271,25 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoTorrentObject { } const url: ActivityUrlObject[] = [] - for (const file of video.VideoFiles) { - url.push({ - type: 'Link', - mimeType: MIMETYPES.VIDEO.EXT_MIMETYPE[ file.extname ] as any, - mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ file.extname ] as any, - href: video.getVideoFileUrl(file, baseUrlHttp), - height: file.resolution, - size: file.size, - fps: file.fps - }) - - url.push({ - type: 'Link', - mimeType: 'application/x-bittorrent' as 'application/x-bittorrent', - mediaType: 'application/x-bittorrent' as 'application/x-bittorrent', - href: video.getTorrentUrl(file, baseUrlHttp), - height: file.resolution - }) - - url.push({ - type: 'Link', - mimeType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet', - mediaType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet', - href: video.generateMagnetUri(file, baseUrlHttp, baseUrlWs), - height: file.resolution - }) - } + addVideoFilesInAPAcc(url, video, baseUrlHttp, baseUrlWs, video.VideoFiles || []) for (const playlist of (video.VideoStreamingPlaylists || [])) { - let tag: (ActivityPlaylistSegmentHashesObject | ActivityPlaylistInfohashesObject)[] + let tag: ActivityTagObject[] tag = playlist.p2pMediaLoaderInfohashes .map(i => ({ type: 'Infohash' as 'Infohash', name: i })) tag.push({ type: 'Link', name: 'sha256', - mimeType: 'application/json' as 'application/json', mediaType: 'application/json' as 'application/json', href: playlist.segmentsSha256Url }) + const playlistWithVideo = Object.assign(playlist, { Video: video }) + addVideoFilesInAPAcc(tag, playlistWithVideo, baseUrlHttp, baseUrlWs, playlist.VideoFiles || []) + url.push({ type: 'Link', - mimeType: 'application/x-mpegURL' as 'application/x-mpegURL', mediaType: 'application/x-mpegURL' as 'application/x-mpegURL', href: playlist.playlistUrl, tag @@ -277,7 +299,6 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoTorrentObject { // Add video url too url.push({ type: 'Link', - mimeType: 'text/html', mediaType: 'text/html', href: WEBSERVER.URL + '/videos/watch/' + video.uuid }) diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts index 0ea90d28c..faad4cc2d 100644 --- a/server/models/video/video-streaming-playlist.ts +++ b/server/models/video/video-streaming-playlist.ts @@ -5,12 +5,14 @@ import { VideoModel } from './video' import { VideoRedundancyModel } from '../redundancy/video-redundancy' import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { CONSTRAINTS_FIELDS, P2P_MEDIA_LOADER_PEER_VERSION, STATIC_PATHS } from '../../initializers/constants' +import { CONSTRAINTS_FIELDS, P2P_MEDIA_LOADER_PEER_VERSION, STATIC_DOWNLOAD_PATHS, STATIC_PATHS } from '../../initializers/constants' import { join } from 'path' import { sha1 } from '../../helpers/core-utils' import { isArrayOf } from '../../helpers/custom-validators/misc' import { Op, QueryTypes } from 'sequelize' import { MStreamingPlaylist, MVideoFile } from '@server/typings/models' +import { VideoFileModel } from '@server/models/video/video-file' +import { getTorrentFileName, getVideoFilename } from '@server/lib/video-paths' @Table({ tableName: 'videoStreamingPlaylist', @@ -70,6 +72,14 @@ export class VideoStreamingPlaylistModel extends Model VideoFileModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'CASCADE' + }) + VideoFiles: VideoFileModel[] + @HasMany(() => VideoRedundancyModel, { foreignKey: { allowNull: false @@ -91,11 +101,11 @@ export class VideoStreamingPlaylistModel extends Model results.length === 1) } - static buildP2PMediaLoaderInfoHashes (playlistUrl: string, videoFiles: MVideoFile[]) { + static buildP2PMediaLoaderInfoHashes (playlistUrl: string, files: unknown[]) { const hashes: string[] = [] // https://github.com/Novage/p2p-media-loader/blob/master/p2p-media-loader-core/lib/p2p-media-manager.ts#L115 - for (let i = 0; i < videoFiles.length; i++) { + for (let i = 0; i < files.length; i++) { hashes.push(sha1(`${P2P_MEDIA_LOADER_PEER_VERSION}${playlistUrl}+V${i}`)) } @@ -139,10 +149,6 @@ export class VideoStreamingPlaylistModel extends Model { + [ ScopeNames.WITH_WEBTORRENT_FILES ]: (withRedundancies = false) => { let subInclude: any[] = [] if (withRedundancies === true) { @@ -691,16 +693,19 @@ export type AvailableForListIDsOptions = { } }, [ ScopeNames.WITH_STREAMING_PLAYLISTS ]: (withRedundancies = false) => { - let subInclude: any[] = [] + const subInclude: IncludeOptions[] = [ + { + model: VideoFileModel.unscoped(), + required: false + } + ] if (withRedundancies === true) { - subInclude = [ - { - attributes: [ 'fileUrl' ], - model: VideoRedundancyModel.unscoped(), - required: false - } - ] + subInclude.push({ + attributes: [ 'fileUrl' ], + model: VideoRedundancyModel.unscoped(), + required: false + }) } return { @@ -913,7 +918,7 @@ export class VideoModel extends Model { @HasMany(() => VideoFileModel, { foreignKey: { name: 'videoId', - allowNull: false + allowNull: true }, hooks: true, onDelete: 'cascade' @@ -1071,7 +1076,7 @@ export class VideoModel extends Model { } return VideoModel.scope([ - ScopeNames.WITH_FILES, + ScopeNames.WITH_WEBTORRENT_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS, ScopeNames.WITH_THUMBNAILS ]).findAll(query) @@ -1463,7 +1468,7 @@ export class VideoModel extends Model { } return VideoModel.scope([ - ScopeNames.WITH_FILES, + ScopeNames.WITH_WEBTORRENT_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS, ScopeNames.WITH_THUMBNAILS ]).findOne(query) @@ -1500,7 +1505,7 @@ export class VideoModel extends Model { return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, - ScopeNames.WITH_FILES, + ScopeNames.WITH_WEBTORRENT_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS, ScopeNames.WITH_THUMBNAILS, ScopeNames.WITH_BLACKLISTED @@ -1521,7 +1526,7 @@ export class VideoModel extends Model { ScopeNames.WITH_BLACKLISTED, ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_SCHEDULED_UPDATE, - ScopeNames.WITH_FILES, + ScopeNames.WITH_WEBTORRENT_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS, ScopeNames.WITH_THUMBNAILS ] @@ -1555,7 +1560,7 @@ export class VideoModel extends Model { ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_SCHEDULED_UPDATE, ScopeNames.WITH_THUMBNAILS, - { method: [ ScopeNames.WITH_FILES, true ] }, + { method: [ ScopeNames.WITH_WEBTORRENT_FILES, true ] }, { method: [ ScopeNames.WITH_STREAMING_PLAYLISTS, true ] } ] @@ -1787,17 +1792,31 @@ export class VideoModel extends Model { this.VideoChannel.Account.isBlocked() } - getOriginalFile (this: T) { - if (Array.isArray(this.VideoFiles) === false) return undefined + getMaxQualityFile (this: T): MVideoFileVideo | MVideoFileStreamingPlaylistVideo { + if (Array.isArray(this.VideoFiles) && this.VideoFiles.length !== 0) { + const file = maxBy(this.VideoFiles, file => file.resolution) - // The original file is the file that have the higher resolution - return maxBy(this.VideoFiles, file => file.resolution) + return Object.assign(file, { Video: this }) + } + + // No webtorrent files, try with streaming playlist files + if (Array.isArray(this.VideoStreamingPlaylists) && this.VideoStreamingPlaylists.length !== 0) { + const streamingPlaylistWithVideo = Object.assign(this.VideoStreamingPlaylists[0], { Video: this }) + + const file = maxBy(streamingPlaylistWithVideo.VideoFiles, file => file.resolution) + return Object.assign(file, { VideoStreamingPlaylist: streamingPlaylistWithVideo }) + } + + return undefined } - getFile (this: T, resolution: number) { + getWebTorrentFile (this: T, resolution: number): MVideoFileVideo { if (Array.isArray(this.VideoFiles) === false) return undefined - return this.VideoFiles.find(f => f.resolution === resolution) + const file = this.VideoFiles.find(f => f.resolution === resolution) + if (!file) return undefined + + return Object.assign(file, { Video: this }) } async addAndSaveThumbnail (thumbnail: MThumbnail, transaction: Transaction) { @@ -1813,10 +1832,6 @@ export class VideoModel extends Model { this.Thumbnails.push(savedThumbnail) } - getVideoFilename (videoFile: MVideoFile) { - return this.uuid + '-' + videoFile.resolution + videoFile.extname - } - generateThumbnailName () { return this.uuid + '.jpg' } @@ -1837,46 +1852,10 @@ export class VideoModel extends Model { return this.Thumbnails.find(t => t.type === ThumbnailType.PREVIEW) } - getTorrentFileName (videoFile: MVideoFile) { - const extension = '.torrent' - return this.uuid + '-' + videoFile.resolution + extension - } - isOwned () { return this.remote === false } - getTorrentFilePath (videoFile: MVideoFile) { - return join(CONFIG.STORAGE.TORRENTS_DIR, this.getTorrentFileName(videoFile)) - } - - getVideoFilePath (videoFile: MVideoFile) { - return join(CONFIG.STORAGE.VIDEOS_DIR, this.getVideoFilename(videoFile)) - } - - async createTorrentAndSetInfoHash (videoFile: MVideoFile) { - const options = { - // Keep the extname, it's used by the client to stream the file inside a web browser - name: `${this.name} ${videoFile.resolution}p${videoFile.extname}`, - createdBy: 'PeerTube', - announceList: [ - [ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ], - [ WEBSERVER.URL + '/tracker/announce' ] - ], - urlList: [ WEBSERVER.URL + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) ] - } - - const torrent = await createTorrentPromise(this.getVideoFilePath(videoFile), options) - - const filePath = join(CONFIG.STORAGE.TORRENTS_DIR, this.getTorrentFileName(videoFile)) - logger.info('Creating torrent %s.', filePath) - - await writeFile(filePath, torrent) - - const parsedTorrent = parseTorrent(torrent) - videoFile.infoHash = parsedTorrent.infoHash - } - getWatchStaticPath () { return '/videos/watch/' + this.uuid } @@ -1909,7 +1888,8 @@ export class VideoModel extends Model { } getFormattedVideoFilesJSON (): VideoFile[] { - return videoFilesModelToFormattedJSON(this, this.VideoFiles) + const { baseUrlHttp, baseUrlWs } = this.getBaseUrls() + return videoFilesModelToFormattedJSON(this, baseUrlHttp, baseUrlWs, this.VideoFiles) } toActivityPubObject (this: MVideoAP): VideoTorrentObject { @@ -1923,8 +1903,10 @@ export class VideoModel extends Model { return peertubeTruncate(this.description, { length: maxLength }) } - getOriginalFileResolution () { - const originalFilePath = this.getVideoFilePath(this.getOriginalFile()) + getMaxQualityResolution () { + const file = this.getMaxQualityFile() + const videoOrPlaylist = file.getVideoOrStreamingPlaylist() + const originalFilePath = getVideoFilePath(videoOrPlaylist, file) return getVideoFileResolution(originalFilePath) } @@ -1933,22 +1915,36 @@ export class VideoModel extends Model { return `/api/${API_VERSION}/videos/${this.uuid}/description` } - getHLSPlaylist () { + getHLSPlaylist (): MStreamingPlaylistFilesVideo { if (!this.VideoStreamingPlaylists) return undefined - return this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) + const playlist = this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) + playlist.Video = this + + return playlist + } + + setHLSPlaylist (playlist: MStreamingPlaylist) { + const toAdd = [ playlist ] as [ VideoStreamingPlaylistModel ] + + if (Array.isArray(this.VideoStreamingPlaylists) === false || this.VideoStreamingPlaylists.length === 0) { + this.VideoStreamingPlaylists = toAdd + return + } + + this.VideoStreamingPlaylists = this.VideoStreamingPlaylists + .filter(s => s.type !== VideoStreamingPlaylistType.HLS) + .concat(toAdd) } removeFile (videoFile: MVideoFile, isRedundancy = false) { - const baseDir = isRedundancy ? CONFIG.STORAGE.REDUNDANCY_DIR : CONFIG.STORAGE.VIDEOS_DIR - - const filePath = join(baseDir, this.getVideoFilename(videoFile)) + const filePath = getVideoFilePath(this, videoFile, isRedundancy) return remove(filePath) .catch(err => logger.warn('Cannot delete file %s.', filePath, { err })) } removeTorrent (videoFile: MVideoFile) { - const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, this.getTorrentFileName(videoFile)) + const torrentPath = getTorrentFilePath(this, videoFile) return remove(torrentPath) .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) } @@ -1973,38 +1969,30 @@ export class VideoModel extends Model { return this.save() } - getBaseUrls () { - let baseUrlHttp - let baseUrlWs + async publishIfNeededAndSave (t: Transaction) { + if (this.state !== VideoState.PUBLISHED) { + this.state = VideoState.PUBLISHED + this.publishedAt = new Date() + await this.save({ transaction: t }) - if (this.isOwned()) { - baseUrlHttp = WEBSERVER.URL - baseUrlWs = WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT - } else { - baseUrlHttp = REMOTE_SCHEME.HTTP + '://' + this.VideoChannel.Account.Actor.Server.host - baseUrlWs = REMOTE_SCHEME.WS + '://' + this.VideoChannel.Account.Actor.Server.host + return true } - return { baseUrlHttp, baseUrlWs } + return false } - generateMagnetUri (videoFile: MVideoFileRedundanciesOpt, baseUrlHttp: string, baseUrlWs: string) { - const xs = this.getTorrentUrl(videoFile, baseUrlHttp) - const announce = this.getTrackerUrls(baseUrlHttp, baseUrlWs) - let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ] - - const redundancies = videoFile.RedundancyVideos - if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl)) - - const magnetHash = { - xs, - announce, - urlList, - infoHash: videoFile.infoHash, - name: this.name + getBaseUrls () { + if (this.isOwned()) { + return { + baseUrlHttp: WEBSERVER.URL, + baseUrlWs: WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + } } - return magnetUtil.encode(magnetHash) + return { + baseUrlHttp: REMOTE_SCHEME.HTTP + '://' + this.VideoChannel.Account.Actor.Server.host, + baseUrlWs: REMOTE_SCHEME.WS + '://' + this.VideoChannel.Account.Actor.Server.host + } } getTrackerUrls (baseUrlHttp: string, baseUrlWs: string) { @@ -2012,23 +2000,23 @@ export class VideoModel extends Model { } getTorrentUrl (videoFile: MVideoFile, baseUrlHttp: string) { - return baseUrlHttp + STATIC_PATHS.TORRENTS + this.getTorrentFileName(videoFile) + return baseUrlHttp + STATIC_PATHS.TORRENTS + getTorrentFileName(this, videoFile) } getTorrentDownloadUrl (videoFile: MVideoFile, baseUrlHttp: string) { - return baseUrlHttp + STATIC_DOWNLOAD_PATHS.TORRENTS + this.getTorrentFileName(videoFile) + return baseUrlHttp + STATIC_DOWNLOAD_PATHS.TORRENTS + getTorrentFileName(this, videoFile) } getVideoFileUrl (videoFile: MVideoFile, baseUrlHttp: string) { - return baseUrlHttp + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) + return baseUrlHttp + STATIC_PATHS.WEBSEED + getVideoFilename(this, videoFile) } getVideoRedundancyUrl (videoFile: MVideoFile, baseUrlHttp: string) { - return baseUrlHttp + STATIC_PATHS.REDUNDANCY + this.getVideoFilename(videoFile) + return baseUrlHttp + STATIC_PATHS.REDUNDANCY + getVideoFilename(this, videoFile) } getVideoFileDownloadUrl (videoFile: MVideoFile, baseUrlHttp: string) { - return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile) + return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + getVideoFilename(this, videoFile) } getBandwidthBits (videoFile: MVideoFile) { diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index 9435bb1e8..3c558d4ea 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts @@ -92,6 +92,9 @@ describe('Test config API validators', function () { '1080p': false, '2160p': false }, + webtorrent: { + enabled: true + }, hls: { enabled: false } @@ -235,6 +238,27 @@ describe('Test config API validators', function () { }) }) + it('Should fail with a disabled webtorrent & hls transcoding', async function () { + const newUpdateParams = immutableAssign(updateParams, { + transcoding: { + hls: { + enabled: false + }, + webtorrent: { + enabled: false + } + } + }) + + await makePutBodyRequest({ + url: server.url, + path, + fields: newUpdateParams, + token: server.accessToken, + statusCodeExpected: 400 + }) + }) + it('Should success with the correct parameters', async function () { await makePutBodyRequest({ url: server.url, diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index 97cc99eea..a494858b3 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts @@ -72,6 +72,7 @@ function checkInitialConfig (server: ServerInfo, data: CustomConfig) { expect(data.transcoding.resolutions['720p']).to.be.true expect(data.transcoding.resolutions['1080p']).to.be.true expect(data.transcoding.resolutions['2160p']).to.be.true + expect(data.transcoding.webtorrent.enabled).to.be.true expect(data.transcoding.hls.enabled).to.be.true expect(data.import.videos.http.enabled).to.be.true @@ -140,6 +141,7 @@ function checkUpdatedConfig (data: CustomConfig) { expect(data.transcoding.resolutions['1080p']).to.be.false expect(data.transcoding.resolutions['2160p']).to.be.false expect(data.transcoding.hls.enabled).to.be.false + expect(data.transcoding.webtorrent.enabled).to.be.true expect(data.import.videos.http.enabled).to.be.false expect(data.import.videos.torrent.enabled).to.be.false @@ -279,6 +281,9 @@ describe('Test config', function () { '1080p': false, '2160p': false }, + webtorrent: { + enabled: true + }, hls: { enabled: false } diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts index 39178bb1a..289209177 100644 --- a/server/tests/api/videos/video-hls.ts +++ b/server/tests/api/videos/video-hls.ts @@ -10,13 +10,13 @@ import { doubleFollow, flushAndRunMultipleServers, getPlaylist, - getVideo, + getVideo, makeGetRequest, makeRawRequest, removeVideo, ServerInfo, - setAccessTokensToServers, + setAccessTokensToServers, updateCustomSubConfig, updateVideo, uploadVideo, - waitJobs + waitJobs, webtorrentAdd } from '../../../../shared/extra-utils' import { VideoDetails } from '../../../../shared/models/videos' import { VideoStreamingPlaylistType } from '../../../../shared/models/videos/video-streaming-playlist.type' @@ -25,20 +25,45 @@ import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants' const expect = chai.expect -async function checkHlsPlaylist (servers: ServerInfo[], videoUUID: string, resolutions = [ 240, 360, 480, 720 ]) { +async function checkHlsPlaylist (servers: ServerInfo[], videoUUID: string, hlsOnly: boolean, resolutions = [ 240, 360, 480, 720 ]) { for (const server of servers) { - const res = await getVideo(server.url, videoUUID) - const videoDetails: VideoDetails = res.body + const resVideoDetails = await getVideo(server.url, videoUUID) + const videoDetails: VideoDetails = resVideoDetails.body + const baseUrl = `http://${videoDetails.account.host}` expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) expect(hlsPlaylist).to.not.be.undefined - { - const res2 = await getPlaylist(hlsPlaylist.playlistUrl) + const hlsFiles = hlsPlaylist.files + expect(hlsFiles).to.have.lengthOf(resolutions.length) - const masterPlaylist = res2.text + if (hlsOnly) expect(videoDetails.files).to.have.lengthOf(0) + else expect(videoDetails.files).to.have.lengthOf(resolutions.length) + + for (const resolution of resolutions) { + const file = hlsFiles.find(f => f.resolution.id === resolution) + expect(file).to.not.be.undefined + + expect(file.magnetUri).to.have.lengthOf.above(2) + expect(file.torrentUrl).to.equal(`${baseUrl}/static/torrents/${videoDetails.uuid}-${file.resolution.id}-hls.torrent`) + expect(file.fileUrl).to.equal(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${videoDetails.uuid}-${file.resolution.id}-fragmented.mp4`) + expect(file.resolution.label).to.equal(resolution + 'p') + + await makeRawRequest(file.torrentUrl, 200) + await makeRawRequest(file.fileUrl, 200) + + const torrent = await webtorrentAdd(file.magnetUri, true) + expect(torrent.files).to.be.an('array') + expect(torrent.files.length).to.equal(1) + expect(torrent.files[0].path).to.exist.and.to.not.equal('') + } + + { + const res = await getPlaylist(hlsPlaylist.playlistUrl) + + const masterPlaylist = res.text for (const resolution of resolutions) { expect(masterPlaylist).to.match(new RegExp('#EXT-X-STREAM-INF:BANDWIDTH=\\d+,RESOLUTION=\\d+x' + resolution + ',FRAME-RATE=\\d+')) @@ -48,18 +73,18 @@ async function checkHlsPlaylist (servers: ServerInfo[], videoUUID: string, resol { for (const resolution of resolutions) { - const res2 = await getPlaylist(`http://localhost:${servers[0].port}/static/streaming-playlists/hls/${videoUUID}/${resolution}.m3u8`) + const res = await getPlaylist(`${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${resolution}.m3u8`) - const subPlaylist = res2.text + const subPlaylist = res.text expect(subPlaylist).to.contain(`${videoUUID}-${resolution}-fragmented.mp4`) } } { - const baseUrl = 'http://localhost:' + servers[0].port + '/static/streaming-playlists/hls' + const baseUrlAndPath = baseUrl + '/static/streaming-playlists/hls' for (const resolution of resolutions) { - await checkSegmentHash(baseUrl, baseUrl, videoUUID, resolution, hlsPlaylist) + await checkSegmentHash(baseUrlAndPath, baseUrlAndPath, videoUUID, resolution, hlsPlaylist) } } } @@ -70,6 +95,67 @@ describe('Test HLS videos', function () { let videoUUID = '' let videoAudioUUID = '' + function runTestSuite (hlsOnly: boolean) { + it('Should upload a video and transcode it to HLS', async function () { + this.timeout(120000) + + const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, { name: 'video 1', fixture: 'video_short.webm' }) + videoUUID = res.body.video.uuid + + await waitJobs(servers) + + await checkHlsPlaylist(servers, videoUUID, hlsOnly) + }) + + it('Should upload an audio file and transcode it to HLS', async function () { + this.timeout(120000) + + const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, { name: 'video audio', fixture: 'sample.ogg' }) + videoAudioUUID = res.body.video.uuid + + await waitJobs(servers) + + await checkHlsPlaylist(servers, videoAudioUUID, hlsOnly, [ DEFAULT_AUDIO_RESOLUTION ]) + }) + + it('Should update the video', async function () { + this.timeout(10000) + + await updateVideo(servers[ 0 ].url, servers[ 0 ].accessToken, videoUUID, { name: 'video 1 updated' }) + + await waitJobs(servers) + + await checkHlsPlaylist(servers, videoUUID, hlsOnly) + }) + + it('Should delete videos', async function () { + this.timeout(10000) + + await removeVideo(servers[ 0 ].url, servers[ 0 ].accessToken, videoUUID) + await removeVideo(servers[ 0 ].url, servers[ 0 ].accessToken, videoAudioUUID) + + await waitJobs(servers) + + for (const server of servers) { + await getVideo(server.url, videoUUID, 404) + await getVideo(server.url, videoAudioUUID, 404) + } + }) + + it('Should have the playlists/segment deleted from the disk', async function () { + for (const server of servers) { + await checkDirectoryIsEmpty(server, 'videos') + await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls')) + } + }) + + it('Should have an empty tmp directory', async function () { + for (const server of servers) { + await checkTmpIsEmpty(server) + } + }) + } + before(async function () { this.timeout(120000) @@ -91,63 +177,36 @@ describe('Test HLS videos', function () { await doubleFollow(servers[0], servers[1]) }) - it('Should upload a video and transcode it to HLS', async function () { - this.timeout(120000) - - const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, { name: 'video 1', fixture: 'video_short.webm' }) - videoUUID = res.body.video.uuid - - await waitJobs(servers) - - await checkHlsPlaylist(servers, videoUUID) + describe('With WebTorrent & HLS enabled', function () { + runTestSuite(false) }) - it('Should upload an audio file and transcode it to HLS', async function () { - this.timeout(120000) + describe('With only HLS enabled', function () { - const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, { name: 'video audio', fixture: 'sample.ogg' }) - videoAudioUUID = res.body.video.uuid + before(async function () { + await updateCustomSubConfig(servers[0].url, servers[0].accessToken, { + transcoding: { + enabled: true, + allowAudioFiles: true, + resolutions: { + '240p': true, + '360p': true, + '480p': true, + '720p': true, + '1080p': true, + '2160p': true + }, + hls: { + enabled: true + }, + webtorrent: { + enabled: false + } + } + }) + }) - await waitJobs(servers) - - await checkHlsPlaylist(servers, videoAudioUUID, [ DEFAULT_AUDIO_RESOLUTION ]) - }) - - it('Should update the video', async function () { - this.timeout(10000) - - await updateVideo(servers[0].url, servers[0].accessToken, videoUUID, { name: 'video 1 updated' }) - - await waitJobs(servers) - - await checkHlsPlaylist(servers, videoUUID) - }) - - it('Should delete videos', async function () { - this.timeout(10000) - - await removeVideo(servers[0].url, servers[0].accessToken, videoUUID) - await removeVideo(servers[0].url, servers[0].accessToken, videoAudioUUID) - - await waitJobs(servers) - - for (const server of servers) { - await getVideo(server.url, videoUUID, 404) - await getVideo(server.url, videoAudioUUID, 404) - } - }) - - it('Should have the playlists/segment deleted from the disk', async function () { - for (const server of servers) { - await checkDirectoryIsEmpty(server, 'videos') - await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls')) - } - }) - - it('Should have an empty tmp directory', async function () { - for (const server of servers) { - await checkTmpIsEmpty(server) - } + runTestSuite(true) }) after(async function () { diff --git a/server/tests/cli/create-import-video-file-job.ts b/server/tests/cli/create-import-video-file-job.ts index 0d378c1aa..aca3216bb 100644 --- a/server/tests/cli/create-import-video-file-job.ts +++ b/server/tests/cli/create-import-video-file-job.ts @@ -2,22 +2,21 @@ import 'mocha' import * as chai from 'chai' -import { VideoDetails, VideoFile } from '../../../shared/models/videos' +import { VideoDetails } from '../../../shared/models/videos' import { cleanupTests, doubleFollow, execCLI, flushAndRunMultipleServers, - flushTests, getEnvCli, getVideo, getVideosList, - killallServers, ServerInfo, setAccessTokensToServers, uploadVideo } from '../../../shared/extra-utils' import { waitJobs } from '../../../shared/extra-utils/server/jobs' +import { VideoFile } from '@shared/models/videos/video-file.model' const expect = chai.expect diff --git a/server/typings/models/account/account.ts b/server/typings/models/account/account.ts index ec78fece8..adb1f3689 100644 --- a/server/typings/models/account/account.ts +++ b/server/typings/models/account/account.ts @@ -15,7 +15,7 @@ import { } from './actor' import { FunctionProperties, PickWith } from '../../utils' import { MAccountBlocklistId } from './account-blocklist' -import { MChannelDefault } from '@server/typings/models' +import { MChannelDefault } from '../video/video-channels' type Use = PickWith diff --git a/server/typings/models/account/actor-follow.ts b/server/typings/models/account/actor-follow.ts index 1c66eb0a0..f44157eba 100644 --- a/server/typings/models/account/actor-follow.ts +++ b/server/typings/models/account/actor-follow.ts @@ -1,17 +1,16 @@ import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { MActor, - MActorAccount, - MActorDefaultAccountChannel, MActorChannelAccountActor, MActorDefault, + MActorDefaultAccountChannel, MActorFormattable, MActorHost, MActorUsername } from './actor' import { PickWith } from '../../utils' import { ActorModel } from '@server/models/activitypub/actor' -import { MChannelDefault } from '@server/typings/models' +import { MChannelDefault } from '../video/video-channels' type Use = PickWith diff --git a/server/typings/models/account/index.d.ts b/server/typings/models/account/index.ts similarity index 100% rename from server/typings/models/account/index.d.ts rename to server/typings/models/account/index.ts diff --git a/server/typings/models/index.d.ts b/server/typings/models/index.ts similarity index 100% rename from server/typings/models/index.d.ts rename to server/typings/models/index.ts diff --git a/server/typings/models/oauth/index.d.ts b/server/typings/models/oauth/index.ts similarity index 100% rename from server/typings/models/oauth/index.d.ts rename to server/typings/models/oauth/index.ts diff --git a/server/typings/models/oauth/oauth-token.ts b/server/typings/models/oauth/oauth-token.ts index af3412925..8ef042d4e 100644 --- a/server/typings/models/oauth/oauth-token.ts +++ b/server/typings/models/oauth/oauth-token.ts @@ -1,6 +1,6 @@ import { OAuthTokenModel } from '@server/models/oauth/oauth-token' import { PickWith } from '@server/typings/utils' -import { MUserAccountUrl } from '@server/typings/models' +import { MUserAccountUrl } from '../user/user' type Use = PickWith diff --git a/server/typings/models/server/index.d.ts b/server/typings/models/server/index.ts similarity index 100% rename from server/typings/models/server/index.d.ts rename to server/typings/models/server/index.ts diff --git a/server/typings/models/server/server-blocklist.ts b/server/typings/models/server/server-blocklist.ts index c81f604f5..c3e6230f2 100644 --- a/server/typings/models/server/server-blocklist.ts +++ b/server/typings/models/server/server-blocklist.ts @@ -1,6 +1,7 @@ import { ServerBlocklistModel } from '@server/models/server/server-blocklist' import { PickWith } from '@server/typings/utils' -import { MAccountDefault, MAccountFormattable, MServer, MServerFormattable } from '@server/typings/models' +import { MAccountDefault, MAccountFormattable } from '../account/account' +import { MServer, MServerFormattable } from './server' type Use = PickWith diff --git a/server/typings/models/user/index.d.ts b/server/typings/models/user/index.ts similarity index 100% rename from server/typings/models/user/index.d.ts rename to server/typings/models/user/index.ts diff --git a/server/typings/models/user/user.ts b/server/typings/models/user/user.ts index 52d6d4a05..a2750adc7 100644 --- a/server/typings/models/user/user.ts +++ b/server/typings/models/user/user.ts @@ -11,7 +11,7 @@ import { } from '../account' import { MNotificationSetting, MNotificationSettingFormattable } from './user-notification-setting' import { AccountModel } from '@server/models/account/account' -import { MChannelFormattable } from '@server/typings/models' +import { MChannelFormattable } from '../video/video-channels' type Use = PickWith diff --git a/server/typings/models/video/index.d.ts b/server/typings/models/video/index.ts similarity index 100% rename from server/typings/models/video/index.d.ts rename to server/typings/models/video/index.ts diff --git a/server/typings/models/video/schedule-video-update.ts b/server/typings/models/video/schedule-video-update.ts index ada9af06e..e6f478cdf 100644 --- a/server/typings/models/video/schedule-video-update.ts +++ b/server/typings/models/video/schedule-video-update.ts @@ -1,9 +1,18 @@ import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' +import { PickWith } from '@server/typings/utils' +import { MVideoAPWithoutCaption, MVideoWithBlacklistLight } from './video' + +type Use = PickWith + +// ############################################################################ export type MScheduleVideoUpdate = Omit // ############################################################################ +export type MScheduleVideoUpdateVideoAll = MScheduleVideoUpdate & + Use<'Video', MVideoAPWithoutCaption & MVideoWithBlacklistLight> + // Format for API or AP object export type MScheduleVideoUpdateFormattable = Pick diff --git a/server/typings/models/video/video-blacklist.ts b/server/typings/models/video/video-blacklist.ts index e12880454..7122a9dc0 100644 --- a/server/typings/models/video/video-blacklist.ts +++ b/server/typings/models/video/video-blacklist.ts @@ -1,6 +1,6 @@ import { VideoBlacklistModel } from '../../../models/video/video-blacklist' import { PickWith } from '@server/typings/utils' -import { MVideo, MVideoFormattable } from '@server/typings/models' +import { MVideo, MVideoFormattable } from './video' type Use = PickWith diff --git a/server/typings/models/video/video-caption.ts b/server/typings/models/video/video-caption.ts index 7cb2a2ad3..ffa56f544 100644 --- a/server/typings/models/video/video-caption.ts +++ b/server/typings/models/video/video-caption.ts @@ -1,6 +1,6 @@ import { VideoCaptionModel } from '../../../models/video/video-caption' import { FunctionProperties, PickWith } from '@server/typings/utils' -import { MVideo, MVideoUUID } from '@server/typings/models' +import { MVideo, MVideoUUID } from './video' type Use = PickWith diff --git a/server/typings/models/video/video-change-ownership.ts b/server/typings/models/video/video-change-ownership.ts index 72634cdb2..e5b5bbc1d 100644 --- a/server/typings/models/video/video-change-ownership.ts +++ b/server/typings/models/video/video-change-ownership.ts @@ -1,6 +1,7 @@ import { VideoChangeOwnershipModel } from '@server/models/video/video-change-ownership' import { PickWith } from '@server/typings/utils' -import { MAccountDefault, MAccountFormattable, MVideo, MVideoWithFileThumbnail } from '@server/typings/models' +import { MAccountDefault, MAccountFormattable } from '../account/account' +import { MVideo, MVideoWithAllFiles } from './video' type Use = PickWith @@ -11,7 +12,7 @@ export type MVideoChangeOwnership = Omit & Use<'NextOwner', MAccountDefault> & - Use<'Video', MVideoWithFileThumbnail> + Use<'Video', MVideoWithAllFiles> // ############################################################################ diff --git a/server/typings/models/video/video-comment.ts b/server/typings/models/video/video-comment.ts index 4fd1c29e8..d693f9186 100644 --- a/server/typings/models/video/video-comment.ts +++ b/server/typings/models/video/video-comment.ts @@ -1,6 +1,6 @@ import { VideoCommentModel } from '../../../models/video/video-comment' import { PickWith, PickWithOpt } from '../../utils' -import { MAccountDefault, MAccountFormattable, MAccountUrl, MActorUrl } from '../account' +import { MAccountDefault, MAccountFormattable, MAccountUrl } from '../account' import { MVideoAccountLight, MVideoFeed, MVideoIdUrl, MVideoUrl } from './video' type Use = PickWith diff --git a/server/typings/models/video/video-file.ts b/server/typings/models/video/video-file.ts index 484351a8d..352fe3d32 100644 --- a/server/typings/models/video/video-file.ts +++ b/server/typings/models/video/video-file.ts @@ -2,18 +2,33 @@ import { VideoFileModel } from '../../../models/video/video-file' import { PickWith, PickWithOpt } from '../../utils' import { MVideo, MVideoUUID } from './video' import { MVideoRedundancyFileUrl } from './video-redundancy' +import { MStreamingPlaylistVideo, MStreamingPlaylist } from './video-streaming-playlist' type Use = PickWith // ############################################################################ -export type MVideoFile = Omit +export type MVideoFile = Omit export type MVideoFileVideo = MVideoFile & Use<'Video', MVideo> +export type MVideoFileStreamingPlaylist = MVideoFile & + Use<'VideoStreamingPlaylist', MStreamingPlaylist> + +export type MVideoFileStreamingPlaylistVideo = MVideoFile & + Use<'VideoStreamingPlaylist', MStreamingPlaylistVideo> + export type MVideoFileVideoUUID = MVideoFile & Use<'Video', MVideoUUID> export type MVideoFileRedundanciesOpt = MVideoFile & PickWithOpt + +export function isStreamingPlaylistFile (file: any): file is MVideoFileStreamingPlaylist { + return !!file.videoStreamingPlaylistId +} + +export function isWebtorrentFile (file: any): file is MVideoFileVideo { + return !!file.videoId +} diff --git a/server/typings/models/video/video-import.ts b/server/typings/models/video/video-import.ts index c6a1c5b66..e119f17f9 100644 --- a/server/typings/models/video/video-import.ts +++ b/server/typings/models/video/video-import.ts @@ -1,6 +1,7 @@ import { VideoImportModel } from '@server/models/video/video-import' import { PickWith, PickWithOpt } from '@server/typings/utils' -import { MUser, MVideo, MVideoAccountLight, MVideoFormattable, MVideoTag, MVideoThumbnail, MVideoWithFile } from '@server/typings/models' +import { MVideo, MVideoAccountLight, MVideoFormattable, MVideoTag, MVideoThumbnail, MVideoWithFile } from './video' +import { MUser } from '../user/user' type Use = PickWith diff --git a/server/typings/models/video/video-playlist-element.ts b/server/typings/models/video/video-playlist-element.ts index 7b1b993ce..1aeff78d8 100644 --- a/server/typings/models/video/video-playlist-element.ts +++ b/server/typings/models/video/video-playlist-element.ts @@ -1,6 +1,7 @@ import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element' import { PickWith } from '@server/typings/utils' -import { MVideoFormattable, MVideoPlaylistPrivacy, MVideoThumbnail, MVideoUrl } from '@server/typings/models' +import { MVideoFormattable, MVideoThumbnail, MVideoUrl } from './video' +import { MVideoPlaylistPrivacy } from './video-playlist' type Use = PickWith diff --git a/server/typings/models/video/video-rate.ts b/server/typings/models/video/video-rate.ts index 2ff8a625b..f6bb527fc 100644 --- a/server/typings/models/video/video-rate.ts +++ b/server/typings/models/video/video-rate.ts @@ -1,6 +1,7 @@ import { AccountVideoRateModel } from '@server/models/account/account-video-rate' import { PickWith } from '@server/typings/utils' -import { MAccountAudience, MAccountUrl, MVideo, MVideoFormattable } from '..' +import { MAccountAudience, MAccountUrl } from '../account/account' +import { MVideo, MVideoFormattable } from './video' type Use = PickWith diff --git a/server/typings/models/video/video-redundancy.ts b/server/typings/models/video/video-redundancy.ts index f3846afd7..25bdac057 100644 --- a/server/typings/models/video/video-redundancy.ts +++ b/server/typings/models/video/video-redundancy.ts @@ -1,10 +1,10 @@ import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' import { PickWith, PickWithOpt } from '@server/typings/utils' -import { MStreamingPlaylistVideo, MVideoFile, MVideoFileVideo, MVideoUrl } from '@server/typings/models' -import { VideoStreamingPlaylist } from '../../../../shared/models/videos/video-streaming-playlist.model' import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { VideoFile } from '../../../../shared/models/videos' import { VideoFileModel } from '@server/models/video/video-file' +import { MVideoFile, MVideoFileVideo } from './video-file' +import { MStreamingPlaylistVideo } from './video-streaming-playlist' +import { MVideoUrl } from './video' type Use = PickWith diff --git a/server/typings/models/video/video-streaming-playlist.ts b/server/typings/models/video/video-streaming-playlist.ts index 79696bcff..436c0c072 100644 --- a/server/typings/models/video/video-streaming-playlist.ts +++ b/server/typings/models/video/video-streaming-playlist.ts @@ -1,19 +1,33 @@ import { VideoStreamingPlaylistModel } from '../../../models/video/video-streaming-playlist' import { PickWith, PickWithOpt } from '../../utils' import { MVideoRedundancyFileUrl } from './video-redundancy' -import { MVideo, MVideoUrl } from '@server/typings/models' +import { MVideo } from './video' +import { MVideoFile } from './video-file' type Use = PickWith // ############################################################################ -export type MStreamingPlaylist = Omit +export type MStreamingPlaylist = Omit + +export type MStreamingPlaylistFiles = MStreamingPlaylist & + Use<'VideoFiles', MVideoFile[]> export type MStreamingPlaylistVideo = MStreamingPlaylist & Use<'Video', MVideo> +export type MStreamingPlaylistFilesVideo = MStreamingPlaylist & + Use<'VideoFiles', MVideoFile[]> & + Use<'Video', MVideo> + export type MStreamingPlaylistRedundancies = MStreamingPlaylist & + Use<'VideoFiles', MVideoFile[]> & Use<'RedundancyVideos', MVideoRedundancyFileUrl[]> export type MStreamingPlaylistRedundanciesOpt = MStreamingPlaylist & + Use<'VideoFiles', MVideoFile[]> & PickWithOpt + +export function isStreamingPlaylist (value: MVideo | MStreamingPlaylistVideo): value is MStreamingPlaylistVideo { + return !!(value as MStreamingPlaylist).playlistUrl +} diff --git a/server/typings/models/video/video.ts b/server/typings/models/video/video.ts index 9a53bd337..7f69a91de 100644 --- a/server/typings/models/video/video.ts +++ b/server/typings/models/video/video.ts @@ -10,7 +10,7 @@ import { } from './video-channels' import { MTag } from './tag' import { MVideoCaptionLanguage } from './video-caption' -import { MStreamingPlaylist, MStreamingPlaylistRedundancies, MStreamingPlaylistRedundanciesOpt } from './video-streaming-playlist' +import { MStreamingPlaylistFiles, MStreamingPlaylistRedundancies, MStreamingPlaylistRedundanciesOpt } from './video-streaming-playlist' import { MVideoFile, MVideoFileRedundanciesOpt } from './video-file' import { MThumbnail } from './thumbnail' import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist' @@ -40,7 +40,8 @@ export type MVideoFeed = Pick // "With" to not confuse with the VideoFile model export type MVideoWithFile = MVideo & - Use<'VideoFiles', MVideoFile[]> + Use<'VideoFiles', MVideoFile[]> & + Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> export type MVideoThumbnail = MVideo & Use<'Thumbnails', MThumbnail[]> @@ -66,7 +67,7 @@ export type MVideoWithCaptions = MVideo & Use<'VideoCaptions', MVideoCaptionLanguage[]> export type MVideoWithStreamingPlaylist = MVideo & - Use<'VideoStreamingPlaylists', MStreamingPlaylist[]> + Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> // ############################################################################ @@ -93,12 +94,12 @@ export type MVideoWithRights = MVideo & export type MVideoWithAllFiles = MVideo & Use<'VideoFiles', MVideoFile[]> & Use<'Thumbnails', MThumbnail[]> & - Use<'VideoStreamingPlaylists', MStreamingPlaylist[]> + Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> export type MVideoAccountLightBlacklistAllFiles = MVideo & Use<'VideoFiles', MVideoFile[]> & Use<'Thumbnails', MThumbnail[]> & - Use<'VideoStreamingPlaylists', MStreamingPlaylist[]> & + Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> & Use<'VideoChannel', MChannelAccountLight> & Use<'VideoBlacklist', MVideoBlacklistLight> @@ -124,7 +125,7 @@ export type MVideoFullLight = MVideo & Use<'UserVideoHistories', MUserVideoHistoryTime[]> & Use<'VideoFiles', MVideoFile[]> & Use<'ScheduleVideoUpdate', MScheduleVideoUpdate> & - Use<'VideoStreamingPlaylists', MStreamingPlaylist[]> + Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> // ############################################################################ @@ -133,10 +134,11 @@ export type MVideoFullLight = MVideo & export type MVideoAP = MVideo & Use<'Tags', MTag[]> & Use<'VideoChannel', MChannelAccountLight> & - Use<'VideoStreamingPlaylists', MStreamingPlaylist[]> & + Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> & Use<'VideoCaptions', MVideoCaptionLanguage[]> & Use<'VideoBlacklist', MVideoBlacklistUnfederated> & - Use<'VideoFiles', MVideoFileRedundanciesOpt[]> + Use<'VideoFiles', MVideoFileRedundanciesOpt[]> & + Use<'Thumbnails', MThumbnail[]> export type MVideoAPWithoutCaption = Omit diff --git a/shared/extra-utils/server/config.ts b/shared/extra-utils/server/config.ts index 578dd35cf..ada173313 100644 --- a/shared/extra-utils/server/config.ts +++ b/shared/extra-utils/server/config.ts @@ -118,6 +118,9 @@ function updateCustomSubConfig (url: string, token: string, newConfig: DeepParti '1080p': false, '2160p': false }, + webtorrent: { + enabled: true + }, hls: { enabled: false } diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts index 75f7d58d7..1fcc949da 100644 --- a/shared/extra-utils/videos/videos.ts +++ b/shared/extra-utils/videos/videos.ts @@ -573,7 +573,6 @@ async function completeVideoCheck ( // Transcoding enabled: extension will always be .mp4 if (attributes.files.length > 1) extension = '.mp4' - const magnetUri = file.magnetUri expect(file.magnetUri).to.have.lengthOf.above(2) expect(file.torrentUrl).to.equal(`http://${attributes.account.host}/static/torrents/${videoDetails.uuid}-${file.resolution.id}.torrent`) expect(file.fileUrl).to.equal(`http://${attributes.account.host}/static/webseed/${videoDetails.uuid}-${file.resolution.id}${extension}`) @@ -594,7 +593,7 @@ async function completeVideoCheck ( await testImage(url, attributes.previewfile, videoDetails.previewPath) } - const torrent = await webtorrentAdd(magnetUri, true) + const torrent = await webtorrentAdd(file.magnetUri, true) expect(torrent.files).to.be.an('array') expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') diff --git a/shared/models/activitypub/objects/common-objects.ts b/shared/models/activitypub/objects/common-objects.ts index 8c89810d6..2a6529fed 100644 --- a/shared/models/activitypub/objects/common-objects.ts +++ b/shared/models/activitypub/objects/common-objects.ts @@ -3,12 +3,6 @@ export interface ActivityIdentifierObject { name: string } -export interface ActivityTagObject { - type: 'Hashtag' | 'Mention' - href?: string - name: string -} - export interface ActivityIconObject { type: 'Image' url: string @@ -19,8 +13,6 @@ export interface ActivityIconObject { export type ActivityVideoUrlObject = { type: 'Link' - // TODO: remove mimeType (backward compatibility, introduced in v1.1.0) - mimeType?: 'video/mp4' | 'video/webm' | 'video/ogg' mediaType: 'video/mp4' | 'video/webm' | 'video/ogg' href: string height: number @@ -31,8 +23,6 @@ export type ActivityVideoUrlObject = { export type ActivityPlaylistSegmentHashesObject = { type: 'Link' name: 'sha256' - // TODO: remove mimeType (backward compatibility, introduced in v1.1.0) - mimeType?: 'application/json' mediaType: 'application/json' href: string } @@ -44,31 +34,56 @@ export type ActivityPlaylistInfohashesObject = { export type ActivityPlaylistUrlObject = { type: 'Link' - // TODO: remove mimeType (backward compatibility, introduced in v1.1.0) - mimeType?: 'application/x-mpegURL' mediaType: 'application/x-mpegURL' href: string - tag?: (ActivityPlaylistSegmentHashesObject | ActivityPlaylistInfohashesObject)[] + tag?: ActivityTagObject[] } export type ActivityBitTorrentUrlObject = { type: 'Link' - // TODO: remove mimeType (backward compatibility, introduced in v1.1.0) - mimeType?: 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet' mediaType: 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet' href: string height: number } +export type ActivityMagnetUrlObject = { + type: 'Link' + mediaType: 'application/x-bittorrent;x-scheme-handler/magnet' + href: string + height: number +} + export type ActivityHtmlUrlObject = { type: 'Link' - // TODO: remove mimeType (backward compatibility, introduced in v1.1.0) - mimeType?: 'text/html' mediaType: 'text/html' href: string } -export type ActivityUrlObject = ActivityVideoUrlObject | ActivityPlaylistUrlObject | ActivityBitTorrentUrlObject | ActivityHtmlUrlObject +export interface ActivityHashTagObject { + type: 'Hashtag' | 'Mention' + href?: string + name: string +} + +export interface ActivityMentionObject { + type: 'Hashtag' | 'Mention' + href?: string + name: string +} + +export type ActivityTagObject = ActivityPlaylistSegmentHashesObject | + ActivityPlaylistInfohashesObject | + ActivityVideoUrlObject | + ActivityHashTagObject | + ActivityMentionObject | + ActivityBitTorrentUrlObject | + ActivityMagnetUrlObject + +export type ActivityUrlObject = ActivityVideoUrlObject | + ActivityPlaylistUrlObject | + ActivityBitTorrentUrlObject | + ActivityMagnetUrlObject | + ActivityHtmlUrlObject export interface ActivityPubAttributedTo { type: 'Group' | 'Person' diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts index c9957f825..97972b759 100644 --- a/shared/models/server/custom-config.model.ts +++ b/shared/models/server/custom-config.model.ts @@ -69,8 +69,10 @@ export interface CustomConfig { transcoding: { enabled: boolean + allowAdditionalExtensions: boolean allowAudioFiles: boolean + threads: number resolutions: { '240p': boolean @@ -80,6 +82,11 @@ export interface CustomConfig { '1080p': boolean '2160p': boolean } + + webtorrent: { + enabled: boolean + } + hls: { enabled: boolean } diff --git a/shared/models/server/server-config.model.ts b/shared/models/server/server-config.model.ts index 3498f86d7..6d1072333 100644 --- a/shared/models/server/server-config.model.ts +++ b/shared/models/server/server-config.model.ts @@ -56,6 +56,10 @@ export interface ServerConfig { enabled: boolean } + webtorrent: { + enabled: boolean + } + enabledResolutions: number[] } diff --git a/shared/models/videos/index.ts b/shared/models/videos/index.ts index 194ae1b96..51ccb9fbd 100644 --- a/shared/models/videos/index.ts +++ b/shared/models/videos/index.ts @@ -23,6 +23,7 @@ export * from './playlist/video-playlist-element.model' export * from './video-change-ownership.model' export * from './video-change-ownership-create.model' export * from './video-create.model' +export * from './video-file.model' export * from './video-privacy.enum' export * from './video-rate.type' export * from './video-resolution.enum' diff --git a/shared/models/videos/video-file.model.ts b/shared/models/videos/video-file.model.ts new file mode 100644 index 000000000..04da0627e --- /dev/null +++ b/shared/models/videos/video-file.model.ts @@ -0,0 +1,12 @@ +import { VideoConstant, VideoResolution } from '@shared/models' + +export interface VideoFile { + magnetUri: string + resolution: VideoConstant + size: number // Bytes + torrentUrl: string + torrentDownloadUrl: string + fileUrl: string + fileDownloadUrl: string + fps: number +} diff --git a/shared/models/videos/video-streaming-playlist.model.ts b/shared/models/videos/video-streaming-playlist.model.ts index 17f8fe865..42fce4bdc 100644 --- a/shared/models/videos/video-streaming-playlist.model.ts +++ b/shared/models/videos/video-streaming-playlist.model.ts @@ -1,4 +1,5 @@ import { VideoStreamingPlaylistType } from './video-streaming-playlist.type' +import { VideoFile } from '@shared/models/videos/video-file.model' export class VideoStreamingPlaylist { id: number @@ -9,4 +10,6 @@ export class VideoStreamingPlaylist { redundancies: { baseUrl: string }[] + + files: VideoFile[] } diff --git a/shared/models/videos/video.model.ts b/shared/models/videos/video.model.ts index e057b3e06..7576439fe 100644 --- a/shared/models/videos/video.model.ts +++ b/shared/models/videos/video.model.ts @@ -5,17 +5,7 @@ import { VideoPrivacy } from './video-privacy.enum' import { VideoScheduleUpdate } from './video-schedule-update.model' import { VideoConstant } from './video-constant.model' import { VideoStreamingPlaylist } from './video-streaming-playlist.model' - -export interface VideoFile { - magnetUri: string - resolution: VideoConstant - size: number // Bytes - torrentUrl: string - torrentDownloadUrl: string - fileUrl: string - fileDownloadUrl: string - fps: number -} +import { VideoFile } from './video-file.model' export interface Video { id: number diff --git a/tsconfig.json b/tsconfig.json index e23c8eed1..7eed7d0cd 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -16,8 +16,7 @@ ], "typeRoots": [ "node_modules/sitemap/node_modules/@types", - "node_modules/@types", - "server/typings" + "node_modules/@types" ], "baseUrl": "./", "paths": { diff --git a/yarn.lock b/yarn.lock index a8d942233..175008d8b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7240,10 +7240,10 @@ typedarray@^0.0.6: resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= -typescript@^3.4.3: - version "3.6.4" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.6.4.tgz#b18752bb3792bc1a0281335f7f6ebf1bbfc5b91d" - integrity sha512-unoCll1+l+YK4i4F8f22TaNVPRHcD9PA3yCuZ8g5e0qGqlVlJ/8FSateOLLSagn+Yg5+ZwuPkL8LFUc0Jcvksg== +typescript@^3.7.2: + version "3.7.2" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.7.2.tgz#27e489b95fa5909445e9fef5ee48d81697ad18fb" + integrity sha512-ml7V7JfiN2Xwvcer+XAf2csGO1bPBdRbFCkYBczNZggrBZ9c7G3riSUeJmqEU5uOtXNPMhE3n+R4FA/3YOAWOQ== uint64be@^2.0.2: version "2.0.2"