From 0c9668f77901e7540e2c7045eb0f2974a4842a69 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 21 Apr 2023 14:55:10 +0200 Subject: [PATCH] Implement remote runner jobs in server Move ffmpeg functions to @shared --- config/default.yaml | 30 +- config/production.yaml.example | 34 +- server.ts | 2 + server/controllers/api/config.ts | 6 + server/controllers/api/index.ts | 2 + server/controllers/api/jobs.ts | 3 + server/controllers/api/runners/index.ts | 18 + server/controllers/api/runners/jobs-files.ts | 84 ++++ server/controllers/api/runners/jobs.ts | 352 +++++++++++++ .../controllers/api/runners/manage-runners.ts | 107 ++++ .../api/runners/registration-tokens.ts | 87 ++++ server/controllers/api/videos/transcoding.ts | 87 +--- server/controllers/api/videos/upload.ts | 71 +-- server/controllers/bots.ts | 6 +- server/controllers/object-storage-proxy.ts | 87 +--- server/helpers/core-utils.ts | 56 +-- server/helpers/custom-validators/misc.ts | 5 + .../helpers/custom-validators/runners/jobs.ts | 166 +++++++ .../custom-validators/runners/runners.ts | 30 ++ server/helpers/debounce.ts | 16 + server/helpers/ffmpeg/codecs.ts | 64 +++ server/helpers/ffmpeg/ffmpeg-commons.ts | 114 ----- server/helpers/ffmpeg/ffmpeg-edition.ts | 258 ---------- server/helpers/ffmpeg/ffmpeg-encoders.ts | 116 ----- server/helpers/ffmpeg/ffmpeg-image.ts | 14 + server/helpers/ffmpeg/ffmpeg-images.ts | 46 -- server/helpers/ffmpeg/ffmpeg-live.ts | 204 -------- server/helpers/ffmpeg/ffmpeg-options.ts | 45 ++ server/helpers/ffmpeg/ffmpeg-presets.ts | 156 ------ server/helpers/ffmpeg/ffmpeg-vod.ts | 267 ---------- server/helpers/ffmpeg/ffprobe-utils.ts | 254 ---------- server/helpers/ffmpeg/framerate.ts | 44 ++ server/helpers/ffmpeg/index.ts | 12 +- server/helpers/image-utils.ts | 8 +- server/helpers/peertube-crypto.ts | 3 +- server/helpers/token-generator.ts | 19 + server/helpers/webtorrent.ts | 2 +- server/initializers/checker-after-init.ts | 2 +- server/initializers/checker-before-init.ts | 10 +- server/initializers/config.ts | 12 + server/initializers/constants.ts | 78 ++- server/initializers/database.ts | 10 +- server/initializers/installer.ts | 16 +- .../migrations/0765-remote-transcoding.ts | 78 +++ server/lib/hls.ts | 18 +- .../handlers/transcoding-job-builder.ts | 47 ++ .../job-queue/handlers/video-file-import.ts | 2 +- server/lib/job-queue/handlers/video-import.ts | 12 +- .../job-queue/handlers/video-live-ending.ts | 10 +- .../handlers/video-studio-edition.ts | 68 +-- .../job-queue/handlers/video-transcoding.ts | 282 ++--------- server/lib/job-queue/job-queue.ts | 65 +-- server/lib/live/live-manager.ts | 94 ++-- server/lib/live/live-segment-sha-store.ts | 5 +- server/lib/live/live-utils.ts | 12 +- server/lib/live/shared/muxing-session.ts | 191 ++++--- .../abstract-transcoding-wrapper.ts | 101 ++++ .../ffmpeg-transcoding-wrapper.ts | 95 ++++ .../live/shared/transcoding-wrapper/index.ts | 3 + .../remote-transcoding-wrapper.ts | 20 + server/lib/object-storage/index.ts | 1 + server/lib/object-storage/proxy.ts | 97 ++++ server/lib/peertube-socket.ts | 32 +- server/lib/plugins/plugin-helpers-builder.ts | 2 +- server/lib/runners/index.ts | 3 + .../job-handlers/abstract-job-handler.ts | 271 ++++++++++ .../abstract-vod-transcoding-job-handler.ts | 71 +++ server/lib/runners/job-handlers/index.ts | 6 + .../live-rtmp-hls-transcoding-job-handler.ts | 170 +++++++ .../job-handlers/runner-job-handlers.ts | 18 + .../lib/runners/job-handlers/shared/index.ts | 1 + .../job-handlers/shared/vod-helpers.ts | 44 ++ ...vod-audio-merge-transcoding-job-handler.ts | 97 ++++ .../vod-hls-transcoding-job-handler.ts | 114 +++++ .../vod-web-video-transcoding-job-handler.ts | 84 ++++ server/lib/runners/runner-urls.ts | 9 + server/lib/runners/runner.ts | 36 ++ .../runner-job-watch-dog-scheduler.ts | 42 ++ server/lib/server-config-manager.ts | 10 +- .../lib/transcoding/create-transcoding-job.ts | 36 ++ .../default-transcoding-profiles.ts | 16 +- server/lib/transcoding/ended-transcoding.ts | 18 + server/lib/transcoding/hls-transcoding.ts | 181 +++++++ .../lib/transcoding/shared/ffmpeg-builder.ts | 18 + server/lib/transcoding/shared/index.ts | 2 + .../job-builders/abstract-job-builder.ts | 38 ++ .../transcoding/shared/job-builders/index.ts | 2 + .../transcoding-job-queue-builder.ts | 308 ++++++++++++ .../transcoding-runner-job-builder.ts | 189 +++++++ .../transcoding-quick-transcode.ts | 61 +++ .../transcoding/transcoding-resolutions.ts | 52 ++ server/lib/transcoding/transcoding.ts | 465 ------------------ server/lib/transcoding/web-transcoding.ts | 273 ++++++++++ server/lib/uploadx.ts | 5 +- server/lib/video-blacklist.ts | 2 +- server/lib/video-file.ts | 54 +- server/lib/video-studio.ts | 2 +- server/lib/video.ts | 63 +-- server/middlewares/auth.ts | 27 +- server/middlewares/doc.ts | 2 +- server/middlewares/error.ts | 6 +- server/middlewares/rate-limiter.ts | 28 +- server/middlewares/validators/config.ts | 2 + .../middlewares/validators/runners/index.ts | 3 + .../validators/runners/job-files.ts | 27 + server/middlewares/validators/runners/jobs.ts | 156 ++++++ .../validators/runners/registration-token.ts | 37 ++ .../middlewares/validators/runners/runners.ts | 95 ++++ server/middlewares/validators/sort.ts | 4 + .../validators/videos/video-live.ts | 9 + .../validators/videos/video-studio.ts | 2 +- .../middlewares/validators/videos/videos.ts | 2 +- server/models/runner/runner-job.ts | 347 +++++++++++++ .../runner/runner-registration-token.ts | 103 ++++ server/models/runner/runner.ts | 112 +++++ server/models/shared/update.ts | 28 +- server/models/video/video-job-info.ts | 16 +- server/models/video/video-live-session.ts | 13 +- server/models/video/video.ts | 10 +- server/types/express.d.ts | 6 + server/types/models/runners/index.ts | 3 + server/types/models/runners/runner-job.ts | 20 + .../runners/runner-registration-token.ts | 5 + server/types/models/runners/runner.ts | 5 + shared/core-utils/common/number.ts | 12 +- shared/core-utils/common/promises.ts | 47 +- shared/extra-utils/index.ts | 1 - shared/ffmpeg/ffmpeg-command-wrapper.ts | 234 +++++++++ shared/ffmpeg/ffmpeg-edition.ts | 239 +++++++++ shared/ffmpeg/ffmpeg-images.ts | 59 +++ shared/ffmpeg/ffmpeg-live.ts | 184 +++++++ shared/ffmpeg/ffmpeg-utils.ts | 17 + shared/ffmpeg/ffmpeg-version.ts | 24 + shared/ffmpeg/ffmpeg-vod.ts | 256 ++++++++++ shared/{extra-utils => ffmpeg}/ffprobe.ts | 19 +- shared/ffmpeg/index.ts | 8 + shared/ffmpeg/shared/encoder-options.ts | 39 ++ shared/ffmpeg/shared/index.ts | 2 + shared/ffmpeg/shared/presets.ts | 93 ++++ shared/models/index.ts | 1 + .../runners/abort-runner-job-body.model.ts | 6 + .../runners/accept-runner-job-body.model.ts | 3 + .../runners/accept-runner-job-result.model.ts | 6 + .../runners/error-runner-job-body.model.ts | 6 + shared/models/runners/index.ts | 21 + .../runners/list-runner-jobs-query.model.ts | 6 + .../list-runner-registration-tokens.model.ts | 5 + .../runners/list-runners-query.model.ts | 5 + .../runners/register-runner-body.model.ts | 6 + .../runners/register-runner-result.model.ts | 4 + .../runners/request-runner-job-body.model.ts | 3 + .../request-runner-job-result.model.ts | 10 + .../runners/runner-job-payload.model.ts | 68 +++ .../runner-job-private-payload.model.ts | 34 ++ .../models/runners/runner-job-state.model.ts | 10 + .../runners/runner-job-success-body.model.ts | 41 ++ shared/models/runners/runner-job-type.type.ts | 5 + .../runners/runner-job-update-body.model.ts | 28 ++ shared/models/runners/runner-job.model.ts | 45 ++ .../runners/runner-registration-token.ts | 10 + shared/models/runners/runner.model.ts | 12 + .../runners/unregister-runner-body.model.ts | 3 + shared/models/server/custom-config.model.ts | 7 + shared/models/server/job.model.ts | 43 +- shared/models/server/server-config.model.ts | 8 + .../models/server/server-error-code.enum.ts | 5 +- shared/models/users/user-right.enum.ts | 4 +- .../videos/live/live-video-error.enum.ts | 4 +- 168 files changed, 6907 insertions(+), 2803 deletions(-) create mode 100644 server/controllers/api/runners/index.ts create mode 100644 server/controllers/api/runners/jobs-files.ts create mode 100644 server/controllers/api/runners/jobs.ts create mode 100644 server/controllers/api/runners/manage-runners.ts create mode 100644 server/controllers/api/runners/registration-tokens.ts create mode 100644 server/helpers/custom-validators/runners/jobs.ts create mode 100644 server/helpers/custom-validators/runners/runners.ts create mode 100644 server/helpers/debounce.ts create mode 100644 server/helpers/ffmpeg/codecs.ts delete mode 100644 server/helpers/ffmpeg/ffmpeg-commons.ts delete mode 100644 server/helpers/ffmpeg/ffmpeg-edition.ts delete mode 100644 server/helpers/ffmpeg/ffmpeg-encoders.ts create mode 100644 server/helpers/ffmpeg/ffmpeg-image.ts delete mode 100644 server/helpers/ffmpeg/ffmpeg-images.ts delete mode 100644 server/helpers/ffmpeg/ffmpeg-live.ts create mode 100644 server/helpers/ffmpeg/ffmpeg-options.ts delete mode 100644 server/helpers/ffmpeg/ffmpeg-presets.ts delete mode 100644 server/helpers/ffmpeg/ffmpeg-vod.ts delete mode 100644 server/helpers/ffmpeg/ffprobe-utils.ts create mode 100644 server/helpers/ffmpeg/framerate.ts create mode 100644 server/helpers/token-generator.ts create mode 100644 server/initializers/migrations/0765-remote-transcoding.ts create mode 100644 server/lib/job-queue/handlers/transcoding-job-builder.ts create mode 100644 server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts create mode 100644 server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts create mode 100644 server/lib/live/shared/transcoding-wrapper/index.ts create mode 100644 server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts create mode 100644 server/lib/object-storage/proxy.ts create mode 100644 server/lib/runners/index.ts create mode 100644 server/lib/runners/job-handlers/abstract-job-handler.ts create mode 100644 server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts create mode 100644 server/lib/runners/job-handlers/index.ts create mode 100644 server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts create mode 100644 server/lib/runners/job-handlers/runner-job-handlers.ts create mode 100644 server/lib/runners/job-handlers/shared/index.ts create mode 100644 server/lib/runners/job-handlers/shared/vod-helpers.ts create mode 100644 server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts create mode 100644 server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts create mode 100644 server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts create mode 100644 server/lib/runners/runner-urls.ts create mode 100644 server/lib/runners/runner.ts create mode 100644 server/lib/schedulers/runner-job-watch-dog-scheduler.ts create mode 100644 server/lib/transcoding/create-transcoding-job.ts create mode 100644 server/lib/transcoding/ended-transcoding.ts create mode 100644 server/lib/transcoding/hls-transcoding.ts create mode 100644 server/lib/transcoding/shared/ffmpeg-builder.ts create mode 100644 server/lib/transcoding/shared/index.ts create mode 100644 server/lib/transcoding/shared/job-builders/abstract-job-builder.ts create mode 100644 server/lib/transcoding/shared/job-builders/index.ts create mode 100644 server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts create mode 100644 server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts create mode 100644 server/lib/transcoding/transcoding-quick-transcode.ts create mode 100644 server/lib/transcoding/transcoding-resolutions.ts delete mode 100644 server/lib/transcoding/transcoding.ts create mode 100644 server/lib/transcoding/web-transcoding.ts create mode 100644 server/middlewares/validators/runners/index.ts create mode 100644 server/middlewares/validators/runners/job-files.ts create mode 100644 server/middlewares/validators/runners/jobs.ts create mode 100644 server/middlewares/validators/runners/registration-token.ts create mode 100644 server/middlewares/validators/runners/runners.ts create mode 100644 server/models/runner/runner-job.ts create mode 100644 server/models/runner/runner-registration-token.ts create mode 100644 server/models/runner/runner.ts create mode 100644 server/types/models/runners/index.ts create mode 100644 server/types/models/runners/runner-job.ts create mode 100644 server/types/models/runners/runner-registration-token.ts create mode 100644 server/types/models/runners/runner.ts create mode 100644 shared/ffmpeg/ffmpeg-command-wrapper.ts create mode 100644 shared/ffmpeg/ffmpeg-edition.ts create mode 100644 shared/ffmpeg/ffmpeg-images.ts create mode 100644 shared/ffmpeg/ffmpeg-live.ts create mode 100644 shared/ffmpeg/ffmpeg-utils.ts create mode 100644 shared/ffmpeg/ffmpeg-version.ts create mode 100644 shared/ffmpeg/ffmpeg-vod.ts rename shared/{extra-utils => ffmpeg}/ffprobe.ts (91%) create mode 100644 shared/ffmpeg/index.ts create mode 100644 shared/ffmpeg/shared/encoder-options.ts create mode 100644 shared/ffmpeg/shared/index.ts create mode 100644 shared/ffmpeg/shared/presets.ts create mode 100644 shared/models/runners/abort-runner-job-body.model.ts create mode 100644 shared/models/runners/accept-runner-job-body.model.ts create mode 100644 shared/models/runners/accept-runner-job-result.model.ts create mode 100644 shared/models/runners/error-runner-job-body.model.ts create mode 100644 shared/models/runners/index.ts create mode 100644 shared/models/runners/list-runner-jobs-query.model.ts create mode 100644 shared/models/runners/list-runner-registration-tokens.model.ts create mode 100644 shared/models/runners/list-runners-query.model.ts create mode 100644 shared/models/runners/register-runner-body.model.ts create mode 100644 shared/models/runners/register-runner-result.model.ts create mode 100644 shared/models/runners/request-runner-job-body.model.ts create mode 100644 shared/models/runners/request-runner-job-result.model.ts create mode 100644 shared/models/runners/runner-job-payload.model.ts create mode 100644 shared/models/runners/runner-job-private-payload.model.ts create mode 100644 shared/models/runners/runner-job-state.model.ts create mode 100644 shared/models/runners/runner-job-success-body.model.ts create mode 100644 shared/models/runners/runner-job-type.type.ts create mode 100644 shared/models/runners/runner-job-update-body.model.ts create mode 100644 shared/models/runners/runner-job.model.ts create mode 100644 shared/models/runners/runner-registration-token.ts create mode 100644 shared/models/runners/runner.model.ts create mode 100644 shared/models/runners/unregister-runner-body.model.ts diff --git a/config/default.yaml b/config/default.yaml index dfa43a0aa..986b2e999 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -375,6 +375,12 @@ feeds: # Default number of comments displayed in feeds count: 20 +remote_runners: + # Consider jobs that are processed by a remote runner as stalled after this period of time without any update + stalled_jobs: + live: '30 seconds' + vod: '2 minutes' + cache: previews: size: 500 # Max number of previews you want to cache @@ -433,12 +439,18 @@ transcoding: # If a user uploads an audio file, PeerTube will create a video by merging the preview file and the audio file allow_audio_files: true - # Amount of threads used by ffmpeg for 1 transcoding job + # Enable remote runners to transcode your videos + # If enabled, your instance won't transcode the videos itself + # At least 1 remote runner must be configured to transcode your videos + remote_runners: + enabled: false + + # Amount of threads used by ffmpeg for 1 local transcoding job threads: 1 - # Amount of transcoding jobs to execute in parallel + # Amount of local transcoding jobs to execute in parallel concurrency: 1 - # Choose the transcoding profile + # Choose the local transcoding profile # New profiles can be added by plugins # Available in core PeerTube: 'default' profile: 'default' @@ -533,9 +545,17 @@ live: # Allow to transcode the live streaming in multiple live resolutions transcoding: enabled: true + + # Enable remote runners to transcode your videos + # If enabled, your instance won't transcode the videos itself + # At least 1 remote runner must be configured to transcode your videos + remote_runners: + enabled: false + + # Amount of threads used by ffmpeg per live when using local transcoding threads: 2 - # Choose the transcoding profile + # Choose the local transcoding profile # New profiles can be added by plugins # Available in core PeerTube: 'default' profile: 'default' @@ -754,7 +774,7 @@ search: search_index: enabled: false # URL of the search index, that should use the same search API and routes - # than PeerTube: https://docs.joinpeertube.org/api/rest-reference.html + # than PeerTube: https://docs.joinpeertube.org/api-rest-reference.html # You should deploy your own with https://framagit.org/framasoft/peertube/search-index, # and can use https://search.joinpeertube.org/ for tests, but keep in mind the latter is an unmoderated search index url: '' diff --git a/config/production.yaml.example b/config/production.yaml.example index 0fb6ababc..bd01375cd 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -373,6 +373,12 @@ feeds: # Default number of comments displayed in feeds count: 20 +remote_runners: + # Consider jobs that are processed by a remote runner as stalled after this period of time without any update + stalled_jobs: + live: '30 seconds' + vod: '2 minutes' + ############################################################################### # # From this point, almost all following keys can be overridden by the web interface @@ -443,12 +449,18 @@ transcoding: # If a user uploads an audio file, PeerTube will create a video by merging the preview file and the audio file allow_audio_files: true - # Amount of threads used by ffmpeg for 1 transcoding job + # Enable remote runners to transcode your videos + # If enabled, your instance won't transcode the videos itself + # At least 1 remote runner must be configured to transcode your videos + remote_runners: + enabled: false + + # Amount of threads used by ffmpeg for 1 local transcoding job threads: 1 - # Amount of transcoding jobs to execute in parallel + # Amount of local transcoding jobs to execute in parallel concurrency: 1 - # Choose the transcoding profile + # Choose the local transcoding profile # New profiles can be added by plugins # Available in core PeerTube: 'default' profile: 'default' @@ -543,9 +555,17 @@ live: # Allow to transcode the live streaming in multiple live resolutions transcoding: enabled: true + + # Enable remote runners to transcode your videos + # If enabled, your instance won't transcode the videos itself + # At least 1 remote runner must be configured to transcode your videos + remote_runners: + enabled: false + + # Amount of threads used by ffmpeg per live when using local transcoding threads: 2 - # Choose the transcoding profile + # Choose the local transcoding profile # New profiles can be added by plugins # Available in core PeerTube: 'default' profile: 'default' @@ -607,7 +627,7 @@ import: # See https://docs.joinpeertube.org/maintain/configuration#security for more information enabled: false - # Add ability for your users to synchronize their channels with external channels, playlists, etc. + # Add ability for your users to synchronize their channels with external channels, playlists, etc video_channel_synchronization: enabled: false @@ -768,9 +788,9 @@ search: # You should deploy your own with https://framagit.org/framasoft/peertube/search-index, # and can use https://search.joinpeertube.org/ for tests, but keep in mind the latter is an unmoderated search index url: '' - # You can disable local search, so users only use the search index + # You can disable local search in the client, so users only use the search index disable_local_search: false - # If you did not disable local search, you can decide to use the search index by default + # If you did not disable local search in the client, you can decide to use the search index by default is_default_search: false # PeerTube client/interface configuration diff --git a/server.ts b/server.ts index 7bab18b0c..a7a723b24 100644 --- a/server.ts +++ b/server.ts @@ -133,6 +133,7 @@ import { AutoFollowIndexInstances } from './server/lib/schedulers/auto-follow-in import { RemoveDanglingResumableUploadsScheduler } from './server/lib/schedulers/remove-dangling-resumable-uploads-scheduler' import { VideoViewsBufferScheduler } from './server/lib/schedulers/video-views-buffer-scheduler' import { GeoIPUpdateScheduler } from './server/lib/schedulers/geo-ip-update-scheduler' +import { RunnerJobWatchDogScheduler } from './server/lib/schedulers/runner-job-watch-dog-scheduler' import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto' import { PeerTubeSocket } from './server/lib/peertube-socket' import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls' @@ -331,6 +332,7 @@ async function startApplication () { VideoChannelSyncLatestScheduler.Instance.enable() VideoViewsBufferScheduler.Instance.enable() GeoIPUpdateScheduler.Instance.enable() + RunnerJobWatchDogScheduler.Instance.enable() OpenTelemetryMetrics.Instance.registerMetrics({ trackerServer }) diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index 60d168d12..0b9aaffda 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -217,6 +217,9 @@ function customConfig (): CustomConfig { }, transcoding: { enabled: CONFIG.TRANSCODING.ENABLED, + remoteRunners: { + enabled: CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED + }, allowAdditionalExtensions: CONFIG.TRANSCODING.ALLOW_ADDITIONAL_EXTENSIONS, allowAudioFiles: CONFIG.TRANSCODING.ALLOW_AUDIO_FILES, threads: CONFIG.TRANSCODING.THREADS, @@ -252,6 +255,9 @@ function customConfig (): CustomConfig { maxUserLives: CONFIG.LIVE.MAX_USER_LIVES, transcoding: { enabled: CONFIG.LIVE.TRANSCODING.ENABLED, + remoteRunners: { + enabled: CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED + }, threads: CONFIG.LIVE.TRANSCODING.THREADS, profile: CONFIG.LIVE.TRANSCODING.PROFILE, resolutions: { diff --git a/server/controllers/api/index.ts b/server/controllers/api/index.ts index e1d197c8a..646f9597e 100644 --- a/server/controllers/api/index.ts +++ b/server/controllers/api/index.ts @@ -15,6 +15,7 @@ import { metricsRouter } from './metrics' import { oauthClientsRouter } from './oauth-clients' import { overviewsRouter } from './overviews' import { pluginRouter } from './plugins' +import { runnersRouter } from './runners' import { searchRouter } from './search' import { serverRouter } from './server' import { usersRouter } from './users' @@ -55,6 +56,7 @@ apiRouter.use('/overviews', overviewsRouter) apiRouter.use('/plugins', pluginRouter) apiRouter.use('/custom-pages', customPageRouter) apiRouter.use('/blocklist', blocklistRouter) +apiRouter.use('/runners', runnersRouter) apiRouter.use('/ping', pong) apiRouter.use('/*', badRequest) diff --git a/server/controllers/api/jobs.ts b/server/controllers/api/jobs.ts index 6a53e3083..b63e2f962 100644 --- a/server/controllers/api/jobs.ts +++ b/server/controllers/api/jobs.ts @@ -93,6 +93,9 @@ async function formatJob (job: BullJob, state?: JobState): Promise { state: state || await job.getState(), type: job.queueName as JobType, data: job.data, + parent: job.parent + ? { id: job.parent.id } + : undefined, progress: job.progress as number, priority: job.opts.priority, error, diff --git a/server/controllers/api/runners/index.ts b/server/controllers/api/runners/index.ts new file mode 100644 index 000000000..c98ded354 --- /dev/null +++ b/server/controllers/api/runners/index.ts @@ -0,0 +1,18 @@ +import express from 'express' +import { runnerJobsRouter } from './jobs' +import { runnerJobFilesRouter } from './jobs-files' +import { manageRunnersRouter } from './manage-runners' +import { runnerRegistrationTokensRouter } from './registration-tokens' + +const runnersRouter = express.Router() + +runnersRouter.use('/', manageRunnersRouter) +runnersRouter.use('/', runnerJobsRouter) +runnersRouter.use('/', runnerJobFilesRouter) +runnersRouter.use('/', runnerRegistrationTokensRouter) + +// --------------------------------------------------------------------------- + +export { + runnersRouter +} diff --git a/server/controllers/api/runners/jobs-files.ts b/server/controllers/api/runners/jobs-files.ts new file mode 100644 index 000000000..e43ce35f5 --- /dev/null +++ b/server/controllers/api/runners/jobs-files.ts @@ -0,0 +1,84 @@ +import express from 'express' +import { logger, loggerTagsFactory } from '@server/helpers/logger' +import { proxifyHLS, proxifyWebTorrentFile } from '@server/lib/object-storage' +import { VideoPathManager } from '@server/lib/video-path-manager' +import { asyncMiddleware } from '@server/middlewares' +import { jobOfRunnerGetValidator } from '@server/middlewares/validators/runners' +import { runnerJobGetVideoTranscodingFileValidator } from '@server/middlewares/validators/runners/job-files' +import { VideoStorage } from '@shared/models' + +const lTags = loggerTagsFactory('api', 'runner') + +const runnerJobFilesRouter = express.Router() + +runnerJobFilesRouter.post('/jobs/:jobUUID/files/videos/:videoId/max-quality', + asyncMiddleware(jobOfRunnerGetValidator), + asyncMiddleware(runnerJobGetVideoTranscodingFileValidator), + asyncMiddleware(getMaxQualityVideoFile) +) + +runnerJobFilesRouter.post('/jobs/:jobUUID/files/videos/:videoId/previews/max-quality', + asyncMiddleware(jobOfRunnerGetValidator), + asyncMiddleware(runnerJobGetVideoTranscodingFileValidator), + getMaxQualityVideoPreview +) + +// --------------------------------------------------------------------------- + +export { + runnerJobFilesRouter +} + +// --------------------------------------------------------------------------- + +async function getMaxQualityVideoFile (req: express.Request, res: express.Response) { + const runnerJob = res.locals.runnerJob + const runner = runnerJob.Runner + const video = res.locals.videoAll + + logger.info( + 'Get max quality file of video %s of job %s for runner %s', video.uuid, runnerJob.uuid, runner.name, + lTags(runner.name, runnerJob.id, runnerJob.type) + ) + + const file = video.getMaxQualityFile() + + if (file.storage === VideoStorage.OBJECT_STORAGE) { + if (file.isHLS()) { + return proxifyHLS({ + req, + res, + filename: file.filename, + playlist: video.getHLSPlaylist(), + reinjectVideoFileToken: false, + video + }) + } + + // Web video + return proxifyWebTorrentFile({ + req, + res, + filename: file.filename + }) + } + + return VideoPathManager.Instance.makeAvailableVideoFile(file, videoPath => { + return res.sendFile(videoPath) + }) +} + +function getMaxQualityVideoPreview (req: express.Request, res: express.Response) { + const runnerJob = res.locals.runnerJob + const runner = runnerJob.Runner + const video = res.locals.videoAll + + logger.info( + 'Get max quality preview file of video %s of job %s for runner %s', video.uuid, runnerJob.uuid, runner.name, + lTags(runner.name, runnerJob.id, runnerJob.type) + ) + + const file = video.getPreview() + + return res.sendFile(file.getPath()) +} diff --git a/server/controllers/api/runners/jobs.ts b/server/controllers/api/runners/jobs.ts new file mode 100644 index 000000000..7d488ec11 --- /dev/null +++ b/server/controllers/api/runners/jobs.ts @@ -0,0 +1,352 @@ +import express, { UploadFiles } from 'express' +import { createReqFiles } from '@server/helpers/express-utils' +import { logger, loggerTagsFactory } from '@server/helpers/logger' +import { generateRunnerJobToken } from '@server/helpers/token-generator' +import { MIMETYPES } from '@server/initializers/constants' +import { sequelizeTypescript } from '@server/initializers/database' +import { getRunnerJobHandlerClass, updateLastRunnerContact } from '@server/lib/runners' +import { + asyncMiddleware, + authenticate, + ensureUserHasRight, + paginationValidator, + runnerJobsSortValidator, + setDefaultPagination, + setDefaultSort +} from '@server/middlewares' +import { + abortRunnerJobValidator, + acceptRunnerJobValidator, + errorRunnerJobValidator, + getRunnerFromTokenValidator, + jobOfRunnerGetValidator, + runnerJobGetValidator, + successRunnerJobValidator, + updateRunnerJobValidator +} from '@server/middlewares/validators/runners' +import { RunnerModel } from '@server/models/runner/runner' +import { RunnerJobModel } from '@server/models/runner/runner-job' +import { + AbortRunnerJobBody, + AcceptRunnerJobResult, + ErrorRunnerJobBody, + HttpStatusCode, + ListRunnerJobsQuery, + LiveRTMPHLSTranscodingUpdatePayload, + RequestRunnerJobResult, + RunnerJobState, + RunnerJobSuccessBody, + RunnerJobSuccessPayload, + RunnerJobType, + RunnerJobUpdateBody, + RunnerJobUpdatePayload, + UserRight, + VODAudioMergeTranscodingSuccess, + VODHLSTranscodingSuccess, + VODWebVideoTranscodingSuccess +} from '@shared/models' + +const postRunnerJobSuccessVideoFiles = createReqFiles( + [ 'payload[videoFile]', 'payload[resolutionPlaylistFile]' ], + { ...MIMETYPES.VIDEO.MIMETYPE_EXT, ...MIMETYPES.M3U8.MIMETYPE_EXT } +) + +const runnerJobUpdateVideoFiles = createReqFiles( + [ 'payload[videoChunkFile]', 'payload[resolutionPlaylistFile]', 'payload[masterPlaylistFile]' ], + { ...MIMETYPES.VIDEO.MIMETYPE_EXT, ...MIMETYPES.M3U8.MIMETYPE_EXT } +) + +const lTags = loggerTagsFactory('api', 'runner') + +const runnerJobsRouter = express.Router() + +// --------------------------------------------------------------------------- +// Controllers for runners +// --------------------------------------------------------------------------- + +runnerJobsRouter.post('/jobs/request', + asyncMiddleware(getRunnerFromTokenValidator), + asyncMiddleware(requestRunnerJob) +) + +runnerJobsRouter.post('/jobs/:jobUUID/accept', + asyncMiddleware(runnerJobGetValidator), + acceptRunnerJobValidator, + asyncMiddleware(getRunnerFromTokenValidator), + asyncMiddleware(acceptRunnerJob) +) + +runnerJobsRouter.post('/jobs/:jobUUID/abort', + asyncMiddleware(jobOfRunnerGetValidator), + abortRunnerJobValidator, + asyncMiddleware(abortRunnerJob) +) + +runnerJobsRouter.post('/jobs/:jobUUID/update', + runnerJobUpdateVideoFiles, + asyncMiddleware(jobOfRunnerGetValidator), + updateRunnerJobValidator, + asyncMiddleware(updateRunnerJobController) +) + +runnerJobsRouter.post('/jobs/:jobUUID/error', + asyncMiddleware(jobOfRunnerGetValidator), + errorRunnerJobValidator, + asyncMiddleware(errorRunnerJob) +) + +runnerJobsRouter.post('/jobs/:jobUUID/success', + postRunnerJobSuccessVideoFiles, + asyncMiddleware(jobOfRunnerGetValidator), + successRunnerJobValidator, + asyncMiddleware(postRunnerJobSuccess) +) + +// --------------------------------------------------------------------------- +// Controllers for admins +// --------------------------------------------------------------------------- + +runnerJobsRouter.post('/jobs/:jobUUID/cancel', + authenticate, + ensureUserHasRight(UserRight.MANAGE_RUNNERS), + asyncMiddleware(runnerJobGetValidator), + asyncMiddleware(cancelRunnerJob) +) + +runnerJobsRouter.get('/jobs', + authenticate, + ensureUserHasRight(UserRight.MANAGE_RUNNERS), + paginationValidator, + runnerJobsSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listRunnerJobs) +) + +// --------------------------------------------------------------------------- + +export { + runnerJobsRouter +} + +// --------------------------------------------------------------------------- + +// --------------------------------------------------------------------------- +// Controllers for runners +// --------------------------------------------------------------------------- + +async function requestRunnerJob (req: express.Request, res: express.Response) { + const runner = res.locals.runner + const availableJobs = await RunnerJobModel.listAvailableJobs() + + logger.debug('Runner %s requests for a job.', runner.name, { availableJobs, ...lTags(runner.name) }) + + const result: RequestRunnerJobResult = { + availableJobs: availableJobs.map(j => ({ + uuid: j.uuid, + type: j.type, + payload: j.payload + })) + } + + updateLastRunnerContact(req, runner) + + return res.json(result) +} + +async function acceptRunnerJob (req: express.Request, res: express.Response) { + const runner = res.locals.runner + const runnerJob = res.locals.runnerJob + + runnerJob.state = RunnerJobState.PROCESSING + runnerJob.processingJobToken = generateRunnerJobToken() + runnerJob.startedAt = new Date() + runnerJob.runnerId = runner.id + + const newRunnerJob = await sequelizeTypescript.transaction(transaction => { + return runnerJob.save({ transaction }) + }) + newRunnerJob.Runner = runner as RunnerModel + + const result: AcceptRunnerJobResult = { + job: { + ...newRunnerJob.toFormattedJSON(), + + jobToken: newRunnerJob.processingJobToken + } + } + + updateLastRunnerContact(req, runner) + + logger.info( + 'Remote runner %s has accepted job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type, + lTags(runner.name, runnerJob.uuid, runnerJob.type) + ) + + return res.json(result) +} + +async function abortRunnerJob (req: express.Request, res: express.Response) { + const runnerJob = res.locals.runnerJob + const runner = runnerJob.Runner + const body: AbortRunnerJobBody = req.body + + logger.info( + 'Remote runner %s is aborting job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type, + { reason: body.reason, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) } + ) + + const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) + await new RunnerJobHandler().abort({ runnerJob }) + + updateLastRunnerContact(req, runnerJob.Runner) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function errorRunnerJob (req: express.Request, res: express.Response) { + const runnerJob = res.locals.runnerJob + const runner = runnerJob.Runner + const body: ErrorRunnerJobBody = req.body + + runnerJob.failures += 1 + + logger.error( + 'Remote runner %s had an error with job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type, + { errorMessage: body.message, totalFailures: runnerJob.failures, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) } + ) + + const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) + await new RunnerJobHandler().error({ runnerJob, message: body.message }) + + updateLastRunnerContact(req, runnerJob.Runner) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +// --------------------------------------------------------------------------- + +const jobUpdateBuilders: { + [id in RunnerJobType]?: (payload: RunnerJobUpdatePayload, files?: UploadFiles) => RunnerJobUpdatePayload +} = { + 'live-rtmp-hls-transcoding': (payload: LiveRTMPHLSTranscodingUpdatePayload, files) => { + return { + ...payload, + + masterPlaylistFile: files['payload[masterPlaylistFile]']?.[0].path, + resolutionPlaylistFile: files['payload[resolutionPlaylistFile]']?.[0].path, + videoChunkFile: files['payload[videoChunkFile]']?.[0].path + } + } +} + +async function updateRunnerJobController (req: express.Request, res: express.Response) { + const runnerJob = res.locals.runnerJob + const runner = runnerJob.Runner + const body: RunnerJobUpdateBody = req.body + + const payloadBuilder = jobUpdateBuilders[runnerJob.type] + const updatePayload = payloadBuilder + ? payloadBuilder(body.payload, req.files as UploadFiles) + : undefined + + logger.debug( + 'Remote runner %s is updating job %s (%s)', runnerJob.Runner.name, runnerJob.uuid, runnerJob.type, + { body, updatePayload, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) } + ) + + const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) + await new RunnerJobHandler().update({ + runnerJob, + progress: req.body.progress, + updatePayload + }) + + updateLastRunnerContact(req, runnerJob.Runner) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +// --------------------------------------------------------------------------- + +const jobSuccessPayloadBuilders: { + [id in RunnerJobType]: (payload: RunnerJobSuccessPayload, files?: UploadFiles) => RunnerJobSuccessPayload +} = { + 'vod-web-video-transcoding': (payload: VODWebVideoTranscodingSuccess, files) => { + return { + ...payload, + + videoFile: files['payload[videoFile]'][0].path + } + }, + + 'vod-hls-transcoding': (payload: VODHLSTranscodingSuccess, files) => { + return { + ...payload, + + videoFile: files['payload[videoFile]'][0].path, + resolutionPlaylistFile: files['payload[resolutionPlaylistFile]'][0].path + } + }, + + 'vod-audio-merge-transcoding': (payload: VODAudioMergeTranscodingSuccess, files) => { + return { + ...payload, + + videoFile: files['payload[videoFile]'][0].path + } + }, + + 'live-rtmp-hls-transcoding': () => ({}) +} + +async function postRunnerJobSuccess (req: express.Request, res: express.Response) { + const runnerJob = res.locals.runnerJob + const runner = runnerJob.Runner + const body: RunnerJobSuccessBody = req.body + + const resultPayload = jobSuccessPayloadBuilders[runnerJob.type](body.payload, req.files as UploadFiles) + + logger.info( + 'Remote runner %s is sending success result for job %s (%s)', runnerJob.Runner.name, runnerJob.uuid, runnerJob.type, + { resultPayload, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) } + ) + + const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) + await new RunnerJobHandler().complete({ runnerJob, resultPayload }) + + updateLastRunnerContact(req, runnerJob.Runner) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +// --------------------------------------------------------------------------- +// Controllers for admins +// --------------------------------------------------------------------------- + +async function cancelRunnerJob (req: express.Request, res: express.Response) { + const runnerJob = res.locals.runnerJob + + logger.info('Cancelling job %s (%s)', runnerJob.type, lTags(runnerJob.uuid, runnerJob.type)) + + const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) + await new RunnerJobHandler().cancel({ runnerJob }) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function listRunnerJobs (req: express.Request, res: express.Response) { + const query: ListRunnerJobsQuery = req.query + + const resultList = await RunnerJobModel.listForApi({ + start: query.start, + count: query.count, + sort: query.sort, + search: query.search + }) + + return res.json({ + total: resultList.total, + data: resultList.data.map(d => d.toFormattedAdminJSON()) + }) +} diff --git a/server/controllers/api/runners/manage-runners.ts b/server/controllers/api/runners/manage-runners.ts new file mode 100644 index 000000000..eb08c4b1d --- /dev/null +++ b/server/controllers/api/runners/manage-runners.ts @@ -0,0 +1,107 @@ +import express from 'express' +import { logger, loggerTagsFactory } from '@server/helpers/logger' +import { generateRunnerToken } from '@server/helpers/token-generator' +import { + asyncMiddleware, + authenticate, + ensureUserHasRight, + paginationValidator, + runnersSortValidator, + setDefaultPagination, + setDefaultSort +} from '@server/middlewares' +import { deleteRunnerValidator, getRunnerFromTokenValidator, registerRunnerValidator } from '@server/middlewares/validators/runners' +import { RunnerModel } from '@server/models/runner/runner' +import { HttpStatusCode, ListRunnersQuery, RegisterRunnerBody, UserRight } from '@shared/models' + +const lTags = loggerTagsFactory('api', 'runner') + +const manageRunnersRouter = express.Router() + +manageRunnersRouter.post('/register', + asyncMiddleware(registerRunnerValidator), + asyncMiddleware(registerRunner) +) +manageRunnersRouter.post('/unregister', + asyncMiddleware(getRunnerFromTokenValidator), + asyncMiddleware(unregisterRunner) +) + +manageRunnersRouter.delete('/:runnerId', + authenticate, + ensureUserHasRight(UserRight.MANAGE_RUNNERS), + asyncMiddleware(deleteRunnerValidator), + asyncMiddleware(deleteRunner) +) + +manageRunnersRouter.get('/', + authenticate, + ensureUserHasRight(UserRight.MANAGE_RUNNERS), + paginationValidator, + runnersSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listRunners) +) + +// --------------------------------------------------------------------------- + +export { + manageRunnersRouter +} + +// --------------------------------------------------------------------------- + +async function registerRunner (req: express.Request, res: express.Response) { + const body: RegisterRunnerBody = req.body + + const runnerToken = generateRunnerToken() + + const runner = new RunnerModel({ + runnerToken, + name: body.name, + description: body.description, + lastContact: new Date(), + ip: req.ip, + runnerRegistrationTokenId: res.locals.runnerRegistrationToken.id + }) + + await runner.save() + + logger.info('Registered new runner %s', runner.name, { ...lTags(runner.name) }) + + return res.json({ id: runner.id, runnerToken }) +} +async function unregisterRunner (req: express.Request, res: express.Response) { + const runner = res.locals.runner + await runner.destroy() + + logger.info('Unregistered runner %s', runner.name, { ...lTags(runner.name) }) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function deleteRunner (req: express.Request, res: express.Response) { + const runner = res.locals.runner + + await runner.destroy() + + logger.info('Deleted runner %s', runner.name, { ...lTags(runner.name) }) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function listRunners (req: express.Request, res: express.Response) { + const query: ListRunnersQuery = req.query + + const resultList = await RunnerModel.listForApi({ + start: query.start, + count: query.count, + sort: query.sort + }) + + return res.json({ + total: resultList.total, + data: resultList.data.map(d => d.toFormattedJSON()) + }) +} diff --git a/server/controllers/api/runners/registration-tokens.ts b/server/controllers/api/runners/registration-tokens.ts new file mode 100644 index 000000000..5ac3773fe --- /dev/null +++ b/server/controllers/api/runners/registration-tokens.ts @@ -0,0 +1,87 @@ +import express from 'express' +import { generateRunnerRegistrationToken } from '@server/helpers/token-generator' +import { + asyncMiddleware, + authenticate, + ensureUserHasRight, + paginationValidator, + runnerRegistrationTokensSortValidator, + setDefaultPagination, + setDefaultSort +} from '@server/middlewares' +import { deleteRegistrationTokenValidator } from '@server/middlewares/validators/runners' +import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token' +import { HttpStatusCode, ListRunnerRegistrationTokensQuery, UserRight } from '@shared/models' +import { logger, loggerTagsFactory } from '@server/helpers/logger' + +const lTags = loggerTagsFactory('api', 'runner') + +const runnerRegistrationTokensRouter = express.Router() + +runnerRegistrationTokensRouter.post('/registration-tokens/generate', + authenticate, + ensureUserHasRight(UserRight.MANAGE_RUNNERS), + asyncMiddleware(generateRegistrationToken) +) + +runnerRegistrationTokensRouter.delete('/registration-tokens/:id', + authenticate, + ensureUserHasRight(UserRight.MANAGE_RUNNERS), + asyncMiddleware(deleteRegistrationTokenValidator), + asyncMiddleware(deleteRegistrationToken) +) + +runnerRegistrationTokensRouter.get('/registration-tokens', + authenticate, + ensureUserHasRight(UserRight.MANAGE_RUNNERS), + paginationValidator, + runnerRegistrationTokensSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listRegistrationTokens) +) + +// --------------------------------------------------------------------------- + +export { + runnerRegistrationTokensRouter +} + +// --------------------------------------------------------------------------- + +async function generateRegistrationToken (req: express.Request, res: express.Response) { + logger.info('Generating new runner registration token.', lTags()) + + const registrationToken = new RunnerRegistrationTokenModel({ + registrationToken: generateRunnerRegistrationToken() + }) + + await registrationToken.save() + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function deleteRegistrationToken (req: express.Request, res: express.Response) { + logger.info('Removing runner registration token.', lTags()) + + const runnerRegistrationToken = res.locals.runnerRegistrationToken + + await runnerRegistrationToken.destroy() + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function listRegistrationTokens (req: express.Request, res: express.Response) { + const query: ListRunnerRegistrationTokensQuery = req.query + + const resultList = await RunnerRegistrationTokenModel.listForApi({ + start: query.start, + count: query.count, + sort: query.sort + }) + + return res.json({ + total: resultList.total, + data: resultList.data.map(d => d.toFormattedJSON()) + }) +} diff --git a/server/controllers/api/videos/transcoding.ts b/server/controllers/api/videos/transcoding.ts index 8c9a5322b..54f484b2b 100644 --- a/server/controllers/api/videos/transcoding.ts +++ b/server/controllers/api/videos/transcoding.ts @@ -1,10 +1,8 @@ -import Bluebird from 'bluebird' import express from 'express' -import { computeResolutionsToTranscode } from '@server/helpers/ffmpeg' import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { JobQueue } from '@server/lib/job-queue' import { Hooks } from '@server/lib/plugins/hooks' -import { buildTranscodingJob } from '@server/lib/video' +import { createTranscodingJobs } from '@server/lib/transcoding/create-transcoding-job' +import { computeResolutionsToTranscode } from '@server/lib/transcoding/transcoding-resolutions' import { HttpStatusCode, UserRight, VideoState, VideoTranscodingCreate } from '@shared/models' import { asyncMiddleware, authenticate, createTranscodingValidator, ensureUserHasRight } from '../../../middlewares' @@ -47,82 +45,13 @@ async function createTranscoding (req: express.Request, res: express.Response) { video.state = VideoState.TO_TRANSCODE await video.save() - const childrenResolutions = resolutions.filter(r => r !== maxResolution) - - logger.info('Manually creating transcoding jobs for %s.', body.transcodingType, { childrenResolutions, maxResolution }) - - const children = await Bluebird.mapSeries(childrenResolutions, resolution => { - if (body.transcodingType === 'hls') { - return buildHLSJobOption({ - videoUUID: video.uuid, - hasAudio, - resolution, - isMaxQuality: false - }) - } - - if (body.transcodingType === 'webtorrent') { - return buildWebTorrentJobOption({ - videoUUID: video.uuid, - hasAudio, - resolution - }) - } + await createTranscodingJobs({ + video, + resolutions, + transcodingType: body.transcodingType, + isNewVideo: false, + user: null // Don't specify priority since these transcoding jobs are fired by the admin }) - const parent = body.transcodingType === 'hls' - ? await buildHLSJobOption({ - videoUUID: video.uuid, - hasAudio, - resolution: maxResolution, - isMaxQuality: false - }) - : await buildWebTorrentJobOption({ - videoUUID: video.uuid, - hasAudio, - resolution: maxResolution - }) - - // Porcess the last resolution after the other ones to prevent concurrency issue - // Because low resolutions use the biggest one as ffmpeg input - await JobQueue.Instance.createJobWithChildren(parent, children) - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) } - -function buildHLSJobOption (options: { - videoUUID: string - hasAudio: boolean - resolution: number - isMaxQuality: boolean -}) { - const { videoUUID, hasAudio, resolution, isMaxQuality } = options - - return buildTranscodingJob({ - type: 'new-resolution-to-hls', - videoUUID, - resolution, - hasAudio, - copyCodecs: false, - isNewVideo: false, - autoDeleteWebTorrentIfNeeded: false, - isMaxQuality - }) -} - -function buildWebTorrentJobOption (options: { - videoUUID: string - hasAudio: boolean - resolution: number -}) { - const { videoUUID, hasAudio, resolution } = options - - return buildTranscodingJob({ - type: 'new-resolution-to-webtorrent', - videoUUID, - isNewVideo: false, - resolution, - hasAudio, - createHLSIfNeeded: false - }) -} diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts index 43313a143..885ac8b81 100644 --- a/server/controllers/api/videos/upload.ts +++ b/server/controllers/api/videos/upload.ts @@ -3,28 +3,20 @@ import { move } from 'fs-extra' import { basename } from 'path' import { getResumableUploadPath } from '@server/helpers/upload' import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' -import { JobQueue } from '@server/lib/job-queue' -import { generateWebTorrentVideoFilename } from '@server/lib/paths' +import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue' import { Redis } from '@server/lib/redis' import { uploadx } from '@server/lib/uploadx' -import { - buildLocalVideoFromReq, - buildMoveToObjectStorageJob, - buildOptimizeOrMergeAudioJob, - buildVideoThumbnailsFromReq, - setVideoTags -} from '@server/lib/video' +import { buildLocalVideoFromReq, buildMoveToObjectStorageJob, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video' +import { buildNewFile } from '@server/lib/video-file' import { VideoPathManager } from '@server/lib/video-path-manager' import { buildNextVideoState } from '@server/lib/video-state' import { openapiOperationDoc } from '@server/middlewares/doc' import { VideoSourceModel } from '@server/models/video/video-source' import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' -import { getLowercaseExtension } from '@shared/core-utils' -import { isAudioFile, uuidToShort } from '@shared/extra-utils' -import { HttpStatusCode, VideoCreate, VideoResolution, VideoState } from '@shared/models' +import { uuidToShort } from '@shared/extra-utils' +import { HttpStatusCode, VideoCreate, VideoState } from '@shared/models' import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' import { createReqFiles } from '../../../helpers/express-utils' -import { buildFileMetadata, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '../../../helpers/ffmpeg' import { logger, loggerTagsFactory } from '../../../helpers/logger' import { MIMETYPES } from '../../../initializers/constants' import { sequelizeTypescript } from '../../../initializers/database' @@ -41,7 +33,6 @@ import { } from '../../../middlewares' import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' import { VideoModel } from '../../../models/video/video' -import { VideoFileModel } from '../../../models/video/video-file' const lTags = loggerTagsFactory('api', 'video') const auditLogger = auditLoggerFactory('videos') @@ -148,7 +139,7 @@ async function addVideo (options: { video.VideoChannel = videoChannel video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object - const videoFile = await buildNewFile(videoPhysicalFile) + const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video' }) const originalFilename = videoPhysicalFile.originalname // Move physical file @@ -227,30 +218,8 @@ async function addVideo (options: { } } -async function buildNewFile (videoPhysicalFile: express.VideoUploadFile) { - const videoFile = new VideoFileModel({ - extname: getLowercaseExtension(videoPhysicalFile.filename), - size: videoPhysicalFile.size, - videoStreamingPlaylistId: null, - metadata: await buildFileMetadata(videoPhysicalFile.path) - }) - - const probe = await ffprobePromise(videoPhysicalFile.path) - - if (await isAudioFile(videoPhysicalFile.path, probe)) { - videoFile.resolution = VideoResolution.H_NOVIDEO - } else { - videoFile.fps = await getVideoStreamFPS(videoPhysicalFile.path, probe) - videoFile.resolution = (await getVideoStreamDimensionsInfo(videoPhysicalFile.path, probe)).resolution - } - - videoFile.filename = generateWebTorrentVideoFilename(videoFile.resolution, videoFile.extname) - - return videoFile -} - async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile, user: MUserId) { - return JobQueue.Instance.createSequentialJobFlow( + const jobs: (CreateJobArgument & CreateJobOptions)[] = [ { type: 'manage-video-torrent' as 'manage-video-torrent', payload: { @@ -274,16 +243,26 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide videoUUID: video.uuid, isNewVideo: true } - }, + } + ] - video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE - ? await buildMoveToObjectStorageJob({ video, previousVideoState: undefined }) - : undefined, + if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { + jobs.push(await buildMoveToObjectStorageJob({ video, previousVideoState: undefined })) + } - video.state === VideoState.TO_TRANSCODE - ? await buildOptimizeOrMergeAudioJob({ video, videoFile, user }) - : undefined - ) + if (video.state === VideoState.TO_TRANSCODE) { + jobs.push({ + type: 'transcoding-job-builder' as 'transcoding-job-builder', + payload: { + videoUUID: video.uuid, + optimizeJob: { + isNewVideo: true + } + } + }) + } + + return JobQueue.Instance.createSequentialJobFlow(...jobs) } async function deleteUploadResumableCache (req: express.Request, res: express.Response, next: express.NextFunction) { diff --git a/server/controllers/bots.ts b/server/controllers/bots.ts index a5ce1d79f..2b825a730 100644 --- a/server/controllers/bots.ts +++ b/server/controllers/bots.ts @@ -1,8 +1,8 @@ -import { getServerActor } from '@server/models/application/application' -import { logger } from '@uploadx/core' import express from 'express' import { truncate } from 'lodash' -import { SitemapStream, streamToPromise, ErrorLevel } from 'sitemap' +import { ErrorLevel, SitemapStream, streamToPromise } from 'sitemap' +import { logger } from '@server/helpers/logger' +import { getServerActor } from '@server/models/application/application' import { buildNSFWFilter } from '../helpers/express-utils' import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants' import { asyncMiddleware } from '../middlewares' diff --git a/server/controllers/object-storage-proxy.ts b/server/controllers/object-storage-proxy.ts index c530b57f8..8e2cc4af9 100644 --- a/server/controllers/object-storage-proxy.ts +++ b/server/controllers/object-storage-proxy.ts @@ -1,11 +1,7 @@ import cors from 'cors' import express from 'express' -import { PassThrough, pipeline } from 'stream' -import { logger } from '@server/helpers/logger' -import { StreamReplacer } from '@server/helpers/stream-replacer' import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants' -import { injectQueryToPlaylistUrls } from '@server/lib/hls' -import { getHLSFileReadStream, getWebTorrentFileReadStream } from '@server/lib/object-storage' +import { proxifyHLS, proxifyWebTorrentFile } from '@server/lib/object-storage' import { asyncMiddleware, ensureCanAccessPrivateVideoHLSFiles, @@ -13,9 +9,7 @@ import { ensurePrivateObjectStorageProxyIsEnabled, optionalAuthenticate } from '@server/middlewares' -import { HttpStatusCode } from '@shared/models' -import { buildReinjectVideoFileTokenQuery, doReinjectVideoFileToken } from './shared/m3u8-playlist' -import { GetObjectCommandOutput } from '@aws-sdk/client-s3' +import { doReinjectVideoFileToken } from './shared/m3u8-playlist' const objectStorageProxyRouter = express.Router() @@ -25,14 +19,14 @@ objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + ':file ensurePrivateObjectStorageProxyIsEnabled, optionalAuthenticate, asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles), - asyncMiddleware(proxifyWebTorrent) + asyncMiddleware(proxifyWebTorrentController) ) objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:filename', ensurePrivateObjectStorageProxyIsEnabled, optionalAuthenticate, asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles), - asyncMiddleware(proxifyHLS) + asyncMiddleware(proxifyHLSController) ) // --------------------------------------------------------------------------- @@ -41,76 +35,25 @@ export { objectStorageProxyRouter } -async function proxifyWebTorrent (req: express.Request, res: express.Response) { +function proxifyWebTorrentController (req: express.Request, res: express.Response) { const filename = req.params.filename - logger.debug('Proxifying WebTorrent file %s from object storage.', filename) - - try { - const { response: s3Response, stream } = await getWebTorrentFileReadStream({ - filename, - rangeHeader: req.header('range') - }) - - setS3Headers(res, s3Response) - - return stream.pipe(res) - } catch (err) { - return handleObjectStorageFailure(res, err) - } + return proxifyWebTorrentFile({ req, res, filename }) } -async function proxifyHLS (req: express.Request, res: express.Response) { +function proxifyHLSController (req: express.Request, res: express.Response) { const playlist = res.locals.videoStreamingPlaylist const video = res.locals.onlyVideo const filename = req.params.filename - logger.debug('Proxifying HLS file %s from object storage.', filename) + const reinjectVideoFileToken = filename.endsWith('.m3u8') && doReinjectVideoFileToken(req) - try { - const { response: s3Response, stream } = await getHLSFileReadStream({ - playlist: playlist.withVideo(video), - filename, - rangeHeader: req.header('range') - }) - - setS3Headers(res, s3Response) - - const streamReplacer = filename.endsWith('.m3u8') && doReinjectVideoFileToken(req) - ? new StreamReplacer(line => injectQueryToPlaylistUrls(line, buildReinjectVideoFileTokenQuery(req, filename.endsWith('master.m3u8')))) - : new PassThrough() - - return pipeline( - stream, - streamReplacer, - res, - err => { - if (!err) return - - handleObjectStorageFailure(res, err) - } - ) - } catch (err) { - return handleObjectStorageFailure(res, err) - } -} - -function handleObjectStorageFailure (res: express.Response, err: Error) { - if (err.name === 'NoSuchKey') { - logger.debug('Could not find key in object storage to proxify private HLS video file.', { err }) - return res.sendStatus(HttpStatusCode.NOT_FOUND_404) - } - - return res.fail({ - status: HttpStatusCode.INTERNAL_SERVER_ERROR_500, - message: err.message, - type: err.name + return proxifyHLS({ + req, + res, + playlist, + video, + filename, + reinjectVideoFileToken }) } - -function setS3Headers (res: express.Response, s3Response: GetObjectCommandOutput) { - if (s3Response.$metadata.httpStatusCode === HttpStatusCode.PARTIAL_CONTENT_206) { - res.setHeader('Content-Range', s3Response.ContentRange) - res.status(HttpStatusCode.PARTIAL_CONTENT_206) - } -} diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 73bd994c1..242c49e89 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts @@ -11,6 +11,7 @@ import { truncate } from 'lodash' import { pipeline } from 'stream' import { URL } from 'url' import { promisify } from 'util' +import { promisify1, promisify2, promisify3 } from '@shared/core-utils' const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => { if (!oldObject || typeof oldObject !== 'object') { @@ -229,18 +230,6 @@ function execShell (command: string, options?: ExecOptions) { // --------------------------------------------------------------------------- -function isOdd (num: number) { - return (num % 2) !== 0 -} - -function toEven (num: number) { - if (isOdd(num)) return num + 1 - - return num -} - -// --------------------------------------------------------------------------- - function generateRSAKeyPairPromise (size: number) { return new Promise<{ publicKey: string, privateKey: string }>((res, rej) => { const options: RSAKeyPairOptions<'pem', 'pem'> = { @@ -286,40 +275,6 @@ function generateED25519KeyPairPromise () { // --------------------------------------------------------------------------- -function promisify0 (func: (cb: (err: any, result: A) => void) => void): () => Promise { - return function promisified (): Promise { - return new Promise((resolve: (arg: A) => void, reject: (err: any) => void) => { - func.apply(null, [ (err: any, res: A) => err ? reject(err) : resolve(res) ]) - }) - } -} - -// Thanks to https://gist.github.com/kumasento/617daa7e46f13ecdd9b2 -function promisify1 (func: (arg: T, cb: (err: any, result: A) => void) => void): (arg: T) => Promise { - return function promisified (arg: T): Promise { - return new Promise((resolve: (arg: A) => void, reject: (err: any) => void) => { - func.apply(null, [ arg, (err: any, res: A) => err ? reject(err) : resolve(res) ]) - }) - } -} - -function promisify2 (func: (arg1: T, arg2: U, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U) => Promise { - return function promisified (arg1: T, arg2: U): Promise { - return new Promise((resolve: (arg: A) => void, reject: (err: any) => void) => { - func.apply(null, [ arg1, arg2, (err: any, res: A) => err ? reject(err) : resolve(res) ]) - }) - } -} - -// eslint-disable-next-line max-len -function promisify3 (func: (arg1: T, arg2: U, arg3: V, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U, arg3: V) => Promise { - return function promisified (arg1: T, arg2: U, arg3: V): Promise { - return new Promise((resolve: (arg: A) => void, reject: (err: any) => void) => { - func.apply(null, [ arg1, arg2, arg3, (err: any, res: A) => err ? reject(err) : resolve(res) ]) - }) - } -} - const randomBytesPromise = promisify1(randomBytes) const scryptPromise = promisify3(scrypt) const execPromise2 = promisify2(exec) @@ -345,10 +300,6 @@ export { pageToStartAndCount, peertubeTruncate, - promisify0, - promisify1, - promisify2, - scryptPromise, randomBytesPromise, @@ -360,8 +311,5 @@ export { execPromise, pipelinePromise, - parseSemVersion, - - isOdd, - toEven + parseSemVersion } diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts index ebab4c6b2..fa0f469f6 100644 --- a/server/helpers/custom-validators/misc.ts +++ b/server/helpers/custom-validators/misc.ts @@ -15,6 +15,10 @@ function isSafePath (p: string) { }) } +function isSafeFilename (filename: string, extension: string) { + return typeof filename === 'string' && !!filename.match(new RegExp(`^[a-z0-9-]+\\.${extension}$`)) +} + function isSafePeerTubeFilenameWithoutExtension (filename: string) { return filename.match(/^[a-z0-9-]+$/) } @@ -177,5 +181,6 @@ export { toIntArray, isFileValid, isSafePeerTubeFilenameWithoutExtension, + isSafeFilename, checkMimetypeRegex } diff --git a/server/helpers/custom-validators/runners/jobs.ts b/server/helpers/custom-validators/runners/jobs.ts new file mode 100644 index 000000000..5f755d5bb --- /dev/null +++ b/server/helpers/custom-validators/runners/jobs.ts @@ -0,0 +1,166 @@ +import { UploadFilesForCheck } from 'express' +import validator from 'validator' +import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' +import { + LiveRTMPHLSTranscodingSuccess, + RunnerJobSuccessPayload, + RunnerJobType, + RunnerJobUpdatePayload, + VODAudioMergeTranscodingSuccess, + VODHLSTranscodingSuccess, + VODWebVideoTranscodingSuccess +} from '@shared/models' +import { exists, isFileValid, isSafeFilename } from '../misc' + +const RUNNER_JOBS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNER_JOBS + +const runnerJobTypes = new Set([ 'vod-hls-transcoding', 'vod-web-video-transcoding', 'vod-audio-merge-transcoding' ]) +function isRunnerJobTypeValid (value: RunnerJobType) { + return runnerJobTypes.has(value) +} + +function isRunnerJobSuccessPayloadValid (value: RunnerJobSuccessPayload, type: RunnerJobType, files: UploadFilesForCheck) { + return isRunnerJobVODWebVideoResultPayloadValid(value as VODWebVideoTranscodingSuccess, type, files) || + isRunnerJobVODHLSResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) || + isRunnerJobVODAudioMergeResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) || + isRunnerJobLiveRTMPHLSResultPayloadValid(value as LiveRTMPHLSTranscodingSuccess, type) +} + +// --------------------------------------------------------------------------- + +function isRunnerJobProgressValid (value: string) { + return validator.isInt(value + '', RUNNER_JOBS_CONSTRAINTS_FIELDS.PROGRESS) +} + +function isRunnerJobUpdatePayloadValid (value: RunnerJobUpdatePayload, type: RunnerJobType, files: UploadFilesForCheck) { + return isRunnerJobVODWebVideoUpdatePayloadValid(value, type, files) || + isRunnerJobVODHLSUpdatePayloadValid(value, type, files) || + isRunnerJobVODAudioMergeUpdatePayloadValid(value, type, files) || + isRunnerJobLiveRTMPHLSUpdatePayloadValid(value, type, files) +} + +// --------------------------------------------------------------------------- + +function isRunnerJobTokenValid (value: string) { + return exists(value) && validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.TOKEN) +} + +function isRunnerJobAbortReasonValid (value: string) { + return validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.REASON) +} + +function isRunnerJobErrorMessageValid (value: string) { + return validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.ERROR_MESSAGE) +} + +// --------------------------------------------------------------------------- + +export { + isRunnerJobTypeValid, + isRunnerJobSuccessPayloadValid, + isRunnerJobUpdatePayloadValid, + isRunnerJobTokenValid, + isRunnerJobErrorMessageValid, + isRunnerJobProgressValid, + isRunnerJobAbortReasonValid +} + +// --------------------------------------------------------------------------- + +function isRunnerJobVODWebVideoResultPayloadValid ( + _value: VODWebVideoTranscodingSuccess, + type: RunnerJobType, + files: UploadFilesForCheck +) { + return type === 'vod-web-video-transcoding' && + isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) +} + +function isRunnerJobVODHLSResultPayloadValid ( + _value: VODHLSTranscodingSuccess, + type: RunnerJobType, + files: UploadFilesForCheck +) { + return type === 'vod-hls-transcoding' && + isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) && + isFileValid({ files, field: 'payload[resolutionPlaylistFile]', mimeTypeRegex: null, maxSize: null }) +} + +function isRunnerJobVODAudioMergeResultPayloadValid ( + _value: VODAudioMergeTranscodingSuccess, + type: RunnerJobType, + files: UploadFilesForCheck +) { + return type === 'vod-audio-merge-transcoding' && + isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) +} + +function isRunnerJobLiveRTMPHLSResultPayloadValid ( + value: LiveRTMPHLSTranscodingSuccess, + type: RunnerJobType +) { + return type === 'live-rtmp-hls-transcoding' && (!value || (typeof value === 'object' && Object.keys(value).length === 0)) +} + +// --------------------------------------------------------------------------- + +function isRunnerJobVODWebVideoUpdatePayloadValid ( + value: RunnerJobUpdatePayload, + type: RunnerJobType, + _files: UploadFilesForCheck +) { + return type === 'vod-web-video-transcoding' && + (!value || (typeof value === 'object' && Object.keys(value).length === 0)) +} + +function isRunnerJobVODHLSUpdatePayloadValid ( + value: RunnerJobUpdatePayload, + type: RunnerJobType, + _files: UploadFilesForCheck +) { + return type === 'vod-hls-transcoding' && + (!value || (typeof value === 'object' && Object.keys(value).length === 0)) +} + +function isRunnerJobVODAudioMergeUpdatePayloadValid ( + value: RunnerJobUpdatePayload, + type: RunnerJobType, + _files: UploadFilesForCheck +) { + return type === 'vod-audio-merge-transcoding' && + (!value || (typeof value === 'object' && Object.keys(value).length === 0)) +} + +function isRunnerJobLiveRTMPHLSUpdatePayloadValid ( + value: RunnerJobUpdatePayload, + type: RunnerJobType, + files: UploadFilesForCheck +) { + let result = type === 'live-rtmp-hls-transcoding' && !!value && !!files + + result &&= isFileValid({ files, field: 'payload[masterPlaylistFile]', mimeTypeRegex: null, maxSize: null, optional: true }) + + result &&= isFileValid({ + files, + field: 'payload[resolutionPlaylistFile]', + mimeTypeRegex: null, + maxSize: null, + optional: !value.resolutionPlaylistFilename + }) + + if (files['payload[resolutionPlaylistFile]']) { + result &&= isSafeFilename(value.resolutionPlaylistFilename, 'm3u8') + } + + return result && + isSafeFilename(value.videoChunkFilename, 'ts') && + ( + ( + value.type === 'remove-chunk' + ) || + ( + value.type === 'add-chunk' && + isFileValid({ files, field: 'payload[videoChunkFile]', mimeTypeRegex: null, maxSize: null }) + ) + ) +} diff --git a/server/helpers/custom-validators/runners/runners.ts b/server/helpers/custom-validators/runners/runners.ts new file mode 100644 index 000000000..953fac3b5 --- /dev/null +++ b/server/helpers/custom-validators/runners/runners.ts @@ -0,0 +1,30 @@ +import validator from 'validator' +import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' +import { exists } from '../misc' + +const RUNNERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNERS + +function isRunnerRegistrationTokenValid (value: string) { + return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.TOKEN) +} + +function isRunnerTokenValid (value: string) { + return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.TOKEN) +} + +function isRunnerNameValid (value: string) { + return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.NAME) +} + +function isRunnerDescriptionValid (value: string) { + return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.DESCRIPTION) +} + +// --------------------------------------------------------------------------- + +export { + isRunnerRegistrationTokenValid, + isRunnerTokenValid, + isRunnerNameValid, + isRunnerDescriptionValid +} diff --git a/server/helpers/debounce.ts b/server/helpers/debounce.ts new file mode 100644 index 000000000..77d99a894 --- /dev/null +++ b/server/helpers/debounce.ts @@ -0,0 +1,16 @@ +export function Debounce (config: { timeoutMS: number }) { + let timeoutRef: NodeJS.Timeout + + return function (_target, _key, descriptor: PropertyDescriptor) { + const original = descriptor.value + + descriptor.value = function (...args: any[]) { + clearTimeout(timeoutRef) + + timeoutRef = setTimeout(() => { + original.apply(this, args) + + }, config.timeoutMS) + } + } +} diff --git a/server/helpers/ffmpeg/codecs.ts b/server/helpers/ffmpeg/codecs.ts new file mode 100644 index 000000000..3bd7db396 --- /dev/null +++ b/server/helpers/ffmpeg/codecs.ts @@ -0,0 +1,64 @@ +import { FfprobeData } from 'fluent-ffmpeg' +import { getAudioStream, getVideoStream } from '@shared/ffmpeg' +import { logger } from '../logger' +import { forceNumber } from '@shared/core-utils' + +export async function getVideoStreamCodec (path: string) { + const videoStream = await getVideoStream(path) + if (!videoStream) return '' + + const videoCodec = videoStream.codec_tag_string + + if (videoCodec === 'vp09') return 'vp09.00.50.08' + if (videoCodec === 'hev1') return 'hev1.1.6.L93.B0' + + const baseProfileMatrix = { + avc1: { + High: '6400', + Main: '4D40', + Baseline: '42E0' + }, + av01: { + High: '1', + Main: '0', + Professional: '2' + } + } + + let baseProfile = baseProfileMatrix[videoCodec][videoStream.profile] + if (!baseProfile) { + logger.warn('Cannot get video profile codec of %s.', path, { videoStream }) + baseProfile = baseProfileMatrix[videoCodec]['High'] // Fallback + } + + if (videoCodec === 'av01') { + let level = videoStream.level.toString() + if (level.length === 1) level = `0${level}` + + // Guess the tier indicator and bit depth + return `${videoCodec}.${baseProfile}.${level}M.08` + } + + let level = forceNumber(videoStream.level).toString(16) + if (level.length === 1) level = `0${level}` + + // Default, h264 codec + return `${videoCodec}.${baseProfile}${level}` +} + +export async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) { + const { audioStream } = await getAudioStream(path, existingProbe) + + if (!audioStream) return '' + + const audioCodecName = audioStream.codec_name + + if (audioCodecName === 'opus') return 'opus' + if (audioCodecName === 'vorbis') return 'vorbis' + if (audioCodecName === 'aac') return 'mp4a.40.2' + if (audioCodecName === 'mp3') return 'mp4a.40.34' + + logger.warn('Cannot get audio codec of %s.', path, { audioStream }) + + return 'mp4a.40.2' // Fallback +} diff --git a/server/helpers/ffmpeg/ffmpeg-commons.ts b/server/helpers/ffmpeg/ffmpeg-commons.ts deleted file mode 100644 index 3906a2089..000000000 --- a/server/helpers/ffmpeg/ffmpeg-commons.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { Job } from 'bullmq' -import ffmpeg, { FfmpegCommand } from 'fluent-ffmpeg' -import { execPromise } from '@server/helpers/core-utils' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { FFMPEG_NICE } from '@server/initializers/constants' -import { EncoderOptions } from '@shared/models' - -const lTags = loggerTagsFactory('ffmpeg') - -type StreamType = 'audio' | 'video' - -function getFFmpeg (input: string, type: 'live' | 'vod') { - // We set cwd explicitly because ffmpeg appears to create temporary files when trancoding which fails in read-only file systems - const command = ffmpeg(input, { - niceness: type === 'live' ? FFMPEG_NICE.LIVE : FFMPEG_NICE.VOD, - cwd: CONFIG.STORAGE.TMP_DIR - }) - - const threads = type === 'live' - ? CONFIG.LIVE.TRANSCODING.THREADS - : CONFIG.TRANSCODING.THREADS - - if (threads > 0) { - // If we don't set any threads ffmpeg will chose automatically - command.outputOption('-threads ' + threads) - } - - return command -} - -function getFFmpegVersion () { - return new Promise((res, rej) => { - (ffmpeg() as any)._getFfmpegPath((err, ffmpegPath) => { - if (err) return rej(err) - if (!ffmpegPath) return rej(new Error('Could not find ffmpeg path')) - - return execPromise(`${ffmpegPath} -version`) - .then(stdout => { - const parsed = stdout.match(/ffmpeg version .?(\d+\.\d+(\.\d+)?)/) - if (!parsed?.[1]) return rej(new Error(`Could not find ffmpeg version in ${stdout}`)) - - // Fix ffmpeg version that does not include patch version (4.4 for example) - let version = parsed[1] - if (version.match(/^\d+\.\d+$/)) { - version += '.0' - } - - return res(version) - }) - .catch(err => rej(err)) - }) - }) -} - -async function runCommand (options: { - command: FfmpegCommand - silent?: boolean // false by default - job?: Job -}) { - const { command, silent = false, job } = options - - return new Promise((res, rej) => { - let shellCommand: string - - command.on('start', cmdline => { shellCommand = cmdline }) - - command.on('error', (err, stdout, stderr) => { - if (silent !== true) logger.error('Error in ffmpeg.', { stdout, stderr, shellCommand, ...lTags() }) - - rej(err) - }) - - command.on('end', (stdout, stderr) => { - logger.debug('FFmpeg command ended.', { stdout, stderr, shellCommand, ...lTags() }) - - res() - }) - - if (job) { - command.on('progress', progress => { - if (!progress.percent) return - - job.updateProgress(Math.round(progress.percent)) - .catch(err => logger.warn('Cannot set ffmpeg job progress.', { err, ...lTags() })) - }) - } - - command.run() - }) -} - -function buildStreamSuffix (base: string, streamNum?: number) { - if (streamNum !== undefined) { - return `${base}:${streamNum}` - } - - return base -} - -function getScaleFilter (options: EncoderOptions): string { - if (options.scaleFilter) return options.scaleFilter.name - - return 'scale' -} - -export { - getFFmpeg, - getFFmpegVersion, - runCommand, - StreamType, - buildStreamSuffix, - getScaleFilter -} diff --git a/server/helpers/ffmpeg/ffmpeg-edition.ts b/server/helpers/ffmpeg/ffmpeg-edition.ts deleted file mode 100644 index 02c5ea8de..000000000 --- a/server/helpers/ffmpeg/ffmpeg-edition.ts +++ /dev/null @@ -1,258 +0,0 @@ -import { FilterSpecification } from 'fluent-ffmpeg' -import { VIDEO_FILTERS } from '@server/initializers/constants' -import { AvailableEncoders } from '@shared/models' -import { logger, loggerTagsFactory } from '../logger' -import { getFFmpeg, runCommand } from './ffmpeg-commons' -import { presetVOD } from './ffmpeg-presets' -import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS, hasAudioStream } from './ffprobe-utils' - -const lTags = loggerTagsFactory('ffmpeg') - -async function cutVideo (options: { - inputPath: string - outputPath: string - start?: number - end?: number - - availableEncoders: AvailableEncoders - profile: string -}) { - const { inputPath, outputPath, availableEncoders, profile } = options - - logger.debug('Will cut the video.', { options, ...lTags() }) - - const mainProbe = await ffprobePromise(inputPath) - const fps = await getVideoStreamFPS(inputPath, mainProbe) - const { resolution } = await getVideoStreamDimensionsInfo(inputPath, mainProbe) - - let command = getFFmpeg(inputPath, 'vod') - .output(outputPath) - - command = await presetVOD({ - command, - input: inputPath, - availableEncoders, - profile, - resolution, - fps, - canCopyAudio: false, - canCopyVideo: false - }) - - if (options.start) { - command.outputOption('-ss ' + options.start) - } - - if (options.end) { - command.outputOption('-to ' + options.end) - } - - await runCommand({ command }) -} - -async function addWatermark (options: { - inputPath: string - watermarkPath: string - outputPath: string - - availableEncoders: AvailableEncoders - profile: string -}) { - const { watermarkPath, inputPath, outputPath, availableEncoders, profile } = options - - logger.debug('Will add watermark to the video.', { options, ...lTags() }) - - const videoProbe = await ffprobePromise(inputPath) - const fps = await getVideoStreamFPS(inputPath, videoProbe) - const { resolution } = await getVideoStreamDimensionsInfo(inputPath, videoProbe) - - let command = getFFmpeg(inputPath, 'vod') - .output(outputPath) - command.input(watermarkPath) - - command = await presetVOD({ - command, - input: inputPath, - availableEncoders, - profile, - resolution, - fps, - canCopyAudio: true, - canCopyVideo: false - }) - - const complexFilter: FilterSpecification[] = [ - // Scale watermark - { - inputs: [ '[1]', '[0]' ], - filter: 'scale2ref', - options: { - w: 'oh*mdar', - h: `ih*${VIDEO_FILTERS.WATERMARK.SIZE_RATIO}` - }, - outputs: [ '[watermark]', '[video]' ] - }, - - { - inputs: [ '[video]', '[watermark]' ], - filter: 'overlay', - options: { - x: `main_w - overlay_w - (main_h * ${VIDEO_FILTERS.WATERMARK.HORIZONTAL_MARGIN_RATIO})`, - y: `main_h * ${VIDEO_FILTERS.WATERMARK.VERTICAL_MARGIN_RATIO}` - } - } - ] - - command.complexFilter(complexFilter) - - await runCommand({ command }) -} - -async function addIntroOutro (options: { - inputPath: string - introOutroPath: string - outputPath: string - type: 'intro' | 'outro' - - availableEncoders: AvailableEncoders - profile: string -}) { - const { introOutroPath, inputPath, outputPath, availableEncoders, profile, type } = options - - logger.debug('Will add intro/outro to the video.', { options, ...lTags() }) - - const mainProbe = await ffprobePromise(inputPath) - const fps = await getVideoStreamFPS(inputPath, mainProbe) - const { resolution } = await getVideoStreamDimensionsInfo(inputPath, mainProbe) - const mainHasAudio = await hasAudioStream(inputPath, mainProbe) - - const introOutroProbe = await ffprobePromise(introOutroPath) - const introOutroHasAudio = await hasAudioStream(introOutroPath, introOutroProbe) - - let command = getFFmpeg(inputPath, 'vod') - .output(outputPath) - - command.input(introOutroPath) - - if (!introOutroHasAudio && mainHasAudio) { - const duration = await getVideoStreamDuration(introOutroPath, introOutroProbe) - - command.input('anullsrc') - command.withInputFormat('lavfi') - command.withInputOption('-t ' + duration) - } - - command = await presetVOD({ - command, - input: inputPath, - availableEncoders, - profile, - resolution, - fps, - canCopyAudio: false, - canCopyVideo: false - }) - - // Add black background to correctly scale intro/outro with padding - const complexFilter: FilterSpecification[] = [ - { - inputs: [ '1', '0' ], - filter: 'scale2ref', - options: { - w: 'iw', - h: `ih` - }, - outputs: [ 'intro-outro', 'main' ] - }, - { - inputs: [ 'intro-outro', 'main' ], - filter: 'scale2ref', - options: { - w: 'iw', - h: `ih` - }, - outputs: [ 'to-scale', 'main' ] - }, - { - inputs: 'to-scale', - filter: 'drawbox', - options: { - t: 'fill' - }, - outputs: [ 'to-scale-bg' ] - }, - { - inputs: [ '1', 'to-scale-bg' ], - filter: 'scale2ref', - options: { - w: 'iw', - h: 'ih', - force_original_aspect_ratio: 'decrease', - flags: 'spline' - }, - outputs: [ 'to-scale', 'to-scale-bg' ] - }, - { - inputs: [ 'to-scale-bg', 'to-scale' ], - filter: 'overlay', - options: { - x: '(main_w - overlay_w)/2', - y: '(main_h - overlay_h)/2' - }, - outputs: 'intro-outro-resized' - } - ] - - const concatFilter = { - inputs: [], - filter: 'concat', - options: { - n: 2, - v: 1, - unsafe: 1 - }, - outputs: [ 'v' ] - } - - const introOutroFilterInputs = [ 'intro-outro-resized' ] - const mainFilterInputs = [ 'main' ] - - if (mainHasAudio) { - mainFilterInputs.push('0:a') - - if (introOutroHasAudio) { - introOutroFilterInputs.push('1:a') - } else { - // Silent input - introOutroFilterInputs.push('2:a') - } - } - - if (type === 'intro') { - concatFilter.inputs = [ ...introOutroFilterInputs, ...mainFilterInputs ] - } else { - concatFilter.inputs = [ ...mainFilterInputs, ...introOutroFilterInputs ] - } - - if (mainHasAudio) { - concatFilter.options['a'] = 1 - concatFilter.outputs.push('a') - - command.outputOption('-map [a]') - } - - command.outputOption('-map [v]') - - complexFilter.push(concatFilter) - command.complexFilter(complexFilter) - - await runCommand({ command }) -} - -// --------------------------------------------------------------------------- - -export { - cutVideo, - addIntroOutro, - addWatermark -} diff --git a/server/helpers/ffmpeg/ffmpeg-encoders.ts b/server/helpers/ffmpeg/ffmpeg-encoders.ts deleted file mode 100644 index 5bd80ba05..000000000 --- a/server/helpers/ffmpeg/ffmpeg-encoders.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { getAvailableEncoders } from 'fluent-ffmpeg' -import { pick } from '@shared/core-utils' -import { AvailableEncoders, EncoderOptionsBuilder, EncoderOptionsBuilderParams, EncoderProfile } from '@shared/models' -import { promisify0 } from '../core-utils' -import { logger, loggerTagsFactory } from '../logger' - -const lTags = loggerTagsFactory('ffmpeg') - -// Detect supported encoders by ffmpeg -let supportedEncoders: Map -async function checkFFmpegEncoders (peertubeAvailableEncoders: AvailableEncoders): Promise> { - if (supportedEncoders !== undefined) { - return supportedEncoders - } - - const getAvailableEncodersPromise = promisify0(getAvailableEncoders) - const availableFFmpegEncoders = await getAvailableEncodersPromise() - - const searchEncoders = new Set() - for (const type of [ 'live', 'vod' ]) { - for (const streamType of [ 'audio', 'video' ]) { - for (const encoder of peertubeAvailableEncoders.encodersToTry[type][streamType]) { - searchEncoders.add(encoder) - } - } - } - - supportedEncoders = new Map() - - for (const searchEncoder of searchEncoders) { - supportedEncoders.set(searchEncoder, availableFFmpegEncoders[searchEncoder] !== undefined) - } - - logger.info('Built supported ffmpeg encoders.', { supportedEncoders, searchEncoders, ...lTags() }) - - return supportedEncoders -} - -function resetSupportedEncoders () { - supportedEncoders = undefined -} - -// Run encoder builder depending on available encoders -// Try encoders by priority: if the encoder is available, run the chosen profile or fallback to the default one -// If the default one does not exist, check the next encoder -async function getEncoderBuilderResult (options: EncoderOptionsBuilderParams & { - streamType: 'video' | 'audio' - input: string - - availableEncoders: AvailableEncoders - profile: string - - videoType: 'vod' | 'live' -}) { - const { availableEncoders, profile, streamType, videoType } = options - - const encodersToTry = availableEncoders.encodersToTry[videoType][streamType] - const encoders = availableEncoders.available[videoType] - - for (const encoder of encodersToTry) { - if (!(await checkFFmpegEncoders(availableEncoders)).get(encoder)) { - logger.debug('Encoder %s not available in ffmpeg, skipping.', encoder, lTags()) - continue - } - - if (!encoders[encoder]) { - logger.debug('Encoder %s not available in peertube encoders, skipping.', encoder, lTags()) - continue - } - - // An object containing available profiles for this encoder - const builderProfiles: EncoderProfile = encoders[encoder] - let builder = builderProfiles[profile] - - if (!builder) { - logger.debug('Profile %s for encoder %s not available. Fallback to default.', profile, encoder, lTags()) - builder = builderProfiles.default - - if (!builder) { - logger.debug('Default profile for encoder %s not available. Try next available encoder.', encoder, lTags()) - continue - } - } - - const result = await builder( - pick(options, [ - 'input', - 'canCopyAudio', - 'canCopyVideo', - 'resolution', - 'inputBitrate', - 'fps', - 'inputRatio', - 'streamNum' - ]) - ) - - return { - result, - - // If we don't have output options, then copy the input stream - encoder: result.copy === true - ? 'copy' - : encoder - } - } - - return null -} - -export { - checkFFmpegEncoders, - resetSupportedEncoders, - - getEncoderBuilderResult -} diff --git a/server/helpers/ffmpeg/ffmpeg-image.ts b/server/helpers/ffmpeg/ffmpeg-image.ts new file mode 100644 index 000000000..0bb0ff2c0 --- /dev/null +++ b/server/helpers/ffmpeg/ffmpeg-image.ts @@ -0,0 +1,14 @@ +import { FFmpegImage } from '@shared/ffmpeg' +import { getFFmpegCommandWrapperOptions } from './ffmpeg-options' + +export function processGIF (options: Parameters[0]) { + return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).processGIF(options) +} + +export function generateThumbnailFromVideo (options: Parameters[0]) { + return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).generateThumbnailFromVideo(options) +} + +export function convertWebPToJPG (options: Parameters[0]) { + return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).convertWebPToJPG(options) +} diff --git a/server/helpers/ffmpeg/ffmpeg-images.ts b/server/helpers/ffmpeg/ffmpeg-images.ts deleted file mode 100644 index 7f64c6d0a..000000000 --- a/server/helpers/ffmpeg/ffmpeg-images.ts +++ /dev/null @@ -1,46 +0,0 @@ -import ffmpeg from 'fluent-ffmpeg' -import { FFMPEG_NICE } from '@server/initializers/constants' -import { runCommand } from './ffmpeg-commons' - -function convertWebPToJPG (path: string, destination: string): Promise { - const command = ffmpeg(path, { niceness: FFMPEG_NICE.THUMBNAIL }) - .output(destination) - - return runCommand({ command, silent: true }) -} - -function processGIF ( - path: string, - destination: string, - newSize: { width: number, height: number } -): Promise { - const command = ffmpeg(path, { niceness: FFMPEG_NICE.THUMBNAIL }) - .fps(20) - .size(`${newSize.width}x${newSize.height}`) - .output(destination) - - return runCommand({ command }) -} - -async function generateThumbnailFromVideo (fromPath: string, folder: string, imageName: string) { - const pendingImageName = 'pending-' + imageName - - const options = { - filename: pendingImageName, - count: 1, - folder - } - - return new Promise((res, rej) => { - ffmpeg(fromPath, { niceness: FFMPEG_NICE.THUMBNAIL }) - .on('error', rej) - .on('end', () => res(imageName)) - .thumbnail(options) - }) -} - -export { - convertWebPToJPG, - processGIF, - generateThumbnailFromVideo -} diff --git a/server/helpers/ffmpeg/ffmpeg-live.ts b/server/helpers/ffmpeg/ffmpeg-live.ts deleted file mode 100644 index 379d7b1ad..000000000 --- a/server/helpers/ffmpeg/ffmpeg-live.ts +++ /dev/null @@ -1,204 +0,0 @@ -import { FfmpegCommand, FilterSpecification } from 'fluent-ffmpeg' -import { join } from 'path' -import { VIDEO_LIVE } from '@server/initializers/constants' -import { AvailableEncoders, LiveVideoLatencyMode } from '@shared/models' -import { logger, loggerTagsFactory } from '../logger' -import { buildStreamSuffix, getFFmpeg, getScaleFilter, StreamType } from './ffmpeg-commons' -import { getEncoderBuilderResult } from './ffmpeg-encoders' -import { addDefaultEncoderGlobalParams, addDefaultEncoderParams, applyEncoderOptions } from './ffmpeg-presets' -import { computeFPS } from './ffprobe-utils' - -const lTags = loggerTagsFactory('ffmpeg') - -async function getLiveTranscodingCommand (options: { - inputUrl: string - - outPath: string - masterPlaylistName: string - latencyMode: LiveVideoLatencyMode - - resolutions: number[] - - // Input information - fps: number - bitrate: number - ratio: number - hasAudio: boolean - - availableEncoders: AvailableEncoders - profile: string -}) { - const { - inputUrl, - outPath, - resolutions, - fps, - bitrate, - availableEncoders, - profile, - masterPlaylistName, - ratio, - latencyMode, - hasAudio - } = options - - const command = getFFmpeg(inputUrl, 'live') - - const varStreamMap: string[] = [] - - const complexFilter: FilterSpecification[] = [ - { - inputs: '[v:0]', - filter: 'split', - options: resolutions.length, - outputs: resolutions.map(r => `vtemp${r}`) - } - ] - - command.outputOption('-sc_threshold 0') - - addDefaultEncoderGlobalParams(command) - - for (let i = 0; i < resolutions.length; i++) { - const streamMap: string[] = [] - const resolution = resolutions[i] - const resolutionFPS = computeFPS(fps, resolution) - - const baseEncoderBuilderParams = { - input: inputUrl, - - availableEncoders, - profile, - - canCopyAudio: true, - canCopyVideo: true, - - inputBitrate: bitrate, - inputRatio: ratio, - - resolution, - fps: resolutionFPS, - - streamNum: i, - videoType: 'live' as 'live' - } - - { - const streamType: StreamType = 'video' - const builderResult = await getEncoderBuilderResult({ ...baseEncoderBuilderParams, streamType }) - if (!builderResult) { - throw new Error('No available live video encoder found') - } - - command.outputOption(`-map [vout${resolution}]`) - - addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps: resolutionFPS, streamNum: i }) - - logger.debug( - 'Apply ffmpeg live video params from %s using %s profile.', builderResult.encoder, profile, - { builderResult, fps: resolutionFPS, resolution, ...lTags() } - ) - - command.outputOption(`${buildStreamSuffix('-c:v', i)} ${builderResult.encoder}`) - applyEncoderOptions(command, builderResult.result) - - complexFilter.push({ - inputs: `vtemp${resolution}`, - filter: getScaleFilter(builderResult.result), - options: `w=-2:h=${resolution}`, - outputs: `vout${resolution}` - }) - - streamMap.push(`v:${i}`) - } - - if (hasAudio) { - const streamType: StreamType = 'audio' - const builderResult = await getEncoderBuilderResult({ ...baseEncoderBuilderParams, streamType }) - if (!builderResult) { - throw new Error('No available live audio encoder found') - } - - command.outputOption('-map a:0') - - addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps: resolutionFPS, streamNum: i }) - - logger.debug( - 'Apply ffmpeg live audio params from %s using %s profile.', builderResult.encoder, profile, - { builderResult, fps: resolutionFPS, resolution, ...lTags() } - ) - - command.outputOption(`${buildStreamSuffix('-c:a', i)} ${builderResult.encoder}`) - applyEncoderOptions(command, builderResult.result) - - streamMap.push(`a:${i}`) - } - - varStreamMap.push(streamMap.join(',')) - } - - command.complexFilter(complexFilter) - - addDefaultLiveHLSParams({ command, outPath, masterPlaylistName, latencyMode }) - - command.outputOption('-var_stream_map', varStreamMap.join(' ')) - - return command -} - -function getLiveMuxingCommand (options: { - inputUrl: string - outPath: string - masterPlaylistName: string - latencyMode: LiveVideoLatencyMode -}) { - const { inputUrl, outPath, masterPlaylistName, latencyMode } = options - - const command = getFFmpeg(inputUrl, 'live') - - command.outputOption('-c:v copy') - command.outputOption('-c:a copy') - command.outputOption('-map 0:a?') - command.outputOption('-map 0:v?') - - addDefaultLiveHLSParams({ command, outPath, masterPlaylistName, latencyMode }) - - return command -} - -function getLiveSegmentTime (latencyMode: LiveVideoLatencyMode) { - if (latencyMode === LiveVideoLatencyMode.SMALL_LATENCY) { - return VIDEO_LIVE.SEGMENT_TIME_SECONDS.SMALL_LATENCY - } - - return VIDEO_LIVE.SEGMENT_TIME_SECONDS.DEFAULT_LATENCY -} - -// --------------------------------------------------------------------------- - -export { - getLiveSegmentTime, - - getLiveTranscodingCommand, - getLiveMuxingCommand -} - -// --------------------------------------------------------------------------- - -function addDefaultLiveHLSParams (options: { - command: FfmpegCommand - outPath: string - masterPlaylistName: string - latencyMode: LiveVideoLatencyMode -}) { - const { command, outPath, masterPlaylistName, latencyMode } = options - - command.outputOption('-hls_time ' + getLiveSegmentTime(latencyMode)) - command.outputOption('-hls_list_size ' + VIDEO_LIVE.SEGMENTS_LIST_SIZE) - command.outputOption('-hls_flags delete_segments+independent_segments+program_date_time') - command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%06d.ts')}`) - command.outputOption('-master_pl_name ' + masterPlaylistName) - command.outputOption(`-f hls`) - - command.output(join(outPath, '%v.m3u8')) -} diff --git a/server/helpers/ffmpeg/ffmpeg-options.ts b/server/helpers/ffmpeg/ffmpeg-options.ts new file mode 100644 index 000000000..db6350d39 --- /dev/null +++ b/server/helpers/ffmpeg/ffmpeg-options.ts @@ -0,0 +1,45 @@ +import { logger } from '@server/helpers/logger' +import { CONFIG } from '@server/initializers/config' +import { FFMPEG_NICE } from '@server/initializers/constants' +import { FFmpegCommandWrapperOptions } from '@shared/ffmpeg' +import { AvailableEncoders } from '@shared/models' + +type CommandType = 'live' | 'vod' | 'thumbnail' + +export function getFFmpegCommandWrapperOptions (type: CommandType, availableEncoders?: AvailableEncoders): FFmpegCommandWrapperOptions { + return { + availableEncoders, + profile: getProfile(type), + + niceness: FFMPEG_NICE[type], + tmpDirectory: CONFIG.STORAGE.TMP_DIR, + threads: getThreads(type), + + logger: { + debug: logger.debug.bind(logger), + info: logger.info.bind(logger), + warn: logger.warn.bind(logger), + error: logger.error.bind(logger) + }, + lTags: { tags: [ 'ffmpeg' ] } + } +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +function getThreads (type: CommandType) { + if (type === 'live') return CONFIG.LIVE.TRANSCODING.THREADS + if (type === 'vod') return CONFIG.TRANSCODING.THREADS + + // Auto + return 0 +} + +function getProfile (type: CommandType) { + if (type === 'live') return CONFIG.LIVE.TRANSCODING.PROFILE + if (type === 'vod') return CONFIG.TRANSCODING.PROFILE + + return undefined +} diff --git a/server/helpers/ffmpeg/ffmpeg-presets.ts b/server/helpers/ffmpeg/ffmpeg-presets.ts deleted file mode 100644 index d1160a4a2..000000000 --- a/server/helpers/ffmpeg/ffmpeg-presets.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { FfmpegCommand } from 'fluent-ffmpeg' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { pick } from '@shared/core-utils' -import { AvailableEncoders, EncoderOptions } from '@shared/models' -import { buildStreamSuffix, getScaleFilter, StreamType } from './ffmpeg-commons' -import { getEncoderBuilderResult } from './ffmpeg-encoders' -import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, hasAudioStream } from './ffprobe-utils' - -const lTags = loggerTagsFactory('ffmpeg') - -// --------------------------------------------------------------------------- - -function addDefaultEncoderGlobalParams (command: FfmpegCommand) { - // avoid issues when transcoding some files: https://trac.ffmpeg.org/ticket/6375 - command.outputOption('-max_muxing_queue_size 1024') - // strip all metadata - .outputOption('-map_metadata -1') - // allows import of source material with incompatible pixel formats (e.g. MJPEG video) - .outputOption('-pix_fmt yuv420p') -} - -function addDefaultEncoderParams (options: { - command: FfmpegCommand - encoder: 'libx264' | string - fps: number - - streamNum?: number -}) { - const { command, encoder, fps, streamNum } = options - - if (encoder === 'libx264') { - // 3.1 is the minimal resource allocation for our highest supported resolution - command.outputOption(buildStreamSuffix('-level:v', streamNum) + ' 3.1') - - if (fps) { - // Keyframe interval of 2 seconds for faster seeking and resolution switching. - // https://streaminglearningcenter.com/blogs/whats-the-right-keyframe-interval.html - // https://superuser.com/a/908325 - command.outputOption(buildStreamSuffix('-g:v', streamNum) + ' ' + (fps * 2)) - } - } -} - -// --------------------------------------------------------------------------- - -async function presetVOD (options: { - command: FfmpegCommand - input: string - - availableEncoders: AvailableEncoders - profile: string - - canCopyAudio: boolean - canCopyVideo: boolean - - resolution: number - fps: number - - scaleFilterValue?: string -}) { - const { command, input, profile, resolution, fps, scaleFilterValue } = options - - let localCommand = command - .format('mp4') - .outputOption('-movflags faststart') - - addDefaultEncoderGlobalParams(command) - - const probe = await ffprobePromise(input) - - // Audio encoder - const bitrate = await getVideoStreamBitrate(input, probe) - const videoStreamDimensions = await getVideoStreamDimensionsInfo(input, probe) - - let streamsToProcess: StreamType[] = [ 'audio', 'video' ] - - if (!await hasAudioStream(input, probe)) { - localCommand = localCommand.noAudio() - streamsToProcess = [ 'video' ] - } - - for (const streamType of streamsToProcess) { - const builderResult = await getEncoderBuilderResult({ - ...pick(options, [ 'availableEncoders', 'canCopyAudio', 'canCopyVideo' ]), - - input, - inputBitrate: bitrate, - inputRatio: videoStreamDimensions?.ratio || 0, - - profile, - resolution, - fps, - streamType, - - videoType: 'vod' as 'vod' - }) - - if (!builderResult) { - throw new Error('No available encoder found for stream ' + streamType) - } - - logger.debug( - 'Apply ffmpeg params from %s for %s stream of input %s using %s profile.', - builderResult.encoder, streamType, input, profile, - { builderResult, resolution, fps, ...lTags() } - ) - - if (streamType === 'video') { - localCommand.videoCodec(builderResult.encoder) - - if (scaleFilterValue) { - localCommand.outputOption(`-vf ${getScaleFilter(builderResult.result)}=${scaleFilterValue}`) - } - } else if (streamType === 'audio') { - localCommand.audioCodec(builderResult.encoder) - } - - applyEncoderOptions(localCommand, builderResult.result) - addDefaultEncoderParams({ command: localCommand, encoder: builderResult.encoder, fps }) - } - - return localCommand -} - -function presetCopy (command: FfmpegCommand): FfmpegCommand { - return command - .format('mp4') - .videoCodec('copy') - .audioCodec('copy') -} - -function presetOnlyAudio (command: FfmpegCommand): FfmpegCommand { - return command - .format('mp4') - .audioCodec('copy') - .noVideo() -} - -function applyEncoderOptions (command: FfmpegCommand, options: EncoderOptions): FfmpegCommand { - return command - .inputOptions(options.inputOptions ?? []) - .outputOptions(options.outputOptions ?? []) -} - -// --------------------------------------------------------------------------- - -export { - presetVOD, - presetCopy, - presetOnlyAudio, - - addDefaultEncoderGlobalParams, - addDefaultEncoderParams, - - applyEncoderOptions -} diff --git a/server/helpers/ffmpeg/ffmpeg-vod.ts b/server/helpers/ffmpeg/ffmpeg-vod.ts deleted file mode 100644 index d84703eb9..000000000 --- a/server/helpers/ffmpeg/ffmpeg-vod.ts +++ /dev/null @@ -1,267 +0,0 @@ -import { MutexInterface } from 'async-mutex' -import { Job } from 'bullmq' -import { FfmpegCommand } from 'fluent-ffmpeg' -import { readFile, writeFile } from 'fs-extra' -import { dirname } from 'path' -import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' -import { pick } from '@shared/core-utils' -import { AvailableEncoders, VideoResolution } from '@shared/models' -import { logger, loggerTagsFactory } from '../logger' -import { getFFmpeg, runCommand } from './ffmpeg-commons' -import { presetCopy, presetOnlyAudio, presetVOD } from './ffmpeg-presets' -import { computeFPS, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS } from './ffprobe-utils' - -const lTags = loggerTagsFactory('ffmpeg') - -// --------------------------------------------------------------------------- - -type TranscodeVODOptionsType = 'hls' | 'hls-from-ts' | 'quick-transcode' | 'video' | 'merge-audio' | 'only-audio' - -interface BaseTranscodeVODOptions { - type: TranscodeVODOptionsType - - inputPath: string - outputPath: string - - // Will be released after the ffmpeg started - // To prevent a bug where the input file does not exist anymore when running ffmpeg - inputFileMutexReleaser: MutexInterface.Releaser - - availableEncoders: AvailableEncoders - profile: string - - resolution: number - - job?: Job -} - -interface HLSTranscodeOptions extends BaseTranscodeVODOptions { - type: 'hls' - copyCodecs: boolean - hlsPlaylist: { - videoFilename: string - } -} - -interface HLSFromTSTranscodeOptions extends BaseTranscodeVODOptions { - type: 'hls-from-ts' - - isAAC: boolean - - hlsPlaylist: { - videoFilename: string - } -} - -interface QuickTranscodeOptions extends BaseTranscodeVODOptions { - type: 'quick-transcode' -} - -interface VideoTranscodeOptions extends BaseTranscodeVODOptions { - type: 'video' -} - -interface MergeAudioTranscodeOptions extends BaseTranscodeVODOptions { - type: 'merge-audio' - audioPath: string -} - -interface OnlyAudioTranscodeOptions extends BaseTranscodeVODOptions { - type: 'only-audio' -} - -type TranscodeVODOptions = - HLSTranscodeOptions - | HLSFromTSTranscodeOptions - | VideoTranscodeOptions - | MergeAudioTranscodeOptions - | OnlyAudioTranscodeOptions - | QuickTranscodeOptions - -// --------------------------------------------------------------------------- - -const builders: { - [ type in TranscodeVODOptionsType ]: (c: FfmpegCommand, o?: TranscodeVODOptions) => Promise | FfmpegCommand -} = { - 'quick-transcode': buildQuickTranscodeCommand, - 'hls': buildHLSVODCommand, - 'hls-from-ts': buildHLSVODFromTSCommand, - 'merge-audio': buildAudioMergeCommand, - 'only-audio': buildOnlyAudioCommand, - 'video': buildVODCommand -} - -async function transcodeVOD (options: TranscodeVODOptions) { - logger.debug('Will run transcode.', { options, ...lTags() }) - - let command = getFFmpeg(options.inputPath, 'vod') - .output(options.outputPath) - - command = await builders[options.type](command, options) - - command.on('start', () => { - setTimeout(() => { - options.inputFileMutexReleaser() - }, 1000) - }) - - await runCommand({ command, job: options.job }) - - await fixHLSPlaylistIfNeeded(options) -} - -// --------------------------------------------------------------------------- - -export { - transcodeVOD, - - buildVODCommand, - - TranscodeVODOptions, - TranscodeVODOptionsType -} - -// --------------------------------------------------------------------------- - -async function buildVODCommand (command: FfmpegCommand, options: TranscodeVODOptions) { - const probe = await ffprobePromise(options.inputPath) - - let fps = await getVideoStreamFPS(options.inputPath, probe) - fps = computeFPS(fps, options.resolution) - - let scaleFilterValue: string - - if (options.resolution !== undefined) { - const videoStreamInfo = await getVideoStreamDimensionsInfo(options.inputPath, probe) - - scaleFilterValue = videoStreamInfo?.isPortraitMode === true - ? `w=${options.resolution}:h=-2` - : `w=-2:h=${options.resolution}` - } - - command = await presetVOD({ - ...pick(options, [ 'resolution', 'availableEncoders', 'profile' ]), - - command, - input: options.inputPath, - canCopyAudio: true, - canCopyVideo: true, - fps, - scaleFilterValue - }) - - return command -} - -function buildQuickTranscodeCommand (command: FfmpegCommand) { - command = presetCopy(command) - - command = command.outputOption('-map_metadata -1') // strip all metadata - .outputOption('-movflags faststart') - - return command -} - -// --------------------------------------------------------------------------- -// Audio transcoding -// --------------------------------------------------------------------------- - -async function buildAudioMergeCommand (command: FfmpegCommand, options: MergeAudioTranscodeOptions) { - command = command.loop(undefined) - - const scaleFilterValue = getMergeAudioScaleFilterValue() - command = await presetVOD({ - ...pick(options, [ 'resolution', 'availableEncoders', 'profile' ]), - - command, - input: options.audioPath, - canCopyAudio: true, - canCopyVideo: true, - fps: VIDEO_TRANSCODING_FPS.AUDIO_MERGE, - scaleFilterValue - }) - - command.outputOption('-preset:v veryfast') - - command = command.input(options.audioPath) - .outputOption('-tune stillimage') - .outputOption('-shortest') - - return command -} - -function buildOnlyAudioCommand (command: FfmpegCommand, _options: OnlyAudioTranscodeOptions) { - command = presetOnlyAudio(command) - - return command -} - -// --------------------------------------------------------------------------- -// HLS transcoding -// --------------------------------------------------------------------------- - -async function buildHLSVODCommand (command: FfmpegCommand, options: HLSTranscodeOptions) { - const videoPath = getHLSVideoPath(options) - - if (options.copyCodecs) command = presetCopy(command) - else if (options.resolution === VideoResolution.H_NOVIDEO) command = presetOnlyAudio(command) - else command = await buildVODCommand(command, options) - - addCommonHLSVODCommandOptions(command, videoPath) - - return command -} - -function buildHLSVODFromTSCommand (command: FfmpegCommand, options: HLSFromTSTranscodeOptions) { - const videoPath = getHLSVideoPath(options) - - command.outputOption('-c copy') - - if (options.isAAC) { - // Required for example when copying an AAC stream from an MPEG-TS - // Since it's a bitstream filter, we don't need to reencode the audio - command.outputOption('-bsf:a aac_adtstoasc') - } - - addCommonHLSVODCommandOptions(command, videoPath) - - return command -} - -function addCommonHLSVODCommandOptions (command: FfmpegCommand, outputPath: string) { - return command.outputOption('-hls_time 4') - .outputOption('-hls_list_size 0') - .outputOption('-hls_playlist_type vod') - .outputOption('-hls_segment_filename ' + outputPath) - .outputOption('-hls_segment_type fmp4') - .outputOption('-f hls') - .outputOption('-hls_flags single_file') -} - -async function fixHLSPlaylistIfNeeded (options: TranscodeVODOptions) { - if (options.type !== 'hls' && options.type !== 'hls-from-ts') return - - const fileContent = await readFile(options.outputPath) - - const videoFileName = options.hlsPlaylist.videoFilename - const videoFilePath = getHLSVideoPath(options) - - // Fix wrong mapping with some ffmpeg versions - const newContent = fileContent.toString() - .replace(`#EXT-X-MAP:URI="${videoFilePath}",`, `#EXT-X-MAP:URI="${videoFileName}",`) - - await writeFile(options.outputPath, newContent) -} - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -function getHLSVideoPath (options: HLSTranscodeOptions | HLSFromTSTranscodeOptions) { - return `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}` -} - -// Avoid "height not divisible by 2" error -function getMergeAudioScaleFilterValue () { - return 'trunc(iw/2)*2:trunc(ih/2)*2' -} diff --git a/server/helpers/ffmpeg/ffprobe-utils.ts b/server/helpers/ffmpeg/ffprobe-utils.ts deleted file mode 100644 index fb270b3cb..000000000 --- a/server/helpers/ffmpeg/ffprobe-utils.ts +++ /dev/null @@ -1,254 +0,0 @@ -import { FfprobeData } from 'fluent-ffmpeg' -import { getMaxBitrate } from '@shared/core-utils' -import { - buildFileMetadata, - ffprobePromise, - getAudioStream, - getMaxAudioBitrate, - getVideoStream, - getVideoStreamBitrate, - getVideoStreamDimensionsInfo, - getVideoStreamDuration, - getVideoStreamFPS, - hasAudioStream -} from '@shared/extra-utils/ffprobe' -import { VideoResolution, VideoTranscodingFPS } from '@shared/models' -import { CONFIG } from '../../initializers/config' -import { VIDEO_TRANSCODING_FPS } from '../../initializers/constants' -import { toEven } from '../core-utils' -import { logger } from '../logger' - -/** - * - * Helpers to run ffprobe and extract data from the JSON output - * - */ - -// --------------------------------------------------------------------------- -// Codecs -// --------------------------------------------------------------------------- - -async function getVideoStreamCodec (path: string) { - const videoStream = await getVideoStream(path) - if (!videoStream) return '' - - const videoCodec = videoStream.codec_tag_string - - if (videoCodec === 'vp09') return 'vp09.00.50.08' - if (videoCodec === 'hev1') return 'hev1.1.6.L93.B0' - - const baseProfileMatrix = { - avc1: { - High: '6400', - Main: '4D40', - Baseline: '42E0' - }, - av01: { - High: '1', - Main: '0', - Professional: '2' - } - } - - let baseProfile = baseProfileMatrix[videoCodec][videoStream.profile] - if (!baseProfile) { - logger.warn('Cannot get video profile codec of %s.', path, { videoStream }) - baseProfile = baseProfileMatrix[videoCodec]['High'] // Fallback - } - - if (videoCodec === 'av01') { - let level = videoStream.level.toString() - if (level.length === 1) level = `0${level}` - - // Guess the tier indicator and bit depth - return `${videoCodec}.${baseProfile}.${level}M.08` - } - - let level = videoStream.level.toString(16) - if (level.length === 1) level = `0${level}` - - // Default, h264 codec - return `${videoCodec}.${baseProfile}${level}` -} - -async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) { - const { audioStream } = await getAudioStream(path, existingProbe) - - if (!audioStream) return '' - - const audioCodecName = audioStream.codec_name - - if (audioCodecName === 'opus') return 'opus' - if (audioCodecName === 'vorbis') return 'vorbis' - if (audioCodecName === 'aac') return 'mp4a.40.2' - if (audioCodecName === 'mp3') return 'mp4a.40.34' - - logger.warn('Cannot get audio codec of %s.', path, { audioStream }) - - return 'mp4a.40.2' // Fallback -} - -// --------------------------------------------------------------------------- -// Resolutions -// --------------------------------------------------------------------------- - -function computeResolutionsToTranscode (options: { - input: number - type: 'vod' | 'live' - includeInput: boolean - strictLower: boolean - hasAudio: boolean -}) { - const { input, type, includeInput, strictLower, hasAudio } = options - - const configResolutions = type === 'vod' - ? CONFIG.TRANSCODING.RESOLUTIONS - : CONFIG.LIVE.TRANSCODING.RESOLUTIONS - - const resolutionsEnabled = new Set() - - // Put in the order we want to proceed jobs - const availableResolutions: VideoResolution[] = [ - VideoResolution.H_NOVIDEO, - VideoResolution.H_480P, - VideoResolution.H_360P, - VideoResolution.H_720P, - VideoResolution.H_240P, - VideoResolution.H_144P, - VideoResolution.H_1080P, - VideoResolution.H_1440P, - VideoResolution.H_4K - ] - - for (const resolution of availableResolutions) { - // Resolution not enabled - if (configResolutions[resolution + 'p'] !== true) continue - // Too big resolution for input file - if (input < resolution) continue - // We only want lower resolutions than input file - if (strictLower && input === resolution) continue - // Audio resolutio but no audio in the video - if (resolution === VideoResolution.H_NOVIDEO && !hasAudio) continue - - resolutionsEnabled.add(resolution) - } - - if (includeInput) { - // Always use an even resolution to avoid issues with ffmpeg - resolutionsEnabled.add(toEven(input)) - } - - return Array.from(resolutionsEnabled) -} - -// --------------------------------------------------------------------------- -// Can quick transcode -// --------------------------------------------------------------------------- - -async function canDoQuickTranscode (path: string): Promise { - if (CONFIG.TRANSCODING.PROFILE !== 'default') return false - - const probe = await ffprobePromise(path) - - return await canDoQuickVideoTranscode(path, probe) && - await canDoQuickAudioTranscode(path, probe) -} - -async function canDoQuickAudioTranscode (path: string, probe?: FfprobeData): Promise { - const parsedAudio = await getAudioStream(path, probe) - - if (!parsedAudio.audioStream) return true - - if (parsedAudio.audioStream['codec_name'] !== 'aac') return false - - const audioBitrate = parsedAudio.bitrate - if (!audioBitrate) return false - - const maxAudioBitrate = getMaxAudioBitrate('aac', audioBitrate) - if (maxAudioBitrate !== -1 && audioBitrate > maxAudioBitrate) return false - - const channelLayout = parsedAudio.audioStream['channel_layout'] - // Causes playback issues with Chrome - if (!channelLayout || channelLayout === 'unknown' || channelLayout === 'quad') return false - - return true -} - -async function canDoQuickVideoTranscode (path: string, probe?: FfprobeData): Promise { - const videoStream = await getVideoStream(path, probe) - const fps = await getVideoStreamFPS(path, probe) - const bitRate = await getVideoStreamBitrate(path, probe) - const resolutionData = await getVideoStreamDimensionsInfo(path, probe) - - // If ffprobe did not manage to guess the bitrate - if (!bitRate) return false - - // check video params - if (!videoStream) return false - if (videoStream['codec_name'] !== 'h264') return false - if (videoStream['pix_fmt'] !== 'yuv420p') return false - if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false - if (bitRate > getMaxBitrate({ ...resolutionData, fps })) return false - - return true -} - -// --------------------------------------------------------------------------- -// Framerate -// --------------------------------------------------------------------------- - -function getClosestFramerateStandard > (fps: number, type: K) { - return VIDEO_TRANSCODING_FPS[type].slice(0) - .sort((a, b) => fps % a - fps % b)[0] -} - -function computeFPS (fpsArg: number, resolution: VideoResolution) { - let fps = fpsArg - - if ( - // On small/medium resolutions, limit FPS - resolution !== undefined && - resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN && - fps > VIDEO_TRANSCODING_FPS.AVERAGE - ) { - // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value - fps = getClosestFramerateStandard(fps, 'STANDARD') - } - - // Hard FPS limits - if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard(fps, 'HD_STANDARD') - - if (fps < VIDEO_TRANSCODING_FPS.MIN) { - throw new Error(`Cannot compute FPS because ${fps} is lower than our minimum value ${VIDEO_TRANSCODING_FPS.MIN}`) - } - - return fps -} - -// --------------------------------------------------------------------------- - -export { - // Re export ffprobe utils - getVideoStreamDimensionsInfo, - buildFileMetadata, - getMaxAudioBitrate, - getVideoStream, - getVideoStreamDuration, - getAudioStream, - hasAudioStream, - getVideoStreamFPS, - ffprobePromise, - getVideoStreamBitrate, - - getVideoStreamCodec, - getAudioStreamCodec, - - computeFPS, - getClosestFramerateStandard, - - computeResolutionsToTranscode, - - canDoQuickTranscode, - canDoQuickVideoTranscode, - canDoQuickAudioTranscode -} diff --git a/server/helpers/ffmpeg/framerate.ts b/server/helpers/ffmpeg/framerate.ts new file mode 100644 index 000000000..18cb0e0e2 --- /dev/null +++ b/server/helpers/ffmpeg/framerate.ts @@ -0,0 +1,44 @@ +import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' +import { VideoResolution } from '@shared/models' + +export function computeOutputFPS (options: { + inputFPS: number + resolution: VideoResolution +}) { + const { resolution } = options + + let fps = options.inputFPS + + if ( + // On small/medium resolutions, limit FPS + resolution !== undefined && + resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN && + fps > VIDEO_TRANSCODING_FPS.AVERAGE + ) { + // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value + fps = getClosestFramerateStandard({ fps, type: 'STANDARD' }) + } + + // Hard FPS limits + if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard({ fps, type: 'HD_STANDARD' }) + + if (fps < VIDEO_TRANSCODING_FPS.MIN) { + throw new Error(`Cannot compute FPS because ${fps} is lower than our minimum value ${VIDEO_TRANSCODING_FPS.MIN}`) + } + + return fps +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +function getClosestFramerateStandard (options: { + fps: number + type: 'HD_STANDARD' | 'STANDARD' +}) { + const { fps, type } = options + + return VIDEO_TRANSCODING_FPS[type].slice(0) + .sort((a, b) => fps % a - fps % b)[0] +} diff --git a/server/helpers/ffmpeg/index.ts b/server/helpers/ffmpeg/index.ts index e3bb2013f..bf1c73fb6 100644 --- a/server/helpers/ffmpeg/index.ts +++ b/server/helpers/ffmpeg/index.ts @@ -1,8 +1,4 @@ -export * from './ffmpeg-commons' -export * from './ffmpeg-edition' -export * from './ffmpeg-encoders' -export * from './ffmpeg-images' -export * from './ffmpeg-live' -export * from './ffmpeg-presets' -export * from './ffmpeg-vod' -export * from './ffprobe-utils' +export * from './codecs' +export * from './ffmpeg-image' +export * from './ffmpeg-options' +export * from './framerate' diff --git a/server/helpers/image-utils.ts b/server/helpers/image-utils.ts index bbd4692ef..05b258d8a 100644 --- a/server/helpers/image-utils.ts +++ b/server/helpers/image-utils.ts @@ -3,7 +3,7 @@ import Jimp, { read as jimpRead } from 'jimp' import { join } from 'path' import { getLowercaseExtension } from '@shared/core-utils' import { buildUUID } from '@shared/extra-utils' -import { convertWebPToJPG, generateThumbnailFromVideo, processGIF } from './ffmpeg/ffmpeg-images' +import { convertWebPToJPG, generateThumbnailFromVideo, processGIF } from './ffmpeg' import { logger, loggerTagsFactory } from './logger' const lTags = loggerTagsFactory('image-utils') @@ -30,7 +30,7 @@ async function processImage (options: { // Use FFmpeg to process GIF if (extension === '.gif') { - await processGIF(path, destination, newSize) + await processGIF({ path, destination, newSize }) } else { await jimpProcessor(path, destination, newSize, extension) } @@ -50,7 +50,7 @@ async function generateImageFromVideoFile (options: { const pendingImagePath = join(folder, pendingImageName) try { - await generateThumbnailFromVideo(fromPath, folder, imageName) + await generateThumbnailFromVideo({ fromPath, folder, imageName }) const destination = join(folder, imageName) await processImage({ path: pendingImagePath, destination, newSize: size }) @@ -99,7 +99,7 @@ async function jimpProcessor (path: string, destination: string, newSize: { widt logger.debug('Cannot read %s with jimp. Try to convert the image using ffmpeg first.', path, { err }) const newName = path + '.jpg' - await convertWebPToJPG(path, newName) + await convertWebPToJPG({ path, destination: newName }) await rename(newName, path) sourceImage = await jimpRead(path) diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts index ae7d11800..95e78a904 100644 --- a/server/helpers/peertube-crypto.ts +++ b/server/helpers/peertube-crypto.ts @@ -2,10 +2,11 @@ import { compare, genSalt, hash } from 'bcrypt' import { createCipheriv, createDecipheriv, createSign, createVerify } from 'crypto' import { Request } from 'express' import { cloneDeep } from 'lodash' +import { promisify1, promisify2 } from '@shared/core-utils' import { sha256 } from '@shared/extra-utils' import { BCRYPT_SALT_SIZE, ENCRYPTION, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants' import { MActor } from '../types/models' -import { generateRSAKeyPairPromise, promisify1, promisify2, randomBytesPromise, scryptPromise } from './core-utils' +import { generateRSAKeyPairPromise, randomBytesPromise, scryptPromise } from './core-utils' import { jsonld } from './custom-jsonld-signature' import { logger } from './logger' diff --git a/server/helpers/token-generator.ts b/server/helpers/token-generator.ts new file mode 100644 index 000000000..16313b818 --- /dev/null +++ b/server/helpers/token-generator.ts @@ -0,0 +1,19 @@ +import { buildUUID } from '@shared/extra-utils' + +function generateRunnerRegistrationToken () { + return 'ptrrt-' + buildUUID() +} + +function generateRunnerToken () { + return 'ptrt-' + buildUUID() +} + +function generateRunnerJobToken () { + return 'ptrjt-' + buildUUID() +} + +export { + generateRunnerRegistrationToken, + generateRunnerToken, + generateRunnerJobToken +} diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index a3c93e6fe..e690e3890 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -13,9 +13,9 @@ import { VideoPathManager } from '@server/lib/video-path-manager' import { MVideo } from '@server/types/models/video/video' import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file' import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist' +import { promisify2 } from '@shared/core-utils' import { sha1 } from '@shared/extra-utils' import { CONFIG } from '../initializers/config' -import { promisify2 } from './core-utils' import { logger } from './logger' import { generateVideoImportTmpPath } from './utils' import { extractVideo } from './video' diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts index 14ed82cb4..68dea909d 100644 --- a/server/initializers/checker-after-init.ts +++ b/server/initializers/checker-after-init.ts @@ -1,7 +1,7 @@ import config from 'config' import { URL } from 'url' -import { getFFmpegVersion } from '@server/helpers/ffmpeg' import { uniqify } from '@shared/core-utils' +import { getFFmpegVersion } from '@shared/ffmpeg' import { VideoRedundancyConfigFilter } from '@shared/models/redundancy/video-redundancy-config-filter.type' import { RecentlyAddedStrategy } from '../../shared/models/redundancy' import { isProdInstance, parseBytes, parseSemVersion } from '../helpers/core-utils' diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index 49010c059..2361aa1eb 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts @@ -1,5 +1,6 @@ import { IConfig } from 'config' -import { parseSemVersion, promisify0 } from '../helpers/core-utils' +import { promisify0 } from '@shared/core-utils' +import { parseSemVersion } from '../helpers/core-utils' import { logger } from '../helpers/logger' // Special behaviour for config because we can reload it @@ -36,7 +37,9 @@ function checkMissedConfig () { 'transcoding.profile', 'transcoding.concurrency', 'transcoding.resolutions.0p', 'transcoding.resolutions.144p', 'transcoding.resolutions.240p', 'transcoding.resolutions.360p', 'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p', - 'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'video_studio.enabled', + 'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'transcoding.remote_runners.enabled', + 'video_studio.enabled', + 'remote_runners.stalled_jobs.vod', 'remote_runners.stalled_jobs.live', 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout', 'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user', 'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization', @@ -74,7 +77,8 @@ function checkMissedConfig () { 'live.transcoding.enabled', 'live.transcoding.threads', 'live.transcoding.profile', 'live.transcoding.resolutions.144p', 'live.transcoding.resolutions.240p', 'live.transcoding.resolutions.360p', 'live.transcoding.resolutions.480p', 'live.transcoding.resolutions.720p', 'live.transcoding.resolutions.1080p', - 'live.transcoding.resolutions.1440p', 'live.transcoding.resolutions.2160p', 'live.transcoding.always_transcode_original_resolution' + 'live.transcoding.resolutions.1440p', 'live.transcoding.resolutions.2160p', 'live.transcoding.always_transcode_original_resolution', + 'live.transcoding.remote_runners.enabled' ] const requiredAlternatives = [ diff --git a/server/initializers/config.ts b/server/initializers/config.ts index e2442213c..699dd4704 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -304,6 +304,12 @@ const CONFIG = { COUNT: config.get('feeds.comments.count') } }, + REMOTE_RUNNERS: { + STALLED_JOBS: { + LIVE: parseDurationToMs(config.get('remote_runners.stalled_jobs.live')), + VOD: parseDurationToMs(config.get('remote_runners.stalled_jobs.vod')) + } + }, ADMIN: { get EMAIL () { return config.get('admin.email') } }, @@ -359,6 +365,9 @@ const CONFIG = { }, WEBTORRENT: { get ENABLED () { return config.get('transcoding.webtorrent.enabled') } + }, + REMOTE_RUNNERS: { + get ENABLED () { return config.get('transcoding.remote_runners.enabled') } } }, LIVE: { @@ -406,6 +415,9 @@ const CONFIG = { get '1080p' () { return config.get('live.transcoding.resolutions.1080p') }, get '1440p' () { return config.get('live.transcoding.resolutions.1440p') }, get '2160p' () { return config.get('live.transcoding.resolutions.2160p') } + }, + REMOTE_RUNNERS: { + get ENABLED () { return config.get('live.transcoding.remote_runners.enabled') } } } }, diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 6cad4eb23..279e77421 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -6,6 +6,7 @@ import { randomInt, root } from '@shared/core-utils' import { AbuseState, JobType, + RunnerJobState, UserRegistrationState, VideoChannelSyncState, VideoImportState, @@ -26,7 +27,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 760 +const LAST_MIGRATION_VERSION = 765 // --------------------------------------------------------------------------- @@ -81,6 +82,10 @@ const SORTABLE_COLUMNS = { USER_REGISTRATIONS: [ 'createdAt', 'state' ], + RUNNERS: [ 'createdAt' ], + RUNNER_REGISTRATION_TOKENS: [ 'createdAt' ], + RUNNER_JOBS: [ 'updatedAt', 'createdAt', 'priority', 'state', 'progress' ], + VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'trending', 'hot', 'best' ], // Don't forget to update peertube-search-index with the same values @@ -139,6 +144,8 @@ const REMOTE_SCHEME = { WS: 'wss' } +// --------------------------------------------------------------------------- + const JOB_ATTEMPTS: { [id in JobType]: number } = { 'activitypub-http-broadcast': 1, 'activitypub-http-broadcast-parallel': 1, @@ -160,6 +167,7 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = { 'video-channel-import': 1, 'after-video-channel-import': 1, 'move-to-object-storage': 3, + 'transcoding-job-builder': 1, 'notify': 1, 'federate-video': 1 } @@ -183,6 +191,7 @@ const JOB_CONCURRENCY: { [id in Exclude { + { + const query = ` + CREATE TABLE IF NOT EXISTS "runnerRegistrationToken"( + "id" serial, + "registrationToken" varchar(255) NOT NULL, + "createdAt" timestamp with time zone NOT NULL, + "updatedAt" timestamp with time zone NOT NULL, + PRIMARY KEY ("id") + ); + ` + + await utils.sequelize.query(query, { transaction : utils.transaction }) + } + + { + const query = ` + CREATE TABLE IF NOT EXISTS "runner"( + "id" serial, + "runnerToken" varchar(255) NOT NULL, + "name" varchar(255) NOT NULL, + "description" varchar(1000), + "lastContact" timestamp with time zone NOT NULL, + "ip" varchar(255) NOT NULL, + "runnerRegistrationTokenId" integer REFERENCES "runnerRegistrationToken"("id") ON DELETE CASCADE ON UPDATE CASCADE, + "createdAt" timestamp with time zone NOT NULL, + "updatedAt" timestamp with time zone NOT NULL, + PRIMARY KEY ("id") + ); + ` + + await utils.sequelize.query(query, { transaction : utils.transaction }) + } + + { + const query = ` + CREATE TABLE IF NOT EXISTS "runnerJob"( + "id" serial, + "uuid" uuid NOT NULL, + "type" varchar(255) NOT NULL, + "payload" jsonb NOT NULL, + "privatePayload" jsonb NOT NULL, + "state" integer NOT NULL, + "failures" integer NOT NULL DEFAULT 0, + "error" varchar(5000), + "priority" integer NOT NULL, + "processingJobToken" varchar(255), + "progress" integer, + "startedAt" timestamp with time zone, + "finishedAt" timestamp with time zone, + "dependsOnRunnerJobId" integer REFERENCES "runnerJob"("id") ON DELETE CASCADE ON UPDATE CASCADE, + "runnerId" integer REFERENCES "runner"("id") ON DELETE SET NULL ON UPDATE CASCADE, + "createdAt" timestamp with time zone NOT NULL, + "updatedAt" timestamp with time zone NOT NULL, + PRIMARY KEY ("id") + ); + + + ` + + await utils.sequelize.query(query, { transaction : utils.transaction }) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 053b5d326..fc1d7e1b0 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -3,10 +3,11 @@ import { flatten } from 'lodash' import PQueue from 'p-queue' import { basename, dirname, join } from 'path' import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models' -import { uniqify } from '@shared/core-utils' +import { uniqify, uuidRegex } from '@shared/core-utils' import { sha256 } from '@shared/extra-utils' +import { getVideoStreamDimensionsInfo } from '@shared/ffmpeg' import { VideoStorage } from '@shared/models' -import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamDimensionsInfo } from '../helpers/ffmpeg' +import { getAudioStreamCodec, getVideoStreamCodec } from '../helpers/ffmpeg' import { logger } from '../helpers/logger' import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' import { generateRandomString } from '../helpers/utils' @@ -234,6 +235,16 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, // --------------------------------------------------------------------------- +async function renameVideoFileInPlaylist (playlistPath: string, newVideoFilename: string) { + const content = await readFile(playlistPath, 'utf8') + + const newContent = content.replace(new RegExp(`${uuidRegex}-\\d+-fragmented.mp4`, 'g'), newVideoFilename) + + await writeFile(playlistPath, newContent, 'utf8') +} + +// --------------------------------------------------------------------------- + function injectQueryToPlaylistUrls (content: string, queryString: string) { return content.replace(/\.(m3u8|ts|mp4)/gm, '.$1?' + queryString) } @@ -247,7 +258,8 @@ export { downloadPlaylistSegments, updateStreamingPlaylistsInfohashesIfNeeded, updatePlaylistAfterFileChange, - injectQueryToPlaylistUrls + injectQueryToPlaylistUrls, + renameVideoFileInPlaylist } // --------------------------------------------------------------------------- diff --git a/server/lib/job-queue/handlers/transcoding-job-builder.ts b/server/lib/job-queue/handlers/transcoding-job-builder.ts new file mode 100644 index 000000000..8b4a877d7 --- /dev/null +++ b/server/lib/job-queue/handlers/transcoding-job-builder.ts @@ -0,0 +1,47 @@ +import { Job } from 'bullmq' +import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job' +import { UserModel } from '@server/models/user/user' +import { VideoModel } from '@server/models/video/video' +import { VideoJobInfoModel } from '@server/models/video/video-job-info' +import { pick } from '@shared/core-utils' +import { TranscodingJobBuilderPayload } from '@shared/models' +import { logger } from '../../../helpers/logger' +import { JobQueue } from '../job-queue' + +async function processTranscodingJobBuilder (job: Job) { + const payload = job.data as TranscodingJobBuilderPayload + + logger.info('Processing transcoding job builder in job %s.', job.id) + + if (payload.optimizeJob) { + const video = await VideoModel.loadFull(payload.videoUUID) + const user = await UserModel.loadByVideoId(video.id) + const videoFile = video.getMaxQualityFile() + + await createOptimizeOrMergeAudioJobs({ + ...pick(payload.optimizeJob, [ 'isNewVideo' ]), + + video, + videoFile, + user + }) + } + + for (const job of (payload.jobs || [])) { + await JobQueue.Instance.createJob(job) + + await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode') + } + + for (const sequentialJobs of (payload.sequentialJobs || [])) { + await JobQueue.Instance.createSequentialJobFlow(...sequentialJobs) + + await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode', sequentialJobs.length) + } +} + +// --------------------------------------------------------------------------- + +export { + processTranscodingJobBuilder +} diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts index d950f6407..9a4550e4d 100644 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ b/server/lib/job-queue/handlers/video-file-import.ts @@ -10,8 +10,8 @@ import { VideoModel } from '@server/models/video/video' import { VideoFileModel } from '@server/models/video/video-file' import { MVideoFullLight } from '@server/types/models' import { getLowercaseExtension } from '@shared/core-utils' +import { getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg' import { VideoFileImportPayload, VideoStorage } from '@shared/models' -import { getVideoStreamFPS, getVideoStreamDimensionsInfo } from '../../../helpers/ffmpeg' import { logger } from '../../../helpers/logger' import { JobQueue } from '../job-queue' diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 4d361c7b9..2a063282c 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -7,15 +7,16 @@ import { isPostImportVideoAccepted } from '@server/lib/moderation' import { generateWebTorrentVideoFilename } from '@server/lib/paths' import { Hooks } from '@server/lib/plugins/hooks' import { ServerConfigManager } from '@server/lib/server-config-manager' +import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job' import { isAbleToUploadVideo } from '@server/lib/user' -import { buildMoveToObjectStorageJob, buildOptimizeOrMergeAudioJob } from '@server/lib/video' +import { buildMoveToObjectStorageJob } from '@server/lib/video' import { VideoPathManager } from '@server/lib/video-path-manager' import { buildNextVideoState } from '@server/lib/video-state' import { ThumbnailModel } from '@server/models/video/thumbnail' import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' import { getLowercaseExtension } from '@shared/core-utils' -import { isAudioFile } from '@shared/extra-utils' +import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS, isAudioFile } from '@shared/ffmpeg' import { ThumbnailType, VideoImportPayload, @@ -28,7 +29,6 @@ import { VideoResolution, VideoState } from '@shared/models' -import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS } from '../../../helpers/ffmpeg' import { logger } from '../../../helpers/logger' import { getSecureTorrentName } from '../../../helpers/utils' import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' @@ -137,7 +137,7 @@ async function processFile (downloader: () => Promise, videoImport: MVid const { resolution } = await isAudioFile(tempVideoPath, probe) ? { resolution: VideoResolution.H_NOVIDEO } - : await getVideoStreamDimensionsInfo(tempVideoPath) + : await getVideoStreamDimensionsInfo(tempVideoPath, probe) const fps = await getVideoStreamFPS(tempVideoPath, probe) const duration = await getVideoStreamDuration(tempVideoPath, probe) @@ -313,9 +313,7 @@ async function afterImportSuccess (options: { } if (video.state === VideoState.TO_TRANSCODE) { // Create transcoding jobs? - await JobQueue.Instance.createJob( - await buildOptimizeOrMergeAudioJob({ video, videoFile, user }) - ) + await createOptimizeOrMergeAudioJobs({ video, videoFile, isNewVideo: true, user }) } } diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts index 2f3a971bd..1bf43f592 100644 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ b/server/lib/job-queue/handlers/video-live-ending.ts @@ -1,25 +1,25 @@ import { Job } from 'bullmq' import { readdir, remove } from 'fs-extra' import { join } from 'path' -import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo } from '@server/helpers/ffmpeg' import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths' import { generateVideoMiniature } from '@server/lib/thumbnail' -import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/transcoding' +import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding' +import { VideoPathManager } from '@server/lib/video-path-manager' import { moveToNextState } from '@server/lib/video-state' import { VideoModel } from '@server/models/video/video' import { VideoBlacklistModel } from '@server/models/video/video-blacklist' import { VideoFileModel } from '@server/models/video/video-file' import { VideoLiveModel } from '@server/models/video/video-live' +import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' import { VideoLiveSessionModel } from '@server/models/video/video-live-session' import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models' +import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg' import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' import { logger, loggerTagsFactory } from '../../../helpers/logger' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' const lTags = loggerTagsFactory('live', 'job') @@ -224,6 +224,7 @@ async function assignReplayFilesToVideo (options: { const probe = await ffprobePromise(concatenatedTsFilePath) const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe) const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe) + const fps = await getVideoStreamFPS(concatenatedTsFilePath, probe) try { await generateHlsPlaylistResolutionFromTS({ @@ -231,6 +232,7 @@ async function assignReplayFilesToVideo (options: { inputFileMutexReleaser, concatenatedTsFilePath, resolution, + fps, isAAC: audioStream?.codec_name === 'aac' }) } catch (err) { diff --git a/server/lib/job-queue/handlers/video-studio-edition.ts b/server/lib/job-queue/handlers/video-studio-edition.ts index 3e208d83d..991d11ef1 100644 --- a/server/lib/job-queue/handlers/video-studio-edition.ts +++ b/server/lib/job-queue/handlers/video-studio-edition.ts @@ -1,15 +1,16 @@ import { Job } from 'bullmq' import { move, remove } from 'fs-extra' import { join } from 'path' -import { addIntroOutro, addWatermark, cutVideo } from '@server/helpers/ffmpeg' +import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' import { createTorrentAndSetInfoHashFromPath } from '@server/helpers/webtorrent' import { CONFIG } from '@server/initializers/config' +import { VIDEO_FILTERS } from '@server/initializers/constants' import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' import { generateWebTorrentVideoFilename } from '@server/lib/paths' +import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job' import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles' import { isAbleToUploadVideo } from '@server/lib/user' -import { buildOptimizeOrMergeAudioJob } from '@server/lib/video' -import { removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file' +import { buildFileMetadata, removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file' import { VideoPathManager } from '@server/lib/video-path-manager' import { approximateIntroOutroAdditionalSize } from '@server/lib/video-studio' import { UserModel } from '@server/models/user/user' @@ -17,15 +18,8 @@ import { VideoModel } from '@server/models/video/video' import { VideoFileModel } from '@server/models/video/video-file' import { MVideo, MVideoFile, MVideoFullLight, MVideoId, MVideoWithAllFiles } from '@server/types/models' import { getLowercaseExtension, pick } from '@shared/core-utils' -import { - buildFileMetadata, - buildUUID, - ffprobePromise, - getFileSize, - getVideoStreamDimensionsInfo, - getVideoStreamDuration, - getVideoStreamFPS -} from '@shared/extra-utils' +import { buildUUID, getFileSize } from '@shared/extra-utils' +import { FFmpegEdition, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS } from '@shared/ffmpeg' import { VideoStudioEditionPayload, VideoStudioTask, @@ -36,7 +30,6 @@ import { VideoStudioTaskWatermarkPayload } from '@shared/models' import { logger, loggerTagsFactory } from '../../../helpers/logger' -import { JobQueue } from '../job-queue' const lTagsBase = loggerTagsFactory('video-edition') @@ -102,9 +95,7 @@ async function processVideoStudioEdition (job: Job) { const user = await UserModel.loadByVideoId(video.id) - await JobQueue.Instance.createJob( - await buildOptimizeOrMergeAudioJob({ video, videoFile: newFile, user, isNewVideo: false }) - ) + await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user }) } // --------------------------------------------------------------------------- @@ -131,9 +122,9 @@ const taskProcessors: { [id in VideoStudioTask['name']]: (options: TaskProcessor } async function processTask (options: TaskProcessorOptions) { - const { video, task } = options + const { video, task, lTags } = options - logger.info('Processing %s task for video %s.', task.name, video.uuid, { task, ...options.lTags }) + logger.info('Processing %s task for video %s.', task.name, video.uuid, { task, ...lTags }) const processor = taskProcessors[options.task.name] if (!process) throw new Error('Unknown task ' + task.name) @@ -142,48 +133,53 @@ async function processTask (options: TaskProcessorOptions) { } function processAddIntroOutro (options: TaskProcessorOptions) { - const { task } = options + const { task, lTags } = options - return addIntroOutro({ + logger.debug('Will add intro/outro to the video.', { options, ...lTags }) + + return buildFFmpegEdition().addIntroOutro({ ...pick(options, [ 'inputPath', 'outputPath' ]), introOutroPath: task.options.file, type: task.name === 'add-intro' ? 'intro' - : 'outro', - - availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), - profile: CONFIG.TRANSCODING.PROFILE + : 'outro' }) } function processCut (options: TaskProcessorOptions) { - const { task } = options + const { task, lTags } = options - return cutVideo({ + logger.debug('Will cut the video.', { options, ...lTags }) + + return buildFFmpegEdition().cutVideo({ ...pick(options, [ 'inputPath', 'outputPath' ]), start: task.options.start, - end: task.options.end, - - availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), - profile: CONFIG.TRANSCODING.PROFILE + end: task.options.end }) } function processAddWatermark (options: TaskProcessorOptions) { - const { task } = options + const { task, lTags } = options - return addWatermark({ + logger.debug('Will add watermark to the video.', { options, ...lTags }) + + return buildFFmpegEdition().addWatermark({ ...pick(options, [ 'inputPath', 'outputPath' ]), watermarkPath: task.options.file, - availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), - profile: CONFIG.TRANSCODING.PROFILE + videoFilters: { + watermarkSizeRatio: VIDEO_FILTERS.WATERMARK.SIZE_RATIO, + horitonzalMarginRatio: VIDEO_FILTERS.WATERMARK.HORIZONTAL_MARGIN_RATIO, + verticalMarginRatio: VIDEO_FILTERS.WATERMARK.VERTICAL_MARGIN_RATIO + } }) } +// --------------------------------------------------------------------------- + async function buildNewFile (video: MVideoId, path: string) { const videoFile = new VideoFileModel({ extname: getLowercaseExtension(path), @@ -223,3 +219,7 @@ async function checkUserQuotaOrThrow (video: MVideoFullLight, payload: VideoStud throw new Error('Quota exceeded for this user to edit the video') } } + +function buildFFmpegEdition () { + return new FFmpegEdition(getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders())) +} diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index 3e6d23363..17b717275 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts @@ -1,13 +1,13 @@ import { Job } from 'bullmq' -import { TranscodeVODOptionsType } from '@server/helpers/ffmpeg' -import { Hooks } from '@server/lib/plugins/hooks' -import { buildTranscodingJob, getTranscodingJobPriority } from '@server/lib/video' +import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' +import { generateHlsPlaylistResolution } from '@server/lib/transcoding/hls-transcoding' +import { mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewWebTorrentResolution } from '@server/lib/transcoding/web-transcoding' +import { removeAllWebTorrentFiles } from '@server/lib/video-file' import { VideoPathManager } from '@server/lib/video-path-manager' -import { moveToFailedTranscodingState, moveToNextState } from '@server/lib/video-state' +import { moveToFailedTranscodingState } from '@server/lib/video-state' import { UserModel } from '@server/models/user/user' import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { MUser, MUserId, MVideo, MVideoFullLight, MVideoWithFile } from '@server/types/models' -import { pick } from '@shared/core-utils' +import { MUser, MUserId, MVideoFullLight } from '@server/types/models' import { HLSTranscodingPayload, MergeAudioTranscodingPayload, @@ -15,18 +15,8 @@ import { OptimizeTranscodingPayload, VideoTranscodingPayload } from '@shared/models' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg' import { logger, loggerTagsFactory } from '../../../helpers/logger' -import { CONFIG } from '../../../initializers/config' import { VideoModel } from '../../../models/video/video' -import { - generateHlsPlaylistResolution, - mergeAudioVideofile, - optimizeOriginalVideofile, - transcodeNewWebTorrentResolution -} from '../../transcoding/transcoding' -import { JobQueue } from '../job-queue' type HandlerFunction = (job: Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise @@ -84,7 +74,37 @@ export { // Job handlers // --------------------------------------------------------------------------- -async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MVideoFullLight, user: MUser) { +async function handleWebTorrentMergeAudioJob (job: Job, payload: MergeAudioTranscodingPayload, video: MVideoFullLight, user: MUserId) { + logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid)) + + await mergeAudioVideofile({ video, resolution: payload.resolution, fps: payload.fps, job }) + + logger.info('Merge audio transcoding job for %s ended.', video.uuid, lTags(video.uuid)) + + await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video }) +} + +async function handleWebTorrentOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) { + logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid)) + + await optimizeOriginalVideofile({ video, inputVideoFile: video.getMaxQualityFile(), quickTranscode: payload.quickTranscode, job }) + + logger.info('Optimize transcoding job for %s ended.', video.uuid, lTags(video.uuid)) + + await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video }) +} + +async function handleNewWebTorrentResolutionJob (job: Job, payload: NewWebTorrentResolutionTranscodingPayload, video: MVideoFullLight) { + logger.info('Handling WebTorrent transcoding job for %s.', video.uuid, lTags(video.uuid)) + + await transcodeNewWebTorrentResolution({ video, resolution: payload.resolution, fps: payload.fps, job }) + + logger.info('WebTorrent transcoding job for %s ended.', video.uuid, lTags(video.uuid)) + + await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video }) +} + +async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MVideoFullLight) { logger.info('Handling HLS transcoding job for %s.', video.uuid, lTags(video.uuid)) const videoFileInput = payload.copyCodecs @@ -104,6 +124,7 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MV videoInputPath, inputFileMutexReleaser, resolution: payload.resolution, + fps: payload.fps, copyCodecs: payload.copyCodecs, job }) @@ -114,230 +135,11 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MV logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid)) - await onHlsPlaylistGeneration(video, user, payload) -} + if (payload.deleteWebTorrentFiles === true) { + logger.info('Removing WebTorrent files of %s now we have a HLS version of it.', video.uuid, lTags(video.uuid)) -async function handleNewWebTorrentResolutionJob ( - job: Job, - payload: NewWebTorrentResolutionTranscodingPayload, - video: MVideoFullLight, - user: MUserId -) { - logger.info('Handling WebTorrent transcoding job for %s.', video.uuid, lTags(video.uuid)) - - await transcodeNewWebTorrentResolution({ video, resolution: payload.resolution, job }) - - logger.info('WebTorrent transcoding job for %s ended.', video.uuid, lTags(video.uuid)) - - await onNewWebTorrentFileResolution(video, user, payload) -} - -async function handleWebTorrentMergeAudioJob (job: Job, payload: MergeAudioTranscodingPayload, video: MVideoFullLight, user: MUserId) { - logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid)) - - await mergeAudioVideofile({ video, resolution: payload.resolution, job }) - - logger.info('Merge audio transcoding job for %s ended.', video.uuid, lTags(video.uuid)) - - await onVideoFirstWebTorrentTranscoding(video, payload, 'video', user) -} - -async function handleWebTorrentOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) { - logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid)) - - const { transcodeType } = await optimizeOriginalVideofile({ video, inputVideoFile: video.getMaxQualityFile(), job }) - - logger.info('Optimize transcoding job for %s ended.', video.uuid, lTags(video.uuid)) - - await onVideoFirstWebTorrentTranscoding(video, payload, transcodeType, user) -} - -// --------------------------------------------------------------------------- - -async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, payload: HLSTranscodingPayload) { - if (payload.isMaxQuality && payload.autoDeleteWebTorrentIfNeeded && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { - // Remove webtorrent files if not enabled - for (const file of video.VideoFiles) { - await video.removeWebTorrentFile(file) - await file.destroy() - } - - video.VideoFiles = [] - - // Create HLS new resolution jobs - await createLowerResolutionsJobs({ - video, - user, - videoFileResolution: payload.resolution, - hasAudio: payload.hasAudio, - isNewVideo: payload.isNewVideo ?? true, - type: 'hls' - }) + await removeAllWebTorrentFiles(video) } - await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') - await retryTransactionWrapper(moveToNextState, { video, isNewVideo: payload.isNewVideo }) -} - -async function onVideoFirstWebTorrentTranscoding ( - videoArg: MVideoWithFile, - payload: OptimizeTranscodingPayload | MergeAudioTranscodingPayload, - transcodeType: TranscodeVODOptionsType, - user: MUserId -) { - const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid) - - try { - // Maybe the video changed in database, refresh it - const videoDatabase = await VideoModel.loadFull(videoArg.uuid) - // Video does not exist anymore - if (!videoDatabase) return undefined - - const { resolution, audioStream } = await videoDatabase.probeMaxQualityFile() - - // Generate HLS version of the original file - const originalFileHLSPayload = { - ...payload, - - hasAudio: !!audioStream, - resolution: videoDatabase.getMaxQualityFile().resolution, - // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues - copyCodecs: transcodeType !== 'quick-transcode', - isMaxQuality: true - } - const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload) - const hasNewResolutions = await createLowerResolutionsJobs({ - video: videoDatabase, - user, - videoFileResolution: resolution, - hasAudio: !!audioStream, - type: 'webtorrent', - isNewVideo: payload.isNewVideo ?? true - }) - - await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode') - - // Move to next state if there are no other resolutions to generate - if (!hasHls && !hasNewResolutions) { - await retryTransactionWrapper(moveToNextState, { video: videoDatabase, isNewVideo: payload.isNewVideo }) - } - } finally { - mutexReleaser() - } -} - -async function onNewWebTorrentFileResolution ( - video: MVideo, - user: MUserId, - payload: NewWebTorrentResolutionTranscodingPayload | MergeAudioTranscodingPayload -) { - if (payload.createHLSIfNeeded) { - await createHlsJobIfEnabled(user, { hasAudio: true, copyCodecs: true, isMaxQuality: false, ...payload }) - } - - await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') - - await retryTransactionWrapper(moveToNextState, { video, isNewVideo: payload.isNewVideo }) -} - -// --------------------------------------------------------------------------- - -async function createHlsJobIfEnabled (user: MUserId, payload: { - videoUUID: string - resolution: number - hasAudio: boolean - copyCodecs: boolean - isMaxQuality: boolean - isNewVideo?: boolean -}) { - if (!payload || CONFIG.TRANSCODING.ENABLED !== true || CONFIG.TRANSCODING.HLS.ENABLED !== true) return false - - const jobOptions = { - priority: await getTranscodingJobPriority(user) - } - - const hlsTranscodingPayload: HLSTranscodingPayload = { - type: 'new-resolution-to-hls', - autoDeleteWebTorrentIfNeeded: true, - - ...pick(payload, [ 'videoUUID', 'resolution', 'copyCodecs', 'isMaxQuality', 'isNewVideo', 'hasAudio' ]) - } - - await JobQueue.Instance.createJob(await buildTranscodingJob(hlsTranscodingPayload, jobOptions)) - - return true -} - -async function createLowerResolutionsJobs (options: { - video: MVideoFullLight - user: MUserId - videoFileResolution: number - hasAudio: boolean - isNewVideo: boolean - type: 'hls' | 'webtorrent' -}) { - const { video, user, videoFileResolution, isNewVideo, hasAudio, type } = options - - // Create transcoding jobs if there are enabled resolutions - const resolutionsEnabled = await Hooks.wrapObject( - computeResolutionsToTranscode({ input: videoFileResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }), - 'filter:transcoding.auto.resolutions-to-transcode.result', - options - ) - - const resolutionCreated: string[] = [] - - for (const resolution of resolutionsEnabled) { - let dataInput: VideoTranscodingPayload - - if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED && type === 'webtorrent') { - // WebTorrent will create subsequent HLS job - dataInput = { - type: 'new-resolution-to-webtorrent', - videoUUID: video.uuid, - resolution, - hasAudio, - createHLSIfNeeded: true, - isNewVideo - } - - resolutionCreated.push('webtorrent-' + resolution) - } - - if (CONFIG.TRANSCODING.HLS.ENABLED && type === 'hls') { - dataInput = { - type: 'new-resolution-to-hls', - videoUUID: video.uuid, - resolution, - hasAudio, - copyCodecs: false, - isMaxQuality: false, - autoDeleteWebTorrentIfNeeded: true, - isNewVideo - } - - resolutionCreated.push('hls-' + resolution) - } - - if (!dataInput) continue - - const jobOptions = { - priority: await getTranscodingJobPriority(user) - } - - await JobQueue.Instance.createJob(await buildTranscodingJob(dataInput, jobOptions)) - } - - if (resolutionCreated.length === 0) { - logger.info('No transcoding jobs created for video %s (no resolutions).', video.uuid, lTags(video.uuid)) - - return false - } - - logger.info( - 'New resolutions %s transcoding jobs created for video %s and origin file resolution of %d.', type, video.uuid, videoFileResolution, - { resolutionCreated, ...lTags(video.uuid) } - ) - - return true + await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video }) } diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index cc6be0bd8..21bf0f226 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts @@ -31,6 +31,7 @@ import { MoveObjectStoragePayload, NotifyPayload, RefreshPayload, + TranscodingJobBuilderPayload, VideoChannelImportPayload, VideoFileImportPayload, VideoImportPayload, @@ -56,6 +57,7 @@ import { processFederateVideo } from './handlers/federate-video' import { processManageVideoTorrent } from './handlers/manage-video-torrent' import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage' import { processNotify } from './handlers/notify' +import { processTranscodingJobBuilder } from './handlers/transcoding-job-builder' import { processVideoChannelImport } from './handlers/video-channel-import' import { processVideoFileImport } from './handlers/video-file-import' import { processVideoImport } from './handlers/video-import' @@ -69,11 +71,12 @@ export type CreateJobArgument = { type: 'activitypub-http-broadcast-parallel', payload: ActivitypubHttpBroadcastPayload } | { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | - { type: 'activitypub-http-cleaner', payload: {} } | + { type: 'activitypub-cleaner', payload: {} } | { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | { type: 'video-file-import', payload: VideoFileImportPayload } | { type: 'video-transcoding', payload: VideoTranscodingPayload } | { type: 'email', payload: EmailPayload } | + { type: 'transcoding-job-builder', payload: TranscodingJobBuilderPayload } | { type: 'video-import', payload: VideoImportPayload } | { type: 'activitypub-refresher', payload: RefreshPayload } | { type: 'videos-views-stats', payload: {} } | @@ -96,28 +99,29 @@ export type CreateJobOptions = { } const handlers: { [id in JobType]: (job: Job) => Promise } = { - 'activitypub-http-broadcast': processActivityPubHttpSequentialBroadcast, - 'activitypub-http-broadcast-parallel': processActivityPubParallelHttpBroadcast, - 'activitypub-http-unicast': processActivityPubHttpUnicast, - 'activitypub-http-fetcher': processActivityPubHttpFetcher, 'activitypub-cleaner': processActivityPubCleaner, 'activitypub-follow': processActivityPubFollow, - 'video-file-import': processVideoFileImport, - 'video-transcoding': processVideoTranscoding, - 'email': processEmail, - 'video-import': processVideoImport, - 'videos-views-stats': processVideosViewsStats, + 'activitypub-http-broadcast-parallel': processActivityPubParallelHttpBroadcast, + 'activitypub-http-broadcast': processActivityPubHttpSequentialBroadcast, + 'activitypub-http-fetcher': processActivityPubHttpFetcher, + 'activitypub-http-unicast': processActivityPubHttpUnicast, 'activitypub-refresher': refreshAPObject, - 'video-live-ending': processVideoLiveEnding, 'actor-keys': processActorKeys, - 'video-redundancy': processVideoRedundancy, - 'move-to-object-storage': processMoveToObjectStorage, - 'manage-video-torrent': processManageVideoTorrent, - 'video-studio-edition': processVideoStudioEdition, - 'video-channel-import': processVideoChannelImport, 'after-video-channel-import': processAfterVideoChannelImport, + 'email': processEmail, + 'federate-video': processFederateVideo, + 'transcoding-job-builder': processTranscodingJobBuilder, + 'manage-video-torrent': processManageVideoTorrent, + 'move-to-object-storage': processMoveToObjectStorage, 'notify': processNotify, - 'federate-video': processFederateVideo + 'video-channel-import': processVideoChannelImport, + 'video-file-import': processVideoFileImport, + 'video-import': processVideoImport, + 'video-live-ending': processVideoLiveEnding, + 'video-redundancy': processVideoRedundancy, + 'video-studio-edition': processVideoStudioEdition, + 'video-transcoding': processVideoTranscoding, + 'videos-views-stats': processVideosViewsStats } const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise } = { @@ -125,28 +129,29 @@ const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise } } const jobTypes: JobType[] = [ + 'activitypub-cleaner', 'activitypub-follow', - 'activitypub-http-broadcast', 'activitypub-http-broadcast-parallel', + 'activitypub-http-broadcast', 'activitypub-http-fetcher', 'activitypub-http-unicast', - 'activitypub-cleaner', + 'activitypub-refresher', + 'actor-keys', + 'after-video-channel-import', 'email', - 'video-transcoding', + 'federate-video', + 'transcoding-job-builder', + 'manage-video-torrent', + 'move-to-object-storage', + 'notify', + 'video-channel-import', 'video-file-import', 'video-import', - 'videos-views-stats', - 'activitypub-refresher', - 'video-redundancy', - 'actor-keys', 'video-live-ending', - 'move-to-object-storage', - 'manage-video-torrent', + 'video-redundancy', 'video-studio-edition', - 'video-channel-import', - 'after-video-channel-import', - 'notify', - 'federate-video' + 'video-transcoding', + 'videos-views-stats' ] const silentFailure = new Set([ 'activitypub-http-unicast' ]) diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts index 05274955d..aa32a9d52 100644 --- a/server/lib/live/live-manager.ts +++ b/server/lib/live/live-manager.ts @@ -2,36 +2,30 @@ import { readdir, readFile } from 'fs-extra' import { createServer, Server } from 'net' import { join } from 'path' import { createServer as createServerTLS, Server as ServerTLS } from 'tls' -import { - computeResolutionsToTranscode, - ffprobePromise, - getLiveSegmentTime, - getVideoStreamBitrate, - getVideoStreamDimensionsInfo, - getVideoStreamFPS, - hasAudioStream -} from '@server/helpers/ffmpeg' import { logger, loggerTagsFactory } from '@server/helpers/logger' import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config' import { VIDEO_LIVE } from '@server/initializers/constants' +import { sequelizeTypescript } from '@server/initializers/database' import { UserModel } from '@server/models/user/user' import { VideoModel } from '@server/models/video/video' import { VideoLiveModel } from '@server/models/video/video-live' +import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' import { VideoLiveSessionModel } from '@server/models/video/video-live-session' import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' import { MVideo, MVideoLiveSession, MVideoLiveVideo, MVideoLiveVideoWithSetting } from '@server/types/models' import { pick, wait } from '@shared/core-utils' +import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream } from '@shared/ffmpeg' import { LiveVideoError, VideoState } from '@shared/models' import { federateVideoIfNeeded } from '../activitypub/videos' import { JobQueue } from '../job-queue' import { getLiveReplayBaseDirectory } from '../paths' import { PeerTubeSocket } from '../peertube-socket' import { Hooks } from '../plugins/hooks' +import { computeResolutionsToTranscode } from '../transcoding/transcoding-resolutions' import { LiveQuotaStore } from './live-quota-store' -import { cleanupAndDestroyPermanentLive } from './live-utils' +import { cleanupAndDestroyPermanentLive, getLiveSegmentTime } from './live-utils' import { MuxingSession } from './shared' -import { sequelizeTypescript } from '@server/initializers/database' -import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' +import { RunnerJobModel } from '@server/models/runner/runner-job' const NodeRtmpSession = require('node-media-server/src/node_rtmp_session') const context = require('node-media-server/src/node_core_ctx') @@ -57,7 +51,7 @@ class LiveManager { private static instance: LiveManager private readonly muxingSessions = new Map() - private readonly videoSessions = new Map() + private readonly videoSessions = new Map() private rtmpServer: Server private rtmpsServer: ServerTLS @@ -177,14 +171,19 @@ class LiveManager { return !!this.rtmpServer } - stopSessionOf (videoId: number, error: LiveVideoError | null) { - const sessionId = this.videoSessions.get(videoId) - if (!sessionId) return + stopSessionOf (videoUUID: string, error: LiveVideoError | null) { + const sessionId = this.videoSessions.get(videoUUID) + if (!sessionId) { + logger.debug('No live session to stop for video %s', videoUUID, lTags(sessionId, videoUUID)) + return + } - this.saveEndingSession(videoId, error) - .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId) })) + logger.info('Stopping live session of video %s', videoUUID, { error, ...lTags(sessionId, videoUUID) }) - this.videoSessions.delete(videoId) + this.saveEndingSession(videoUUID, error) + .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId, videoUUID) })) + + this.videoSessions.delete(videoUUID) this.abortSession(sessionId) } @@ -221,6 +220,11 @@ class LiveManager { return this.abortSession(sessionId) } + if (this.videoSessions.has(video.uuid)) { + logger.warn('Video %s has already a live session. Refusing stream %s.', video.uuid, streamKey, lTags(sessionId, video.uuid)) + return this.abortSession(sessionId) + } + // Cleanup old potential live (could happen with a permanent live) const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id) if (oldStreamingPlaylist) { @@ -229,7 +233,7 @@ class LiveManager { await cleanupAndDestroyPermanentLive(video, oldStreamingPlaylist) } - this.videoSessions.set(video.id, sessionId) + this.videoSessions.set(video.uuid, sessionId) const now = Date.now() const probe = await ffprobePromise(inputUrl) @@ -253,7 +257,7 @@ class LiveManager { ) logger.info( - 'Will mux/transcode live video of original resolution %d.', resolution, + 'Handling live video of original resolution %d.', resolution, { allResolutions, ...lTags(sessionId, video.uuid) } ) @@ -301,44 +305,44 @@ class LiveManager { muxingSession.on('live-ready', () => this.publishAndFederateLive(videoLive, localLTags)) - muxingSession.on('bad-socket-health', ({ videoId }) => { + muxingSession.on('bad-socket-health', ({ videoUUID }) => { logger.error( 'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' + ' Stopping session of video %s.', videoUUID, localLTags ) - this.stopSessionOf(videoId, LiveVideoError.BAD_SOCKET_HEALTH) + this.stopSessionOf(videoUUID, LiveVideoError.BAD_SOCKET_HEALTH) }) - muxingSession.on('duration-exceeded', ({ videoId }) => { + muxingSession.on('duration-exceeded', ({ videoUUID }) => { logger.info('Stopping session of %s: max duration exceeded.', videoUUID, localLTags) - this.stopSessionOf(videoId, LiveVideoError.DURATION_EXCEEDED) + this.stopSessionOf(videoUUID, LiveVideoError.DURATION_EXCEEDED) }) - muxingSession.on('quota-exceeded', ({ videoId }) => { + muxingSession.on('quota-exceeded', ({ videoUUID }) => { logger.info('Stopping session of %s: user quota exceeded.', videoUUID, localLTags) - this.stopSessionOf(videoId, LiveVideoError.QUOTA_EXCEEDED) + this.stopSessionOf(videoUUID, LiveVideoError.QUOTA_EXCEEDED) }) - muxingSession.on('ffmpeg-error', ({ videoId }) => { - this.stopSessionOf(videoId, LiveVideoError.FFMPEG_ERROR) + muxingSession.on('transcoding-error', ({ videoUUID }) => { + this.stopSessionOf(videoUUID, LiveVideoError.FFMPEG_ERROR) }) - muxingSession.on('ffmpeg-end', ({ videoId }) => { - this.onMuxingFFmpegEnd(videoId, sessionId) + muxingSession.on('transcoding-end', ({ videoUUID }) => { + this.onMuxingFFmpegEnd(videoUUID, sessionId) }) - muxingSession.on('after-cleanup', ({ videoId }) => { + muxingSession.on('after-cleanup', ({ videoUUID }) => { this.muxingSessions.delete(sessionId) LiveQuotaStore.Instance.removeLive(user.id, videoLive.id) muxingSession.destroy() - return this.onAfterMuxingCleanup({ videoId, liveSession }) + return this.onAfterMuxingCleanup({ videoUUID, liveSession }) .catch(err => logger.error('Error in end transmuxing.', { err, ...localLTags })) }) @@ -379,22 +383,24 @@ class LiveManager { } } - private onMuxingFFmpegEnd (videoId: number, sessionId: string) { - this.videoSessions.delete(videoId) + private onMuxingFFmpegEnd (videoUUID: string, sessionId: string) { + this.videoSessions.delete(videoUUID) - this.saveEndingSession(videoId, null) + this.saveEndingSession(videoUUID, null) .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId) })) } private async onAfterMuxingCleanup (options: { - videoId: number | string + videoUUID: string liveSession?: MVideoLiveSession cleanupNow?: boolean // Default false }) { - const { videoId, liveSession: liveSessionArg, cleanupNow = false } = options + const { videoUUID, liveSession: liveSessionArg, cleanupNow = false } = options + + logger.debug('Live of video %s has been cleaned up. Moving to its next state.', videoUUID, lTags(videoUUID)) try { - const fullVideo = await VideoModel.loadFull(videoId) + const fullVideo = await VideoModel.loadFull(videoUUID) if (!fullVideo) return const live = await VideoLiveModel.loadByVideoId(fullVideo.id) @@ -437,15 +443,17 @@ class LiveManager { await federateVideoIfNeeded(fullVideo, false) } catch (err) { - logger.error('Cannot save/federate new video state of live streaming of video %d.', videoId, { err, ...lTags(videoId + '') }) + logger.error('Cannot save/federate new video state of live streaming of video %s.', videoUUID, { err, ...lTags(videoUUID) }) } } private async handleBrokenLives () { + await RunnerJobModel.cancelAllJobs({ type: 'live-rtmp-hls-transcoding' }) + const videoUUIDs = await VideoModel.listPublishedLiveUUIDs() for (const uuid of videoUUIDs) { - await this.onAfterMuxingCleanup({ videoId: uuid, cleanupNow: true }) + await this.onAfterMuxingCleanup({ videoUUID: uuid, cleanupNow: true }) } } @@ -494,8 +502,8 @@ class LiveManager { }) } - private async saveEndingSession (videoId: number, error: LiveVideoError | null) { - const liveSession = await VideoLiveSessionModel.findCurrentSessionOf(videoId) + private async saveEndingSession (videoUUID: string, error: LiveVideoError | null) { + const liveSession = await VideoLiveSessionModel.findCurrentSessionOf(videoUUID) if (!liveSession) return liveSession.endDate = new Date() diff --git a/server/lib/live/live-segment-sha-store.ts b/server/lib/live/live-segment-sha-store.ts index 4d03754a9..251301141 100644 --- a/server/lib/live/live-segment-sha-store.ts +++ b/server/lib/live/live-segment-sha-store.ts @@ -52,7 +52,10 @@ class LiveSegmentShaStore { logger.debug('Removing live sha segment %s.', segmentPath, lTags(this.videoUUID)) if (!this.segmentsSha256.has(segmentName)) { - logger.warn('Unknown segment in files map for video %s and segment %s.', this.videoUUID, segmentPath, lTags(this.videoUUID)) + logger.warn( + 'Unknown segment in live segment hash store for video %s and segment %s.', + this.videoUUID, segmentPath, lTags(this.videoUUID) + ) return } diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts index c0dec9829..3fb3ce1ce 100644 --- a/server/lib/live/live-utils.ts +++ b/server/lib/live/live-utils.ts @@ -1,8 +1,9 @@ import { pathExists, readdir, remove } from 'fs-extra' import { basename, join } from 'path' import { logger } from '@server/helpers/logger' +import { VIDEO_LIVE } from '@server/initializers/constants' import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models' -import { VideoStorage } from '@shared/models' +import { LiveVideoLatencyMode, VideoStorage } from '@shared/models' import { listHLSFileKeysOf, removeHLSFileObjectStorageByFullKey, removeHLSObjectStorage } from '../object-storage' import { getLiveDirectory } from '../paths' @@ -37,10 +38,19 @@ async function cleanupTMPLiveFiles (video: MVideo, streamingPlaylist: MStreaming await cleanupTMPLiveFilesFromFilesystem(video) } +function getLiveSegmentTime (latencyMode: LiveVideoLatencyMode) { + if (latencyMode === LiveVideoLatencyMode.SMALL_LATENCY) { + return VIDEO_LIVE.SEGMENT_TIME_SECONDS.SMALL_LATENCY + } + + return VIDEO_LIVE.SEGMENT_TIME_SECONDS.DEFAULT_LATENCY +} + export { cleanupAndDestroyPermanentLive, cleanupUnsavedNormalLive, cleanupTMPLiveFiles, + getLiveSegmentTime, buildConcatenatedName } diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts index 2727fc4a7..f3f8fc886 100644 --- a/server/lib/live/shared/muxing-session.ts +++ b/server/lib/live/shared/muxing-session.ts @@ -1,11 +1,10 @@ import { mapSeries } from 'bluebird' import { FSWatcher, watch } from 'chokidar' -import { FfmpegCommand } from 'fluent-ffmpeg' +import { EventEmitter } from 'events' import { appendFile, ensureDir, readFile, stat } from 'fs-extra' import PQueue from 'p-queue' import { basename, join } from 'path' -import { EventEmitter } from 'stream' -import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg' +import { computeOutputFPS } from '@server/helpers/ffmpeg' import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' import { CONFIG } from '@server/initializers/config' import { MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE } from '@server/initializers/constants' @@ -20,24 +19,24 @@ import { getLiveDirectory, getLiveReplayBaseDirectory } from '../../paths' -import { VideoTranscodingProfilesManager } from '../../transcoding/default-transcoding-profiles' import { isAbleToUploadVideo } from '../../user' import { LiveQuotaStore } from '../live-quota-store' import { LiveSegmentShaStore } from '../live-segment-sha-store' -import { buildConcatenatedName } from '../live-utils' +import { buildConcatenatedName, getLiveSegmentTime } from '../live-utils' +import { AbstractTranscodingWrapper, FFmpegTranscodingWrapper, RemoteTranscodingWrapper } from './transcoding-wrapper' import memoizee = require('memoizee') interface MuxingSessionEvents { - 'live-ready': (options: { videoId: number }) => void + 'live-ready': (options: { videoUUID: string }) => void - 'bad-socket-health': (options: { videoId: number }) => void - 'duration-exceeded': (options: { videoId: number }) => void - 'quota-exceeded': (options: { videoId: number }) => void + 'bad-socket-health': (options: { videoUUID: string }) => void + 'duration-exceeded': (options: { videoUUID: string }) => void + 'quota-exceeded': (options: { videoUUID: string }) => void - 'ffmpeg-end': (options: { videoId: number }) => void - 'ffmpeg-error': (options: { videoId: number }) => void + 'transcoding-end': (options: { videoUUID: string }) => void + 'transcoding-error': (options: { videoUUID: string }) => void - 'after-cleanup': (options: { videoId: number }) => void + 'after-cleanup': (options: { videoUUID: string }) => void } declare interface MuxingSession { @@ -52,7 +51,7 @@ declare interface MuxingSession { class MuxingSession extends EventEmitter { - private ffmpegCommand: FfmpegCommand + private transcodingWrapper: AbstractTranscodingWrapper private readonly context: any private readonly user: MUserId @@ -67,7 +66,6 @@ class MuxingSession extends EventEmitter { private readonly hasAudio: boolean - private readonly videoId: number private readonly videoUUID: string private readonly saveReplay: boolean @@ -126,7 +124,6 @@ class MuxingSession extends EventEmitter { this.allResolutions = options.allResolutions - this.videoId = this.videoLive.Video.id this.videoUUID = this.videoLive.Video.uuid this.saveReplay = this.videoLive.saveReplay @@ -145,63 +142,23 @@ class MuxingSession extends EventEmitter { await this.prepareDirectories() - this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED - ? await getLiveTranscodingCommand({ - inputUrl: this.inputUrl, + this.transcodingWrapper = this.buildTranscodingWrapper() - outPath: this.outDirectory, - masterPlaylistName: this.streamingPlaylist.playlistFilename, + this.transcodingWrapper.on('end', () => this.onTranscodedEnded()) + this.transcodingWrapper.on('error', () => this.onTranscodingError()) - latencyMode: this.videoLive.latencyMode, - - resolutions: this.allResolutions, - fps: this.fps, - bitrate: this.bitrate, - ratio: this.ratio, - - hasAudio: this.hasAudio, - - availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), - profile: CONFIG.LIVE.TRANSCODING.PROFILE - }) - : getLiveMuxingCommand({ - inputUrl: this.inputUrl, - outPath: this.outDirectory, - masterPlaylistName: this.streamingPlaylist.playlistFilename, - latencyMode: this.videoLive.latencyMode - }) - - logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags()) + await this.transcodingWrapper.run() this.watchMasterFile() this.watchTSFiles() this.watchM3U8File() - - let ffmpegShellCommand: string - this.ffmpegCommand.on('start', cmdline => { - ffmpegShellCommand = cmdline - - logger.debug('Running ffmpeg command for live', { ffmpegShellCommand, ...this.lTags() }) - }) - - this.ffmpegCommand.on('error', (err, stdout, stderr) => { - this.onFFmpegError({ err, stdout, stderr, ffmpegShellCommand }) - }) - - this.ffmpegCommand.on('end', () => { - this.emit('ffmpeg-end', ({ videoId: this.videoId })) - - this.onFFmpegEnded() - }) - - this.ffmpegCommand.run() } abort () { - if (!this.ffmpegCommand) return + if (!this.transcodingWrapper) return this.aborted = true - this.ffmpegCommand.kill('SIGINT') + this.transcodingWrapper.abort() } destroy () { @@ -210,48 +167,6 @@ class MuxingSession extends EventEmitter { this.hasClientSocketInBadHealthWithCache.clear() } - private onFFmpegError (options: { - err: any - stdout: string - stderr: string - ffmpegShellCommand: string - }) { - const { err, stdout, stderr, ffmpegShellCommand } = options - - this.onFFmpegEnded() - - // Don't care that we killed the ffmpeg process - if (err?.message?.includes('Exiting normally')) return - - logger.error('Live transcoding error.', { err, stdout, stderr, ffmpegShellCommand, ...this.lTags() }) - - this.emit('ffmpeg-error', ({ videoId: this.videoId })) - } - - private onFFmpegEnded () { - logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.inputUrl, this.lTags()) - - setTimeout(() => { - // Wait latest segments generation, and close watchers - - Promise.all([ this.tsWatcher.close(), this.masterWatcher.close(), this.m3u8Watcher.close() ]) - .then(() => { - // Process remaining segments hash - for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) { - this.processSegments(this.segmentsToProcessPerPlaylist[key]) - } - }) - .catch(err => { - logger.error( - 'Cannot close watchers of %s or process remaining hash segments.', this.outDirectory, - { err, ...this.lTags() } - ) - }) - - this.emit('after-cleanup', { videoId: this.videoId }) - }, 1000) - } - private watchMasterFile () { this.masterWatcher = watch(this.outDirectory + '/' + this.streamingPlaylist.playlistFilename) @@ -272,6 +187,8 @@ class MuxingSession extends EventEmitter { this.masterPlaylistCreated = true + logger.info('Master playlist file for %s has been created', this.videoUUID, this.lTags()) + this.masterWatcher.close() .catch(err => logger.error('Cannot close master watcher of %s.', this.outDirectory, { err, ...this.lTags() })) }) @@ -318,19 +235,19 @@ class MuxingSession extends EventEmitter { this.segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ] if (this.hasClientSocketInBadHealthWithCache(this.sessionId)) { - this.emit('bad-socket-health', { videoId: this.videoId }) + this.emit('bad-socket-health', { videoUUID: this.videoUUID }) return } // Duration constraint check if (this.isDurationConstraintValid(startStreamDateTime) !== true) { - this.emit('duration-exceeded', { videoId: this.videoId }) + this.emit('duration-exceeded', { videoUUID: this.videoUUID }) return } // Check user quota if the user enabled replay saving if (await this.isQuotaExceeded(segmentPath) === true) { - this.emit('quota-exceeded', { videoId: this.videoId }) + this.emit('quota-exceeded', { videoUUID: this.videoUUID }) } } @@ -438,10 +355,40 @@ class MuxingSession extends EventEmitter { if (this.masterPlaylistCreated && !this.liveReady) { this.liveReady = true - this.emit('live-ready', { videoId: this.videoId }) + this.emit('live-ready', { videoUUID: this.videoUUID }) } } + private onTranscodingError () { + this.emit('transcoding-error', ({ videoUUID: this.videoUUID })) + } + + private onTranscodedEnded () { + this.emit('transcoding-end', ({ videoUUID: this.videoUUID })) + + logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.inputUrl, this.lTags()) + + setTimeout(() => { + // Wait latest segments generation, and close watchers + + Promise.all([ this.tsWatcher.close(), this.masterWatcher.close(), this.m3u8Watcher.close() ]) + .then(() => { + // Process remaining segments hash + for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) { + this.processSegments(this.segmentsToProcessPerPlaylist[key]) + } + }) + .catch(err => { + logger.error( + 'Cannot close watchers of %s or process remaining hash segments.', this.outDirectory, + { err, ...this.lTags() } + ) + }) + + this.emit('after-cleanup', { videoUUID: this.videoUUID }) + }, 1000) + } + private hasClientSocketInBadHealth (sessionId: string) { const rtmpSession = this.context.sessions.get(sessionId) @@ -503,6 +450,36 @@ class MuxingSession extends EventEmitter { sendToObjectStorage: CONFIG.OBJECT_STORAGE.ENABLED }) } + + private buildTranscodingWrapper () { + const options = { + streamingPlaylist: this.streamingPlaylist, + videoLive: this.videoLive, + + lTags: this.lTags, + + inputUrl: this.inputUrl, + + toTranscode: this.allResolutions.map(resolution => ({ + resolution, + fps: computeOutputFPS({ inputFPS: this.fps, resolution }) + })), + + fps: this.fps, + bitrate: this.bitrate, + ratio: this.ratio, + hasAudio: this.hasAudio, + + segmentListSize: VIDEO_LIVE.SEGMENTS_LIST_SIZE, + segmentDuration: getLiveSegmentTime(this.videoLive.latencyMode), + + outDirectory: this.outDirectory + } + + return CONFIG.LIVE.TRANSCODING.ENABLED && CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED + ? new RemoteTranscodingWrapper(options) + : new FFmpegTranscodingWrapper(options) + } } // --------------------------------------------------------------------------- diff --git a/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts b/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts new file mode 100644 index 000000000..226ba4573 --- /dev/null +++ b/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts @@ -0,0 +1,101 @@ +import EventEmitter from 'events' +import { LoggerTagsFn } from '@server/helpers/logger' +import { MStreamingPlaylistVideo, MVideoLiveVideo } from '@server/types/models' +import { LiveVideoError } from '@shared/models' + +interface TranscodingWrapperEvents { + 'end': () => void + + 'error': (options: { err: Error }) => void +} + +declare interface AbstractTranscodingWrapper { + on( + event: U, listener: TranscodingWrapperEvents[U] + ): this + + emit( + event: U, ...args: Parameters + ): boolean +} + +interface AbstractTranscodingWrapperOptions { + streamingPlaylist: MStreamingPlaylistVideo + videoLive: MVideoLiveVideo + + lTags: LoggerTagsFn + + inputUrl: string + fps: number + toTranscode: { + resolution: number + fps: number + }[] + + bitrate: number + ratio: number + hasAudio: boolean + + segmentListSize: number + segmentDuration: number + + outDirectory: string +} + +abstract class AbstractTranscodingWrapper extends EventEmitter { + protected readonly videoLive: MVideoLiveVideo + + protected readonly toTranscode: { + resolution: number + fps: number + }[] + + protected readonly inputUrl: string + protected readonly fps: number + protected readonly bitrate: number + protected readonly ratio: number + protected readonly hasAudio: boolean + + protected readonly segmentListSize: number + protected readonly segmentDuration: number + + protected readonly videoUUID: string + + protected readonly outDirectory: string + + protected readonly lTags: LoggerTagsFn + + protected readonly streamingPlaylist: MStreamingPlaylistVideo + + constructor (options: AbstractTranscodingWrapperOptions) { + super() + + this.lTags = options.lTags + + this.videoLive = options.videoLive + this.videoUUID = options.videoLive.Video.uuid + this.streamingPlaylist = options.streamingPlaylist + + this.inputUrl = options.inputUrl + this.fps = options.fps + this.toTranscode = options.toTranscode + + this.bitrate = options.bitrate + this.ratio = options.ratio + this.hasAudio = options.hasAudio + + this.segmentListSize = options.segmentListSize + this.segmentDuration = options.segmentDuration + + this.outDirectory = options.outDirectory + } + + abstract run (): Promise + + abstract abort (error?: LiveVideoError): void +} + +export { + AbstractTranscodingWrapper, + AbstractTranscodingWrapperOptions +} diff --git a/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts b/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts new file mode 100644 index 000000000..1f4c12bd4 --- /dev/null +++ b/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts @@ -0,0 +1,95 @@ +import { FfmpegCommand } from 'fluent-ffmpeg' +import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' +import { logger } from '@server/helpers/logger' +import { CONFIG } from '@server/initializers/config' +import { VIDEO_LIVE } from '@server/initializers/constants' +import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles' +import { FFmpegLive } from '@shared/ffmpeg' +import { getLiveSegmentTime } from '../../live-utils' +import { AbstractTranscodingWrapper } from './abstract-transcoding-wrapper' + +export class FFmpegTranscodingWrapper extends AbstractTranscodingWrapper { + private ffmpegCommand: FfmpegCommand + private ended = false + + async run () { + this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED + ? await this.buildFFmpegLive().getLiveTranscodingCommand({ + inputUrl: this.inputUrl, + + outPath: this.outDirectory, + masterPlaylistName: this.streamingPlaylist.playlistFilename, + + segmentListSize: this.segmentListSize, + segmentDuration: this.segmentDuration, + + toTranscode: this.toTranscode, + + bitrate: this.bitrate, + ratio: this.ratio, + + hasAudio: this.hasAudio + }) + : this.buildFFmpegLive().getLiveMuxingCommand({ + inputUrl: this.inputUrl, + outPath: this.outDirectory, + + masterPlaylistName: this.streamingPlaylist.playlistFilename, + + segmentListSize: VIDEO_LIVE.SEGMENTS_LIST_SIZE, + segmentDuration: getLiveSegmentTime(this.videoLive.latencyMode) + }) + + logger.info('Running local live muxing/transcoding for %s.', this.videoUUID, this.lTags()) + + this.ffmpegCommand.run() + + let ffmpegShellCommand: string + this.ffmpegCommand.on('start', cmdline => { + ffmpegShellCommand = cmdline + + logger.debug('Running ffmpeg command for live', { ffmpegShellCommand, ...this.lTags() }) + }) + + this.ffmpegCommand.on('error', (err, stdout, stderr) => { + this.onFFmpegError({ err, stdout, stderr, ffmpegShellCommand }) + }) + + this.ffmpegCommand.on('end', () => { + this.onFFmpegEnded() + }) + + this.ffmpegCommand.run() + } + + abort () { + // Nothing to do, ffmpeg will automatically exit + } + + private onFFmpegError (options: { + err: any + stdout: string + stderr: string + ffmpegShellCommand: string + }) { + const { err, stdout, stderr, ffmpegShellCommand } = options + + // Don't care that we killed the ffmpeg process + if (err?.message?.includes('Exiting normally')) return + + logger.error('FFmpeg transcoding error.', { err, stdout, stderr, ffmpegShellCommand, ...this.lTags() }) + + this.emit('error', { err }) + } + + private onFFmpegEnded () { + if (this.ended) return + + this.ended = true + this.emit('end') + } + + private buildFFmpegLive () { + return new FFmpegLive(getFFmpegCommandWrapperOptions('live', VideoTranscodingProfilesManager.Instance.getAvailableEncoders())) + } +} diff --git a/server/lib/live/shared/transcoding-wrapper/index.ts b/server/lib/live/shared/transcoding-wrapper/index.ts new file mode 100644 index 000000000..ae28fa1ca --- /dev/null +++ b/server/lib/live/shared/transcoding-wrapper/index.ts @@ -0,0 +1,3 @@ +export * from './abstract-transcoding-wrapper' +export * from './ffmpeg-transcoding-wrapper' +export * from './remote-transcoding-wrapper' diff --git a/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts b/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts new file mode 100644 index 000000000..345eaf442 --- /dev/null +++ b/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts @@ -0,0 +1,20 @@ +import { LiveRTMPHLSTranscodingJobHandler } from '@server/lib/runners' +import { AbstractTranscodingWrapper } from './abstract-transcoding-wrapper' + +export class RemoteTranscodingWrapper extends AbstractTranscodingWrapper { + async run () { + await new LiveRTMPHLSTranscodingJobHandler().create({ + rtmpUrl: this.inputUrl, + toTranscode: this.toTranscode, + video: this.videoLive.Video, + outputDirectory: this.outDirectory, + playlist: this.streamingPlaylist, + segmentListSize: this.segmentListSize, + segmentDuration: this.segmentDuration + }) + } + + abort () { + this.emit('end') + } +} diff --git a/server/lib/object-storage/index.ts b/server/lib/object-storage/index.ts index 8b413a40e..6525f8dfb 100644 --- a/server/lib/object-storage/index.ts +++ b/server/lib/object-storage/index.ts @@ -1,3 +1,4 @@ export * from './keys' +export * from './proxy' export * from './urls' export * from './videos' diff --git a/server/lib/object-storage/proxy.ts b/server/lib/object-storage/proxy.ts new file mode 100644 index 000000000..c782a8a25 --- /dev/null +++ b/server/lib/object-storage/proxy.ts @@ -0,0 +1,97 @@ +import express from 'express' +import { PassThrough, pipeline } from 'stream' +import { GetObjectCommandOutput } from '@aws-sdk/client-s3' +import { buildReinjectVideoFileTokenQuery } from '@server/controllers/shared/m3u8-playlist' +import { logger } from '@server/helpers/logger' +import { StreamReplacer } from '@server/helpers/stream-replacer' +import { MStreamingPlaylist, MVideo } from '@server/types/models' +import { HttpStatusCode } from '@shared/models' +import { injectQueryToPlaylistUrls } from '../hls' +import { getHLSFileReadStream, getWebTorrentFileReadStream } from './videos' + +export async function proxifyWebTorrentFile (options: { + req: express.Request + res: express.Response + filename: string +}) { + const { req, res, filename } = options + + logger.debug('Proxifying WebTorrent file %s from object storage.', filename) + + try { + const { response: s3Response, stream } = await getWebTorrentFileReadStream({ + filename, + rangeHeader: req.header('range') + }) + + setS3Headers(res, s3Response) + + return stream.pipe(res) + } catch (err) { + return handleObjectStorageFailure(res, err) + } +} + +export async function proxifyHLS (options: { + req: express.Request + res: express.Response + playlist: MStreamingPlaylist + video: MVideo + filename: string + reinjectVideoFileToken: boolean +}) { + const { req, res, playlist, video, filename, reinjectVideoFileToken } = options + + logger.debug('Proxifying HLS file %s from object storage.', filename) + + try { + const { response: s3Response, stream } = await getHLSFileReadStream({ + playlist: playlist.withVideo(video), + filename, + rangeHeader: req.header('range') + }) + + setS3Headers(res, s3Response) + + const streamReplacer = reinjectVideoFileToken + ? new StreamReplacer(line => injectQueryToPlaylistUrls(line, buildReinjectVideoFileTokenQuery(req, filename.endsWith('master.m3u8')))) + : new PassThrough() + + return pipeline( + stream, + streamReplacer, + res, + err => { + if (!err) return + + handleObjectStorageFailure(res, err) + } + ) + } catch (err) { + return handleObjectStorageFailure(res, err) + } +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +function handleObjectStorageFailure (res: express.Response, err: Error) { + if (err.name === 'NoSuchKey') { + logger.debug('Could not find key in object storage to proxify private HLS video file.', { err }) + return res.sendStatus(HttpStatusCode.NOT_FOUND_404) + } + + return res.fail({ + status: HttpStatusCode.INTERNAL_SERVER_ERROR_500, + message: err.message, + type: err.name + }) +} + +function setS3Headers (res: express.Response, s3Response: GetObjectCommandOutput) { + if (s3Response.$metadata.httpStatusCode === HttpStatusCode.PARTIAL_CONTENT_206) { + res.setHeader('Content-Range', s3Response.ContentRange) + res.status(HttpStatusCode.PARTIAL_CONTENT_206) + } +} diff --git a/server/lib/peertube-socket.ts b/server/lib/peertube-socket.ts index 0398ca61d..ded7e9743 100644 --- a/server/lib/peertube-socket.ts +++ b/server/lib/peertube-socket.ts @@ -2,10 +2,12 @@ import { Server as HTTPServer } from 'http' import { Namespace, Server as SocketServer, Socket } from 'socket.io' import { isIdValid } from '@server/helpers/custom-validators/misc' import { MVideo, MVideoImmutable } from '@server/types/models' +import { MRunner } from '@server/types/models/runners' import { UserNotificationModelForApi } from '@server/types/models/user' import { LiveVideoEventPayload, LiveVideoEventType } from '@shared/models' import { logger } from '../helpers/logger' -import { authenticateSocket } from '../middlewares' +import { authenticateRunnerSocket, authenticateSocket } from '../middlewares' +import { Debounce } from '@server/helpers/debounce' class PeerTubeSocket { @@ -13,6 +15,7 @@ class PeerTubeSocket { private userNotificationSockets: { [ userId: number ]: Socket[] } = {} private liveVideosNamespace: Namespace + private readonly runnerSockets = new Set() private constructor () {} @@ -24,7 +27,7 @@ class PeerTubeSocket { .on('connection', socket => { const userId = socket.handshake.auth.user.id - logger.debug('User %d connected on the notification system.', userId) + logger.debug('User %d connected to the notification system.', userId) if (!this.userNotificationSockets[userId]) this.userNotificationSockets[userId] = [] @@ -53,6 +56,22 @@ class PeerTubeSocket { socket.leave(videoId) }) }) + + io.of('/runners') + .use(authenticateRunnerSocket) + .on('connection', socket => { + const runner: MRunner = socket.handshake.auth.runner + + logger.debug(`New runner "${runner.name}" connected to the notification system.`) + + this.runnerSockets.add(socket) + + socket.on('disconnect', () => { + logger.debug(`Runner "${runner.name}" disconnected from the notification system.`) + + this.runnerSockets.delete(socket) + }) + }) } sendNotification (userId: number, notification: UserNotificationModelForApi) { @@ -89,6 +108,15 @@ class PeerTubeSocket { .emit(type, data) } + @Debounce({ timeoutMS: 1000 }) + sendAvailableJobsPingToRunners () { + logger.debug(`Sending available-jobs notification to ${this.runnerSockets.size} runner sockets`) + + for (const runners of this.runnerSockets) { + runners.emit('available-jobs') + } + } + static get Instance () { return this.instance || (this.instance = new this()) } diff --git a/server/lib/plugins/plugin-helpers-builder.ts b/server/lib/plugins/plugin-helpers-builder.ts index 66383af46..92ef87cca 100644 --- a/server/lib/plugins/plugin-helpers-builder.ts +++ b/server/lib/plugins/plugin-helpers-builder.ts @@ -1,7 +1,6 @@ import express from 'express' import { Server } from 'http' import { join } from 'path' -import { ffprobePromise } from '@server/helpers/ffmpeg/ffprobe-utils' import { buildLogger } from '@server/helpers/logger' import { CONFIG } from '@server/initializers/config' import { WEBSERVER } from '@server/initializers/constants' @@ -16,6 +15,7 @@ import { VideoModel } from '@server/models/video/video' import { VideoBlacklistModel } from '@server/models/video/video-blacklist' import { MPlugin, MVideo, UserNotificationModelForApi } from '@server/types/models' import { PeerTubeHelpers } from '@server/types/plugins' +import { ffprobePromise } from '@shared/ffmpeg' import { VideoBlacklistCreate, VideoStorage } from '@shared/models' import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist' import { PeerTubeSocket } from '../peertube-socket' diff --git a/server/lib/runners/index.ts b/server/lib/runners/index.ts new file mode 100644 index 000000000..a737c7b59 --- /dev/null +++ b/server/lib/runners/index.ts @@ -0,0 +1,3 @@ +export * from './job-handlers' +export * from './runner' +export * from './runner-urls' diff --git a/server/lib/runners/job-handlers/abstract-job-handler.ts b/server/lib/runners/job-handlers/abstract-job-handler.ts new file mode 100644 index 000000000..73fc14574 --- /dev/null +++ b/server/lib/runners/job-handlers/abstract-job-handler.ts @@ -0,0 +1,271 @@ +import { retryTransactionWrapper } from '@server/helpers/database-utils' +import { logger, loggerTagsFactory } from '@server/helpers/logger' +import { RUNNER_JOBS } from '@server/initializers/constants' +import { sequelizeTypescript } from '@server/initializers/database' +import { PeerTubeSocket } from '@server/lib/peertube-socket' +import { RunnerJobModel } from '@server/models/runner/runner-job' +import { setAsUpdated } from '@server/models/shared' +import { MRunnerJob } from '@server/types/models/runners' +import { pick } from '@shared/core-utils' +import { + RunnerJobLiveRTMPHLSTranscodingPayload, + RunnerJobLiveRTMPHLSTranscodingPrivatePayload, + RunnerJobState, + RunnerJobSuccessPayload, + RunnerJobType, + RunnerJobUpdatePayload, + RunnerJobVODAudioMergeTranscodingPayload, + RunnerJobVODAudioMergeTranscodingPrivatePayload, + RunnerJobVODHLSTranscodingPayload, + RunnerJobVODHLSTranscodingPrivatePayload, + RunnerJobVODWebVideoTranscodingPayload, + RunnerJobVODWebVideoTranscodingPrivatePayload +} from '@shared/models' + +type CreateRunnerJobArg = + { + type: Extract + payload: RunnerJobVODWebVideoTranscodingPayload + privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload + } | + { + type: Extract + payload: RunnerJobVODHLSTranscodingPayload + privatePayload: RunnerJobVODHLSTranscodingPrivatePayload + } | + { + type: Extract + payload: RunnerJobVODAudioMergeTranscodingPayload + privatePayload: RunnerJobVODAudioMergeTranscodingPrivatePayload + } | + { + type: Extract + payload: RunnerJobLiveRTMPHLSTranscodingPayload + privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload + } + +export abstract class AbstractJobHandler { + + protected readonly lTags = loggerTagsFactory('runner') + + // --------------------------------------------------------------------------- + + abstract create (options: C): Promise + + protected async createRunnerJob (options: CreateRunnerJobArg & { + jobUUID: string + priority: number + dependsOnRunnerJob?: MRunnerJob + }): Promise { + const { priority, dependsOnRunnerJob } = options + + const runnerJob = new RunnerJobModel({ + ...pick(options, [ 'type', 'payload', 'privatePayload' ]), + + uuid: options.jobUUID, + + state: dependsOnRunnerJob + ? RunnerJobState.WAITING_FOR_PARENT_JOB + : RunnerJobState.PENDING, + + dependsOnRunnerJobId: dependsOnRunnerJob?.id, + + priority + }) + + const job = await sequelizeTypescript.transaction(async transaction => { + return runnerJob.save({ transaction }) + }) + + if (runnerJob.state === RunnerJobState.PENDING) { + PeerTubeSocket.Instance.sendAvailableJobsPingToRunners() + } + + return job + } + + // --------------------------------------------------------------------------- + + protected abstract specificUpdate (options: { + runnerJob: MRunnerJob + updatePayload?: U + }): Promise | void + + async update (options: { + runnerJob: MRunnerJob + progress?: number + updatePayload?: U + }) { + const { runnerJob, progress } = options + + await this.specificUpdate(options) + + if (progress) runnerJob.progress = progress + + await retryTransactionWrapper(() => { + return sequelizeTypescript.transaction(async transaction => { + if (runnerJob.changed()) { + return runnerJob.save({ transaction }) + } + + // Don't update the job too often + if (new Date().getTime() - runnerJob.updatedAt.getTime() > 2000) { + await setAsUpdated({ sequelize: sequelizeTypescript, table: 'runnerJob', id: runnerJob.id, transaction }) + } + }) + }) + } + + // --------------------------------------------------------------------------- + + async complete (options: { + runnerJob: MRunnerJob + resultPayload: S + }) { + const { runnerJob } = options + + try { + await this.specificComplete(options) + + runnerJob.state = RunnerJobState.COMPLETED + } catch (err) { + logger.error('Cannot complete runner job', { err, ...this.lTags(runnerJob.id, runnerJob.type) }) + + runnerJob.state = RunnerJobState.ERRORED + runnerJob.error = err.message + } + + runnerJob.progress = null + runnerJob.finishedAt = new Date() + + await retryTransactionWrapper(() => { + return sequelizeTypescript.transaction(async transaction => { + await runnerJob.save({ transaction }) + }) + }) + + const [ affectedCount ] = await RunnerJobModel.updateDependantJobsOf(runnerJob) + + if (affectedCount !== 0) PeerTubeSocket.Instance.sendAvailableJobsPingToRunners() + } + + protected abstract specificComplete (options: { + runnerJob: MRunnerJob + resultPayload: S + }): Promise | void + + // --------------------------------------------------------------------------- + + async cancel (options: { + runnerJob: MRunnerJob + fromParent?: boolean + }) { + const { runnerJob, fromParent } = options + + await this.specificCancel(options) + + const cancelState = fromParent + ? RunnerJobState.PARENT_CANCELLED + : RunnerJobState.CANCELLED + + runnerJob.setToErrorOrCancel(cancelState) + + await retryTransactionWrapper(() => { + return sequelizeTypescript.transaction(async transaction => { + await runnerJob.save({ transaction }) + }) + }) + + const children = await RunnerJobModel.listChildrenOf(runnerJob) + for (const child of children) { + logger.info(`Cancelling child job ${child.uuid} of ${runnerJob.uuid} because of parent cancel`, this.lTags(child.uuid)) + + await this.cancel({ runnerJob: child, fromParent: true }) + } + } + + protected abstract specificCancel (options: { + runnerJob: MRunnerJob + }): Promise | void + + // --------------------------------------------------------------------------- + + protected abstract isAbortSupported (): boolean + + async abort (options: { + runnerJob: MRunnerJob + }) { + const { runnerJob } = options + + if (this.isAbortSupported() !== true) { + return this.error({ runnerJob, message: 'Job has been aborted but it is not supported by this job type' }) + } + + await this.specificAbort(options) + + runnerJob.resetToPending() + + await retryTransactionWrapper(() => { + return sequelizeTypescript.transaction(async transaction => { + await runnerJob.save({ transaction }) + }) + }) + } + + protected setAbortState (runnerJob: MRunnerJob) { + runnerJob.resetToPending() + } + + protected abstract specificAbort (options: { + runnerJob: MRunnerJob + }): Promise | void + + // --------------------------------------------------------------------------- + + async error (options: { + runnerJob: MRunnerJob + message: string + fromParent?: boolean + }) { + const { runnerJob, message, fromParent } = options + + const errorState = fromParent + ? RunnerJobState.PARENT_ERRORED + : RunnerJobState.ERRORED + + const nextState = errorState === RunnerJobState.ERRORED && this.isAbortSupported() && runnerJob.failures < RUNNER_JOBS.MAX_FAILURES + ? RunnerJobState.PENDING + : errorState + + await this.specificError({ ...options, nextState }) + + if (nextState === errorState) { + runnerJob.setToErrorOrCancel(nextState) + runnerJob.error = message + } else { + runnerJob.resetToPending() + } + + await retryTransactionWrapper(() => { + return sequelizeTypescript.transaction(async transaction => { + await runnerJob.save({ transaction }) + }) + }) + + if (runnerJob.state === errorState) { + const children = await RunnerJobModel.listChildrenOf(runnerJob) + + for (const child of children) { + logger.info(`Erroring child job ${child.uuid} of ${runnerJob.uuid} because of parent error`, this.lTags(child.uuid)) + + await this.error({ runnerJob: child, message: 'Parent error', fromParent: true }) + } + } + } + + protected abstract specificError (options: { + runnerJob: MRunnerJob + message: string + nextState: RunnerJobState + }): Promise | void +} diff --git a/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts b/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts new file mode 100644 index 000000000..517645848 --- /dev/null +++ b/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts @@ -0,0 +1,71 @@ + +import { retryTransactionWrapper } from '@server/helpers/database-utils' +import { logger } from '@server/helpers/logger' +import { moveToFailedTranscodingState, moveToNextState } from '@server/lib/video-state' +import { VideoJobInfoModel } from '@server/models/video/video-job-info' +import { MRunnerJob } from '@server/types/models/runners' +import { + LiveRTMPHLSTranscodingUpdatePayload, + RunnerJobSuccessPayload, + RunnerJobUpdatePayload, + RunnerJobVODPrivatePayload +} from '@shared/models' +import { AbstractJobHandler } from './abstract-job-handler' +import { loadTranscodingRunnerVideo } from './shared' + +// eslint-disable-next-line max-len +export abstract class AbstractVODTranscodingJobHandler extends AbstractJobHandler { + + // --------------------------------------------------------------------------- + + protected isAbortSupported () { + return true + } + + protected specificUpdate (_options: { + runnerJob: MRunnerJob + updatePayload?: LiveRTMPHLSTranscodingUpdatePayload + }) { + // empty + } + + protected specificAbort (_options: { + runnerJob: MRunnerJob + }) { + // empty + } + + protected async specificError (options: { + runnerJob: MRunnerJob + }) { + const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags) + if (!video) return + + await moveToFailedTranscodingState(video) + + await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') + } + + protected async specificCancel (options: { + runnerJob: MRunnerJob + }) { + const { runnerJob } = options + + const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags) + if (!video) return + + const pending = await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') + + logger.debug(`Pending transcode decreased to ${pending} after cancel`, this.lTags(video.uuid)) + + if (pending === 0) { + logger.info( + `All transcoding jobs of ${video.uuid} have been processed or canceled, moving it to its next state`, + this.lTags(video.uuid) + ) + + const privatePayload = runnerJob.privatePayload as RunnerJobVODPrivatePayload + await retryTransactionWrapper(moveToNextState, { video, isNewVideo: privatePayload.isNewVideo }) + } + } +} diff --git a/server/lib/runners/job-handlers/index.ts b/server/lib/runners/job-handlers/index.ts new file mode 100644 index 000000000..0fca72b9a --- /dev/null +++ b/server/lib/runners/job-handlers/index.ts @@ -0,0 +1,6 @@ +export * from './abstract-job-handler' +export * from './live-rtmp-hls-transcoding-job-handler' +export * from './vod-audio-merge-transcoding-job-handler' +export * from './vod-hls-transcoding-job-handler' +export * from './vod-web-video-transcoding-job-handler' +export * from './runner-job-handlers' diff --git a/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts b/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts new file mode 100644 index 000000000..c3d0e427d --- /dev/null +++ b/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts @@ -0,0 +1,170 @@ +import { move, remove } from 'fs-extra' +import { join } from 'path' +import { logger } from '@server/helpers/logger' +import { JOB_PRIORITY } from '@server/initializers/constants' +import { LiveManager } from '@server/lib/live' +import { MStreamingPlaylist, MVideo } from '@server/types/models' +import { MRunnerJob } from '@server/types/models/runners' +import { buildUUID } from '@shared/extra-utils' +import { + LiveRTMPHLSTranscodingSuccess, + LiveRTMPHLSTranscodingUpdatePayload, + LiveVideoError, + RunnerJobLiveRTMPHLSTranscodingPayload, + RunnerJobLiveRTMPHLSTranscodingPrivatePayload, + RunnerJobState +} from '@shared/models' +import { AbstractJobHandler } from './abstract-job-handler' + +type CreateOptions = { + video: MVideo + playlist: MStreamingPlaylist + + rtmpUrl: string + + toTranscode: { + resolution: number + fps: number + }[] + + segmentListSize: number + segmentDuration: number + + outputDirectory: string +} + +// eslint-disable-next-line max-len +export class LiveRTMPHLSTranscodingJobHandler extends AbstractJobHandler { + + async create (options: CreateOptions) { + const { video, rtmpUrl, toTranscode, playlist, segmentDuration, segmentListSize, outputDirectory } = options + + const jobUUID = buildUUID() + const payload: RunnerJobLiveRTMPHLSTranscodingPayload = { + input: { + rtmpUrl + }, + output: { + toTranscode, + segmentListSize, + segmentDuration + } + } + + const privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload = { + videoUUID: video.uuid, + masterPlaylistName: playlist.playlistFilename, + outputDirectory + } + + const job = await this.createRunnerJob({ + type: 'live-rtmp-hls-transcoding', + jobUUID, + payload, + privatePayload, + priority: JOB_PRIORITY.TRANSCODING + }) + + return job + } + + // --------------------------------------------------------------------------- + + async specificUpdate (options: { + runnerJob: MRunnerJob + updatePayload: LiveRTMPHLSTranscodingUpdatePayload + }) { + const { runnerJob, updatePayload } = options + + const privatePayload = runnerJob.privatePayload as RunnerJobLiveRTMPHLSTranscodingPrivatePayload + const outputDirectory = privatePayload.outputDirectory + const videoUUID = privatePayload.videoUUID + + if (updatePayload.type === 'add-chunk') { + await move( + updatePayload.videoChunkFile as string, + join(outputDirectory, updatePayload.videoChunkFilename), + { overwrite: true } + ) + } else if (updatePayload.type === 'remove-chunk') { + await remove(join(outputDirectory, updatePayload.videoChunkFilename)) + } + + if (updatePayload.resolutionPlaylistFile && updatePayload.resolutionPlaylistFilename) { + await move( + updatePayload.resolutionPlaylistFile as string, + join(outputDirectory, updatePayload.resolutionPlaylistFilename), + { overwrite: true } + ) + } + + if (updatePayload.masterPlaylistFile) { + await move(updatePayload.masterPlaylistFile as string, join(outputDirectory, privatePayload.masterPlaylistName), { overwrite: true }) + } + + logger.info( + 'Runner live RTMP to HLS job %s for %s updated.', + runnerJob.uuid, videoUUID, { updatePayload, ...this.lTags(videoUUID, runnerJob.uuid) } + ) + } + + // --------------------------------------------------------------------------- + + protected specificComplete (options: { + runnerJob: MRunnerJob + }) { + return this.stopLive({ + runnerJob: options.runnerJob, + type: 'ended' + }) + } + + // --------------------------------------------------------------------------- + + protected isAbortSupported () { + return false + } + + protected specificAbort () { + throw new Error('Not implemented') + } + + protected specificError (options: { + runnerJob: MRunnerJob + nextState: RunnerJobState + }) { + return this.stopLive({ + runnerJob: options.runnerJob, + type: 'errored' + }) + } + + protected specificCancel (options: { + runnerJob: MRunnerJob + }) { + return this.stopLive({ + runnerJob: options.runnerJob, + type: 'cancelled' + }) + } + + private stopLive (options: { + runnerJob: MRunnerJob + type: 'ended' | 'errored' | 'cancelled' + }) { + const { runnerJob, type } = options + + const privatePayload = runnerJob.privatePayload as RunnerJobLiveRTMPHLSTranscodingPrivatePayload + const videoUUID = privatePayload.videoUUID + + const errorType = { + ended: null, + errored: LiveVideoError.RUNNER_JOB_ERROR, + cancelled: LiveVideoError.RUNNER_JOB_CANCEL + } + + LiveManager.Instance.stopSessionOf(privatePayload.videoUUID, errorType[type]) + + logger.info('Runner live RTMP to HLS job %s for video %s %s.', runnerJob.uuid, videoUUID, type, this.lTags(runnerJob.uuid, videoUUID)) + } +} diff --git a/server/lib/runners/job-handlers/runner-job-handlers.ts b/server/lib/runners/job-handlers/runner-job-handlers.ts new file mode 100644 index 000000000..7bad1bc77 --- /dev/null +++ b/server/lib/runners/job-handlers/runner-job-handlers.ts @@ -0,0 +1,18 @@ +import { MRunnerJob } from '@server/types/models/runners' +import { RunnerJobSuccessPayload, RunnerJobType, RunnerJobUpdatePayload } from '@shared/models' +import { AbstractJobHandler } from './abstract-job-handler' +import { LiveRTMPHLSTranscodingJobHandler } from './live-rtmp-hls-transcoding-job-handler' +import { VODAudioMergeTranscodingJobHandler } from './vod-audio-merge-transcoding-job-handler' +import { VODHLSTranscodingJobHandler } from './vod-hls-transcoding-job-handler' +import { VODWebVideoTranscodingJobHandler } from './vod-web-video-transcoding-job-handler' + +const processors: Record AbstractJobHandler> = { + 'vod-web-video-transcoding': VODWebVideoTranscodingJobHandler, + 'vod-hls-transcoding': VODHLSTranscodingJobHandler, + 'vod-audio-merge-transcoding': VODAudioMergeTranscodingJobHandler, + 'live-rtmp-hls-transcoding': LiveRTMPHLSTranscodingJobHandler +} + +export function getRunnerJobHandlerClass (job: MRunnerJob) { + return processors[job.type] +} diff --git a/server/lib/runners/job-handlers/shared/index.ts b/server/lib/runners/job-handlers/shared/index.ts new file mode 100644 index 000000000..348273ae2 --- /dev/null +++ b/server/lib/runners/job-handlers/shared/index.ts @@ -0,0 +1 @@ +export * from './vod-helpers' diff --git a/server/lib/runners/job-handlers/shared/vod-helpers.ts b/server/lib/runners/job-handlers/shared/vod-helpers.ts new file mode 100644 index 000000000..93ae89ff8 --- /dev/null +++ b/server/lib/runners/job-handlers/shared/vod-helpers.ts @@ -0,0 +1,44 @@ +import { move } from 'fs-extra' +import { dirname, join } from 'path' +import { logger, LoggerTagsFn } from '@server/helpers/logger' +import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' +import { onWebTorrentVideoFileTranscoding } from '@server/lib/transcoding/web-transcoding' +import { buildNewFile } from '@server/lib/video-file' +import { VideoModel } from '@server/models/video/video' +import { MVideoFullLight } from '@server/types/models' +import { MRunnerJob } from '@server/types/models/runners' +import { RunnerJobVODAudioMergeTranscodingPrivatePayload, RunnerJobVODWebVideoTranscodingPrivatePayload } from '@shared/models' + +export async function onVODWebVideoOrAudioMergeTranscodingJob (options: { + video: MVideoFullLight + videoFilePath: string + privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload | RunnerJobVODAudioMergeTranscodingPrivatePayload +}) { + const { video, videoFilePath, privatePayload } = options + + const videoFile = await buildNewFile({ path: videoFilePath, mode: 'web-video' }) + videoFile.videoId = video.id + + const newVideoFilePath = join(dirname(videoFilePath), videoFile.filename) + await move(videoFilePath, newVideoFilePath) + + await onWebTorrentVideoFileTranscoding({ + video, + videoFile, + videoOutputPath: newVideoFilePath + }) + + await onTranscodingEnded({ isNewVideo: privatePayload.isNewVideo, moveVideoToNextState: true, video }) +} + +export async function loadTranscodingRunnerVideo (runnerJob: MRunnerJob, lTags: LoggerTagsFn) { + const videoUUID = runnerJob.privatePayload.videoUUID + + const video = await VideoModel.loadFull(videoUUID) + if (!video) { + logger.info('Video %s does not exist anymore after transcoding runner job.', videoUUID, lTags(videoUUID)) + return undefined + } + + return video +} diff --git a/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts new file mode 100644 index 000000000..a7b33f87e --- /dev/null +++ b/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts @@ -0,0 +1,97 @@ +import { pick } from 'lodash' +import { logger } from '@server/helpers/logger' +import { VideoJobInfoModel } from '@server/models/video/video-job-info' +import { MVideo } from '@server/types/models' +import { MRunnerJob } from '@server/types/models/runners' +import { buildUUID } from '@shared/extra-utils' +import { getVideoStreamDuration } from '@shared/ffmpeg' +import { + RunnerJobUpdatePayload, + RunnerJobVODAudioMergeTranscodingPayload, + RunnerJobVODWebVideoTranscodingPrivatePayload, + VODAudioMergeTranscodingSuccess +} from '@shared/models' +import { generateRunnerTranscodingVideoInputFileUrl, generateRunnerTranscodingVideoPreviewFileUrl } from '../runner-urls' +import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler' +import { loadTranscodingRunnerVideo, onVODWebVideoOrAudioMergeTranscodingJob } from './shared' + +type CreateOptions = { + video: MVideo + isNewVideo: boolean + resolution: number + fps: number + priority: number + dependsOnRunnerJob?: MRunnerJob +} + +// eslint-disable-next-line max-len +export class VODAudioMergeTranscodingJobHandler extends AbstractVODTranscodingJobHandler { + + async create (options: CreateOptions) { + const { video, resolution, fps, priority, dependsOnRunnerJob } = options + + const jobUUID = buildUUID() + const payload: RunnerJobVODAudioMergeTranscodingPayload = { + input: { + audioFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid), + previewFileUrl: generateRunnerTranscodingVideoPreviewFileUrl(jobUUID, video.uuid) + }, + output: { + resolution, + fps + } + } + + const privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload = { + ...pick(options, [ 'isNewVideo' ]), + + videoUUID: video.uuid + } + + const job = await this.createRunnerJob({ + type: 'vod-audio-merge-transcoding', + jobUUID, + payload, + privatePayload, + priority, + dependsOnRunnerJob + }) + + await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') + + return job + } + + // --------------------------------------------------------------------------- + + async specificComplete (options: { + runnerJob: MRunnerJob + resultPayload: VODAudioMergeTranscodingSuccess + }) { + const { runnerJob, resultPayload } = options + const privatePayload = runnerJob.privatePayload as RunnerJobVODWebVideoTranscodingPrivatePayload + + const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) + if (!video) return + + const videoFilePath = resultPayload.videoFile as string + + // ffmpeg generated a new video file, so update the video duration + // See https://trac.ffmpeg.org/ticket/5456 + video.duration = await getVideoStreamDuration(videoFilePath) + await video.save() + + // We can remove the old audio file + const oldAudioFile = video.VideoFiles[0] + await video.removeWebTorrentFile(oldAudioFile) + await oldAudioFile.destroy() + video.VideoFiles = [] + + await onVODWebVideoOrAudioMergeTranscodingJob({ video, videoFilePath, privatePayload }) + + logger.info( + 'Runner VOD audio merge transcoding job %s for %s ended.', + runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid) + ) + } +} diff --git a/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts new file mode 100644 index 000000000..02566b9d5 --- /dev/null +++ b/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts @@ -0,0 +1,114 @@ +import { move } from 'fs-extra' +import { dirname, join } from 'path' +import { logger } from '@server/helpers/logger' +import { renameVideoFileInPlaylist } from '@server/lib/hls' +import { getHlsResolutionPlaylistFilename } from '@server/lib/paths' +import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' +import { onHLSVideoFileTranscoding } from '@server/lib/transcoding/hls-transcoding' +import { buildNewFile, removeAllWebTorrentFiles } from '@server/lib/video-file' +import { VideoJobInfoModel } from '@server/models/video/video-job-info' +import { MVideo } from '@server/types/models' +import { MRunnerJob } from '@server/types/models/runners' +import { pick } from '@shared/core-utils' +import { buildUUID } from '@shared/extra-utils' +import { + RunnerJobUpdatePayload, + RunnerJobVODHLSTranscodingPayload, + RunnerJobVODHLSTranscodingPrivatePayload, + VODHLSTranscodingSuccess +} from '@shared/models' +import { generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls' +import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler' +import { loadTranscodingRunnerVideo } from './shared' + +type CreateOptions = { + video: MVideo + isNewVideo: boolean + deleteWebVideoFiles: boolean + resolution: number + fps: number + priority: number + dependsOnRunnerJob?: MRunnerJob +} + +// eslint-disable-next-line max-len +export class VODHLSTranscodingJobHandler extends AbstractVODTranscodingJobHandler { + + async create (options: CreateOptions) { + const { video, resolution, fps, dependsOnRunnerJob, priority } = options + + const jobUUID = buildUUID() + + const payload: RunnerJobVODHLSTranscodingPayload = { + input: { + videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid) + }, + output: { + resolution, + fps + } + } + + const privatePayload: RunnerJobVODHLSTranscodingPrivatePayload = { + ...pick(options, [ 'isNewVideo', 'deleteWebVideoFiles' ]), + + videoUUID: video.uuid + } + + const job = await this.createRunnerJob({ + type: 'vod-hls-transcoding', + jobUUID, + payload, + privatePayload, + priority, + dependsOnRunnerJob + }) + + await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') + + return job + } + + // --------------------------------------------------------------------------- + + async specificComplete (options: { + runnerJob: MRunnerJob + resultPayload: VODHLSTranscodingSuccess + }) { + const { runnerJob, resultPayload } = options + const privatePayload = runnerJob.privatePayload as RunnerJobVODHLSTranscodingPrivatePayload + + const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) + if (!video) return + + const videoFilePath = resultPayload.videoFile as string + const resolutionPlaylistFilePath = resultPayload.resolutionPlaylistFile as string + + const videoFile = await buildNewFile({ path: videoFilePath, mode: 'hls' }) + const newVideoFilePath = join(dirname(videoFilePath), videoFile.filename) + await move(videoFilePath, newVideoFilePath) + + const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFile.filename) + const newResolutionPlaylistFilePath = join(dirname(resolutionPlaylistFilePath), resolutionPlaylistFilename) + await move(resolutionPlaylistFilePath, newResolutionPlaylistFilePath) + + await renameVideoFileInPlaylist(newResolutionPlaylistFilePath, videoFile.filename) + + await onHLSVideoFileTranscoding({ + video, + videoFile, + m3u8OutputPath: newResolutionPlaylistFilePath, + videoOutputPath: newVideoFilePath + }) + + await onTranscodingEnded({ isNewVideo: privatePayload.isNewVideo, moveVideoToNextState: true, video }) + + if (privatePayload.deleteWebVideoFiles === true) { + logger.info('Removing web video files of %s now we have a HLS version of it.', video.uuid, this.lTags(video.uuid)) + + await removeAllWebTorrentFiles(video) + } + + logger.info('Runner VOD HLS job %s for %s ended.', runnerJob.uuid, video.uuid, this.lTags(runnerJob.uuid, video.uuid)) + } +} diff --git a/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts new file mode 100644 index 000000000..57761a7a1 --- /dev/null +++ b/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts @@ -0,0 +1,84 @@ +import { pick } from 'lodash' +import { logger } from '@server/helpers/logger' +import { VideoJobInfoModel } from '@server/models/video/video-job-info' +import { MVideo } from '@server/types/models' +import { MRunnerJob } from '@server/types/models/runners' +import { buildUUID } from '@shared/extra-utils' +import { + RunnerJobUpdatePayload, + RunnerJobVODWebVideoTranscodingPayload, + RunnerJobVODWebVideoTranscodingPrivatePayload, + VODWebVideoTranscodingSuccess +} from '@shared/models' +import { generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls' +import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler' +import { loadTranscodingRunnerVideo, onVODWebVideoOrAudioMergeTranscodingJob } from './shared' + +type CreateOptions = { + video: MVideo + isNewVideo: boolean + resolution: number + fps: number + priority: number + dependsOnRunnerJob?: MRunnerJob +} + +// eslint-disable-next-line max-len +export class VODWebVideoTranscodingJobHandler extends AbstractVODTranscodingJobHandler { + + async create (options: CreateOptions) { + const { video, resolution, fps, priority, dependsOnRunnerJob } = options + + const jobUUID = buildUUID() + const payload: RunnerJobVODWebVideoTranscodingPayload = { + input: { + videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid) + }, + output: { + resolution, + fps + } + } + + const privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload = { + ...pick(options, [ 'isNewVideo' ]), + + videoUUID: video.uuid + } + + const job = await this.createRunnerJob({ + type: 'vod-web-video-transcoding', + jobUUID, + payload, + privatePayload, + dependsOnRunnerJob, + priority + }) + + await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') + + return job + } + + // --------------------------------------------------------------------------- + + async specificComplete (options: { + runnerJob: MRunnerJob + resultPayload: VODWebVideoTranscodingSuccess + }) { + const { runnerJob, resultPayload } = options + const privatePayload = runnerJob.privatePayload as RunnerJobVODWebVideoTranscodingPrivatePayload + + const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) + if (!video) return + + const videoFilePath = resultPayload.videoFile as string + + await onVODWebVideoOrAudioMergeTranscodingJob({ video, videoFilePath, privatePayload }) + + logger.info( + 'Runner VOD web video transcoding job %s for %s ended.', + runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid) + ) + } +} diff --git a/server/lib/runners/runner-urls.ts b/server/lib/runners/runner-urls.ts new file mode 100644 index 000000000..329fb1170 --- /dev/null +++ b/server/lib/runners/runner-urls.ts @@ -0,0 +1,9 @@ +import { WEBSERVER } from '@server/initializers/constants' + +export function generateRunnerTranscodingVideoInputFileUrl (jobUUID: string, videoUUID: string) { + return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/max-quality' +} + +export function generateRunnerTranscodingVideoPreviewFileUrl (jobUUID: string, videoUUID: string) { + return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/previews/max-quality' +} diff --git a/server/lib/runners/runner.ts b/server/lib/runners/runner.ts new file mode 100644 index 000000000..74c814ba1 --- /dev/null +++ b/server/lib/runners/runner.ts @@ -0,0 +1,36 @@ +import express from 'express' +import { retryTransactionWrapper } from '@server/helpers/database-utils' +import { logger, loggerTagsFactory } from '@server/helpers/logger' +import { sequelizeTypescript } from '@server/initializers/database' +import { MRunner } from '@server/types/models/runners' + +const lTags = loggerTagsFactory('runner') + +const updatingRunner = new Set() + +function updateLastRunnerContact (req: express.Request, runner: MRunner) { + const now = new Date() + + // Don't update last runner contact too often + if (now.getTime() - runner.lastContact.getTime() < 2000) return + if (updatingRunner.has(runner.id)) return + + updatingRunner.add(runner.id) + + runner.lastContact = now + runner.ip = req.ip + + logger.debug('Updating last runner contact for %s', runner.name, lTags(runner.name)) + + retryTransactionWrapper(() => { + return sequelizeTypescript.transaction(async transaction => { + return runner.save({ transaction }) + }) + }) + .catch(err => logger.error('Cannot update last runner contact for %s', runner.name, { err, ...lTags(runner.name) })) + .finally(() => updatingRunner.delete(runner.id)) +} + +export { + updateLastRunnerContact +} diff --git a/server/lib/schedulers/runner-job-watch-dog-scheduler.ts b/server/lib/schedulers/runner-job-watch-dog-scheduler.ts new file mode 100644 index 000000000..f7a26d2bc --- /dev/null +++ b/server/lib/schedulers/runner-job-watch-dog-scheduler.ts @@ -0,0 +1,42 @@ +import { CONFIG } from '@server/initializers/config' +import { RunnerJobModel } from '@server/models/runner/runner-job' +import { logger, loggerTagsFactory } from '../../helpers/logger' +import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' +import { getRunnerJobHandlerClass } from '../runners' +import { AbstractScheduler } from './abstract-scheduler' + +const lTags = loggerTagsFactory('runner') + +export class RunnerJobWatchDogScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.RUNNER_JOB_WATCH_DOG + + private constructor () { + super() + } + + protected async internalExecute () { + const vodStalledJobs = await RunnerJobModel.listStalledJobs({ + staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD, + types: [ 'vod-audio-merge-transcoding', 'vod-hls-transcoding', 'vod-web-video-transcoding' ] + }) + + const liveStalledJobs = await RunnerJobModel.listStalledJobs({ + staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE, + types: [ 'live-rtmp-hls-transcoding' ] + }) + + for (const stalled of [ ...vodStalledJobs, ...liveStalledJobs ]) { + logger.info('Abort stalled runner job %s (%s)', stalled.uuid, stalled.type, lTags(stalled.uuid, stalled.type)) + + const Handler = getRunnerJobHandlerClass(stalled) + await new Handler().abort({ runnerJob: stalled }) + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/lib/server-config-manager.ts b/server/lib/server-config-manager.ts index e87e2854f..ba7916363 100644 --- a/server/lib/server-config-manager.ts +++ b/server/lib/server-config-manager.ts @@ -126,11 +126,14 @@ class ServerConfigManager { serverVersion: PEERTUBE_VERSION, serverCommit: this.serverCommit, transcoding: { + remoteRunners: { + enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED + }, hls: { - enabled: CONFIG.TRANSCODING.HLS.ENABLED + enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.HLS.ENABLED }, webtorrent: { - enabled: CONFIG.TRANSCODING.WEBTORRENT.ENABLED + enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.WEBTORRENT.ENABLED }, enabledResolutions: this.getEnabledResolutions('vod'), profile: CONFIG.TRANSCODING.PROFILE, @@ -150,6 +153,9 @@ class ServerConfigManager { transcoding: { enabled: CONFIG.LIVE.TRANSCODING.ENABLED, + remoteRunners: { + enabled: CONFIG.LIVE.TRANSCODING.ENABLED && CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED + }, enabledResolutions: this.getEnabledResolutions('live'), profile: CONFIG.LIVE.TRANSCODING.PROFILE, availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('live') diff --git a/server/lib/transcoding/create-transcoding-job.ts b/server/lib/transcoding/create-transcoding-job.ts new file mode 100644 index 000000000..46831a912 --- /dev/null +++ b/server/lib/transcoding/create-transcoding-job.ts @@ -0,0 +1,36 @@ +import { CONFIG } from '@server/initializers/config' +import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' +import { TranscodingJobQueueBuilder, TranscodingRunnerJobBuilder } from './shared' + +export function createOptimizeOrMergeAudioJobs (options: { + video: MVideoFullLight + videoFile: MVideoFile + isNewVideo: boolean + user: MUserId +}) { + return getJobBuilder().createOptimizeOrMergeAudioJobs(options) +} + +// --------------------------------------------------------------------------- + +export function createTranscodingJobs (options: { + transcodingType: 'hls' | 'webtorrent' + video: MVideoFullLight + resolutions: number[] + isNewVideo: boolean + user: MUserId +}) { + return getJobBuilder().createTranscodingJobs(options) +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +function getJobBuilder () { + if (CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED === true) { + return new TranscodingRunnerJobBuilder() + } + + return new TranscodingJobQueueBuilder() +} diff --git a/server/lib/transcoding/default-transcoding-profiles.ts b/server/lib/transcoding/default-transcoding-profiles.ts index f47718819..5251784ac 100644 --- a/server/lib/transcoding/default-transcoding-profiles.ts +++ b/server/lib/transcoding/default-transcoding-profiles.ts @@ -1,15 +1,9 @@ import { logger } from '@server/helpers/logger' import { getAverageBitrate, getMinLimitBitrate } from '@shared/core-utils' -import { AvailableEncoders, EncoderOptionsBuilder, EncoderOptionsBuilderParams, VideoResolution } from '../../../shared/models/videos' -import { - buildStreamSuffix, - canDoQuickAudioTranscode, - ffprobePromise, - getAudioStream, - getMaxAudioBitrate, - resetSupportedEncoders -} from '../../helpers/ffmpeg' +import { buildStreamSuffix, FFmpegCommandWrapper, ffprobePromise, getAudioStream, getMaxAudioBitrate } from '@shared/ffmpeg' +import { AvailableEncoders, EncoderOptionsBuilder, EncoderOptionsBuilderParams, VideoResolution } from '@shared/models' +import { canDoQuickAudioTranscode } from './transcoding-quick-transcode' /** * @@ -184,14 +178,14 @@ class VideoTranscodingProfilesManager { addEncoderPriority (type: 'vod' | 'live', streamType: 'audio' | 'video', encoder: string, priority: number) { this.encodersPriorities[type][streamType].push({ name: encoder, priority }) - resetSupportedEncoders() + FFmpegCommandWrapper.resetSupportedEncoders() } removeEncoderPriority (type: 'vod' | 'live', streamType: 'audio' | 'video', encoder: string, priority: number) { this.encodersPriorities[type][streamType] = this.encodersPriorities[type][streamType] .filter(o => o.name !== encoder && o.priority !== priority) - resetSupportedEncoders() + FFmpegCommandWrapper.resetSupportedEncoders() } private getEncodersByPriority (type: 'vod' | 'live', streamType: 'audio' | 'video') { diff --git a/server/lib/transcoding/ended-transcoding.ts b/server/lib/transcoding/ended-transcoding.ts new file mode 100644 index 000000000..d31674ede --- /dev/null +++ b/server/lib/transcoding/ended-transcoding.ts @@ -0,0 +1,18 @@ +import { retryTransactionWrapper } from '@server/helpers/database-utils' +import { VideoJobInfoModel } from '@server/models/video/video-job-info' +import { MVideo } from '@server/types/models' +import { moveToNextState } from '../video-state' + +export async function onTranscodingEnded (options: { + video: MVideo + isNewVideo: boolean + moveVideoToNextState: boolean +}) { + const { video, isNewVideo, moveVideoToNextState } = options + + await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') + + if (moveVideoToNextState) { + await retryTransactionWrapper(moveToNextState, { video, isNewVideo }) + } +} diff --git a/server/lib/transcoding/hls-transcoding.ts b/server/lib/transcoding/hls-transcoding.ts new file mode 100644 index 000000000..cffa859c7 --- /dev/null +++ b/server/lib/transcoding/hls-transcoding.ts @@ -0,0 +1,181 @@ +import { MutexInterface } from 'async-mutex' +import { Job } from 'bullmq' +import { ensureDir, move, stat } from 'fs-extra' +import { basename, extname as extnameUtil, join } from 'path' +import { retryTransactionWrapper } from '@server/helpers/database-utils' +import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' +import { sequelizeTypescript } from '@server/initializers/database' +import { MVideo, MVideoFile } from '@server/types/models' +import { pick } from '@shared/core-utils' +import { getVideoStreamDuration, getVideoStreamFPS } from '@shared/ffmpeg' +import { VideoResolution } from '@shared/models' +import { CONFIG } from '../../initializers/config' +import { VideoFileModel } from '../../models/video/video-file' +import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' +import { updatePlaylistAfterFileChange } from '../hls' +import { generateHLSVideoFilename, getHlsResolutionPlaylistFilename } from '../paths' +import { buildFileMetadata } from '../video-file' +import { VideoPathManager } from '../video-path-manager' +import { buildFFmpegVOD } from './shared' + +// Concat TS segments from a live video to a fragmented mp4 HLS playlist +export async function generateHlsPlaylistResolutionFromTS (options: { + video: MVideo + concatenatedTsFilePath: string + resolution: VideoResolution + fps: number + isAAC: boolean + inputFileMutexReleaser: MutexInterface.Releaser +}) { + return generateHlsPlaylistCommon({ + type: 'hls-from-ts' as 'hls-from-ts', + inputPath: options.concatenatedTsFilePath, + + ...pick(options, [ 'video', 'resolution', 'fps', 'inputFileMutexReleaser', 'isAAC' ]) + }) +} + +// Generate an HLS playlist from an input file, and update the master playlist +export function generateHlsPlaylistResolution (options: { + video: MVideo + videoInputPath: string + resolution: VideoResolution + fps: number + copyCodecs: boolean + inputFileMutexReleaser: MutexInterface.Releaser + job?: Job +}) { + return generateHlsPlaylistCommon({ + type: 'hls' as 'hls', + inputPath: options.videoInputPath, + + ...pick(options, [ 'video', 'resolution', 'fps', 'copyCodecs', 'inputFileMutexReleaser', 'job' ]) + }) +} + +export async function onHLSVideoFileTranscoding (options: { + video: MVideo + videoFile: MVideoFile + videoOutputPath: string + m3u8OutputPath: string +}) { + const { video, videoFile, videoOutputPath, m3u8OutputPath } = options + + // Create or update the playlist + const playlist = await retryTransactionWrapper(() => { + return sequelizeTypescript.transaction(async transaction => { + return VideoStreamingPlaylistModel.loadOrGenerate(video, transaction) + }) + }) + videoFile.videoStreamingPlaylistId = playlist.id + + const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) + + try { + // VOD transcoding is a long task, refresh video attributes + await video.reload() + + const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, videoFile) + await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video)) + + // Move playlist file + const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, basename(m3u8OutputPath)) + await move(m3u8OutputPath, resolutionPlaylistPath, { overwrite: true }) + // Move video file + await move(videoOutputPath, videoFilePath, { overwrite: true }) + + // Update video duration if it was not set (in case of a live for example) + if (!video.duration) { + video.duration = await getVideoStreamDuration(videoFilePath) + await video.save() + } + + const stats = await stat(videoFilePath) + + videoFile.size = stats.size + videoFile.fps = await getVideoStreamFPS(videoFilePath) + videoFile.metadata = await buildFileMetadata(videoFilePath) + + await createTorrentAndSetInfoHash(playlist, videoFile) + + const oldFile = await VideoFileModel.loadHLSFile({ + playlistId: playlist.id, + fps: videoFile.fps, + resolution: videoFile.resolution + }) + + if (oldFile) { + await video.removeStreamingPlaylistVideoFile(playlist, oldFile) + await oldFile.destroy() + } + + const savedVideoFile = await VideoFileModel.customUpsert(videoFile, 'streaming-playlist', undefined) + + await updatePlaylistAfterFileChange(video, playlist) + + return { resolutionPlaylistPath, videoFile: savedVideoFile } + } finally { + mutexReleaser() + } +} + +// --------------------------------------------------------------------------- + +async function generateHlsPlaylistCommon (options: { + type: 'hls' | 'hls-from-ts' + video: MVideo + inputPath: string + + resolution: VideoResolution + fps: number + + inputFileMutexReleaser: MutexInterface.Releaser + + copyCodecs?: boolean + isAAC?: boolean + + job?: Job +}) { + const { type, video, inputPath, resolution, fps, copyCodecs, isAAC, job, inputFileMutexReleaser } = options + const transcodeDirectory = CONFIG.STORAGE.TMP_DIR + + const videoTranscodedBasePath = join(transcodeDirectory, type) + await ensureDir(videoTranscodedBasePath) + + const videoFilename = generateHLSVideoFilename(resolution) + const videoOutputPath = join(videoTranscodedBasePath, videoFilename) + + const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename) + const m3u8OutputPath = join(videoTranscodedBasePath, resolutionPlaylistFilename) + + const transcodeOptions = { + type, + + inputPath, + outputPath: m3u8OutputPath, + + resolution, + fps, + copyCodecs, + + isAAC, + + inputFileMutexReleaser, + + hlsPlaylist: { + videoFilename + } + } + + await buildFFmpegVOD(job).transcode(transcodeOptions) + + const newVideoFile = new VideoFileModel({ + resolution, + extname: extnameUtil(videoFilename), + size: 0, + filename: videoFilename, + fps: -1 + }) + + await onHLSVideoFileTranscoding({ video, videoFile: newVideoFile, videoOutputPath, m3u8OutputPath }) +} diff --git a/server/lib/transcoding/shared/ffmpeg-builder.ts b/server/lib/transcoding/shared/ffmpeg-builder.ts new file mode 100644 index 000000000..441445ec4 --- /dev/null +++ b/server/lib/transcoding/shared/ffmpeg-builder.ts @@ -0,0 +1,18 @@ +import { Job } from 'bullmq' +import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' +import { logger } from '@server/helpers/logger' +import { FFmpegVOD } from '@shared/ffmpeg' +import { VideoTranscodingProfilesManager } from '../default-transcoding-profiles' + +export function buildFFmpegVOD (job?: Job) { + return new FFmpegVOD({ + ...getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()), + + updateJobProgress: progress => { + if (!job) return + + job.updateProgress(progress) + .catch(err => logger.error('Cannot update ffmpeg job progress', { err })) + } + }) +} diff --git a/server/lib/transcoding/shared/index.ts b/server/lib/transcoding/shared/index.ts new file mode 100644 index 000000000..f0b45bcbb --- /dev/null +++ b/server/lib/transcoding/shared/index.ts @@ -0,0 +1,2 @@ +export * from './job-builders' +export * from './ffmpeg-builder' diff --git a/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts b/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts new file mode 100644 index 000000000..f1e9efdcf --- /dev/null +++ b/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts @@ -0,0 +1,38 @@ + +import { JOB_PRIORITY } from '@server/initializers/constants' +import { VideoModel } from '@server/models/video/video' +import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' + +export abstract class AbstractJobBuilder { + + abstract createOptimizeOrMergeAudioJobs (options: { + video: MVideoFullLight + videoFile: MVideoFile + isNewVideo: boolean + user: MUserId + }): Promise + + abstract createTranscodingJobs (options: { + transcodingType: 'hls' | 'webtorrent' + video: MVideoFullLight + resolutions: number[] + isNewVideo: boolean + user: MUserId | null + }): Promise + + protected async getTranscodingJobPriority (options: { + user: MUserId + fallback: number + }) { + const { user, fallback } = options + + if (!user) return fallback + + const now = new Date() + const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7) + + const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek) + + return JOB_PRIORITY.TRANSCODING + videoUploadedByUser + } +} diff --git a/server/lib/transcoding/shared/job-builders/index.ts b/server/lib/transcoding/shared/job-builders/index.ts new file mode 100644 index 000000000..9b1c82adf --- /dev/null +++ b/server/lib/transcoding/shared/job-builders/index.ts @@ -0,0 +1,2 @@ +export * from './transcoding-job-queue-builder' +export * from './transcoding-runner-job-builder' diff --git a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts new file mode 100644 index 000000000..7c892718b --- /dev/null +++ b/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts @@ -0,0 +1,308 @@ +import Bluebird from 'bluebird' +import { computeOutputFPS } from '@server/helpers/ffmpeg' +import { logger } from '@server/helpers/logger' +import { CONFIG } from '@server/initializers/config' +import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' +import { CreateJobArgument, JobQueue } from '@server/lib/job-queue' +import { Hooks } from '@server/lib/plugins/hooks' +import { VideoPathManager } from '@server/lib/video-path-manager' +import { VideoJobInfoModel } from '@server/models/video/video-job-info' +import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models' +import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg' +import { + HLSTranscodingPayload, + MergeAudioTranscodingPayload, + NewWebTorrentResolutionTranscodingPayload, + OptimizeTranscodingPayload, + VideoTranscodingPayload +} from '@shared/models' +import { canDoQuickTranscode } from '../../transcoding-quick-transcode' +import { computeResolutionsToTranscode } from '../../transcoding-resolutions' +import { AbstractJobBuilder } from './abstract-job-builder' + +export class TranscodingJobQueueBuilder extends AbstractJobBuilder { + + async createOptimizeOrMergeAudioJobs (options: { + video: MVideoFullLight + videoFile: MVideoFile + isNewVideo: boolean + user: MUserId + }) { + const { video, videoFile, isNewVideo, user } = options + + let mergeOrOptimizePayload: MergeAudioTranscodingPayload | OptimizeTranscodingPayload + let nextTranscodingSequentialJobPayloads: (NewWebTorrentResolutionTranscodingPayload | HLSTranscodingPayload)[][] = [] + + const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) + + try { + await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => { + const probe = await ffprobePromise(videoFilePath) + + const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe) + const hasAudio = await hasAudioStream(videoFilePath, probe) + const quickTranscode = await canDoQuickTranscode(videoFilePath, probe) + const inputFPS = videoFile.isAudio() + ? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value + : await getVideoStreamFPS(videoFilePath, probe) + + const maxResolution = await isAudioFile(videoFilePath, probe) + ? DEFAULT_AUDIO_RESOLUTION + : resolution + + if (CONFIG.TRANSCODING.HLS.ENABLED === true) { + nextTranscodingSequentialJobPayloads.push([ + this.buildHLSJobPayload({ + deleteWebTorrentFiles: CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false, + + // We had some issues with a web video quick transcoded while producing a HLS version of it + copyCodecs: !quickTranscode, + + resolution: maxResolution, + fps: computeOutputFPS({ inputFPS, resolution: maxResolution }), + videoUUID: video.uuid, + isNewVideo + }) + ]) + } + + const lowerResolutionJobPayloads = await this.buildLowerResolutionJobPayloads({ + video, + inputVideoResolution: maxResolution, + inputVideoFPS: inputFPS, + hasAudio, + isNewVideo + }) + + nextTranscodingSequentialJobPayloads = [ ...nextTranscodingSequentialJobPayloads, ...lowerResolutionJobPayloads ] + + mergeOrOptimizePayload = videoFile.isAudio() + ? this.buildMergeAudioPayload({ videoUUID: video.uuid, isNewVideo }) + : this.buildOptimizePayload({ videoUUID: video.uuid, isNewVideo, quickTranscode }) + }) + } finally { + mutexReleaser() + } + + const nextTranscodingSequentialJobs = await Bluebird.mapSeries(nextTranscodingSequentialJobPayloads, payloads => { + return Bluebird.mapSeries(payloads, payload => { + return this.buildTranscodingJob({ payload, user }) + }) + }) + + const transcodingJobBuilderJob: CreateJobArgument = { + type: 'transcoding-job-builder', + payload: { + videoUUID: video.uuid, + sequentialJobs: nextTranscodingSequentialJobs + } + } + + const mergeOrOptimizeJob = await this.buildTranscodingJob({ payload: mergeOrOptimizePayload, user }) + + return JobQueue.Instance.createSequentialJobFlow(...[ mergeOrOptimizeJob, transcodingJobBuilderJob ]) + } + + // --------------------------------------------------------------------------- + + async createTranscodingJobs (options: { + transcodingType: 'hls' | 'webtorrent' + video: MVideoFullLight + resolutions: number[] + isNewVideo: boolean + user: MUserId | null + }) { + const { video, transcodingType, resolutions, isNewVideo } = options + + const maxResolution = Math.max(...resolutions) + const childrenResolutions = resolutions.filter(r => r !== maxResolution) + + logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution }) + + const { fps: inputFPS } = await video.probeMaxQualityFile() + + const children = childrenResolutions.map(resolution => { + const fps = computeOutputFPS({ inputFPS, resolution }) + + if (transcodingType === 'hls') { + return this.buildHLSJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo }) + } + + if (transcodingType === 'webtorrent') { + return this.buildWebTorrentJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo }) + } + + throw new Error('Unknown transcoding type') + }) + + const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) + + const parent = transcodingType === 'hls' + ? this.buildHLSJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo }) + : this.buildWebTorrentJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo }) + + // Process the last resolution after the other ones to prevent concurrency issue + // Because low resolutions use the biggest one as ffmpeg input + await this.createTranscodingJobsWithChildren({ videoUUID: video.uuid, parent, children, user: null }) + } + + // --------------------------------------------------------------------------- + + private async createTranscodingJobsWithChildren (options: { + videoUUID: string + parent: (HLSTranscodingPayload | NewWebTorrentResolutionTranscodingPayload) + children: (HLSTranscodingPayload | NewWebTorrentResolutionTranscodingPayload)[] + user: MUserId | null + }) { + const { videoUUID, parent, children, user } = options + + const parentJob = await this.buildTranscodingJob({ payload: parent, user }) + const childrenJobs = await Bluebird.mapSeries(children, c => this.buildTranscodingJob({ payload: c, user })) + + await JobQueue.Instance.createJobWithChildren(parentJob, childrenJobs) + + await VideoJobInfoModel.increaseOrCreate(videoUUID, 'pendingTranscode', 1 + children.length) + } + + private async buildTranscodingJob (options: { + payload: VideoTranscodingPayload + user: MUserId | null // null means we don't want priority + }) { + const { user, payload } = options + + return { + type: 'video-transcoding' as 'video-transcoding', + priority: await this.getTranscodingJobPriority({ user, fallback: undefined }), + payload + } + } + + private async buildLowerResolutionJobPayloads (options: { + video: MVideoWithFileThumbnail + inputVideoResolution: number + inputVideoFPS: number + hasAudio: boolean + isNewVideo: boolean + }) { + const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio } = options + + // Create transcoding jobs if there are enabled resolutions + const resolutionsEnabled = await Hooks.wrapObject( + computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }), + 'filter:transcoding.auto.resolutions-to-transcode.result', + options + ) + + const sequentialPayloads: (NewWebTorrentResolutionTranscodingPayload | HLSTranscodingPayload)[][] = [] + + for (const resolution of resolutionsEnabled) { + const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution }) + + if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED) { + const payloads: (NewWebTorrentResolutionTranscodingPayload | HLSTranscodingPayload)[] = [ + this.buildWebTorrentJobPayload({ + videoUUID: video.uuid, + resolution, + fps, + isNewVideo + }) + ] + + // Create a subsequent job to create HLS resolution that will just copy web video codecs + if (CONFIG.TRANSCODING.HLS.ENABLED) { + payloads.push( + this.buildHLSJobPayload({ + videoUUID: video.uuid, + resolution, + fps, + isNewVideo, + copyCodecs: true + }) + ) + } + + sequentialPayloads.push(payloads) + } else if (CONFIG.TRANSCODING.HLS.ENABLED) { + sequentialPayloads.push([ + this.buildHLSJobPayload({ + videoUUID: video.uuid, + resolution, + fps, + copyCodecs: false, + isNewVideo + }) + ]) + } + } + + return sequentialPayloads + } + + private buildHLSJobPayload (options: { + videoUUID: string + resolution: number + fps: number + isNewVideo: boolean + deleteWebTorrentFiles?: boolean // default false + copyCodecs?: boolean // default false + }): HLSTranscodingPayload { + const { videoUUID, resolution, fps, isNewVideo, deleteWebTorrentFiles = false, copyCodecs = false } = options + + return { + type: 'new-resolution-to-hls', + videoUUID, + resolution, + fps, + copyCodecs, + isNewVideo, + deleteWebTorrentFiles + } + } + + private buildWebTorrentJobPayload (options: { + videoUUID: string + resolution: number + fps: number + isNewVideo: boolean + }): NewWebTorrentResolutionTranscodingPayload { + const { videoUUID, resolution, fps, isNewVideo } = options + + return { + type: 'new-resolution-to-webtorrent', + videoUUID, + isNewVideo, + resolution, + fps + } + } + + private buildMergeAudioPayload (options: { + videoUUID: string + isNewVideo: boolean + }): MergeAudioTranscodingPayload { + const { videoUUID, isNewVideo } = options + + return { + type: 'merge-audio-to-webtorrent', + resolution: DEFAULT_AUDIO_RESOLUTION, + fps: VIDEO_TRANSCODING_FPS.AUDIO_MERGE, + videoUUID, + isNewVideo + } + } + + private buildOptimizePayload (options: { + videoUUID: string + quickTranscode: boolean + isNewVideo: boolean + }): OptimizeTranscodingPayload { + const { videoUUID, quickTranscode, isNewVideo } = options + + return { + type: 'optimize-to-webtorrent', + videoUUID, + isNewVideo, + quickTranscode + } + } +} diff --git a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts new file mode 100644 index 000000000..c7a63d2e2 --- /dev/null +++ b/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts @@ -0,0 +1,189 @@ +import { computeOutputFPS } from '@server/helpers/ffmpeg' +import { logger, loggerTagsFactory } from '@server/helpers/logger' +import { CONFIG } from '@server/initializers/config' +import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' +import { Hooks } from '@server/lib/plugins/hooks' +import { VODAudioMergeTranscodingJobHandler, VODHLSTranscodingJobHandler, VODWebVideoTranscodingJobHandler } from '@server/lib/runners' +import { VideoPathManager } from '@server/lib/video-path-manager' +import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models' +import { MRunnerJob } from '@server/types/models/runners' +import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg' +import { computeResolutionsToTranscode } from '../../transcoding-resolutions' +import { AbstractJobBuilder } from './abstract-job-builder' + +/** + * + * Class to build transcoding job in the local job queue + * + */ + +const lTags = loggerTagsFactory('transcoding') + +export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { + + async createOptimizeOrMergeAudioJobs (options: { + video: MVideoFullLight + videoFile: MVideoFile + isNewVideo: boolean + user: MUserId + }) { + const { video, videoFile, isNewVideo, user } = options + + const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) + + try { + await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => { + const probe = await ffprobePromise(videoFilePath) + + const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe) + const hasAudio = await hasAudioStream(videoFilePath, probe) + const inputFPS = videoFile.isAudio() + ? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value + : await getVideoStreamFPS(videoFilePath, probe) + + const maxResolution = await isAudioFile(videoFilePath, probe) + ? DEFAULT_AUDIO_RESOLUTION + : resolution + + const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) + const priority = await this.getTranscodingJobPriority({ user, fallback: 0 }) + + const mainRunnerJob = videoFile.isAudio() + ? await new VODAudioMergeTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority }) + : await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority }) + + if (CONFIG.TRANSCODING.HLS.ENABLED === true) { + await new VODHLSTranscodingJobHandler().create({ + video, + deleteWebVideoFiles: CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false, + resolution: maxResolution, + fps, + isNewVideo, + dependsOnRunnerJob: mainRunnerJob, + priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) + }) + } + + await this.buildLowerResolutionJobPayloads({ + video, + inputVideoResolution: maxResolution, + inputVideoFPS: inputFPS, + hasAudio, + isNewVideo, + mainRunnerJob, + user + }) + }) + } finally { + mutexReleaser() + } + } + + // --------------------------------------------------------------------------- + + async createTranscodingJobs (options: { + transcodingType: 'hls' | 'webtorrent' + video: MVideoFullLight + resolutions: number[] + isNewVideo: boolean + user: MUserId | null + }) { + const { video, transcodingType, resolutions, isNewVideo, user } = options + + const maxResolution = Math.max(...resolutions) + const { fps: inputFPS } = await video.probeMaxQualityFile() + const maxFPS = computeOutputFPS({ inputFPS, resolution: maxResolution }) + const priority = await this.getTranscodingJobPriority({ user, fallback: 0 }) + + const childrenResolutions = resolutions.filter(r => r !== maxResolution) + + logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution }) + + // Process the last resolution before the other ones to prevent concurrency issue + // Because low resolutions use the biggest one as ffmpeg input + const mainJob = transcodingType === 'hls' + // eslint-disable-next-line max-len + ? await new VODHLSTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, deleteWebVideoFiles: false, priority }) + : await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, priority }) + + for (const resolution of childrenResolutions) { + const dependsOnRunnerJob = mainJob + const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) + + if (transcodingType === 'hls') { + await new VODHLSTranscodingJobHandler().create({ + video, + resolution, + fps, + isNewVideo, + deleteWebVideoFiles: false, + dependsOnRunnerJob, + priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) + }) + continue + } + + if (transcodingType === 'webtorrent') { + await new VODWebVideoTranscodingJobHandler().create({ + video, + resolution, + fps, + isNewVideo, + dependsOnRunnerJob, + priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) + }) + continue + } + + throw new Error('Unknown transcoding type') + } + } + + private async buildLowerResolutionJobPayloads (options: { + mainRunnerJob: MRunnerJob + video: MVideoWithFileThumbnail + inputVideoResolution: number + inputVideoFPS: number + hasAudio: boolean + isNewVideo: boolean + user: MUserId + }) { + const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio, mainRunnerJob, user } = options + + // Create transcoding jobs if there are enabled resolutions + const resolutionsEnabled = await Hooks.wrapObject( + computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }), + 'filter:transcoding.auto.resolutions-to-transcode.result', + options + ) + + logger.debug('Lower resolutions build for %s.', video.uuid, { resolutionsEnabled, ...lTags(video.uuid) }) + + for (const resolution of resolutionsEnabled) { + const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution }) + + if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED) { + await new VODWebVideoTranscodingJobHandler().create({ + video, + resolution, + fps, + isNewVideo, + dependsOnRunnerJob: mainRunnerJob, + priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) + }) + } + + if (CONFIG.TRANSCODING.HLS.ENABLED) { + await new VODHLSTranscodingJobHandler().create({ + video, + resolution, + fps, + isNewVideo, + deleteWebVideoFiles: false, + dependsOnRunnerJob: mainRunnerJob, + priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) + }) + } + } + } +} diff --git a/server/lib/transcoding/transcoding-quick-transcode.ts b/server/lib/transcoding/transcoding-quick-transcode.ts new file mode 100644 index 000000000..b7f921890 --- /dev/null +++ b/server/lib/transcoding/transcoding-quick-transcode.ts @@ -0,0 +1,61 @@ +import { FfprobeData } from 'fluent-ffmpeg' +import { CONFIG } from '@server/initializers/config' +import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' +import { getMaxBitrate } from '@shared/core-utils' +import { + ffprobePromise, + getAudioStream, + getMaxAudioBitrate, + getVideoStream, + getVideoStreamBitrate, + getVideoStreamDimensionsInfo, + getVideoStreamFPS +} from '@shared/ffmpeg' + +export async function canDoQuickTranscode (path: string, existingProbe?: FfprobeData): Promise { + if (CONFIG.TRANSCODING.PROFILE !== 'default') return false + + const probe = existingProbe || await ffprobePromise(path) + + return await canDoQuickVideoTranscode(path, probe) && + await canDoQuickAudioTranscode(path, probe) +} + +export async function canDoQuickAudioTranscode (path: string, probe?: FfprobeData): Promise { + const parsedAudio = await getAudioStream(path, probe) + + if (!parsedAudio.audioStream) return true + + if (parsedAudio.audioStream['codec_name'] !== 'aac') return false + + const audioBitrate = parsedAudio.bitrate + if (!audioBitrate) return false + + const maxAudioBitrate = getMaxAudioBitrate('aac', audioBitrate) + if (maxAudioBitrate !== -1 && audioBitrate > maxAudioBitrate) return false + + const channelLayout = parsedAudio.audioStream['channel_layout'] + // Causes playback issues with Chrome + if (!channelLayout || channelLayout === 'unknown' || channelLayout === 'quad') return false + + return true +} + +export async function canDoQuickVideoTranscode (path: string, probe?: FfprobeData): Promise { + const videoStream = await getVideoStream(path, probe) + const fps = await getVideoStreamFPS(path, probe) + const bitRate = await getVideoStreamBitrate(path, probe) + const resolutionData = await getVideoStreamDimensionsInfo(path, probe) + + // If ffprobe did not manage to guess the bitrate + if (!bitRate) return false + + // check video params + if (!videoStream) return false + if (videoStream['codec_name'] !== 'h264') return false + if (videoStream['pix_fmt'] !== 'yuv420p') return false + if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false + if (bitRate > getMaxBitrate({ ...resolutionData, fps })) return false + + return true +} diff --git a/server/lib/transcoding/transcoding-resolutions.ts b/server/lib/transcoding/transcoding-resolutions.ts new file mode 100644 index 000000000..91f4d18d8 --- /dev/null +++ b/server/lib/transcoding/transcoding-resolutions.ts @@ -0,0 +1,52 @@ +import { CONFIG } from '@server/initializers/config' +import { toEven } from '@shared/core-utils' +import { VideoResolution } from '@shared/models' + +export function computeResolutionsToTranscode (options: { + input: number + type: 'vod' | 'live' + includeInput: boolean + strictLower: boolean + hasAudio: boolean +}) { + const { input, type, includeInput, strictLower, hasAudio } = options + + const configResolutions = type === 'vod' + ? CONFIG.TRANSCODING.RESOLUTIONS + : CONFIG.LIVE.TRANSCODING.RESOLUTIONS + + const resolutionsEnabled = new Set() + + // Put in the order we want to proceed jobs + const availableResolutions: VideoResolution[] = [ + VideoResolution.H_NOVIDEO, + VideoResolution.H_480P, + VideoResolution.H_360P, + VideoResolution.H_720P, + VideoResolution.H_240P, + VideoResolution.H_144P, + VideoResolution.H_1080P, + VideoResolution.H_1440P, + VideoResolution.H_4K + ] + + for (const resolution of availableResolutions) { + // Resolution not enabled + if (configResolutions[resolution + 'p'] !== true) continue + // Too big resolution for input file + if (input < resolution) continue + // We only want lower resolutions than input file + if (strictLower && input === resolution) continue + // Audio resolutio but no audio in the video + if (resolution === VideoResolution.H_NOVIDEO && !hasAudio) continue + + resolutionsEnabled.add(resolution) + } + + if (includeInput) { + // Always use an even resolution to avoid issues with ffmpeg + resolutionsEnabled.add(toEven(input)) + } + + return Array.from(resolutionsEnabled) +} diff --git a/server/lib/transcoding/transcoding.ts b/server/lib/transcoding/transcoding.ts deleted file mode 100644 index c7b61e9ba..000000000 --- a/server/lib/transcoding/transcoding.ts +++ /dev/null @@ -1,465 +0,0 @@ -import { MutexInterface } from 'async-mutex' -import { Job } from 'bullmq' -import { copyFile, ensureDir, move, remove, stat } from 'fs-extra' -import { basename, extname as extnameUtil, join } from 'path' -import { toEven } from '@server/helpers/core-utils' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' -import { sequelizeTypescript } from '@server/initializers/database' -import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' -import { pick } from '@shared/core-utils' -import { VideoResolution, VideoStorage } from '../../../shared/models/videos' -import { - buildFileMetadata, - canDoQuickTranscode, - computeResolutionsToTranscode, - ffprobePromise, - getVideoStreamDuration, - getVideoStreamFPS, - transcodeVOD, - TranscodeVODOptions, - TranscodeVODOptionsType -} from '../../helpers/ffmpeg' -import { CONFIG } from '../../initializers/config' -import { VideoFileModel } from '../../models/video/video-file' -import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' -import { updatePlaylistAfterFileChange } from '../hls' -import { generateHLSVideoFilename, generateWebTorrentVideoFilename, getHlsResolutionPlaylistFilename } from '../paths' -import { VideoPathManager } from '../video-path-manager' -import { VideoTranscodingProfilesManager } from './default-transcoding-profiles' - -/** - * - * Functions that run transcoding functions, update the database, cleanup files, create torrent files... - * Mainly called by the job queue - * - */ - -// Optimize the original video file and replace it. The resolution is not changed. -async function optimizeOriginalVideofile (options: { - video: MVideoFullLight - inputVideoFile: MVideoFile - job: Job -}) { - const { video, inputVideoFile, job } = options - - const transcodeDirectory = CONFIG.STORAGE.TMP_DIR - const newExtname = '.mp4' - - // Will be released by our transcodeVOD function once ffmpeg is ran - const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - await video.reload() - - const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) - - const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => { - const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) - - const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath) - ? 'quick-transcode' - : 'video' - - const resolution = buildOriginalFileResolution(inputVideoFile.resolution) - - const transcodeOptions: TranscodeVODOptions = { - type: transcodeType, - - inputPath: videoInputPath, - outputPath: videoTranscodedPath, - - inputFileMutexReleaser, - - availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), - profile: CONFIG.TRANSCODING.PROFILE, - - resolution, - - job - } - - // Could be very long! - await transcodeVOD(transcodeOptions) - - // Important to do this before getVideoFilename() to take in account the new filename - inputVideoFile.resolution = resolution - inputVideoFile.extname = newExtname - inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname) - inputVideoFile.storage = VideoStorage.FILE_SYSTEM - - const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile) - await remove(videoInputPath) - - return { transcodeType, videoFile } - }) - - return result - } finally { - inputFileMutexReleaser() - } -} - -// Transcode the original video file to a lower resolution compatible with WebTorrent -async function transcodeNewWebTorrentResolution (options: { - video: MVideoFullLight - resolution: VideoResolution - job: Job -}) { - const { video, resolution, job } = options - - const transcodeDirectory = CONFIG.STORAGE.TMP_DIR - const newExtname = '.mp4' - - const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - await video.reload() - - const file = video.getMaxQualityFile().withVideoOrPlaylist(video) - - const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => { - const newVideoFile = new VideoFileModel({ - resolution, - extname: newExtname, - filename: generateWebTorrentVideoFilename(resolution, newExtname), - size: 0, - videoId: video.id - }) - - const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) - - const transcodeOptions = resolution === VideoResolution.H_NOVIDEO - ? { - type: 'only-audio' as 'only-audio', - - inputPath: videoInputPath, - outputPath: videoTranscodedPath, - - inputFileMutexReleaser, - - availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), - profile: CONFIG.TRANSCODING.PROFILE, - - resolution, - - job - } - : { - type: 'video' as 'video', - inputPath: videoInputPath, - outputPath: videoTranscodedPath, - - inputFileMutexReleaser, - - availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), - profile: CONFIG.TRANSCODING.PROFILE, - - resolution, - - job - } - - await transcodeVOD(transcodeOptions) - - return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, newVideoFile) - }) - - return result - } finally { - inputFileMutexReleaser() - } -} - -// Merge an image with an audio file to create a video -async function mergeAudioVideofile (options: { - video: MVideoFullLight - resolution: VideoResolution - job: Job -}) { - const { video, resolution, job } = options - - const transcodeDirectory = CONFIG.STORAGE.TMP_DIR - const newExtname = '.mp4' - - const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - await video.reload() - - const inputVideoFile = video.getMinQualityFile() - - const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) - - const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => { - const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) - - // If the user updates the video preview during transcoding - const previewPath = video.getPreview().getPath() - const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) - await copyFile(previewPath, tmpPreviewPath) - - const transcodeOptions = { - type: 'merge-audio' as 'merge-audio', - - inputPath: tmpPreviewPath, - outputPath: videoTranscodedPath, - - inputFileMutexReleaser, - - availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), - profile: CONFIG.TRANSCODING.PROFILE, - - audioPath: audioInputPath, - resolution, - - job - } - - try { - await transcodeVOD(transcodeOptions) - - await remove(audioInputPath) - await remove(tmpPreviewPath) - } catch (err) { - await remove(tmpPreviewPath) - throw err - } - - // Important to do this before getVideoFilename() to take in account the new file extension - inputVideoFile.extname = newExtname - inputVideoFile.resolution = resolution - inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) - - // ffmpeg generated a new video file, so update the video duration - // See https://trac.ffmpeg.org/ticket/5456 - video.duration = await getVideoStreamDuration(videoTranscodedPath) - await video.save() - - return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile) - }) - - return result - } finally { - inputFileMutexReleaser() - } -} - -// Concat TS segments from a live video to a fragmented mp4 HLS playlist -async function generateHlsPlaylistResolutionFromTS (options: { - video: MVideo - concatenatedTsFilePath: string - resolution: VideoResolution - isAAC: boolean - inputFileMutexReleaser: MutexInterface.Releaser -}) { - return generateHlsPlaylistCommon({ - type: 'hls-from-ts' as 'hls-from-ts', - inputPath: options.concatenatedTsFilePath, - - ...pick(options, [ 'video', 'resolution', 'inputFileMutexReleaser', 'isAAC' ]) - }) -} - -// Generate an HLS playlist from an input file, and update the master playlist -function generateHlsPlaylistResolution (options: { - video: MVideo - videoInputPath: string - resolution: VideoResolution - copyCodecs: boolean - inputFileMutexReleaser: MutexInterface.Releaser - job?: Job -}) { - return generateHlsPlaylistCommon({ - type: 'hls' as 'hls', - inputPath: options.videoInputPath, - - ...pick(options, [ 'video', 'resolution', 'copyCodecs', 'inputFileMutexReleaser', 'job' ]) - }) -} - -// --------------------------------------------------------------------------- - -export { - generateHlsPlaylistResolution, - generateHlsPlaylistResolutionFromTS, - optimizeOriginalVideofile, - transcodeNewWebTorrentResolution, - mergeAudioVideofile -} - -// --------------------------------------------------------------------------- - -async function onWebTorrentVideoFileTranscoding ( - video: MVideoFullLight, - videoFile: MVideoFile, - transcodingPath: string, - newVideoFile: MVideoFile -) { - const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - await video.reload() - - const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile) - - const stats = await stat(transcodingPath) - - const probe = await ffprobePromise(transcodingPath) - const fps = await getVideoStreamFPS(transcodingPath, probe) - const metadata = await buildFileMetadata(transcodingPath, probe) - - await move(transcodingPath, outputPath, { overwrite: true }) - - videoFile.size = stats.size - videoFile.fps = fps - videoFile.metadata = metadata - - await createTorrentAndSetInfoHash(video, videoFile) - - const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) - if (oldFile) await video.removeWebTorrentFile(oldFile) - - await VideoFileModel.customUpsert(videoFile, 'video', undefined) - video.VideoFiles = await video.$get('VideoFiles') - - return { video, videoFile } - } finally { - mutexReleaser() - } -} - -async function generateHlsPlaylistCommon (options: { - type: 'hls' | 'hls-from-ts' - video: MVideo - inputPath: string - resolution: VideoResolution - - inputFileMutexReleaser: MutexInterface.Releaser - - copyCodecs?: boolean - isAAC?: boolean - - job?: Job -}) { - const { type, video, inputPath, resolution, copyCodecs, isAAC, job, inputFileMutexReleaser } = options - const transcodeDirectory = CONFIG.STORAGE.TMP_DIR - - const videoTranscodedBasePath = join(transcodeDirectory, type) - await ensureDir(videoTranscodedBasePath) - - const videoFilename = generateHLSVideoFilename(resolution) - const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename) - const resolutionPlaylistFileTranscodePath = join(videoTranscodedBasePath, resolutionPlaylistFilename) - - const transcodeOptions = { - type, - - inputPath, - outputPath: resolutionPlaylistFileTranscodePath, - - availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), - profile: CONFIG.TRANSCODING.PROFILE, - - resolution, - copyCodecs, - - isAAC, - - inputFileMutexReleaser, - - hlsPlaylist: { - videoFilename - }, - - job - } - - await transcodeVOD(transcodeOptions) - - // Create or update the playlist - const playlist = await retryTransactionWrapper(() => { - return sequelizeTypescript.transaction(async transaction => { - return VideoStreamingPlaylistModel.loadOrGenerate(video, transaction) - }) - }) - - const newVideoFile = new VideoFileModel({ - resolution, - extname: extnameUtil(videoFilename), - size: 0, - filename: videoFilename, - fps: -1, - videoStreamingPlaylistId: playlist.id - }) - - const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - // VOD transcoding is a long task, refresh video attributes - await video.reload() - - const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile) - await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video)) - - // Move playlist file - const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename) - await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) - // Move video file - await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true }) - - // Update video duration if it was not set (in case of a live for example) - if (!video.duration) { - video.duration = await getVideoStreamDuration(videoFilePath) - await video.save() - } - - const stats = await stat(videoFilePath) - - newVideoFile.size = stats.size - newVideoFile.fps = await getVideoStreamFPS(videoFilePath) - newVideoFile.metadata = await buildFileMetadata(videoFilePath) - - await createTorrentAndSetInfoHash(playlist, newVideoFile) - - const oldFile = await VideoFileModel.loadHLSFile({ - playlistId: playlist.id, - fps: newVideoFile.fps, - resolution: newVideoFile.resolution - }) - - if (oldFile) { - await video.removeStreamingPlaylistVideoFile(playlist, oldFile) - await oldFile.destroy() - } - - const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) - - await updatePlaylistAfterFileChange(video, playlist) - - return { resolutionPlaylistPath, videoFile: savedVideoFile } - } finally { - mutexReleaser() - } -} - -function buildOriginalFileResolution (inputResolution: number) { - if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) { - return toEven(inputResolution) - } - - const resolutions = computeResolutionsToTranscode({ - input: inputResolution, - type: 'vod', - includeInput: false, - strictLower: false, - // We don't really care about the audio resolution in this context - hasAudio: true - }) - - if (resolutions.length === 0) { - return toEven(inputResolution) - } - - return Math.max(...resolutions) -} diff --git a/server/lib/transcoding/web-transcoding.ts b/server/lib/transcoding/web-transcoding.ts new file mode 100644 index 000000000..d43d03b2a --- /dev/null +++ b/server/lib/transcoding/web-transcoding.ts @@ -0,0 +1,273 @@ +import { Job } from 'bullmq' +import { copyFile, move, remove, stat } from 'fs-extra' +import { basename, join } from 'path' +import { computeOutputFPS } from '@server/helpers/ffmpeg' +import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' +import { MVideoFile, MVideoFullLight } from '@server/types/models' +import { toEven } from '@shared/core-utils' +import { ffprobePromise, getVideoStreamDuration, getVideoStreamFPS, TranscodeVODOptionsType } from '@shared/ffmpeg' +import { VideoResolution, VideoStorage } from '@shared/models' +import { CONFIG } from '../../initializers/config' +import { VideoFileModel } from '../../models/video/video-file' +import { generateWebTorrentVideoFilename } from '../paths' +import { buildFileMetadata } from '../video-file' +import { VideoPathManager } from '../video-path-manager' +import { buildFFmpegVOD } from './shared' +import { computeResolutionsToTranscode } from './transcoding-resolutions' + +// Optimize the original video file and replace it. The resolution is not changed. +export async function optimizeOriginalVideofile (options: { + video: MVideoFullLight + inputVideoFile: MVideoFile + quickTranscode: boolean + job: Job +}) { + const { video, inputVideoFile, quickTranscode, job } = options + + const transcodeDirectory = CONFIG.STORAGE.TMP_DIR + const newExtname = '.mp4' + + // Will be released by our transcodeVOD function once ffmpeg is ran + const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) + + try { + await video.reload() + + const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) + + const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => { + const videoOutputPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) + + const transcodeType: TranscodeVODOptionsType = quickTranscode + ? 'quick-transcode' + : 'video' + + const resolution = buildOriginalFileResolution(inputVideoFile.resolution) + const fps = computeOutputFPS({ inputFPS: inputVideoFile.fps, resolution }) + + // Could be very long! + await buildFFmpegVOD(job).transcode({ + type: transcodeType, + + inputPath: videoInputPath, + outputPath: videoOutputPath, + + inputFileMutexReleaser, + + resolution, + fps + }) + + // Important to do this before getVideoFilename() to take in account the new filename + inputVideoFile.resolution = resolution + inputVideoFile.extname = newExtname + inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname) + inputVideoFile.storage = VideoStorage.FILE_SYSTEM + + const { videoFile } = await onWebTorrentVideoFileTranscoding({ + video, + videoFile: inputVideoFile, + videoOutputPath + }) + + await remove(videoInputPath) + + return { transcodeType, videoFile } + }) + + return result + } finally { + inputFileMutexReleaser() + } +} + +// Transcode the original video file to a lower resolution compatible with WebTorrent +export async function transcodeNewWebTorrentResolution (options: { + video: MVideoFullLight + resolution: VideoResolution + fps: number + job: Job +}) { + const { video, resolution, fps, job } = options + + const transcodeDirectory = CONFIG.STORAGE.TMP_DIR + const newExtname = '.mp4' + + const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) + + try { + await video.reload() + + const file = video.getMaxQualityFile().withVideoOrPlaylist(video) + + const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => { + const newVideoFile = new VideoFileModel({ + resolution, + extname: newExtname, + filename: generateWebTorrentVideoFilename(resolution, newExtname), + size: 0, + videoId: video.id + }) + + const videoOutputPath = join(transcodeDirectory, newVideoFile.filename) + + const transcodeOptions = { + type: 'video' as 'video', + + inputPath: videoInputPath, + outputPath: videoOutputPath, + + inputFileMutexReleaser, + + resolution, + fps + } + + await buildFFmpegVOD(job).transcode(transcodeOptions) + + return onWebTorrentVideoFileTranscoding({ video, videoFile: newVideoFile, videoOutputPath }) + }) + + return result + } finally { + inputFileMutexReleaser() + } +} + +// Merge an image with an audio file to create a video +export async function mergeAudioVideofile (options: { + video: MVideoFullLight + resolution: VideoResolution + fps: number + job: Job +}) { + const { video, resolution, fps, job } = options + + const transcodeDirectory = CONFIG.STORAGE.TMP_DIR + const newExtname = '.mp4' + + const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) + + try { + await video.reload() + + const inputVideoFile = video.getMinQualityFile() + + const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) + + const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => { + const videoOutputPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) + + // If the user updates the video preview during transcoding + const previewPath = video.getPreview().getPath() + const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) + await copyFile(previewPath, tmpPreviewPath) + + const transcodeOptions = { + type: 'merge-audio' as 'merge-audio', + + inputPath: tmpPreviewPath, + outputPath: videoOutputPath, + + inputFileMutexReleaser, + + audioPath: audioInputPath, + resolution, + fps + } + + try { + await buildFFmpegVOD(job).transcode(transcodeOptions) + + await remove(audioInputPath) + await remove(tmpPreviewPath) + } catch (err) { + await remove(tmpPreviewPath) + throw err + } + + // Important to do this before getVideoFilename() to take in account the new file extension + inputVideoFile.extname = newExtname + inputVideoFile.resolution = resolution + inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) + + // ffmpeg generated a new video file, so update the video duration + // See https://trac.ffmpeg.org/ticket/5456 + video.duration = await getVideoStreamDuration(videoOutputPath) + await video.save() + + return onWebTorrentVideoFileTranscoding({ + video, + videoFile: inputVideoFile, + videoOutputPath + }) + }) + + return result + } finally { + inputFileMutexReleaser() + } +} + +export async function onWebTorrentVideoFileTranscoding (options: { + video: MVideoFullLight + videoFile: MVideoFile + videoOutputPath: string +}) { + const { video, videoFile, videoOutputPath } = options + + const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) + + try { + await video.reload() + + const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile) + + const stats = await stat(videoOutputPath) + + const probe = await ffprobePromise(videoOutputPath) + const fps = await getVideoStreamFPS(videoOutputPath, probe) + const metadata = await buildFileMetadata(videoOutputPath, probe) + + await move(videoOutputPath, outputPath, { overwrite: true }) + + videoFile.size = stats.size + videoFile.fps = fps + videoFile.metadata = metadata + + await createTorrentAndSetInfoHash(video, videoFile) + + const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) + if (oldFile) await video.removeWebTorrentFile(oldFile) + + await VideoFileModel.customUpsert(videoFile, 'video', undefined) + video.VideoFiles = await video.$get('VideoFiles') + + return { video, videoFile } + } finally { + mutexReleaser() + } +} + +// --------------------------------------------------------------------------- + +function buildOriginalFileResolution (inputResolution: number) { + if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) { + return toEven(inputResolution) + } + + const resolutions = computeResolutionsToTranscode({ + input: inputResolution, + type: 'vod', + includeInput: false, + strictLower: false, + // We don't really care about the audio resolution in this context + hasAudio: true + }) + + if (resolutions.length === 0) { + return toEven(inputResolution) + } + + return Math.max(...resolutions) +} diff --git a/server/lib/uploadx.ts b/server/lib/uploadx.ts index 58040cb6d..c7e0eb414 100644 --- a/server/lib/uploadx.ts +++ b/server/lib/uploadx.ts @@ -3,6 +3,7 @@ import { buildLogger } from '@server/helpers/logger' import { getResumableUploadPath } from '@server/helpers/upload' import { CONFIG } from '@server/initializers/config' import { LogLevel, Uploadx } from '@uploadx/core' +import { extname } from 'path' const logger = buildLogger('uploadx') @@ -26,7 +27,9 @@ const uploadx = new Uploadx({ if (!res.locals.oauth) return undefined return res.locals.oauth.token.user.id + '' - } + }, + + filename: file => `${file.userId}-${file.id}${extname(file.metadata.filename)}` }) export { diff --git a/server/lib/video-blacklist.ts b/server/lib/video-blacklist.ts index fd5837a3a..cb1ea834c 100644 --- a/server/lib/video-blacklist.ts +++ b/server/lib/video-blacklist.ts @@ -81,7 +81,7 @@ async function blacklistVideo (videoInstance: MVideoAccountLight, options: Video } if (videoInstance.isLive) { - LiveManager.Instance.stopSessionOf(videoInstance.id, LiveVideoError.BLACKLISTED) + LiveManager.Instance.stopSessionOf(videoInstance.uuid, LiveVideoError.BLACKLISTED) } Notifier.Instance.notifyOnVideoBlacklist(blacklist) diff --git a/server/lib/video-file.ts b/server/lib/video-file.ts index 2ab7190f1..8fcc3c253 100644 --- a/server/lib/video-file.ts +++ b/server/lib/video-file.ts @@ -1,6 +1,44 @@ +import { FfprobeData } from 'fluent-ffmpeg' import { logger } from '@server/helpers/logger' +import { VideoFileModel } from '@server/models/video/video-file' import { MVideoWithAllFiles } from '@server/types/models' +import { getLowercaseExtension } from '@shared/core-utils' +import { getFileSize } from '@shared/extra-utils' +import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, isAudioFile } from '@shared/ffmpeg' +import { VideoFileMetadata, VideoResolution } from '@shared/models' import { lTags } from './object-storage/shared' +import { generateHLSVideoFilename, generateWebTorrentVideoFilename } from './paths' + +async function buildNewFile (options: { + path: string + mode: 'web-video' | 'hls' +}) { + const { path, mode } = options + + const probe = await ffprobePromise(path) + const size = await getFileSize(path) + + const videoFile = new VideoFileModel({ + extname: getLowercaseExtension(path), + size, + metadata: await buildFileMetadata(path, probe) + }) + + if (await isAudioFile(path, probe)) { + videoFile.resolution = VideoResolution.H_NOVIDEO + } else { + videoFile.fps = await getVideoStreamFPS(path, probe) + videoFile.resolution = (await getVideoStreamDimensionsInfo(path, probe)).resolution + } + + videoFile.filename = mode === 'web-video' + ? generateWebTorrentVideoFilename(videoFile.resolution, videoFile.extname) + : generateHLSVideoFilename(videoFile.resolution) + + return videoFile +} + +// --------------------------------------------------------------------------- async function removeHLSPlaylist (video: MVideoWithAllFiles) { const hls = video.getHLSPlaylist() @@ -61,9 +99,23 @@ async function removeWebTorrentFile (video: MVideoWithAllFiles, fileToDeleteId: return video } +// --------------------------------------------------------------------------- + +async function buildFileMetadata (path: string, existingProbe?: FfprobeData) { + const metadata = existingProbe || await ffprobePromise(path) + + return new VideoFileMetadata(metadata) +} + +// --------------------------------------------------------------------------- + export { + buildNewFile, + removeHLSPlaylist, removeHLSFile, removeAllWebTorrentFiles, - removeWebTorrentFile + removeWebTorrentFile, + + buildFileMetadata } diff --git a/server/lib/video-studio.ts b/server/lib/video-studio.ts index cdacd35f2..b392bdb00 100644 --- a/server/lib/video-studio.ts +++ b/server/lib/video-studio.ts @@ -1,5 +1,5 @@ import { MVideoFullLight } from '@server/types/models' -import { getVideoStreamDuration } from '@shared/extra-utils' +import { getVideoStreamDuration } from '@shared/ffmpeg' import { VideoStudioTask } from '@shared/models' function buildTaskFileFieldname (indice: number, fieldName = 'file') { diff --git a/server/lib/video.ts b/server/lib/video.ts index aacc41a7a..588dc553f 100644 --- a/server/lib/video.ts +++ b/server/lib/video.ts @@ -2,14 +2,14 @@ import { UploadFiles } from 'express' import memoizee from 'memoizee' import { Transaction } from 'sequelize/types' import { CONFIG } from '@server/initializers/config' -import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY, MEMOIZE_LENGTH, MEMOIZE_TTL } from '@server/initializers/constants' +import { MEMOIZE_LENGTH, MEMOIZE_TTL } from '@server/initializers/constants' import { TagModel } from '@server/models/video/tag' import { VideoModel } from '@server/models/video/video' import { VideoJobInfoModel } from '@server/models/video/video-job-info' import { FilteredModelAttributes } from '@server/types' -import { MThumbnail, MUserId, MVideoFile, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' -import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState, VideoTranscodingPayload } from '@shared/models' -import { CreateJobArgument, CreateJobOptions, JobQueue } from './job-queue/job-queue' +import { MThumbnail, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' +import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState } from '@shared/models' +import { CreateJobArgument, JobQueue } from './job-queue/job-queue' import { updateVideoMiniatureFromExisting } from './thumbnail' import { moveFilesIfPrivacyChanged } from './video-privacy' @@ -87,58 +87,6 @@ async function setVideoTags (options: { // --------------------------------------------------------------------------- -async function buildOptimizeOrMergeAudioJob (options: { - video: MVideoUUID - videoFile: MVideoFile - user: MUserId - isNewVideo?: boolean // Default true -}) { - const { video, videoFile, user, isNewVideo } = options - - let payload: VideoTranscodingPayload - - if (videoFile.isAudio()) { - payload = { - type: 'merge-audio-to-webtorrent', - resolution: DEFAULT_AUDIO_RESOLUTION, - videoUUID: video.uuid, - createHLSIfNeeded: true, - isNewVideo - } - } else { - payload = { - type: 'optimize-to-webtorrent', - videoUUID: video.uuid, - isNewVideo - } - } - - await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode') - - return { - type: 'video-transcoding' as 'video-transcoding', - priority: await getTranscodingJobPriority(user), - payload - } -} - -async function buildTranscodingJob (payload: VideoTranscodingPayload, options: CreateJobOptions = {}) { - await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode') - - return { type: 'video-transcoding' as 'video-transcoding', payload, ...options } -} - -async function getTranscodingJobPriority (user: MUserId) { - const now = new Date() - const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7) - - const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek) - - return JOB_PRIORITY.TRANSCODING + videoUploadedByUser -} - -// --------------------------------------------------------------------------- - async function buildMoveToObjectStorageJob (options: { video: MVideoUUID previousVideoState: VideoState @@ -235,10 +183,7 @@ export { buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags, - buildOptimizeOrMergeAudioJob, - buildTranscodingJob, buildMoveToObjectStorageJob, - getTranscodingJobPriority, addVideoJobsAfterUpdate, getCachedVideoDuration } diff --git a/server/middlewares/auth.ts b/server/middlewares/auth.ts index e6025c8ce..0eefa2a8e 100644 --- a/server/middlewares/auth.ts +++ b/server/middlewares/auth.ts @@ -1,6 +1,7 @@ import express from 'express' import { Socket } from 'socket.io' import { getAccessToken } from '@server/lib/auth/oauth-model' +import { RunnerModel } from '@server/models/runner/runner' import { HttpStatusCode } from '../../shared/models/http/http-error-codes' import { logger } from '../helpers/logger' import { handleOAuthAuthenticate } from '../lib/auth/oauth' @@ -27,7 +28,7 @@ function authenticate (req: express.Request, res: express.Response, next: expres function authenticateSocket (socket: Socket, next: (err?: any) => void) { const accessToken = socket.handshake.query['accessToken'] - logger.debug('Checking socket access token %s.', accessToken) + logger.debug('Checking access token in runner.') if (!accessToken) return next(new Error('No access token provided')) if (typeof accessToken !== 'string') return next(new Error('Access token is invalid')) @@ -73,9 +74,31 @@ function optionalAuthenticate (req: express.Request, res: express.Response, next // --------------------------------------------------------------------------- +function authenticateRunnerSocket (socket: Socket, next: (err?: any) => void) { + const runnerToken = socket.handshake.auth['runnerToken'] + + logger.debug('Checking runner token in socket.') + + if (!runnerToken) return next(new Error('No runner token provided')) + if (typeof runnerToken !== 'string') return next(new Error('Runner token is invalid')) + + RunnerModel.loadByToken(runnerToken) + .then(runner => { + if (!runner) return next(new Error('Invalid runner token.')) + + socket.handshake.auth.runner = runner + + return next() + }) + .catch(err => logger.error('Cannot get runner token.', { err })) +} + +// --------------------------------------------------------------------------- + export { authenticate, authenticateSocket, authenticatePromise, - optionalAuthenticate + optionalAuthenticate, + authenticateRunnerSocket } diff --git a/server/middlewares/doc.ts b/server/middlewares/doc.ts index c43f41977..eef76acaa 100644 --- a/server/middlewares/doc.ts +++ b/server/middlewares/doc.ts @@ -5,7 +5,7 @@ function openapiOperationDoc (options: { operationId?: string }) { return (req: express.Request, res: express.Response, next: express.NextFunction) => { - res.locals.docUrl = options.url || 'https://docs.joinpeertube.org/api/rest-reference.html#operation/' + options.operationId + res.locals.docUrl = options.url || 'https://docs.joinpeertube.org/api-rest-reference.html#operation/' + options.operationId if (next) return next() } diff --git a/server/middlewares/error.ts b/server/middlewares/error.ts index 540edaeeb..94762e355 100644 --- a/server/middlewares/error.ts +++ b/server/middlewares/error.ts @@ -5,7 +5,7 @@ import { HttpStatusCode } from '@shared/models' function apiFailMiddleware (req: express.Request, res: express.Response, next: express.NextFunction) { res.fail = options => { - const { status = HttpStatusCode.BAD_REQUEST_400, message, title, type, data, instance } = options + const { status = HttpStatusCode.BAD_REQUEST_400, message, title, type, data, instance, tags } = options const extension = new ProblemDocumentExtension({ ...data, @@ -31,11 +31,11 @@ function apiFailMiddleware (req: express.Request, res: express.Response, next: e detail: message, type: type - ? `https://docs.joinpeertube.org/api/rest-reference.html#section/Errors/${type}` + ? `https://docs.joinpeertube.org/api-rest-reference.html#section/Errors/${type}` : undefined }, extension) - logger.debug('Bad HTTP request.', { json }) + logger.debug('Bad HTTP request.', { json, tags }) res.json(json) } diff --git a/server/middlewares/rate-limiter.ts b/server/middlewares/rate-limiter.ts index bc9513969..1eef8b360 100644 --- a/server/middlewares/rate-limiter.ts +++ b/server/middlewares/rate-limiter.ts @@ -1,10 +1,12 @@ +import express from 'express' +import RateLimit, { Options as RateLimitHandlerOptions } from 'express-rate-limit' +import { RunnerModel } from '@server/models/runner/runner' import { UserRole } from '@shared/models' -import RateLimit from 'express-rate-limit' import { optionalAuthenticate } from './auth' const whitelistRoles = new Set([ UserRole.ADMINISTRATOR, UserRole.MODERATOR ]) -function buildRateLimiter (options: { +export function buildRateLimiter (options: { windowMs: number max: number skipFailedRequests?: boolean @@ -15,17 +17,33 @@ function buildRateLimiter (options: { skipFailedRequests: options.skipFailedRequests, handler: (req, res, next, options) => { + // Bypass rate limit for registered runners + if (req.body?.runnerToken) { + return RunnerModel.loadByToken(req.body.runnerToken) + .then(runner => { + if (runner) return next() + + return sendRateLimited(res, options) + }) + } + + // Bypass rate limit for admins/moderators return optionalAuthenticate(req, res, () => { if (res.locals.authenticated === true && whitelistRoles.has(res.locals.oauth.token.User.role)) { return next() } - return res.status(options.statusCode).send(options.message) + return sendRateLimited(res, options) }) } }) } -export { - buildRateLimiter +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +function sendRateLimited (res: express.Response, options: RateLimitHandlerOptions) { + return res.status(options.statusCode).send(options.message) + } diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts index 4a9d1cb54..b3e7e5011 100644 --- a/server/middlewares/validators/config.ts +++ b/server/middlewares/validators/config.ts @@ -54,6 +54,7 @@ const customConfigUpdateValidator = [ body('transcoding.resolutions.1080p').isBoolean(), body('transcoding.resolutions.1440p').isBoolean(), body('transcoding.resolutions.2160p').isBoolean(), + body('transcoding.remoteRunners.enabled').isBoolean(), body('transcoding.alwaysTranscodeOriginalResolution').isBoolean(), @@ -97,6 +98,7 @@ const customConfigUpdateValidator = [ body('live.transcoding.resolutions.1440p').isBoolean(), body('live.transcoding.resolutions.2160p').isBoolean(), body('live.transcoding.alwaysTranscodeOriginalResolution').isBoolean(), + body('live.transcoding.remoteRunners.enabled').isBoolean(), body('search.remoteUri.users').isBoolean(), body('search.remoteUri.anonymous').isBoolean(), diff --git a/server/middlewares/validators/runners/index.ts b/server/middlewares/validators/runners/index.ts new file mode 100644 index 000000000..9a9629a80 --- /dev/null +++ b/server/middlewares/validators/runners/index.ts @@ -0,0 +1,3 @@ +export * from './jobs' +export * from './registration-token' +export * from './runners' diff --git a/server/middlewares/validators/runners/job-files.ts b/server/middlewares/validators/runners/job-files.ts new file mode 100644 index 000000000..56afa39aa --- /dev/null +++ b/server/middlewares/validators/runners/job-files.ts @@ -0,0 +1,27 @@ +import express from 'express' +import { HttpStatusCode } from '@shared/models' +import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared' + +const tags = [ 'runner' ] + +export const runnerJobGetVideoTranscodingFileValidator = [ + isValidVideoIdParam('videoId'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (!await doesVideoExist(req.params.videoId, res, 'all')) return + + const runnerJob = res.locals.runnerJob + + if (runnerJob.privatePayload.videoUUID !== res.locals.videoAll.uuid) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Job is not associated to this video', + tags: [ ...tags, res.locals.videoAll.uuid ] + }) + } + + return next() + } +] diff --git a/server/middlewares/validators/runners/jobs.ts b/server/middlewares/validators/runners/jobs.ts new file mode 100644 index 000000000..8cb87e946 --- /dev/null +++ b/server/middlewares/validators/runners/jobs.ts @@ -0,0 +1,156 @@ +import express from 'express' +import { body, param } from 'express-validator' +import { isUUIDValid } from '@server/helpers/custom-validators/misc' +import { + isRunnerJobAbortReasonValid, + isRunnerJobErrorMessageValid, + isRunnerJobProgressValid, + isRunnerJobSuccessPayloadValid, + isRunnerJobTokenValid, + isRunnerJobUpdatePayloadValid +} from '@server/helpers/custom-validators/runners/jobs' +import { isRunnerTokenValid } from '@server/helpers/custom-validators/runners/runners' +import { cleanUpReqFiles } from '@server/helpers/express-utils' +import { RunnerJobModel } from '@server/models/runner/runner-job' +import { HttpStatusCode, RunnerJobState, RunnerJobSuccessBody, RunnerJobUpdateBody, ServerErrorCode } from '@shared/models' +import { areValidationErrors } from '../shared' + +const tags = [ 'runner' ] + +export const acceptRunnerJobValidator = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (res.locals.runnerJob.state !== RunnerJobState.PENDING) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'This runner job is not in pending state', + tags + }) + } + + return next() + } +] + +export const abortRunnerJobValidator = [ + body('reason').custom(isRunnerJobAbortReasonValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return + + return next() + } +] + +export const updateRunnerJobValidator = [ + body('progress').optional().custom(isRunnerJobProgressValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return cleanUpReqFiles(req) + + const body = req.body as RunnerJobUpdateBody + + if (isRunnerJobUpdatePayloadValid(body.payload, res.locals.runnerJob.type, req.files) !== true) { + cleanUpReqFiles(req) + + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Payload is invalid', + tags + }) + } + + return next() + } +] + +export const errorRunnerJobValidator = [ + body('message').custom(isRunnerJobErrorMessageValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return + + return next() + } +] + +export const successRunnerJobValidator = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + const body = req.body as RunnerJobSuccessBody + + if (isRunnerJobSuccessPayloadValid(body.payload, res.locals.runnerJob.type, req.files) !== true) { + cleanUpReqFiles(req) + + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Payload is invalid', + tags + }) + } + + return next() + } +] + +export const runnerJobGetValidator = [ + param('jobUUID').custom(isUUIDValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return + + const runnerJob = await RunnerJobModel.loadWithRunner(req.params.jobUUID) + + if (!runnerJob) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Unknown runner job', + tags + }) + } + + res.locals.runnerJob = runnerJob + + return next() + } +] + +export const jobOfRunnerGetValidator = [ + param('jobUUID').custom(isUUIDValid), + + body('runnerToken').custom(isRunnerTokenValid), + body('jobToken').custom(isRunnerJobTokenValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return cleanUpReqFiles(req) + + const runnerJob = await RunnerJobModel.loadByRunnerAndJobTokensWithRunner({ + uuid: req.params.jobUUID, + runnerToken: req.body.runnerToken, + jobToken: req.body.jobToken + }) + + if (!runnerJob) { + cleanUpReqFiles(req) + + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Unknown runner job', + tags + }) + } + + if (runnerJob.state !== RunnerJobState.PROCESSING) { + cleanUpReqFiles(req) + + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + type: ServerErrorCode.RUNNER_JOB_NOT_IN_PROCESSING_STATE, + message: 'Job is not in "processing" state', + tags + }) + } + + res.locals.runnerJob = runnerJob + + return next() + } +] diff --git a/server/middlewares/validators/runners/registration-token.ts b/server/middlewares/validators/runners/registration-token.ts new file mode 100644 index 000000000..cc31d4a7e --- /dev/null +++ b/server/middlewares/validators/runners/registration-token.ts @@ -0,0 +1,37 @@ +import express from 'express' +import { param } from 'express-validator' +import { isIdValid } from '@server/helpers/custom-validators/misc' +import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token' +import { forceNumber } from '@shared/core-utils' +import { HttpStatusCode } from '@shared/models' +import { areValidationErrors } from '../shared/utils' + +const tags = [ 'runner' ] + +const deleteRegistrationTokenValidator = [ + param('id').custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return + + const registrationToken = await RunnerRegistrationTokenModel.load(forceNumber(req.params.id)) + + if (!registrationToken) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Registration token not found', + tags + }) + } + + res.locals.runnerRegistrationToken = registrationToken + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + deleteRegistrationTokenValidator +} diff --git a/server/middlewares/validators/runners/runners.ts b/server/middlewares/validators/runners/runners.ts new file mode 100644 index 000000000..71a1275d2 --- /dev/null +++ b/server/middlewares/validators/runners/runners.ts @@ -0,0 +1,95 @@ +import express from 'express' +import { body, param } from 'express-validator' +import { isIdValid } from '@server/helpers/custom-validators/misc' +import { + isRunnerDescriptionValid, + isRunnerNameValid, + isRunnerRegistrationTokenValid, + isRunnerTokenValid +} from '@server/helpers/custom-validators/runners/runners' +import { RunnerModel } from '@server/models/runner/runner' +import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token' +import { forceNumber } from '@shared/core-utils' +import { HttpStatusCode, RegisterRunnerBody, ServerErrorCode } from '@shared/models' +import { areValidationErrors } from '../shared/utils' + +const tags = [ 'runner' ] + +const registerRunnerValidator = [ + body('registrationToken').custom(isRunnerRegistrationTokenValid), + body('name').custom(isRunnerNameValid), + body('description').optional().custom(isRunnerDescriptionValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return + + const body: RegisterRunnerBody = req.body + + const runnerRegistrationToken = await RunnerRegistrationTokenModel.loadByRegistrationToken(body.registrationToken) + + if (!runnerRegistrationToken) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Registration token is invalid', + tags + }) + } + + res.locals.runnerRegistrationToken = runnerRegistrationToken + + return next() + } +] + +const deleteRunnerValidator = [ + param('runnerId').custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return + + const runner = await RunnerModel.load(forceNumber(req.params.runnerId)) + + if (!runner) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Runner not found', + tags + }) + } + + res.locals.runner = runner + + return next() + } +] + +const getRunnerFromTokenValidator = [ + body('runnerToken').custom(isRunnerTokenValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return + + const runner = await RunnerModel.loadByToken(req.body.runnerToken) + + if (!runner) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Unknown runner token', + type: ServerErrorCode.UNKNOWN_RUNNER_TOKEN, + tags + }) + } + + res.locals.runner = runner + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + registerRunnerValidator, + deleteRunnerValidator, + getRunnerFromTokenValidator +} diff --git a/server/middlewares/validators/sort.ts b/server/middlewares/validators/sort.ts index e6cc46317..959f663ac 100644 --- a/server/middlewares/validators/sort.ts +++ b/server/middlewares/validators/sort.ts @@ -34,6 +34,10 @@ export const videoChannelsFollowersSortValidator = checkSortFactory(SORTABLE_COL export const userRegistrationsSortValidator = checkSortFactory(SORTABLE_COLUMNS.USER_REGISTRATIONS) +export const runnersSortValidator = checkSortFactory(SORTABLE_COLUMNS.RUNNERS) +export const runnerRegistrationTokensSortValidator = checkSortFactory(SORTABLE_COLUMNS.RUNNER_REGISTRATION_TOKENS) +export const runnerJobsSortValidator = checkSortFactory(SORTABLE_COLUMNS.RUNNER_JOBS) + // --------------------------------------------------------------------------- function checkSortFactory (columns: string[], tags: string[] = []) { diff --git a/server/middlewares/validators/videos/video-live.ts b/server/middlewares/validators/videos/video-live.ts index e80fe1593..2aff831a8 100644 --- a/server/middlewares/validators/videos/video-live.ts +++ b/server/middlewares/validators/videos/video-live.ts @@ -115,6 +115,15 @@ const videoLiveAddValidator = getCommonVideoEditAttributes().concat([ }) } + if (body.saveReplay && !body.replaySettings?.privacy) { + cleanUpReqFiles(req) + + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Live replay is enabled but privacy replay setting is missing' + }) + } + const user = res.locals.oauth.token.User if (!await doesVideoChannelOfAccountExist(body.channelId, user, res)) return cleanUpReqFiles(req) diff --git a/server/middlewares/validators/videos/video-studio.ts b/server/middlewares/validators/videos/video-studio.ts index b3e2d8101..4397e887e 100644 --- a/server/middlewares/validators/videos/video-studio.ts +++ b/server/middlewares/validators/videos/video-studio.ts @@ -10,7 +10,7 @@ import { import { cleanUpReqFiles } from '@server/helpers/express-utils' import { CONFIG } from '@server/initializers/config' import { approximateIntroOutroAdditionalSize, getTaskFile } from '@server/lib/video-studio' -import { isAudioFile } from '@shared/extra-utils' +import { isAudioFile } from '@shared/ffmpeg' import { HttpStatusCode, UserRight, VideoState, VideoStudioCreateEdition, VideoStudioTask } from '@shared/models' import { areValidationErrors, checkUserCanManageVideo, checkUserQuota, doesVideoExist } from '../shared' diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts index d3014e8e7..794e1d4f1 100644 --- a/server/middlewares/validators/videos/videos.ts +++ b/server/middlewares/validators/videos/videos.ts @@ -7,6 +7,7 @@ import { getServerActor } from '@server/models/application/application' import { ExpressPromiseHandler } from '@server/types/express-handler' import { MUserAccountId, MVideoFullLight } from '@server/types/models' import { arrayify, getAllPrivacies } from '@shared/core-utils' +import { getVideoStreamDuration } from '@shared/ffmpeg' import { HttpStatusCode, ServerErrorCode, UserRight, VideoInclude, VideoState } from '@shared/models' import { exists, @@ -37,7 +38,6 @@ import { isVideoSupportValid } from '../../../helpers/custom-validators/videos' import { cleanUpReqFiles } from '../../../helpers/express-utils' -import { getVideoStreamDuration } from '../../../helpers/ffmpeg' import { logger } from '../../../helpers/logger' import { deleteFileAndCatch } from '../../../helpers/utils' import { getVideoWithAttributes } from '../../../helpers/video' diff --git a/server/models/runner/runner-job.ts b/server/models/runner/runner-job.ts new file mode 100644 index 000000000..add6f9a43 --- /dev/null +++ b/server/models/runner/runner-job.ts @@ -0,0 +1,347 @@ +import { FindOptions, Op, Transaction } from 'sequelize' +import { + AllowNull, + BelongsTo, + Column, + CreatedAt, + DataType, + Default, + ForeignKey, + IsUUID, + Model, + Scopes, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { isUUIDValid } from '@server/helpers/custom-validators/misc' +import { CONSTRAINTS_FIELDS, RUNNER_JOB_STATES } from '@server/initializers/constants' +import { MRunnerJob, MRunnerJobRunner, MRunnerJobRunnerParent } from '@server/types/models/runners' +import { RunnerJob, RunnerJobAdmin, RunnerJobPayload, RunnerJobPrivatePayload, RunnerJobState, RunnerJobType } from '@shared/models' +import { AttributesOnly } from '@shared/typescript-utils' +import { getSort, searchAttribute } from '../shared' +import { RunnerModel } from './runner' + +enum ScopeNames { + WITH_RUNNER = 'WITH_RUNNER', + WITH_PARENT = 'WITH_PARENT' +} + +@Scopes(() => ({ + [ScopeNames.WITH_RUNNER]: { + include: [ + { + model: RunnerModel.unscoped(), + required: false + } + ] + }, + [ScopeNames.WITH_PARENT]: { + include: [ + { + model: RunnerJobModel.unscoped(), + required: false + } + ] + } +})) +@Table({ + tableName: 'runnerJob', + indexes: [ + { + fields: [ 'uuid' ], + unique: true + }, + { + fields: [ 'processingJobToken' ], + unique: true + }, + { + fields: [ 'runnerId' ] + } + ] +}) +export class RunnerJobModel extends Model>> { + + @AllowNull(false) + @IsUUID(4) + @Column(DataType.UUID) + uuid: string + + @AllowNull(false) + @Column + type: RunnerJobType + + @AllowNull(false) + @Column(DataType.JSONB) + payload: RunnerJobPayload + + @AllowNull(false) + @Column(DataType.JSONB) + privatePayload: RunnerJobPrivatePayload + + @AllowNull(false) + @Column + state: RunnerJobState + + @AllowNull(false) + @Default(0) + @Column + failures: number + + @AllowNull(true) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.RUNNER_JOBS.ERROR_MESSAGE.max)) + error: string + + // Less has priority + @AllowNull(false) + @Column + priority: number + + // Used to fetch the appropriate job when the runner wants to post the result + @AllowNull(true) + @Column + processingJobToken: string + + @AllowNull(true) + @Column + progress: number + + @AllowNull(true) + @Column + startedAt: Date + + @AllowNull(true) + @Column + finishedAt: Date + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => RunnerJobModel) + @Column + dependsOnRunnerJobId: number + + @BelongsTo(() => RunnerJobModel, { + foreignKey: { + name: 'dependsOnRunnerJobId', + allowNull: true + }, + onDelete: 'cascade' + }) + DependsOnRunnerJob: RunnerJobModel + + @ForeignKey(() => RunnerModel) + @Column + runnerId: number + + @BelongsTo(() => RunnerModel, { + foreignKey: { + name: 'runnerId', + allowNull: true + }, + onDelete: 'SET NULL' + }) + Runner: RunnerModel + + // --------------------------------------------------------------------------- + + static loadWithRunner (uuid: string) { + const query = { + where: { uuid } + } + + return RunnerJobModel.scope(ScopeNames.WITH_RUNNER).findOne(query) + } + + static loadByRunnerAndJobTokensWithRunner (options: { + uuid: string + runnerToken: string + jobToken: string + }) { + const { uuid, runnerToken, jobToken } = options + + const query = { + where: { + uuid, + processingJobToken: jobToken + }, + include: { + model: RunnerModel.unscoped(), + required: true, + where: { + runnerToken + } + } + } + + return RunnerJobModel.findOne(query) + } + + static listAvailableJobs () { + const query = { + limit: 10, + order: getSort('priority'), + where: { + state: RunnerJobState.PENDING + } + } + + return RunnerJobModel.findAll(query) + } + + static listStalledJobs (options: { + staleTimeMS: number + types: RunnerJobType[] + }) { + const before = new Date(Date.now() - options.staleTimeMS) + + return RunnerJobModel.findAll({ + where: { + type: { + [Op.in]: options.types + }, + state: RunnerJobState.PROCESSING, + updatedAt: { + [Op.lt]: before + } + } + }) + } + + static listChildrenOf (job: MRunnerJob, transaction?: Transaction) { + const query = { + where: { + dependsOnRunnerJobId: job.id + }, + transaction + } + + return RunnerJobModel.findAll(query) + } + + static listForApi (options: { + start: number + count: number + sort: string + search?: string + }) { + const { start, count, sort, search } = options + + const query: FindOptions = { + offset: start, + limit: count, + order: getSort(sort) + } + + if (search) { + if (isUUIDValid(search)) { + query.where = { uuid: search } + } else { + query.where = { + [Op.or]: [ + searchAttribute(search, 'type'), + searchAttribute(search, '$Runner.name$') + ] + } + } + } + + return Promise.all([ + RunnerJobModel.scope([ ScopeNames.WITH_RUNNER ]).count(query), + RunnerJobModel.scope([ ScopeNames.WITH_RUNNER, ScopeNames.WITH_PARENT ]).findAll(query) + ]).then(([ total, data ]) => ({ total, data })) + } + + static updateDependantJobsOf (runnerJob: MRunnerJob) { + const where = { + dependsOnRunnerJobId: runnerJob.id + } + + return RunnerJobModel.update({ state: RunnerJobState.PENDING }, { where }) + } + + static cancelAllJobs (options: { type: RunnerJobType }) { + const where = { + type: options.type + } + + return RunnerJobModel.update({ state: RunnerJobState.CANCELLED }, { where }) + } + + // --------------------------------------------------------------------------- + + resetToPending () { + this.state = RunnerJobState.PENDING + this.processingJobToken = null + this.progress = null + this.startedAt = null + this.runnerId = null + } + + setToErrorOrCancel ( + state: RunnerJobState.PARENT_ERRORED | RunnerJobState.ERRORED | RunnerJobState.CANCELLED | RunnerJobState.PARENT_CANCELLED + ) { + this.state = state + this.processingJobToken = null + this.finishedAt = new Date() + } + + toFormattedJSON (this: MRunnerJobRunnerParent): RunnerJob { + const runner = this.Runner + ? { + id: this.Runner.id, + name: this.Runner.name, + description: this.Runner.description + } + : null + + const parent = this.DependsOnRunnerJob + ? { + id: this.DependsOnRunnerJob.id, + uuid: this.DependsOnRunnerJob.uuid, + type: this.DependsOnRunnerJob.type, + state: { + id: this.DependsOnRunnerJob.state, + label: RUNNER_JOB_STATES[this.DependsOnRunnerJob.state] + } + } + : undefined + + return { + uuid: this.uuid, + type: this.type, + + state: { + id: this.state, + label: RUNNER_JOB_STATES[this.state] + }, + + progress: this.progress, + priority: this.priority, + failures: this.failures, + error: this.error, + + payload: this.payload, + + startedAt: this.startedAt?.toISOString(), + finishedAt: this.finishedAt?.toISOString(), + + createdAt: this.createdAt.toISOString(), + updatedAt: this.updatedAt.toISOString(), + + parent, + runner + } + } + + toFormattedAdminJSON (this: MRunnerJobRunnerParent): RunnerJobAdmin { + return { + ...this.toFormattedJSON(), + + privatePayload: this.privatePayload + } + } +} diff --git a/server/models/runner/runner-registration-token.ts b/server/models/runner/runner-registration-token.ts new file mode 100644 index 000000000..b2ae6c9eb --- /dev/null +++ b/server/models/runner/runner-registration-token.ts @@ -0,0 +1,103 @@ +import { FindOptions, literal } from 'sequelize' +import { AllowNull, Column, CreatedAt, HasMany, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { MRunnerRegistrationToken } from '@server/types/models/runners' +import { RunnerRegistrationToken } from '@shared/models' +import { AttributesOnly } from '@shared/typescript-utils' +import { getSort } from '../shared' +import { RunnerModel } from './runner' + +/** + * + * Tokens used by PeerTube runners to register themselves to the PeerTube instance + * + */ + +@Table({ + tableName: 'runnerRegistrationToken', + indexes: [ + { + fields: [ 'registrationToken' ], + unique: true + } + ] +}) +export class RunnerRegistrationTokenModel extends Model>> { + + @AllowNull(false) + @Column + registrationToken: string + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @HasMany(() => RunnerModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade' + }) + Runners: RunnerModel[] + + static load (id: number) { + return RunnerRegistrationTokenModel.findByPk(id) + } + + static loadByRegistrationToken (registrationToken: string) { + const query = { + where: { registrationToken } + } + + return RunnerRegistrationTokenModel.findOne(query) + } + + static countTotal () { + return RunnerRegistrationTokenModel.unscoped().count() + } + + static listForApi (options: { + start: number + count: number + sort: string + }) { + const { start, count, sort } = options + + const query: FindOptions = { + attributes: { + include: [ + [ + literal('(SELECT COUNT(*) FROM "runner" WHERE "runner"."runnerRegistrationTokenId" = "RunnerRegistrationTokenModel"."id")'), + 'registeredRunnersCount' + ] + ] + }, + offset: start, + limit: count, + order: getSort(sort) + } + + return Promise.all([ + RunnerRegistrationTokenModel.count(query), + RunnerRegistrationTokenModel.findAll(query) + ]).then(([ total, data ]) => ({ total, data })) + } + + // --------------------------------------------------------------------------- + + toFormattedJSON (this: MRunnerRegistrationToken): RunnerRegistrationToken { + const registeredRunnersCount = this.get('registeredRunnersCount') as number + + return { + id: this.id, + + registrationToken: this.registrationToken, + + createdAt: this.createdAt, + updatedAt: this.updatedAt, + + registeredRunnersCount + } + } +} diff --git a/server/models/runner/runner.ts b/server/models/runner/runner.ts new file mode 100644 index 000000000..1ef0018b4 --- /dev/null +++ b/server/models/runner/runner.ts @@ -0,0 +1,112 @@ +import { FindOptions } from 'sequelize' +import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { MRunner } from '@server/types/models/runners' +import { Runner } from '@shared/models' +import { AttributesOnly } from '@shared/typescript-utils' +import { getSort } from '../shared' +import { RunnerRegistrationTokenModel } from './runner-registration-token' +import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' + +@Table({ + tableName: 'runner', + indexes: [ + { + fields: [ 'runnerToken' ], + unique: true + }, + { + fields: [ 'runnerRegistrationTokenId' ] + } + ] +}) +export class RunnerModel extends Model>> { + + // Used to identify the appropriate runner when it uses the runner REST API + @AllowNull(false) + @Column + runnerToken: string + + @AllowNull(false) + @Column + name: string + + @AllowNull(true) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.RUNNERS.DESCRIPTION.max)) + description: string + + @AllowNull(false) + @Column + lastContact: Date + + @AllowNull(false) + @Column + ip: string + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => RunnerRegistrationTokenModel) + @Column + runnerRegistrationTokenId: number + + @BelongsTo(() => RunnerRegistrationTokenModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + RunnerRegistrationToken: RunnerRegistrationTokenModel + + // --------------------------------------------------------------------------- + + static load (id: number) { + return RunnerModel.findByPk(id) + } + + static loadByToken (runnerToken: string) { + const query = { + where: { runnerToken } + } + + return RunnerModel.findOne(query) + } + + static listForApi (options: { + start: number + count: number + sort: string + }) { + const { start, count, sort } = options + + const query: FindOptions = { + offset: start, + limit: count, + order: getSort(sort) + } + + return Promise.all([ + RunnerModel.count(query), + RunnerModel.findAll(query) + ]).then(([ total, data ]) => ({ total, data })) + } + + // --------------------------------------------------------------------------- + + toFormattedJSON (this: MRunner): Runner { + return { + id: this.id, + + name: this.name, + description: this.description, + + ip: this.ip, + lastContact: this.lastContact, + + createdAt: this.createdAt, + updatedAt: this.updatedAt + } + } +} diff --git a/server/models/shared/update.ts b/server/models/shared/update.ts index d02c4535d..96db43730 100644 --- a/server/models/shared/update.ts +++ b/server/models/shared/update.ts @@ -1,22 +1,32 @@ import { QueryTypes, Sequelize, Transaction } from 'sequelize' +const updating = new Set() + // Sequelize always skip the update if we only update updatedAt field -function setAsUpdated (options: { +async function setAsUpdated (options: { sequelize: Sequelize table: string id: number transaction?: Transaction }) { const { sequelize, table, id, transaction } = options + const key = table + '-' + id - return sequelize.query( - `UPDATE "${table}" SET "updatedAt" = :updatedAt WHERE id = :id`, - { - replacements: { table, id, updatedAt: new Date() }, - type: QueryTypes.UPDATE, - transaction - } - ) + if (updating.has(key)) return + updating.add(key) + + try { + await sequelize.query( + `UPDATE "${table}" SET "updatedAt" = :updatedAt WHERE id = :id`, + { + replacements: { table, id, updatedAt: new Date() }, + type: QueryTypes.UPDATE, + transaction + } + ) + } finally { + updating.delete(key) + } } export { diff --git a/server/models/video/video-job-info.ts b/server/models/video/video-job-info.ts index 740f6b5c6..5845b8c74 100644 --- a/server/models/video/video-job-info.ts +++ b/server/models/video/video-job-info.ts @@ -1,5 +1,6 @@ import { Op, QueryTypes, Transaction } from 'sequelize' import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, IsInt, Model, Table, Unique, UpdatedAt } from 'sequelize-typescript' +import { forceNumber } from '@shared/core-utils' import { AttributesOnly } from '@shared/typescript-utils' import { VideoModel } from './video' @@ -59,32 +60,33 @@ export class VideoJobInfoModel extends Model { + static async increaseOrCreate (videoUUID: string, column: VideoJobInfoColumnType, amountArg = 1): Promise { const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } + const amount = forceNumber(amountArg) - const [ { pendingMove } ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(` + const [ result ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(` INSERT INTO "videoJobInfo" ("videoId", "${column}", "createdAt", "updatedAt") SELECT - "video"."id" AS "videoId", 1, NOW(), NOW() + "video"."id" AS "videoId", ${amount}, NOW(), NOW() FROM "video" WHERE "video"."uuid" = $videoUUID ON CONFLICT ("videoId") DO UPDATE SET - "${column}" = "videoJobInfo"."${column}" + 1, + "${column}" = "videoJobInfo"."${column}" + ${amount}, "updatedAt" = NOW() RETURNING "${column}" `, options) - return pendingMove + return result[column] } static async decrease (videoUUID: string, column: VideoJobInfoColumnType): Promise { const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } - const result = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(` + const result = await VideoJobInfoModel.sequelize.query(` UPDATE "videoJobInfo" SET @@ -99,7 +101,7 @@ export class VideoJobInfoModel extends Model { diff --git a/server/models/video/video-live-session.ts b/server/models/video/video-live-session.ts index dcded7872..9426f5d11 100644 --- a/server/models/video/video-live-session.ts +++ b/server/models/video/video-live-session.ts @@ -147,12 +147,21 @@ export class VideoLiveSessionModel extends Model>> { logger.info('Stopping live of video %s after video deletion.', instance.uuid) - LiveManager.Instance.stopSessionOf(instance.id, null) + LiveManager.Instance.stopSessionOf(instance.uuid, null) } @BeforeDestroy @@ -1763,10 +1763,12 @@ export class VideoModel extends Model>> { const { audioStream } = await getAudioStream(originalFilePath, probe) const hasAudio = await hasAudioStream(originalFilePath, probe) + const fps = await getVideoStreamFPS(originalFilePath, probe) return { audioStream, hasAudio, + fps, ...await getVideoStreamDimensionsInfo(originalFilePath, probe) } diff --git a/server/types/express.d.ts b/server/types/express.d.ts index a992a9926..a8aeabb3a 100644 --- a/server/types/express.d.ts +++ b/server/types/express.d.ts @@ -44,6 +44,7 @@ import { MVideoShareActor, MVideoThumbnail } from './models' +import { MRunner, MRunnerJobRunner, MRunnerRegistrationToken } from './models/runners' import { MVideoSource } from './models/video/video-source' declare module 'express' { @@ -102,6 +103,8 @@ declare module 'express' { instance?: string data?: PeerTubeProblemDocumentData + + tags?: string[] }) => void locals: { @@ -203,6 +206,9 @@ declare module 'express' { localViewerFull?: MLocalVideoViewerWithWatchSections + runner?: MRunner + runnerRegistrationToken?: MRunnerRegistrationToken + runnerJob?: MRunnerJobRunner } } } diff --git a/server/types/models/runners/index.ts b/server/types/models/runners/index.ts new file mode 100644 index 000000000..e94d4794e --- /dev/null +++ b/server/types/models/runners/index.ts @@ -0,0 +1,3 @@ +export * from './runner' +export * from './runner-job' +export * from './runner-registration-token' diff --git a/server/types/models/runners/runner-job.ts b/server/types/models/runners/runner-job.ts new file mode 100644 index 000000000..ec983ba32 --- /dev/null +++ b/server/types/models/runners/runner-job.ts @@ -0,0 +1,20 @@ +import { RunnerJobModel } from '@server/models/runner/runner-job' +import { PickWith } from '@shared/typescript-utils' +import { MRunner } from './runner' + +type Use = PickWith + +// ############################################################################ + +export type MRunnerJob = Omit + +// ############################################################################ + +export type MRunnerJobRunner = + MRunnerJob & + Use<'Runner', MRunner> + +export type MRunnerJobRunnerParent = + MRunnerJob & + Use<'Runner', MRunner> & + Use<'DependsOnRunnerJob', MRunnerJob> diff --git a/server/types/models/runners/runner-registration-token.ts b/server/types/models/runners/runner-registration-token.ts new file mode 100644 index 000000000..83b8614ad --- /dev/null +++ b/server/types/models/runners/runner-registration-token.ts @@ -0,0 +1,5 @@ +import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token' + +// ############################################################################ + +export type MRunnerRegistrationToken = Omit diff --git a/server/types/models/runners/runner.ts b/server/types/models/runners/runner.ts new file mode 100644 index 000000000..d35356378 --- /dev/null +++ b/server/types/models/runners/runner.ts @@ -0,0 +1,5 @@ +import { RunnerModel } from '@server/models/runner/runner' + +// ############################################################################ + +export type MRunner = Omit diff --git a/shared/core-utils/common/number.ts b/shared/core-utils/common/number.ts index 9a96dcf5c..ce5a6041a 100644 --- a/shared/core-utils/common/number.ts +++ b/shared/core-utils/common/number.ts @@ -1,7 +1,13 @@ -function forceNumber (value: any) { +export function forceNumber (value: any) { return parseInt(value + '') } -export { - forceNumber +export function isOdd (num: number) { + return (num % 2) !== 0 +} + +export function toEven (num: number) { + if (isOdd(num)) return num + 1 + + return num } diff --git a/shared/core-utils/common/promises.ts b/shared/core-utils/common/promises.ts index f17221b97..e3792d12e 100644 --- a/shared/core-utils/common/promises.ts +++ b/shared/core-utils/common/promises.ts @@ -1,12 +1,12 @@ -function isPromise (value: T | Promise): value is Promise { +export function isPromise (value: T | Promise): value is Promise { return value && typeof (value as Promise).then === 'function' } -function isCatchable (value: any) { +export function isCatchable (value: any) { return value && typeof value.catch === 'function' } -function timeoutPromise (promise: Promise, timeoutMs: number) { +export function timeoutPromise (promise: Promise, timeoutMs: number) { let timer: ReturnType return Promise.race([ @@ -18,8 +18,41 @@ function timeoutPromise (promise: Promise, timeoutMs: number) { ]).finally(() => clearTimeout(timer)) } -export { - isPromise, - isCatchable, - timeoutPromise +export function promisify0 (func: (cb: (err: any, result: A) => void) => void): () => Promise { + return function promisified (): Promise { + return new Promise((resolve: (arg: A) => void, reject: (err: any) => void) => { + // eslint-disable-next-line no-useless-call + func.apply(null, [ (err: any, res: A) => err ? reject(err) : resolve(res) ]) + }) + } +} + +// Thanks to https://gist.github.com/kumasento/617daa7e46f13ecdd9b2 +export function promisify1 (func: (arg: T, cb: (err: any, result: A) => void) => void): (arg: T) => Promise { + return function promisified (arg: T): Promise { + return new Promise((resolve: (arg: A) => void, reject: (err: any) => void) => { + // eslint-disable-next-line no-useless-call + func.apply(null, [ arg, (err: any, res: A) => err ? reject(err) : resolve(res) ]) + }) + } +} + +// eslint-disable-next-line max-len +export function promisify2 (func: (arg1: T, arg2: U, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U) => Promise { + return function promisified (arg1: T, arg2: U): Promise { + return new Promise((resolve: (arg: A) => void, reject: (err: any) => void) => { + // eslint-disable-next-line no-useless-call + func.apply(null, [ arg1, arg2, (err: any, res: A) => err ? reject(err) : resolve(res) ]) + }) + } +} + +// eslint-disable-next-line max-len +export function promisify3 (func: (arg1: T, arg2: U, arg3: V, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U, arg3: V) => Promise { + return function promisified (arg1: T, arg2: U, arg3: V): Promise { + return new Promise((resolve: (arg: A) => void, reject: (err: any) => void) => { + // eslint-disable-next-line no-useless-call + func.apply(null, [ arg1, arg2, arg3, (err: any, res: A) => err ? reject(err) : resolve(res) ]) + }) + } } diff --git a/shared/extra-utils/index.ts b/shared/extra-utils/index.ts index e2e161a7b..d4cfcbec8 100644 --- a/shared/extra-utils/index.ts +++ b/shared/extra-utils/index.ts @@ -1,4 +1,3 @@ export * from './crypto' -export * from './ffprobe' export * from './file' export * from './uuid' diff --git a/shared/ffmpeg/ffmpeg-command-wrapper.ts b/shared/ffmpeg/ffmpeg-command-wrapper.ts new file mode 100644 index 000000000..7a8c19d4b --- /dev/null +++ b/shared/ffmpeg/ffmpeg-command-wrapper.ts @@ -0,0 +1,234 @@ +import ffmpeg, { FfmpegCommand, getAvailableEncoders } from 'fluent-ffmpeg' +import { pick, promisify0 } from '@shared/core-utils' +import { AvailableEncoders, EncoderOptionsBuilder, EncoderOptionsBuilderParams, EncoderProfile } from '@shared/models' + +type FFmpegLogger = { + info: (msg: string, obj?: any) => void + debug: (msg: string, obj?: any) => void + warn: (msg: string, obj?: any) => void + error: (msg: string, obj?: any) => void +} + +export interface FFmpegCommandWrapperOptions { + availableEncoders?: AvailableEncoders + profile?: string + + niceness: number + tmpDirectory: string + threads: number + + logger: FFmpegLogger + lTags?: { tags: string[] } + + updateJobProgress?: (progress?: number) => void +} + +export class FFmpegCommandWrapper { + private static supportedEncoders: Map + + private readonly availableEncoders: AvailableEncoders + private readonly profile: string + + private readonly niceness: number + private readonly tmpDirectory: string + private readonly threads: number + + private readonly logger: FFmpegLogger + private readonly lTags: { tags: string[] } + + private readonly updateJobProgress: (progress?: number) => void + + private command: FfmpegCommand + + constructor (options: FFmpegCommandWrapperOptions) { + this.availableEncoders = options.availableEncoders + this.profile = options.profile + this.niceness = options.niceness + this.tmpDirectory = options.tmpDirectory + this.threads = options.threads + this.logger = options.logger + this.lTags = options.lTags || { tags: [] } + this.updateJobProgress = options.updateJobProgress + } + + getAvailableEncoders () { + return this.availableEncoders + } + + getProfile () { + return this.profile + } + + getCommand () { + return this.command + } + + // --------------------------------------------------------------------------- + + debugLog (msg: string, meta: any) { + this.logger.debug(msg, { ...meta, ...this.lTags }) + } + + // --------------------------------------------------------------------------- + + buildCommand (input: string) { + if (this.command) throw new Error('Command is already built') + + // We set cwd explicitly because ffmpeg appears to create temporary files when trancoding which fails in read-only file systems + this.command = ffmpeg(input, { + niceness: this.niceness, + cwd: this.tmpDirectory + }) + + if (this.threads > 0) { + // If we don't set any threads ffmpeg will chose automatically + this.command.outputOption('-threads ' + this.threads) + } + + return this.command + } + + async runCommand (options: { + silent?: boolean // false by default + } = {}) { + const { silent = false } = options + + return new Promise((res, rej) => { + let shellCommand: string + + this.command.on('start', cmdline => { shellCommand = cmdline }) + + this.command.on('error', (err, stdout, stderr) => { + if (silent !== true) this.logger.error('Error in ffmpeg.', { stdout, stderr, shellCommand, ...this.lTags }) + + rej(err) + }) + + this.command.on('end', (stdout, stderr) => { + this.logger.debug('FFmpeg command ended.', { stdout, stderr, shellCommand, ...this.lTags }) + + res() + }) + + if (this.updateJobProgress) { + this.command.on('progress', progress => { + if (!progress.percent) return + + // Sometimes ffmpeg returns an invalid progress + let percent = Math.round(progress.percent) + if (percent < 0) percent = 0 + if (percent > 100) percent = 100 + + this.updateJobProgress(percent) + }) + } + + this.command.run() + }) + } + + // --------------------------------------------------------------------------- + + static resetSupportedEncoders () { + FFmpegCommandWrapper.supportedEncoders = undefined + } + + // Run encoder builder depending on available encoders + // Try encoders by priority: if the encoder is available, run the chosen profile or fallback to the default one + // If the default one does not exist, check the next encoder + async getEncoderBuilderResult (options: EncoderOptionsBuilderParams & { + streamType: 'video' | 'audio' + input: string + + videoType: 'vod' | 'live' + }) { + if (!this.availableEncoders) { + throw new Error('There is no available encoders') + } + + const { streamType, videoType } = options + + const encodersToTry = this.availableEncoders.encodersToTry[videoType][streamType] + const encoders = this.availableEncoders.available[videoType] + + for (const encoder of encodersToTry) { + if (!(await this.checkFFmpegEncoders(this.availableEncoders)).get(encoder)) { + this.logger.debug(`Encoder ${encoder} not available in ffmpeg, skipping.`, this.lTags) + continue + } + + if (!encoders[encoder]) { + this.logger.debug(`Encoder ${encoder} not available in peertube encoders, skipping.`, this.lTags) + continue + } + + // An object containing available profiles for this encoder + const builderProfiles: EncoderProfile = encoders[encoder] + let builder = builderProfiles[this.profile] + + if (!builder) { + this.logger.debug(`Profile ${this.profile} for encoder ${encoder} not available. Fallback to default.`, this.lTags) + builder = builderProfiles.default + + if (!builder) { + this.logger.debug(`Default profile for encoder ${encoder} not available. Try next available encoder.`, this.lTags) + continue + } + } + + const result = await builder( + pick(options, [ + 'input', + 'canCopyAudio', + 'canCopyVideo', + 'resolution', + 'inputBitrate', + 'fps', + 'inputRatio', + 'streamNum' + ]) + ) + + return { + result, + + // If we don't have output options, then copy the input stream + encoder: result.copy === true + ? 'copy' + : encoder + } + } + + return null + } + + // Detect supported encoders by ffmpeg + private async checkFFmpegEncoders (peertubeAvailableEncoders: AvailableEncoders): Promise> { + if (FFmpegCommandWrapper.supportedEncoders !== undefined) { + return FFmpegCommandWrapper.supportedEncoders + } + + const getAvailableEncodersPromise = promisify0(getAvailableEncoders) + const availableFFmpegEncoders = await getAvailableEncodersPromise() + + const searchEncoders = new Set() + for (const type of [ 'live', 'vod' ]) { + for (const streamType of [ 'audio', 'video' ]) { + for (const encoder of peertubeAvailableEncoders.encodersToTry[type][streamType]) { + searchEncoders.add(encoder) + } + } + } + + const supportedEncoders = new Map() + + for (const searchEncoder of searchEncoders) { + supportedEncoders.set(searchEncoder, availableFFmpegEncoders[searchEncoder] !== undefined) + } + + this.logger.info('Built supported ffmpeg encoders.', { supportedEncoders, searchEncoders, ...this.lTags }) + + FFmpegCommandWrapper.supportedEncoders = supportedEncoders + return supportedEncoders + } +} diff --git a/shared/ffmpeg/ffmpeg-edition.ts b/shared/ffmpeg/ffmpeg-edition.ts new file mode 100644 index 000000000..724ca1ea9 --- /dev/null +++ b/shared/ffmpeg/ffmpeg-edition.ts @@ -0,0 +1,239 @@ +import { FilterSpecification } from 'fluent-ffmpeg' +import { FFmpegCommandWrapper, FFmpegCommandWrapperOptions } from './ffmpeg-command-wrapper' +import { presetVOD } from './shared/presets' +import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS, hasAudioStream } from './ffprobe' + +export class FFmpegEdition { + private readonly commandWrapper: FFmpegCommandWrapper + + constructor (options: FFmpegCommandWrapperOptions) { + this.commandWrapper = new FFmpegCommandWrapper(options) + } + + async cutVideo (options: { + inputPath: string + outputPath: string + start?: number + end?: number + }) { + const { inputPath, outputPath } = options + + const mainProbe = await ffprobePromise(inputPath) + const fps = await getVideoStreamFPS(inputPath, mainProbe) + const { resolution } = await getVideoStreamDimensionsInfo(inputPath, mainProbe) + + const command = this.commandWrapper.buildCommand(inputPath) + .output(outputPath) + + await presetVOD({ + commandWrapper: this.commandWrapper, + input: inputPath, + resolution, + fps, + canCopyAudio: false, + canCopyVideo: false + }) + + if (options.start) { + command.outputOption('-ss ' + options.start) + } + + if (options.end) { + command.outputOption('-to ' + options.end) + } + + await this.commandWrapper.runCommand() + } + + async addWatermark (options: { + inputPath: string + watermarkPath: string + outputPath: string + + videoFilters: { + watermarkSizeRatio: number + horitonzalMarginRatio: number + verticalMarginRatio: number + } + }) { + const { watermarkPath, inputPath, outputPath, videoFilters } = options + + const videoProbe = await ffprobePromise(inputPath) + const fps = await getVideoStreamFPS(inputPath, videoProbe) + const { resolution } = await getVideoStreamDimensionsInfo(inputPath, videoProbe) + + const command = this.commandWrapper.buildCommand(inputPath) + .output(outputPath) + + command.input(watermarkPath) + + await presetVOD({ + commandWrapper: this.commandWrapper, + input: inputPath, + resolution, + fps, + canCopyAudio: true, + canCopyVideo: false + }) + + const complexFilter: FilterSpecification[] = [ + // Scale watermark + { + inputs: [ '[1]', '[0]' ], + filter: 'scale2ref', + options: { + w: 'oh*mdar', + h: `ih*${videoFilters.watermarkSizeRatio}` + }, + outputs: [ '[watermark]', '[video]' ] + }, + + { + inputs: [ '[video]', '[watermark]' ], + filter: 'overlay', + options: { + x: `main_w - overlay_w - (main_h * ${videoFilters.horitonzalMarginRatio})`, + y: `main_h * ${videoFilters.verticalMarginRatio}` + } + } + ] + + command.complexFilter(complexFilter) + + await this.commandWrapper.runCommand() + } + + async addIntroOutro (options: { + inputPath: string + introOutroPath: string + outputPath: string + type: 'intro' | 'outro' + }) { + const { introOutroPath, inputPath, outputPath, type } = options + + const mainProbe = await ffprobePromise(inputPath) + const fps = await getVideoStreamFPS(inputPath, mainProbe) + const { resolution } = await getVideoStreamDimensionsInfo(inputPath, mainProbe) + const mainHasAudio = await hasAudioStream(inputPath, mainProbe) + + const introOutroProbe = await ffprobePromise(introOutroPath) + const introOutroHasAudio = await hasAudioStream(introOutroPath, introOutroProbe) + + const command = this.commandWrapper.buildCommand(inputPath) + .output(outputPath) + + command.input(introOutroPath) + + if (!introOutroHasAudio && mainHasAudio) { + const duration = await getVideoStreamDuration(introOutroPath, introOutroProbe) + + command.input('anullsrc') + command.withInputFormat('lavfi') + command.withInputOption('-t ' + duration) + } + + await presetVOD({ + commandWrapper: this.commandWrapper, + input: inputPath, + resolution, + fps, + canCopyAudio: false, + canCopyVideo: false + }) + + // Add black background to correctly scale intro/outro with padding + const complexFilter: FilterSpecification[] = [ + { + inputs: [ '1', '0' ], + filter: 'scale2ref', + options: { + w: 'iw', + h: `ih` + }, + outputs: [ 'intro-outro', 'main' ] + }, + { + inputs: [ 'intro-outro', 'main' ], + filter: 'scale2ref', + options: { + w: 'iw', + h: `ih` + }, + outputs: [ 'to-scale', 'main' ] + }, + { + inputs: 'to-scale', + filter: 'drawbox', + options: { + t: 'fill' + }, + outputs: [ 'to-scale-bg' ] + }, + { + inputs: [ '1', 'to-scale-bg' ], + filter: 'scale2ref', + options: { + w: 'iw', + h: 'ih', + force_original_aspect_ratio: 'decrease', + flags: 'spline' + }, + outputs: [ 'to-scale', 'to-scale-bg' ] + }, + { + inputs: [ 'to-scale-bg', 'to-scale' ], + filter: 'overlay', + options: { + x: '(main_w - overlay_w)/2', + y: '(main_h - overlay_h)/2' + }, + outputs: 'intro-outro-resized' + } + ] + + const concatFilter = { + inputs: [], + filter: 'concat', + options: { + n: 2, + v: 1, + unsafe: 1 + }, + outputs: [ 'v' ] + } + + const introOutroFilterInputs = [ 'intro-outro-resized' ] + const mainFilterInputs = [ 'main' ] + + if (mainHasAudio) { + mainFilterInputs.push('0:a') + + if (introOutroHasAudio) { + introOutroFilterInputs.push('1:a') + } else { + // Silent input + introOutroFilterInputs.push('2:a') + } + } + + if (type === 'intro') { + concatFilter.inputs = [ ...introOutroFilterInputs, ...mainFilterInputs ] + } else { + concatFilter.inputs = [ ...mainFilterInputs, ...introOutroFilterInputs ] + } + + if (mainHasAudio) { + concatFilter.options['a'] = 1 + concatFilter.outputs.push('a') + + command.outputOption('-map [a]') + } + + command.outputOption('-map [v]') + + complexFilter.push(concatFilter) + command.complexFilter(complexFilter) + + await this.commandWrapper.runCommand() + } +} diff --git a/shared/ffmpeg/ffmpeg-images.ts b/shared/ffmpeg/ffmpeg-images.ts new file mode 100644 index 000000000..2db63bd8b --- /dev/null +++ b/shared/ffmpeg/ffmpeg-images.ts @@ -0,0 +1,59 @@ +import { FFmpegCommandWrapper, FFmpegCommandWrapperOptions } from './ffmpeg-command-wrapper' + +export class FFmpegImage { + private readonly commandWrapper: FFmpegCommandWrapper + + constructor (options: FFmpegCommandWrapperOptions) { + this.commandWrapper = new FFmpegCommandWrapper(options) + } + + convertWebPToJPG (options: { + path: string + destination: string + }): Promise { + const { path, destination } = options + + this.commandWrapper.buildCommand(path) + .output(destination) + + return this.commandWrapper.runCommand({ silent: true }) + } + + processGIF (options: { + path: string + destination: string + newSize: { width: number, height: number } + }): Promise { + const { path, destination, newSize } = options + + this.commandWrapper.buildCommand(path) + .fps(20) + .size(`${newSize.width}x${newSize.height}`) + .output(destination) + + return this.commandWrapper.runCommand() + } + + async generateThumbnailFromVideo (options: { + fromPath: string + folder: string + imageName: string + }) { + const { fromPath, folder, imageName } = options + + const pendingImageName = 'pending-' + imageName + + const thumbnailOptions = { + filename: pendingImageName, + count: 1, + folder + } + + return new Promise((res, rej) => { + this.commandWrapper.buildCommand(fromPath) + .on('error', rej) + .on('end', () => res(imageName)) + .thumbnail(thumbnailOptions) + }) + } +} diff --git a/shared/ffmpeg/ffmpeg-live.ts b/shared/ffmpeg/ffmpeg-live.ts new file mode 100644 index 000000000..cca4c6474 --- /dev/null +++ b/shared/ffmpeg/ffmpeg-live.ts @@ -0,0 +1,184 @@ +import { FilterSpecification } from 'fluent-ffmpeg' +import { join } from 'path' +import { pick } from '@shared/core-utils' +import { FFmpegCommandWrapper, FFmpegCommandWrapperOptions } from './ffmpeg-command-wrapper' +import { buildStreamSuffix, getScaleFilter, StreamType } from './ffmpeg-utils' +import { addDefaultEncoderGlobalParams, addDefaultEncoderParams, applyEncoderOptions } from './shared' + +export class FFmpegLive { + private readonly commandWrapper: FFmpegCommandWrapper + + constructor (options: FFmpegCommandWrapperOptions) { + this.commandWrapper = new FFmpegCommandWrapper(options) + } + + async getLiveTranscodingCommand (options: { + inputUrl: string + + outPath: string + masterPlaylistName: string + + toTranscode: { + resolution: number + fps: number + }[] + + // Input information + bitrate: number + ratio: number + hasAudio: boolean + + segmentListSize: number + segmentDuration: number + }) { + const { + inputUrl, + outPath, + toTranscode, + bitrate, + masterPlaylistName, + ratio, + hasAudio + } = options + const command = this.commandWrapper.buildCommand(inputUrl) + + const varStreamMap: string[] = [] + + const complexFilter: FilterSpecification[] = [ + { + inputs: '[v:0]', + filter: 'split', + options: toTranscode.length, + outputs: toTranscode.map(t => `vtemp${t.resolution}`) + } + ] + + command.outputOption('-sc_threshold 0') + + addDefaultEncoderGlobalParams(command) + + for (let i = 0; i < toTranscode.length; i++) { + const streamMap: string[] = [] + const { resolution, fps } = toTranscode[i] + + const baseEncoderBuilderParams = { + input: inputUrl, + + canCopyAudio: true, + canCopyVideo: true, + + inputBitrate: bitrate, + inputRatio: ratio, + + resolution, + fps, + + streamNum: i, + videoType: 'live' as 'live' + } + + { + const streamType: StreamType = 'video' + const builderResult = await this.commandWrapper.getEncoderBuilderResult({ ...baseEncoderBuilderParams, streamType }) + if (!builderResult) { + throw new Error('No available live video encoder found') + } + + command.outputOption(`-map [vout${resolution}]`) + + addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps, streamNum: i }) + + this.commandWrapper.debugLog( + `Apply ffmpeg live video params from ${builderResult.encoder} using ${this.commandWrapper.getProfile()} profile.`, + { builderResult, fps, toTranscode } + ) + + command.outputOption(`${buildStreamSuffix('-c:v', i)} ${builderResult.encoder}`) + applyEncoderOptions(command, builderResult.result) + + complexFilter.push({ + inputs: `vtemp${resolution}`, + filter: getScaleFilter(builderResult.result), + options: `w=-2:h=${resolution}`, + outputs: `vout${resolution}` + }) + + streamMap.push(`v:${i}`) + } + + if (hasAudio) { + const streamType: StreamType = 'audio' + const builderResult = await this.commandWrapper.getEncoderBuilderResult({ ...baseEncoderBuilderParams, streamType }) + if (!builderResult) { + throw new Error('No available live audio encoder found') + } + + command.outputOption('-map a:0') + + addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps, streamNum: i }) + + this.commandWrapper.debugLog( + `Apply ffmpeg live audio params from ${builderResult.encoder} using ${this.commandWrapper.getProfile()} profile.`, + { builderResult, fps, resolution } + ) + + command.outputOption(`${buildStreamSuffix('-c:a', i)} ${builderResult.encoder}`) + applyEncoderOptions(command, builderResult.result) + + streamMap.push(`a:${i}`) + } + + varStreamMap.push(streamMap.join(',')) + } + + command.complexFilter(complexFilter) + + this.addDefaultLiveHLSParams({ ...pick(options, [ 'segmentDuration', 'segmentListSize' ]), outPath, masterPlaylistName }) + + command.outputOption('-var_stream_map', varStreamMap.join(' ')) + + return command + } + + getLiveMuxingCommand (options: { + inputUrl: string + outPath: string + masterPlaylistName: string + + segmentListSize: number + segmentDuration: number + }) { + const { inputUrl, outPath, masterPlaylistName } = options + + const command = this.commandWrapper.buildCommand(inputUrl) + + command.outputOption('-c:v copy') + command.outputOption('-c:a copy') + command.outputOption('-map 0:a?') + command.outputOption('-map 0:v?') + + this.addDefaultLiveHLSParams({ ...pick(options, [ 'segmentDuration', 'segmentListSize' ]), outPath, masterPlaylistName }) + + return command + } + + private addDefaultLiveHLSParams (options: { + outPath: string + masterPlaylistName: string + segmentListSize: number + segmentDuration: number + }) { + const { outPath, masterPlaylistName, segmentListSize, segmentDuration } = options + + const command = this.commandWrapper.getCommand() + + command.outputOption('-hls_time ' + segmentDuration) + command.outputOption('-hls_list_size ' + segmentListSize) + command.outputOption('-hls_flags delete_segments+independent_segments+program_date_time') + command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%06d.ts')}`) + command.outputOption('-master_pl_name ' + masterPlaylistName) + command.outputOption(`-f hls`) + + command.output(join(outPath, '%v.m3u8')) + } +} diff --git a/shared/ffmpeg/ffmpeg-utils.ts b/shared/ffmpeg/ffmpeg-utils.ts new file mode 100644 index 000000000..7d09c32ca --- /dev/null +++ b/shared/ffmpeg/ffmpeg-utils.ts @@ -0,0 +1,17 @@ +import { EncoderOptions } from '@shared/models' + +export type StreamType = 'audio' | 'video' + +export function buildStreamSuffix (base: string, streamNum?: number) { + if (streamNum !== undefined) { + return `${base}:${streamNum}` + } + + return base +} + +export function getScaleFilter (options: EncoderOptions): string { + if (options.scaleFilter) return options.scaleFilter.name + + return 'scale' +} diff --git a/shared/ffmpeg/ffmpeg-version.ts b/shared/ffmpeg/ffmpeg-version.ts new file mode 100644 index 000000000..41d9b2d89 --- /dev/null +++ b/shared/ffmpeg/ffmpeg-version.ts @@ -0,0 +1,24 @@ +import { exec } from 'child_process' +import ffmpeg from 'fluent-ffmpeg' + +export function getFFmpegVersion () { + return new Promise((res, rej) => { + (ffmpeg() as any)._getFfmpegPath((err, ffmpegPath) => { + if (err) return rej(err) + if (!ffmpegPath) return rej(new Error('Could not find ffmpeg path')) + + return exec(`${ffmpegPath} -version`, (err, stdout) => { + if (err) return rej(err) + + const parsed = stdout.match(/ffmpeg version .?(\d+\.\d+(\.\d+)?)/) + if (!parsed?.[1]) return rej(new Error(`Could not find ffmpeg version in ${stdout}`)) + + // Fix ffmpeg version that does not include patch version (4.4 for example) + let version = parsed[1] + if (version.match(/^\d+\.\d+$/)) { + version += '.0' + } + }) + }) + }) +} diff --git a/shared/ffmpeg/ffmpeg-vod.ts b/shared/ffmpeg/ffmpeg-vod.ts new file mode 100644 index 000000000..e40ca0a1e --- /dev/null +++ b/shared/ffmpeg/ffmpeg-vod.ts @@ -0,0 +1,256 @@ +import { MutexInterface } from 'async-mutex' +import { FfmpegCommand } from 'fluent-ffmpeg' +import { readFile, writeFile } from 'fs-extra' +import { dirname } from 'path' +import { pick } from '@shared/core-utils' +import { VideoResolution } from '@shared/models' +import { FFmpegCommandWrapper, FFmpegCommandWrapperOptions } from './ffmpeg-command-wrapper' +import { ffprobePromise, getVideoStreamDimensionsInfo } from './ffprobe' +import { presetCopy, presetOnlyAudio, presetVOD } from './shared/presets' + +export type TranscodeVODOptionsType = 'hls' | 'hls-from-ts' | 'quick-transcode' | 'video' | 'merge-audio' | 'only-audio' + +export interface BaseTranscodeVODOptions { + type: TranscodeVODOptionsType + + inputPath: string + outputPath: string + + // Will be released after the ffmpeg started + // To prevent a bug where the input file does not exist anymore when running ffmpeg + inputFileMutexReleaser: MutexInterface.Releaser + + resolution: number + fps: number +} + +export interface HLSTranscodeOptions extends BaseTranscodeVODOptions { + type: 'hls' + + copyCodecs: boolean + + hlsPlaylist: { + videoFilename: string + } +} + +export interface HLSFromTSTranscodeOptions extends BaseTranscodeVODOptions { + type: 'hls-from-ts' + + isAAC: boolean + + hlsPlaylist: { + videoFilename: string + } +} + +export interface QuickTranscodeOptions extends BaseTranscodeVODOptions { + type: 'quick-transcode' +} + +export interface VideoTranscodeOptions extends BaseTranscodeVODOptions { + type: 'video' +} + +export interface MergeAudioTranscodeOptions extends BaseTranscodeVODOptions { + type: 'merge-audio' + audioPath: string +} + +export interface OnlyAudioTranscodeOptions extends BaseTranscodeVODOptions { + type: 'only-audio' +} + +export type TranscodeVODOptions = + HLSTranscodeOptions + | HLSFromTSTranscodeOptions + | VideoTranscodeOptions + | MergeAudioTranscodeOptions + | OnlyAudioTranscodeOptions + | QuickTranscodeOptions + +// --------------------------------------------------------------------------- + +export class FFmpegVOD { + private readonly commandWrapper: FFmpegCommandWrapper + + private ended = false + + constructor (options: FFmpegCommandWrapperOptions) { + this.commandWrapper = new FFmpegCommandWrapper(options) + } + + async transcode (options: TranscodeVODOptions) { + const builders: { + [ type in TranscodeVODOptionsType ]: (options: TranscodeVODOptions) => Promise | void + } = { + 'quick-transcode': this.buildQuickTranscodeCommand.bind(this), + 'hls': this.buildHLSVODCommand.bind(this), + 'hls-from-ts': this.buildHLSVODFromTSCommand.bind(this), + 'merge-audio': this.buildAudioMergeCommand.bind(this), + // TODO: remove, we merge this in buildWebVideoCommand + 'only-audio': this.buildOnlyAudioCommand.bind(this), + 'video': this.buildWebVideoCommand.bind(this) + } + + this.commandWrapper.debugLog('Will run transcode.', { options }) + + const command = this.commandWrapper.buildCommand(options.inputPath) + .output(options.outputPath) + + await builders[options.type](options) + + command.on('start', () => { + setTimeout(() => { + options.inputFileMutexReleaser() + }, 1000) + }) + + await this.commandWrapper.runCommand() + + await this.fixHLSPlaylistIfNeeded(options) + + this.ended = true + } + + isEnded () { + return this.ended + } + + private async buildWebVideoCommand (options: TranscodeVODOptions) { + const { resolution, fps, inputPath } = options + + if (resolution === VideoResolution.H_NOVIDEO) { + presetOnlyAudio(this.commandWrapper) + return + } + + let scaleFilterValue: string + + if (resolution !== undefined) { + const probe = await ffprobePromise(inputPath) + const videoStreamInfo = await getVideoStreamDimensionsInfo(inputPath, probe) + + scaleFilterValue = videoStreamInfo?.isPortraitMode === true + ? `w=${resolution}:h=-2` + : `w=-2:h=${resolution}` + } + + await presetVOD({ + commandWrapper: this.commandWrapper, + + resolution, + input: inputPath, + canCopyAudio: true, + canCopyVideo: true, + fps, + scaleFilterValue + }) + } + + private buildQuickTranscodeCommand (_options: TranscodeVODOptions) { + const command = this.commandWrapper.getCommand() + + presetCopy(this.commandWrapper) + + command.outputOption('-map_metadata -1') // strip all metadata + .outputOption('-movflags faststart') + } + + // --------------------------------------------------------------------------- + // Audio transcoding + // --------------------------------------------------------------------------- + + private async buildAudioMergeCommand (options: MergeAudioTranscodeOptions) { + const command = this.commandWrapper.getCommand() + + command.loop(undefined) + + await presetVOD({ + ...pick(options, [ 'resolution' ]), + + commandWrapper: this.commandWrapper, + input: options.audioPath, + canCopyAudio: true, + canCopyVideo: true, + fps: options.fps, + scaleFilterValue: this.getMergeAudioScaleFilterValue() + }) + + command.outputOption('-preset:v veryfast') + + command.input(options.audioPath) + .outputOption('-tune stillimage') + .outputOption('-shortest') + } + + private buildOnlyAudioCommand (_options: OnlyAudioTranscodeOptions) { + presetOnlyAudio(this.commandWrapper) + } + + // Avoid "height not divisible by 2" error + private getMergeAudioScaleFilterValue () { + return 'trunc(iw/2)*2:trunc(ih/2)*2' + } + + // --------------------------------------------------------------------------- + // HLS transcoding + // --------------------------------------------------------------------------- + + private async buildHLSVODCommand (options: HLSTranscodeOptions) { + const command = this.commandWrapper.getCommand() + + const videoPath = this.getHLSVideoPath(options) + + if (options.copyCodecs) presetCopy(this.commandWrapper) + else if (options.resolution === VideoResolution.H_NOVIDEO) presetOnlyAudio(this.commandWrapper) + else await this.buildWebVideoCommand(options) + + this.addCommonHLSVODCommandOptions(command, videoPath) + } + + private buildHLSVODFromTSCommand (options: HLSFromTSTranscodeOptions) { + const command = this.commandWrapper.getCommand() + + const videoPath = this.getHLSVideoPath(options) + + command.outputOption('-c copy') + + if (options.isAAC) { + // Required for example when copying an AAC stream from an MPEG-TS + // Since it's a bitstream filter, we don't need to reencode the audio + command.outputOption('-bsf:a aac_adtstoasc') + } + + this.addCommonHLSVODCommandOptions(command, videoPath) + } + + private addCommonHLSVODCommandOptions (command: FfmpegCommand, outputPath: string) { + return command.outputOption('-hls_time 4') + .outputOption('-hls_list_size 0') + .outputOption('-hls_playlist_type vod') + .outputOption('-hls_segment_filename ' + outputPath) + .outputOption('-hls_segment_type fmp4') + .outputOption('-f hls') + .outputOption('-hls_flags single_file') + } + + private async fixHLSPlaylistIfNeeded (options: TranscodeVODOptions) { + if (options.type !== 'hls' && options.type !== 'hls-from-ts') return + + const fileContent = await readFile(options.outputPath) + + const videoFileName = options.hlsPlaylist.videoFilename + const videoFilePath = this.getHLSVideoPath(options) + + // Fix wrong mapping with some ffmpeg versions + const newContent = fileContent.toString() + .replace(`#EXT-X-MAP:URI="${videoFilePath}",`, `#EXT-X-MAP:URI="${videoFileName}",`) + + await writeFile(options.outputPath, newContent) + } + + private getHLSVideoPath (options: HLSTranscodeOptions | HLSFromTSTranscodeOptions) { + return `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}` + } +} diff --git a/shared/extra-utils/ffprobe.ts b/shared/ffmpeg/ffprobe.ts similarity index 91% rename from shared/extra-utils/ffprobe.ts rename to shared/ffmpeg/ffprobe.ts index 7efc58a0d..fda08c28e 100644 --- a/shared/extra-utils/ffprobe.ts +++ b/shared/ffmpeg/ffprobe.ts @@ -1,6 +1,6 @@ import { ffprobe, FfprobeData } from 'fluent-ffmpeg' import { forceNumber } from '@shared/core-utils' -import { VideoFileMetadata, VideoResolution } from '@shared/models/videos' +import { VideoResolution } from '@shared/models/videos' /** * @@ -141,35 +141,29 @@ async function getVideoStreamFPS (path: string, existingProbe?: FfprobeData) { return 0 } -async function buildFileMetadata (path: string, existingProbe?: FfprobeData) { +async function getVideoStreamBitrate (path: string, existingProbe?: FfprobeData): Promise { const metadata = existingProbe || await ffprobePromise(path) - return new VideoFileMetadata(metadata) -} - -async function getVideoStreamBitrate (path: string, existingProbe?: FfprobeData): Promise { - const metadata = await buildFileMetadata(path, existingProbe) - - let bitrate = metadata.format.bit_rate as number + let bitrate = metadata.format.bit_rate if (bitrate && !isNaN(bitrate)) return bitrate const videoStream = await getVideoStream(path, existingProbe) if (!videoStream) return undefined - bitrate = videoStream?.bit_rate + bitrate = forceNumber(videoStream?.bit_rate) if (bitrate && !isNaN(bitrate)) return bitrate return undefined } async function getVideoStreamDuration (path: string, existingProbe?: FfprobeData) { - const metadata = await buildFileMetadata(path, existingProbe) + const metadata = existingProbe || await ffprobePromise(path) return Math.round(metadata.format.duration) } async function getVideoStream (path: string, existingProbe?: FfprobeData) { - const metadata = await buildFileMetadata(path, existingProbe) + const metadata = existingProbe || await ffprobePromise(path) return metadata.streams.find(s => s.codec_type === 'video') } @@ -178,7 +172,6 @@ async function getVideoStream (path: string, existingProbe?: FfprobeData) { export { getVideoStreamDimensionsInfo, - buildFileMetadata, getMaxAudioBitrate, getVideoStream, getVideoStreamDuration, diff --git a/shared/ffmpeg/index.ts b/shared/ffmpeg/index.ts new file mode 100644 index 000000000..07a7d5402 --- /dev/null +++ b/shared/ffmpeg/index.ts @@ -0,0 +1,8 @@ +export * from './ffmpeg-command-wrapper' +export * from './ffmpeg-edition' +export * from './ffmpeg-images' +export * from './ffmpeg-live' +export * from './ffmpeg-utils' +export * from './ffmpeg-version' +export * from './ffmpeg-vod' +export * from './ffprobe' diff --git a/shared/ffmpeg/shared/encoder-options.ts b/shared/ffmpeg/shared/encoder-options.ts new file mode 100644 index 000000000..9692a6b02 --- /dev/null +++ b/shared/ffmpeg/shared/encoder-options.ts @@ -0,0 +1,39 @@ +import { FfmpegCommand } from 'fluent-ffmpeg' +import { EncoderOptions } from '@shared/models' +import { buildStreamSuffix } from '../ffmpeg-utils' + +export function addDefaultEncoderGlobalParams (command: FfmpegCommand) { + // avoid issues when transcoding some files: https://trac.ffmpeg.org/ticket/6375 + command.outputOption('-max_muxing_queue_size 1024') + // strip all metadata + .outputOption('-map_metadata -1') + // allows import of source material with incompatible pixel formats (e.g. MJPEG video) + .outputOption('-pix_fmt yuv420p') +} + +export function addDefaultEncoderParams (options: { + command: FfmpegCommand + encoder: 'libx264' | string + fps: number + + streamNum?: number +}) { + const { command, encoder, fps, streamNum } = options + + if (encoder === 'libx264') { + // 3.1 is the minimal resource allocation for our highest supported resolution + command.outputOption(buildStreamSuffix('-level:v', streamNum) + ' 3.1') + + if (fps) { + // Keyframe interval of 2 seconds for faster seeking and resolution switching. + // https://streaminglearningcenter.com/blogs/whats-the-right-keyframe-interval.html + // https://superuser.com/a/908325 + command.outputOption(buildStreamSuffix('-g:v', streamNum) + ' ' + (fps * 2)) + } + } +} + +export function applyEncoderOptions (command: FfmpegCommand, options: EncoderOptions) { + command.inputOptions(options.inputOptions ?? []) + .outputOptions(options.outputOptions ?? []) +} diff --git a/shared/ffmpeg/shared/index.ts b/shared/ffmpeg/shared/index.ts new file mode 100644 index 000000000..51de0316f --- /dev/null +++ b/shared/ffmpeg/shared/index.ts @@ -0,0 +1,2 @@ +export * from './encoder-options' +export * from './presets' diff --git a/shared/ffmpeg/shared/presets.ts b/shared/ffmpeg/shared/presets.ts new file mode 100644 index 000000000..dcebdc1cf --- /dev/null +++ b/shared/ffmpeg/shared/presets.ts @@ -0,0 +1,93 @@ +import { pick } from '@shared/core-utils' +import { FFmpegCommandWrapper } from '../ffmpeg-command-wrapper' +import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, hasAudioStream } from '../ffprobe' +import { addDefaultEncoderGlobalParams, addDefaultEncoderParams, applyEncoderOptions } from './encoder-options' +import { getScaleFilter, StreamType } from '../ffmpeg-utils' + +export async function presetVOD (options: { + commandWrapper: FFmpegCommandWrapper + + input: string + + canCopyAudio: boolean + canCopyVideo: boolean + + resolution: number + fps: number + + scaleFilterValue?: string +}) { + const { commandWrapper, input, resolution, fps, scaleFilterValue } = options + const command = commandWrapper.getCommand() + + command.format('mp4') + .outputOption('-movflags faststart') + + addDefaultEncoderGlobalParams(command) + + const probe = await ffprobePromise(input) + + // Audio encoder + const bitrate = await getVideoStreamBitrate(input, probe) + const videoStreamDimensions = await getVideoStreamDimensionsInfo(input, probe) + + let streamsToProcess: StreamType[] = [ 'audio', 'video' ] + + if (!await hasAudioStream(input, probe)) { + command.noAudio() + streamsToProcess = [ 'video' ] + } + + for (const streamType of streamsToProcess) { + const builderResult = await commandWrapper.getEncoderBuilderResult({ + ...pick(options, [ 'canCopyAudio', 'canCopyVideo' ]), + + input, + inputBitrate: bitrate, + inputRatio: videoStreamDimensions?.ratio || 0, + + resolution, + fps, + streamType, + + videoType: 'vod' as 'vod' + }) + + if (!builderResult) { + throw new Error('No available encoder found for stream ' + streamType) + } + + commandWrapper.debugLog( + `Apply ffmpeg params from ${builderResult.encoder} for ${streamType} ` + + `stream of input ${input} using ${commandWrapper.getProfile()} profile.`, + { builderResult, resolution, fps } + ) + + if (streamType === 'video') { + command.videoCodec(builderResult.encoder) + + if (scaleFilterValue) { + command.outputOption(`-vf ${getScaleFilter(builderResult.result)}=${scaleFilterValue}`) + } + } else if (streamType === 'audio') { + command.audioCodec(builderResult.encoder) + } + + applyEncoderOptions(command, builderResult.result) + addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps }) + } +} + +export function presetCopy (commandWrapper: FFmpegCommandWrapper) { + commandWrapper.getCommand() + .format('mp4') + .videoCodec('copy') + .audioCodec('copy') +} + +export function presetOnlyAudio (commandWrapper: FFmpegCommandWrapper) { + commandWrapper.getCommand() + .format('mp4') + .audioCodec('copy') + .noVideo() +} diff --git a/shared/models/index.ts b/shared/models/index.ts index 439e9c8e1..78f6e73e3 100644 --- a/shared/models/index.ts +++ b/shared/models/index.ts @@ -11,6 +11,7 @@ export * from './moderation' export * from './overviews' export * from './plugins' export * from './redundancy' +export * from './runners' export * from './search' export * from './server' export * from './tokens' diff --git a/shared/models/runners/abort-runner-job-body.model.ts b/shared/models/runners/abort-runner-job-body.model.ts new file mode 100644 index 000000000..0b9c46c91 --- /dev/null +++ b/shared/models/runners/abort-runner-job-body.model.ts @@ -0,0 +1,6 @@ +export interface AbortRunnerJobBody { + runnerToken: string + jobToken: string + + reason: string +} diff --git a/shared/models/runners/accept-runner-job-body.model.ts b/shared/models/runners/accept-runner-job-body.model.ts new file mode 100644 index 000000000..cb266c4e6 --- /dev/null +++ b/shared/models/runners/accept-runner-job-body.model.ts @@ -0,0 +1,3 @@ +export interface AcceptRunnerJobBody { + runnerToken: string +} diff --git a/shared/models/runners/accept-runner-job-result.model.ts b/shared/models/runners/accept-runner-job-result.model.ts new file mode 100644 index 000000000..f2094b945 --- /dev/null +++ b/shared/models/runners/accept-runner-job-result.model.ts @@ -0,0 +1,6 @@ +import { RunnerJobPayload } from './runner-job-payload.model' +import { RunnerJob } from './runner-job.model' + +export interface AcceptRunnerJobResult { + job: RunnerJob & { jobToken: string } +} diff --git a/shared/models/runners/error-runner-job-body.model.ts b/shared/models/runners/error-runner-job-body.model.ts new file mode 100644 index 000000000..ac8568409 --- /dev/null +++ b/shared/models/runners/error-runner-job-body.model.ts @@ -0,0 +1,6 @@ +export interface ErrorRunnerJobBody { + runnerToken: string + jobToken: string + + message: string +} diff --git a/shared/models/runners/index.ts b/shared/models/runners/index.ts new file mode 100644 index 000000000..a52b82d2e --- /dev/null +++ b/shared/models/runners/index.ts @@ -0,0 +1,21 @@ +export * from './abort-runner-job-body.model' +export * from './accept-runner-job-body.model' +export * from './accept-runner-job-result.model' +export * from './error-runner-job-body.model' +export * from './list-runner-jobs-query.model' +export * from './list-runner-registration-tokens.model' +export * from './list-runners-query.model' +export * from './register-runner-body.model' +export * from './register-runner-result.model' +export * from './request-runner-job-body.model' +export * from './request-runner-job-result.model' +export * from './runner-job-payload.model' +export * from './runner-job-private-payload.model' +export * from './runner-job-state.model' +export * from './runner-job-success-body.model' +export * from './runner-job-type.type' +export * from './runner-job-update-body.model' +export * from './runner-job.model' +export * from './runner-registration-token' +export * from './runner.model' +export * from './unregister-runner-body.model' diff --git a/shared/models/runners/list-runner-jobs-query.model.ts b/shared/models/runners/list-runner-jobs-query.model.ts new file mode 100644 index 000000000..a5b62c55d --- /dev/null +++ b/shared/models/runners/list-runner-jobs-query.model.ts @@ -0,0 +1,6 @@ +export interface ListRunnerJobsQuery { + start?: number + count?: number + sort?: string + search?: string +} diff --git a/shared/models/runners/list-runner-registration-tokens.model.ts b/shared/models/runners/list-runner-registration-tokens.model.ts new file mode 100644 index 000000000..872e059cf --- /dev/null +++ b/shared/models/runners/list-runner-registration-tokens.model.ts @@ -0,0 +1,5 @@ +export interface ListRunnerRegistrationTokensQuery { + start?: number + count?: number + sort?: string +} diff --git a/shared/models/runners/list-runners-query.model.ts b/shared/models/runners/list-runners-query.model.ts new file mode 100644 index 000000000..d4362e4c5 --- /dev/null +++ b/shared/models/runners/list-runners-query.model.ts @@ -0,0 +1,5 @@ +export interface ListRunnersQuery { + start?: number + count?: number + sort?: string +} diff --git a/shared/models/runners/register-runner-body.model.ts b/shared/models/runners/register-runner-body.model.ts new file mode 100644 index 000000000..969bb35e1 --- /dev/null +++ b/shared/models/runners/register-runner-body.model.ts @@ -0,0 +1,6 @@ +export interface RegisterRunnerBody { + registrationToken: string + + name: string + description?: string +} diff --git a/shared/models/runners/register-runner-result.model.ts b/shared/models/runners/register-runner-result.model.ts new file mode 100644 index 000000000..e31776c6a --- /dev/null +++ b/shared/models/runners/register-runner-result.model.ts @@ -0,0 +1,4 @@ +export interface RegisterRunnerResult { + id: number + runnerToken: string +} diff --git a/shared/models/runners/request-runner-job-body.model.ts b/shared/models/runners/request-runner-job-body.model.ts new file mode 100644 index 000000000..0970d9007 --- /dev/null +++ b/shared/models/runners/request-runner-job-body.model.ts @@ -0,0 +1,3 @@ +export interface RequestRunnerJobBody { + runnerToken: string +} diff --git a/shared/models/runners/request-runner-job-result.model.ts b/shared/models/runners/request-runner-job-result.model.ts new file mode 100644 index 000000000..98601c42c --- /dev/null +++ b/shared/models/runners/request-runner-job-result.model.ts @@ -0,0 +1,10 @@ +import { RunnerJobPayload } from './runner-job-payload.model' +import { RunnerJobType } from './runner-job-type.type' + +export interface RequestRunnerJobResult

{ + availableJobs: { + uuid: string + type: RunnerJobType + payload: P + }[] +} diff --git a/shared/models/runners/runner-job-payload.model.ts b/shared/models/runners/runner-job-payload.model.ts new file mode 100644 index 000000000..8f0c17135 --- /dev/null +++ b/shared/models/runners/runner-job-payload.model.ts @@ -0,0 +1,68 @@ +export type RunnerJobVODPayload = + RunnerJobVODWebVideoTranscodingPayload | + RunnerJobVODHLSTranscodingPayload | + RunnerJobVODAudioMergeTranscodingPayload + +export type RunnerJobPayload = + RunnerJobVODPayload | + RunnerJobLiveRTMPHLSTranscodingPayload + +// --------------------------------------------------------------------------- + +export interface RunnerJobVODWebVideoTranscodingPayload { + input: { + videoFileUrl: string + } + + output: { + resolution: number + fps: number + } +} + +export interface RunnerJobVODHLSTranscodingPayload { + input: { + videoFileUrl: string + } + + output: { + resolution: number + fps: number + } +} + +export interface RunnerJobVODAudioMergeTranscodingPayload { + input: { + audioFileUrl: string + previewFileUrl: string + } + + output: { + resolution: number + fps: number + } +} + +// --------------------------------------------------------------------------- + +export function isAudioMergeTranscodingPayload (payload: RunnerJobPayload): payload is RunnerJobVODAudioMergeTranscodingPayload { + return !!(payload as RunnerJobVODAudioMergeTranscodingPayload).input.audioFileUrl +} + +// --------------------------------------------------------------------------- + +export interface RunnerJobLiveRTMPHLSTranscodingPayload { + input: { + rtmpUrl: string + } + + output: { + toTranscode: { + resolution: number + fps: number + }[] + + segmentDuration: number + segmentListSize: number + } +} diff --git a/shared/models/runners/runner-job-private-payload.model.ts b/shared/models/runners/runner-job-private-payload.model.ts new file mode 100644 index 000000000..c1d8d1045 --- /dev/null +++ b/shared/models/runners/runner-job-private-payload.model.ts @@ -0,0 +1,34 @@ +export type RunnerJobVODPrivatePayload = + RunnerJobVODWebVideoTranscodingPrivatePayload | + RunnerJobVODAudioMergeTranscodingPrivatePayload | + RunnerJobVODHLSTranscodingPrivatePayload + +export type RunnerJobPrivatePayload = + RunnerJobVODPrivatePayload | + RunnerJobLiveRTMPHLSTranscodingPrivatePayload + +// --------------------------------------------------------------------------- + +export interface RunnerJobVODWebVideoTranscodingPrivatePayload { + videoUUID: string + isNewVideo: boolean +} + +export interface RunnerJobVODAudioMergeTranscodingPrivatePayload { + videoUUID: string + isNewVideo: boolean +} + +export interface RunnerJobVODHLSTranscodingPrivatePayload { + videoUUID: string + isNewVideo: boolean + deleteWebVideoFiles: boolean +} + +// --------------------------------------------------------------------------- + +export interface RunnerJobLiveRTMPHLSTranscodingPrivatePayload { + videoUUID: string + masterPlaylistName: string + outputDirectory: string +} diff --git a/shared/models/runners/runner-job-state.model.ts b/shared/models/runners/runner-job-state.model.ts new file mode 100644 index 000000000..738db38b7 --- /dev/null +++ b/shared/models/runners/runner-job-state.model.ts @@ -0,0 +1,10 @@ +export enum RunnerJobState { + PENDING = 1, + PROCESSING = 2, + COMPLETED = 3, + ERRORED = 4, + WAITING_FOR_PARENT_JOB = 5, + CANCELLED = 6, + PARENT_ERRORED = 7, + PARENT_CANCELLED = 8 +} diff --git a/shared/models/runners/runner-job-success-body.model.ts b/shared/models/runners/runner-job-success-body.model.ts new file mode 100644 index 000000000..223b7552d --- /dev/null +++ b/shared/models/runners/runner-job-success-body.model.ts @@ -0,0 +1,41 @@ +export interface RunnerJobSuccessBody { + runnerToken: string + jobToken: string + + payload: RunnerJobSuccessPayload +} + +// --------------------------------------------------------------------------- + +export type RunnerJobSuccessPayload = + VODWebVideoTranscodingSuccess | + VODHLSTranscodingSuccess | + VODAudioMergeTranscodingSuccess | + LiveRTMPHLSTranscodingSuccess + +export interface VODWebVideoTranscodingSuccess { + videoFile: Blob | string +} + +export interface VODHLSTranscodingSuccess { + videoFile: Blob | string + resolutionPlaylistFile: Blob | string +} + +export interface VODAudioMergeTranscodingSuccess { + videoFile: Blob | string +} + +export interface LiveRTMPHLSTranscodingSuccess { + +} + +export function isWebVideoOrAudioMergeTranscodingPayloadSuccess ( + payload: RunnerJobSuccessPayload +): payload is VODHLSTranscodingSuccess | VODAudioMergeTranscodingSuccess { + return !!(payload as VODHLSTranscodingSuccess | VODAudioMergeTranscodingSuccess)?.videoFile +} + +export function isHLSTranscodingPayloadSuccess (payload: RunnerJobSuccessPayload): payload is VODHLSTranscodingSuccess { + return !!(payload as VODHLSTranscodingSuccess)?.resolutionPlaylistFile +} diff --git a/shared/models/runners/runner-job-type.type.ts b/shared/models/runners/runner-job-type.type.ts new file mode 100644 index 000000000..36d3b9b25 --- /dev/null +++ b/shared/models/runners/runner-job-type.type.ts @@ -0,0 +1,5 @@ +export type RunnerJobType = + 'vod-web-video-transcoding' | + 'vod-hls-transcoding' | + 'vod-audio-merge-transcoding' | + 'live-rtmp-hls-transcoding' diff --git a/shared/models/runners/runner-job-update-body.model.ts b/shared/models/runners/runner-job-update-body.model.ts new file mode 100644 index 000000000..ed94bbe63 --- /dev/null +++ b/shared/models/runners/runner-job-update-body.model.ts @@ -0,0 +1,28 @@ +export interface RunnerJobUpdateBody { + runnerToken: string + jobToken: string + + progress?: number + payload?: RunnerJobUpdatePayload +} + +// --------------------------------------------------------------------------- + +export type RunnerJobUpdatePayload = LiveRTMPHLSTranscodingUpdatePayload + +export interface LiveRTMPHLSTranscodingUpdatePayload { + type: 'add-chunk' | 'remove-chunk' + + masterPlaylistFile?: Blob | string + + resolutionPlaylistFilename?: string + resolutionPlaylistFile?: Blob | string + + videoChunkFilename: string + videoChunkFile?: Blob | string +} + +export function isLiveRTMPHLSTranscodingUpdatePayload (value: RunnerJobUpdatePayload): value is LiveRTMPHLSTranscodingUpdatePayload { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion + return !!(value as LiveRTMPHLSTranscodingUpdatePayload)?.videoChunkFilename +} diff --git a/shared/models/runners/runner-job.model.ts b/shared/models/runners/runner-job.model.ts new file mode 100644 index 000000000..080093563 --- /dev/null +++ b/shared/models/runners/runner-job.model.ts @@ -0,0 +1,45 @@ +import { VideoConstant } from '../videos' +import { RunnerJobPayload } from './runner-job-payload.model' +import { RunnerJobPrivatePayload } from './runner-job-private-payload.model' +import { RunnerJobState } from './runner-job-state.model' +import { RunnerJobType } from './runner-job-type.type' + +export interface RunnerJob { + uuid: string + + type: RunnerJobType + + state: VideoConstant + + payload: T + + failures: number + error: string | null + + progress: number + priority: number + + startedAt: Date | string + createdAt: Date | string + updatedAt: Date | string + finishedAt: Date | string + + parent?: { + type: RunnerJobType + state: VideoConstant + uuid: string + } + + // If associated to a runner + runner?: { + id: number + name: string + + description: string + } +} + +// eslint-disable-next-line max-len +export interface RunnerJobAdmin extends RunnerJob { + privatePayload: U +} diff --git a/shared/models/runners/runner-registration-token.ts b/shared/models/runners/runner-registration-token.ts new file mode 100644 index 000000000..0a157aa51 --- /dev/null +++ b/shared/models/runners/runner-registration-token.ts @@ -0,0 +1,10 @@ +export interface RunnerRegistrationToken { + id: number + + registrationToken: string + + createdAt: Date + updatedAt: Date + + registeredRunnersCount: number +} diff --git a/shared/models/runners/runner.model.ts b/shared/models/runners/runner.model.ts new file mode 100644 index 000000000..3284f2992 --- /dev/null +++ b/shared/models/runners/runner.model.ts @@ -0,0 +1,12 @@ +export interface Runner { + id: number + + name: string + description: string + + ip: string + lastContact: Date | string + + createdAt: Date | string + updatedAt: Date | string +} diff --git a/shared/models/runners/unregister-runner-body.model.ts b/shared/models/runners/unregister-runner-body.model.ts new file mode 100644 index 000000000..d3465c5d6 --- /dev/null +++ b/shared/models/runners/unregister-runner-body.model.ts @@ -0,0 +1,3 @@ +export interface UnregisterRunnerBody { + runnerToken: string +} diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts index 6ffe3a676..5d2c10278 100644 --- a/shared/models/server/custom-config.model.ts +++ b/shared/models/server/custom-config.model.ts @@ -116,6 +116,10 @@ export interface CustomConfig { allowAdditionalExtensions: boolean allowAudioFiles: boolean + remoteRunners: { + enabled: boolean + } + threads: number concurrency: number @@ -149,6 +153,9 @@ export interface CustomConfig { transcoding: { enabled: boolean + remoteRunners: { + enabled: boolean + } threads: number profile: string resolutions: ConfigResolutions diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index 9c0b5ea56..16187d133 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts @@ -18,6 +18,7 @@ export type JobType = | 'after-video-channel-import' | 'email' | 'federate-video' + | 'transcoding-job-builder' | 'manage-video-torrent' | 'move-to-object-storage' | 'notify' @@ -41,6 +42,10 @@ export interface Job { createdAt: Date | string finishedOn: Date | string processedOn: Date | string + + parent?: { + id: string + } } export type ActivitypubHttpBroadcastPayload = { @@ -139,30 +144,28 @@ interface BaseTranscodingPayload { export interface HLSTranscodingPayload extends BaseTranscodingPayload { type: 'new-resolution-to-hls' resolution: VideoResolution + fps: number copyCodecs: boolean - hasAudio: boolean - - autoDeleteWebTorrentIfNeeded: boolean - isMaxQuality: boolean + deleteWebTorrentFiles: boolean } export interface NewWebTorrentResolutionTranscodingPayload extends BaseTranscodingPayload { type: 'new-resolution-to-webtorrent' resolution: VideoResolution - - hasAudio: boolean - createHLSIfNeeded: boolean + fps: number } export interface MergeAudioTranscodingPayload extends BaseTranscodingPayload { type: 'merge-audio-to-webtorrent' resolution: VideoResolution - createHLSIfNeeded: true + fps: number } export interface OptimizeTranscodingPayload extends BaseTranscodingPayload { type: 'optimize-to-webtorrent' + + quickTranscode: boolean } export type VideoTranscodingPayload = @@ -258,3 +261,27 @@ export interface FederateVideoPayload { videoUUID: string isNewVideo: boolean } + +// --------------------------------------------------------------------------- + +export interface TranscodingJobBuilderPayload { + videoUUID: string + + optimizeJob?: { + isNewVideo: boolean + } + + // Array of jobs to create + jobs?: { + type: 'video-transcoding' + payload: VideoTranscodingPayload + priority?: number + }[] + + // Array of sequential jobs to create + sequentialJobs?: { + type: 'video-transcoding' + payload: VideoTranscodingPayload + priority?: number + }[][] +} diff --git a/shared/models/server/server-config.model.ts b/shared/models/server/server-config.model.ts index d0bd9a00f..38b9d0385 100644 --- a/shared/models/server/server-config.model.ts +++ b/shared/models/server/server-config.model.ts @@ -148,6 +148,10 @@ export interface ServerConfig { profile: string availableProfiles: string[] + + remoteRunners: { + enabled: boolean + } } live: { @@ -165,6 +169,10 @@ export interface ServerConfig { transcoding: { enabled: boolean + remoteRunners: { + enabled: boolean + } + enabledResolutions: number[] profile: string diff --git a/shared/models/server/server-error-code.enum.ts b/shared/models/server/server-error-code.enum.ts index a39cde1b3..24d3c6d21 100644 --- a/shared/models/server/server-error-code.enum.ts +++ b/shared/models/server/server-error-code.enum.ts @@ -45,7 +45,10 @@ export const enum ServerErrorCode { INVALID_TWO_FACTOR = 'invalid_two_factor', ACCOUNT_WAITING_FOR_APPROVAL = 'account_waiting_for_approval', - ACCOUNT_APPROVAL_REJECTED = 'account_approval_rejected' + ACCOUNT_APPROVAL_REJECTED = 'account_approval_rejected', + + RUNNER_JOB_NOT_IN_PROCESSING_STATE = 'runner_job_not_in_processing_state', + UNKNOWN_RUNNER_TOKEN = 'unknown_runner_token' } /** diff --git a/shared/models/users/user-right.enum.ts b/shared/models/users/user-right.enum.ts index 42e5c8cd6..a5a770b75 100644 --- a/shared/models/users/user-right.enum.ts +++ b/shared/models/users/user-right.enum.ts @@ -45,5 +45,7 @@ export const enum UserRight { MANAGE_VIDEO_IMPORTS = 27, - MANAGE_REGISTRATIONS = 28 + MANAGE_REGISTRATIONS = 28, + + MANAGE_RUNNERS = 29 } diff --git a/shared/models/videos/live/live-video-error.enum.ts b/shared/models/videos/live/live-video-error.enum.ts index 3a8e4afa0..a26453505 100644 --- a/shared/models/videos/live/live-video-error.enum.ts +++ b/shared/models/videos/live/live-video-error.enum.ts @@ -3,5 +3,7 @@ export const enum LiveVideoError { DURATION_EXCEEDED = 2, QUOTA_EXCEEDED = 3, FFMPEG_ERROR = 4, - BLACKLISTED = 5 + BLACKLISTED = 5, + RUNNER_JOB_ERROR = 6, + RUNNER_JOB_CANCEL = 7 }