From 18998c45c001869a883ec7a2d286d8170f768381 Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Fri, 6 Aug 2021 15:25:45 +0200
Subject: [PATCH] Correctly handle large HLS files for redundancy

---
 server/lib/hls.ts                                 | 13 +++++++++----
 .../lib/schedulers/videos-redundancy-scheduler.ts | 15 +++++++--------
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index 32b02bc26..0e77ab9fa 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -1,4 +1,4 @@
-import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
+import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, stat, writeFile } from 'fs-extra'
 import { flatten, uniq } from 'lodash'
 import { basename, dirname, join } from 'path'
 import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models'
@@ -108,8 +108,9 @@ async function buildSha256Segment (segmentPath: string) {
   return sha256(buf)
 }
 
-function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) {
+function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number, bodyKBLimit: number) {
   let timer
+  let remainingBodyKBLimit = bodyKBLimit
 
   logger.info('Importing HLS playlist %s', playlistUrl)
 
@@ -136,8 +137,12 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string,
       for (const fileUrl of fileUrls) {
         const destPath = join(tmpDirectory, basename(fileUrl))
 
-        const bodyKBLimit = 10 * 1000 * 1000 // 10GB
-        await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit })
+        await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit })
+
+        const { size } = await stat(destPath)
+        remainingBodyKBLimit -= (size / 1000)
+
+        logger.debug('Downloaded HLS playlist file %s with %d kB remained limit.', fileUrl, Math.floor(remainingBodyKBLimit))
       }
 
       clearTimeout(timer)
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index 103ab1fab..137ae53a0 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -4,9 +4,7 @@ import { getServerActor } from '@server/models/application/application'
 import { TrackerModel } from '@server/models/server/tracker'
 import { VideoModel } from '@server/models/video/video'
 import {
-  MStreamingPlaylist,
   MStreamingPlaylistFiles,
-  MStreamingPlaylistVideo,
   MVideoAccountLight,
   MVideoFile,
   MVideoFileVideo,
@@ -249,7 +247,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
   private async createStreamingPlaylistRedundancy (
     redundancy: VideosRedundancyStrategy,
     video: MVideoAccountLight,
-    playlistArg: MStreamingPlaylist
+    playlistArg: MStreamingPlaylistFiles
   ) {
     let strategy = 'manual'
     let expiresOn: Date = null
@@ -259,16 +257,17 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
       expiresOn = this.buildNewExpiration(redundancy.minLifetime)
     }
 
-    const playlist = playlistArg as MStreamingPlaylistVideo
-    playlist.Video = video
-
+    const playlist = Object.assign(playlistArg, { Video: video })
     const serverActor = await getServerActor()
 
     logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy)
 
     const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
     const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video)
-    await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
+
+    const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000
+    const toleranceKB = maxSizeKB + ((5 * maxSizeKB) / 100) // 5% more tolerance
+    await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT, toleranceKB)
 
     const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({
       expiresOn,
@@ -334,7 +333,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
     return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}`
   }
 
-  private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]) {
+  private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]): number {
     const fileReducer = (previous: number, current: MVideoFile) => previous + current.size
 
     let allFiles = files