Try to have more robust live tests

server/tests/api/transcoding/create-transcoding.ts

@@ -226,7 +226,7 @@ function runTests (objectStorage: boolean) {
         const resolutions = hlsPlaylist.files.map(f => f.resolution.id)
         await checkResolutionsInMasterPlaylist({ server: servers[0], playlistUrl: hlsPlaylist.playlistUrl, resolutions })
 
-        const shaBody = await servers[0].streamingPlaylists.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url })
+        const shaBody = await servers[0].streamingPlaylists.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url, withRetry: true })
         expect(Object.keys(shaBody)).to.have.lengthOf(5)
       }
     }

server/tests/shared/live.ts

@@ -115,7 +115,8 @@ async function testLiveVideoResolutions (options: {
         baseUrlSegment: baseUrl,
         videoUUID: video.uuid,
         segmentName,
-        hlsPlaylist
+        hlsPlaylist,
+        withRetry: objectStorage // With object storage, the request may fail because of inconsistent data in S3
       })
 
       if (originServer.internalServerNumber === server.internalServerNumber) {

server/tests/shared/streaming-playlists.ts

@@ -51,12 +51,13 @@ async function checkLiveSegmentHash (options: {
   videoUUID: string
   segmentName: string
   hlsPlaylist: VideoStreamingPlaylist
+  withRetry?: boolean
 }) {
-  const { server, baseUrlSegment, videoUUID, segmentName, hlsPlaylist } = options
+  const { server, baseUrlSegment, videoUUID, segmentName, hlsPlaylist, withRetry = false } = options
   const command = server.streamingPlaylists
 
-  const segmentBody = await command.getFragmentedSegment({ url: `${baseUrlSegment}/${videoUUID}/${segmentName}` })
-  const shaBody = await command.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url })
+  const segmentBody = await command.getFragmentedSegment({ url: `${baseUrlSegment}/${videoUUID}/${segmentName}`, withRetry })
+  const shaBody = await command.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url, withRetry })
 
   expect(sha256(segmentBody)).to.equal(shaBody[segmentName])
 }

shared/server-commands/videos/live-command.ts

@@ -224,7 +224,7 @@ export class LiveCommand extends AbstractCommand {
         const video = await server.videos.get({ id: videoUUID })
         const hlsPlaylist = video.streamingPlaylists[0]
 
-        const shaBody = await server.streamingPlaylists.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url })
+        const shaBody = await server.streamingPlaylists.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url, withRetry: objectStorage })
 
         if (!shaBody[segmentName]) {
           throw new Error('Segment SHA does not exist')

shared/server-commands/videos/streaming-playlists-command.ts

@@ -44,31 +44,71 @@ export class StreamingPlaylistsCommand extends AbstractCommand {
     }
   }
 
-  getFragmentedSegment (options: OverrideCommandOptions & {
+  async getFragmentedSegment (options: OverrideCommandOptions & {
     url: string
     range?: string
-  }) {
-    return unwrapBody<Buffer>(this.getRawRequest({
-      ...options,
 
-      url: options.url,
-      range: options.range,
-      implicitToken: false,
-      responseType: 'application/octet-stream',
-      defaultExpectedStatus: HttpStatusCode.OK_200
-    }))
+    withRetry?: boolean // default false
+    currentRetry?: number
+  }) {
+    const { withRetry, currentRetry = 1 } = options
+
+    try {
+      const result = await unwrapBody<Buffer>(this.getRawRequest({
+        ...options,
+
+        url: options.url,
+        range: options.range,
+        implicitToken: false,
+        responseType: 'application/octet-stream',
+        defaultExpectedStatus: HttpStatusCode.OK_200
+      }))
+
+      return result
+    } catch (err) {
+      if (!withRetry || currentRetry > 5) throw err
+
+      await wait(100)
+
+      return this.getFragmentedSegment({
+        ...options,
+
+        withRetry,
+        currentRetry: currentRetry + 1
+      })
+    }
   }
 
-  getSegmentSha256 (options: OverrideCommandOptions & {
+  async getSegmentSha256 (options: OverrideCommandOptions & {
     url: string
-  }) {
-    return unwrapBodyOrDecodeToJSON<{ [ id: string ]: string }>(this.getRawRequest({
-      ...options,
 
-      url: options.url,
-      contentType: 'application/json',
-      implicitToken: false,
-      defaultExpectedStatus: HttpStatusCode.OK_200
-    }))
+    withRetry?: boolean // default false
+    currentRetry?: number
+  }) {
+    const { withRetry, currentRetry = 1 } = options
+
+    try {
+      const result = await unwrapBodyOrDecodeToJSON<{ [ id: string ]: string }>(this.getRawRequest({
+        ...options,
+
+        url: options.url,
+        contentType: 'application/json',
+        implicitToken: false,
+        defaultExpectedStatus: HttpStatusCode.OK_200
+      }))
+
+      return result
+    } catch (err) {
+      if (!withRetry || currentRetry > 5) throw err
+
+      await wait(100)
+
+      return this.getSegmentSha256({
+        ...options,
+
+        withRetry,
+        currentRetry: currentRetry + 1
+      })
+    }
   }
 }