Regenerate video filenames on transcoding

In particular when using manual transcoding, to invalidate potential
HTTP caches in front of peertube

server/lib/object-storage/videos.ts

@@ -26,6 +26,10 @@ function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) {
   return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
 }
 
+function removeHLSFileObjectStorage (playlist: MStreamingPlaylistVideo, filename: string) {
+  return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
+}
+
 function removeWebTorrentObjectStorage (videoFile: MVideoFile) {
   return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS)
 }
@@ -63,6 +67,7 @@ export {
   storeHLSFile,
 
   removeHLSObjectStorage,
+  removeHLSFileObjectStorage,
   removeWebTorrentObjectStorage,
 
   makeWebTorrentFileAvailable,

server/lib/transcoding/transcoding.ts

@@ -2,7 +2,9 @@ import { Job } from 'bull'
 import { copyFile, ensureDir, move, remove, stat } from 'fs-extra'
 import { basename, extname as extnameUtil, join } from 'path'
 import { toEven } from '@server/helpers/core-utils'
+import { retryTransactionWrapper } from '@server/helpers/database-utils'
 import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
+import { sequelizeTypescript } from '@server/initializers/database'
 import { MStreamingPlaylistFilesVideo, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
 import { VideoResolution, VideoStorage } from '../../../shared/models/videos'
 import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
@@ -29,8 +31,6 @@ import {
 } from '../paths'
 import { VideoPathManager } from '../video-path-manager'
 import { VideoTranscodingProfilesManager } from './default-transcoding-profiles'
-import { retryTransactionWrapper } from '@server/helpers/database-utils'
-import { sequelizeTypescript } from '@server/initializers/database'
 
 /**
  *
@@ -259,6 +259,9 @@ async function onWebTorrentVideoFileTranscoding (
 
   await createTorrentAndSetInfoHash(video, videoFile)
 
+  const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution })
+  if (oldFile) await video.removeWebTorrentFileAndTorrent(oldFile)
+
   await VideoFileModel.customUpsert(videoFile, 'video', undefined)
   video.VideoFiles = await video.$get('VideoFiles')
 
@@ -311,17 +314,15 @@ async function generateHlsPlaylistCommon (options: {
   await transcodeVOD(transcodeOptions)
 
   // Create or update the playlist
-  const playlist = await retryTransactionWrapper(() => {
+  const { playlist, oldPlaylistFilename, oldSegmentsSha256Filename } = await retryTransactionWrapper(() => {
     return sequelizeTypescript.transaction(async transaction => {
       const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(video, transaction)
 
-      if (!playlist.playlistFilename) {
-        playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive)
-      }
+      const oldPlaylistFilename = playlist.playlistFilename
+      const oldSegmentsSha256Filename = playlist.segmentsSha256Filename
 
-      if (!playlist.segmentsSha256Filename) {
+      playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive)
       playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive)
-      }
 
       playlist.p2pMediaLoaderInfohashes = []
       playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
@@ -330,10 +331,13 @@ async function generateHlsPlaylistCommon (options: {
 
       await playlist.save({ transaction })
 
-      return playlist
+      return { playlist, oldPlaylistFilename, oldSegmentsSha256Filename }
     })
   })
 
+  if (oldPlaylistFilename) await video.removeStreamingPlaylistFile(playlist, oldPlaylistFilename)
+  if (oldSegmentsSha256Filename) await video.removeStreamingPlaylistFile(playlist, oldSegmentsSha256Filename)
+
   // Build the new playlist file
   const extname = extnameUtil(videoFilename)
   const newVideoFile = new VideoFileModel({
@@ -364,11 +368,15 @@ async function generateHlsPlaylistCommon (options: {
 
   await createTorrentAndSetInfoHash(playlist, newVideoFile)
 
+  const oldFile = await VideoFileModel.loadHLSFile({ playlistId: playlist.id, fps: newVideoFile.fps, resolution: newVideoFile.resolution })
+  if (oldFile) await video.removeStreamingPlaylistVideoFile(playlist, oldFile)
+
   const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
 
   const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo
   playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles')
   playlist.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles)
+  playlist.storage = VideoStorage.FILE_SYSTEM
 
   await playlist.save()
 

server/models/video/video-file.ts

@@ -405,15 +405,16 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
     mode: 'streaming-playlist' | 'video',
     transaction: Transaction
   ) {
-    const baseWhere = {
+    const baseFind = {
       fps: videoFile.fps,
-      resolution: videoFile.resolution
+      resolution: videoFile.resolution,
+      transaction
     }
 
-    if (mode === 'streaming-playlist') Object.assign(baseWhere, { videoStreamingPlaylistId: videoFile.videoStreamingPlaylistId })
-    else Object.assign(baseWhere, { videoId: videoFile.videoId })
+    const element = mode === 'streaming-playlist'
+      ? await VideoFileModel.loadHLSFile({ ...baseFind, playlistId: videoFile.videoStreamingPlaylistId })
+      : await VideoFileModel.loadWebTorrentFile({ ...baseFind, videoId: videoFile.videoId })
 
-    const element = await VideoFileModel.findOne({ where: baseWhere, transaction })
     if (!element) return videoFile.save({ transaction })
 
     for (const k of Object.keys(videoFile.toJSON())) {
@@ -423,6 +424,36 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
     return element.save({ transaction })
   }
 
+  static async loadWebTorrentFile (options: {
+    videoId: number
+    fps: number
+    resolution: number
+    transaction?: Transaction
+  }) {
+    const where = {
+      fps: options.fps,
+      resolution: options.resolution,
+      videoId: options.videoId
+    }
+
+    return VideoFileModel.findOne({ where, transaction: options.transaction })
+  }
+
+  static async loadHLSFile (options: {
+    playlistId: number
+    fps: number
+    resolution: number
+    transaction?: Transaction
+  }) {
+    const where = {
+      fps: options.fps,
+      resolution: options.resolution,
+      videoStreamingPlaylistId: options.playlistId
+    }
+
+    return VideoFileModel.findOne({ where, transaction: options.transaction })
+  }
+
   static removeHLSFilesOfVideoId (videoStreamingPlaylistId: number) {
     const options = {
       where: { videoStreamingPlaylistId }

server/models/video/video.ts

@@ -26,7 +26,7 @@ import {
 } from 'sequelize-typescript'
 import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video'
 import { LiveManager } from '@server/lib/live/live-manager'
-import { removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage'
+import { removeHLSFileObjectStorage, removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage'
 import { getHLSDirectory, getHLSRedundancyDirectory } from '@server/lib/paths'
 import { VideoPathManager } from '@server/lib/video-path-manager'
 import { getServerActor } from '@server/models/application/application'
@@ -1816,6 +1816,25 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
     }
   }
 
+  async removeStreamingPlaylistVideoFile (streamingPlaylist: MStreamingPlaylist, videoFile: MVideoFile) {
+    const filePath = VideoPathManager.Instance.getFSHLSOutputPath(this, videoFile.filename)
+    await videoFile.removeTorrent()
+    await remove(filePath)
+
+    if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
+      await removeHLSFileObjectStorage(streamingPlaylist.withVideo(this), videoFile.filename)
+    }
+  }
+
+  async removeStreamingPlaylistFile (streamingPlaylist: MStreamingPlaylist, filename: string) {
+    const filePath = VideoPathManager.Instance.getFSHLSOutputPath(this, filename)
+    await remove(filePath)
+
+    if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
+      await removeHLSFileObjectStorage(streamingPlaylist.withVideo(this), filename)
+    }
+  }
+
   isOutdated () {
     if (this.isOwned()) return false
 

server/tests/api/transcoding/create-transcoding.ts

@@ -46,6 +46,8 @@ function runTests (objectStorage: boolean) {
   let videoUUID: string
   let publishedAt: string
 
+  let shouldBeDeleted: string[]
+
   before(async function () {
     this.timeout(120000)
 
@@ -187,6 +189,12 @@ function runTests (objectStorage: boolean) {
       expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(1)
 
       if (objectStorage) await checkFilesInObjectStorage(videoDetails)
+
+      shouldBeDeleted = [
+        videoDetails.streamingPlaylists[0].files[0].fileUrl,
+        videoDetails.streamingPlaylists[0].playlistUrl,
+        videoDetails.streamingPlaylists[0].segmentsSha256Url
+      ]
     }
 
     await servers[0].config.updateExistingSubConfig({
@@ -227,6 +235,12 @@ function runTests (objectStorage: boolean) {
     }
   })
 
+  it('Should have correctly deleted previous files', async function () {
+    for (const fileUrl of shouldBeDeleted) {
+      await makeRawRequest(fileUrl, HttpStatusCode.NOT_FOUND_404)
+    }
+  })
+
   it('Should not have updated published at attributes', async function () {
     const video = await servers[0].videos.get({ id: videoUUID })