Remove previous thumbnail if needed

.eslintrc.json

@@ -23,6 +23,7 @@
       "consistent-as-needed"
     ],
     "padded-blocks": "off",
+    "prefer-regex-literals": "off",
     "no-async-promise-executor": "off",
     "dot-notation": "off",
     "promise/param-names": "off",

scripts/prune-storage.ts

@@ -95,7 +95,7 @@ function doesVideoExist (keepOnlyOwned: boolean) {
 
 function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType) {
   return async (file: string) => {
-    const thumbnail = await ThumbnailModel.loadWithVideoByName(file, type)
+    const thumbnail = await ThumbnailModel.loadByFilename(file, type)
     if (!thumbnail) return false
 
     if (keepOnlyOwned) {

server/controllers/api/video-playlist.ts

@@ -173,7 +173,11 @@ async function addVideoPlaylist (req: express.Request, res: express.Response) {
 
   const thumbnailField = req.files['thumbnailfile']
   const thumbnailModel = thumbnailField
-    ? await createPlaylistMiniatureFromExisting(thumbnailField[0].path, videoPlaylist, false)
+    ? await createPlaylistMiniatureFromExisting({
+      inputPath: thumbnailField[0].path,
+      playlist: videoPlaylist,
+      automaticallyGenerated: false
+    })
     : undefined
 
   const videoPlaylistCreated = await sequelizeTypescript.transaction(async t => {
@@ -211,7 +215,11 @@ async function updateVideoPlaylist (req: express.Request, res: express.Response)
 
   const thumbnailField = req.files['thumbnailfile']
   const thumbnailModel = thumbnailField
-    ? await createPlaylistMiniatureFromExisting(thumbnailField[0].path, videoPlaylistInstance, false)
+    ? await createPlaylistMiniatureFromExisting({
+      inputPath: thumbnailField[0].path,
+      playlist: videoPlaylistInstance,
+      automaticallyGenerated: false
+    })
     : undefined
 
   try {
@@ -474,7 +482,12 @@ async function generateThumbnailForPlaylist (videoPlaylist: MVideoPlaylistThumbn
   }
 
   const inputPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoMiniature.filename)
-  const thumbnailModel = await createPlaylistMiniatureFromExisting(inputPath, videoPlaylist, true, true)
+  const thumbnailModel = await createPlaylistMiniatureFromExisting({
+    inputPath,
+    playlist: videoPlaylist,
+    automaticallyGenerated: true,
+    keepOriginal: true
+  })
 
   thumbnailModel.videoPlaylistId = videoPlaylist.id
 

server/controllers/api/videos/import.ts

@@ -282,7 +282,7 @@ async function processPreview (req: express.Request, video: MVideoThumbnail): Pr
 
 async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) {
   try {
-    return createVideoMiniatureFromUrl(url, video, ThumbnailType.MINIATURE)
+    return createVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE })
   } catch (err) {
     logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err })
     return undefined
@@ -291,14 +291,14 @@ async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) {
 
 async function processPreviewFromUrl (url: string, video: MVideoThumbnail) {
   try {
-    return createVideoMiniatureFromUrl(url, video, ThumbnailType.PREVIEW)
+    return createVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW })
   } catch (err) {
     logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err })
     return undefined
   }
 }
 
-function insertIntoDB (parameters: {
+async function insertIntoDB (parameters: {
   video: MVideoThumbnail
   thumbnailModel: MThumbnail
   previewModel: MThumbnail
@@ -309,7 +309,7 @@ function insertIntoDB (parameters: {
 }): Promise<MVideoImportFormattable> {
   const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
 
-  return sequelizeTypescript.transaction(async t => {
+  const videoImport = await sequelizeTypescript.transaction(async t => {
     const sequelizeOptions = { transaction: t }
 
     // Save video object in database
@@ -339,4 +339,6 @@ function insertIntoDB (parameters: {
 
     return videoImport
   })
+
+  return videoImport
 }

server/controllers/api/videos/index.ts

@@ -215,7 +215,7 @@ async function addVideo (req: express.Request, res: express.Response) {
   const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
     video,
     files: req.files,
-    fallback: type => generateVideoMiniature(video, videoFile, type)
+    fallback: type => generateVideoMiniature({ video, videoFile, type })
   })
 
   // Create the torrent file

server/lib/activitypub/playlist.ts

@@ -103,7 +103,7 @@ async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAc
 
   if (playlistObject.icon) {
     try {
-      const thumbnailModel = await createPlaylistMiniatureFromUrl(playlistObject.icon.url, refreshedPlaylist)
+      const thumbnailModel = await createPlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist: refreshedPlaylist })
       await refreshedPlaylist.setAndSaveThumbnail(thumbnailModel, undefined)
     } catch (err) {
       logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err })

server/lib/activitypub/videos.ts

@@ -313,7 +313,11 @@ async function updateVideoFromAP (options: {
     let thumbnailModel: MThumbnail
 
     try {
-      thumbnailModel = await createVideoMiniatureFromUrl(getThumbnailFromIcons(videoObject).url, video, ThumbnailType.MINIATURE)
+      thumbnailModel = await createVideoMiniatureFromUrl({
+        downloadUrl: getThumbnailFromIcons(videoObject).url,
+        video,
+        type: ThumbnailType.MINIATURE
+      })
     } catch (err) {
       logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })
     }
@@ -362,7 +366,12 @@ async function updateVideoFromAP (options: {
 
       if (videoUpdated.getPreview()) {
         const previewUrl = getPreviewUrl(getPreviewFromIcons(videoObject), video)
-        const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
+        const previewModel = createPlaceholderThumbnail({
+          fileUrl: previewUrl,
+          video,
+          type: ThumbnailType.PREVIEW,
+          size: PREVIEWS_SIZE
+        })
         await videoUpdated.addAndSaveThumbnail(previewModel, t)
       }
 
@@ -585,8 +594,11 @@ async function createVideo (videoObject: VideoObject, channel: MChannelAccountLi
   const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, videoObject.to)
   const video = VideoModel.build(videoData) as MVideoThumbnail
 
-  const promiseThumbnail = createVideoMiniatureFromUrl(getThumbnailFromIcons(videoObject).url, video, ThumbnailType.MINIATURE)
+  const promiseThumbnail = createVideoMiniatureFromUrl({
-    .catch(err => {
+    downloadUrl: getThumbnailFromIcons(videoObject).url,
+    video,
+    type: ThumbnailType.MINIATURE
+  }).catch(err => {
     logger.error('Cannot create miniature from url.', { err })
     return undefined
   })
@@ -597,6 +609,7 @@ async function createVideo (videoObject: VideoObject, channel: MChannelAccountLi
   }
 
   const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => {
+    try {
       const sequelizeOptions = { transaction: t }
 
       const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight
@@ -604,9 +617,13 @@ async function createVideo (videoObject: VideoObject, channel: MChannelAccountLi
 
       if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
 
-    const previewIcon = getPreviewFromIcons(videoObject)
+      const previewUrl = getPreviewUrl(getPreviewFromIcons(videoObject), videoCreated)
-    const previewUrl = getPreviewUrl(previewIcon, videoCreated)
+      const previewModel = createPlaceholderThumbnail({
-    const previewModel = createPlaceholderThumbnail(previewUrl, videoCreated, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
+        fileUrl: previewUrl,
+        video: videoCreated,
+        type: ThumbnailType.PREVIEW,
+        size: PREVIEWS_SIZE
+      })
 
       if (thumbnailModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
 
@@ -672,6 +689,13 @@ async function createVideo (videoObject: VideoObject, channel: MChannelAccountLi
       logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
 
       return { autoBlacklisted, videoCreated }
+    } catch (err) {
+      // FIXME: Use rollback hook when https://github.com/sequelize/sequelize/pull/13038 is released
+      // Remove thumbnail
+      if (thumbnailModel) await thumbnailModel.removeThumbnail()
+
+      throw err
+    }
   })
 
   if (waitThumbnail === false) {

server/lib/files-cache/videos-preview-cache.ts

@@ -20,7 +20,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
   }
 
   async getFilePathImpl (filename: string) {
-    const thumbnail = await ThumbnailModel.loadWithVideoByName(filename, ThumbnailType.PREVIEW)
+    const thumbnail = await ThumbnailModel.loadWithVideoByFilename(filename, ThumbnailType.PREVIEW)
     if (!thumbnail) return undefined
 
     if (thumbnail.Video.isOwned()) return { isOwned: true, path: thumbnail.getPath() }

server/lib/job-queue/handlers/video-import.ts

@@ -162,7 +162,11 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
     let thumbnailModel: MThumbnail
     let thumbnailSave: object
     if (!videoImportWithFiles.Video.getMiniature()) {
-      thumbnailModel = await generateVideoMiniature(videoImportWithFiles.Video, videoFile, ThumbnailType.MINIATURE)
+      thumbnailModel = await generateVideoMiniature({
+        video: videoImportWithFiles.Video,
+        videoFile,
+        type: ThumbnailType.MINIATURE
+      })
       thumbnailSave = thumbnailModel.toJSON()
     }
 
@@ -170,7 +174,11 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
     let previewModel: MThumbnail
     let previewSave: object
     if (!videoImportWithFiles.Video.getPreview()) {
-      previewModel = await generateVideoMiniature(videoImportWithFiles.Video, videoFile, ThumbnailType.PREVIEW)
+      previewModel = await generateVideoMiniature({
+        video: videoImportWithFiles.Video,
+        videoFile,
+        type: ThumbnailType.PREVIEW
+      })
       previewSave = previewModel.toJSON()
     }
 

server/lib/job-queue/handlers/video-live-ending.ts

@@ -122,11 +122,19 @@ async function saveLive (video: MVideo, live: MVideoLive) {
 
   // Regenerate the thumbnail & preview?
   if (videoWithFiles.getMiniature().automaticallyGenerated === true) {
-    await generateVideoMiniature(videoWithFiles, videoWithFiles.getMaxQualityFile(), ThumbnailType.MINIATURE)
+    await generateVideoMiniature({
+      video: videoWithFiles,
+      videoFile: videoWithFiles.getMaxQualityFile(),
+      type: ThumbnailType.MINIATURE
+    })
   }
 
   if (videoWithFiles.getPreview().automaticallyGenerated === true) {
-    await generateVideoMiniature(videoWithFiles, videoWithFiles.getMaxQualityFile(), ThumbnailType.PREVIEW)
+    await generateVideoMiniature({
+      video: videoWithFiles,
+      videoFile: videoWithFiles.getMaxQualityFile(),
+      type: ThumbnailType.PREVIEW
+    })
   }
 
   await publishAndFederateIfNeeded(videoWithFiles, true)

server/lib/thumbnail.ts

@@ -1,33 +1,48 @@
+import { join } from 'path'
+
+import { ThumbnailType } from '../../shared/models/videos/thumbnail.type'
 import { generateImageFromVideoFile } from '../helpers/ffmpeg-utils'
+import { processImage } from '../helpers/image-utils'
+import { downloadImage } from '../helpers/requests'
 import { CONFIG } from '../initializers/config'
 import { ASSETS_PATH, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '../initializers/constants'
 import { ThumbnailModel } from '../models/video/thumbnail'
-import { ThumbnailType } from '../../shared/models/videos/thumbnail.type'
-import { processImage } from '../helpers/image-utils'
-import { join } from 'path'
-import { downloadImage } from '../helpers/requests'
-import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist'
 import { MVideoFile, MVideoThumbnail } from '../types/models'
 import { MThumbnail } from '../types/models/video/thumbnail'
+import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist'
 import { getVideoFilePath } from './video-paths'
 
 type ImageSize = { height: number, width: number }
 
-function createPlaylistMiniatureFromExisting (
+function createPlaylistMiniatureFromExisting (options: {
-  inputPath: string,
+  inputPath: string
-  playlist: MVideoPlaylistThumbnail,
+  playlist: MVideoPlaylistThumbnail
-  automaticallyGenerated: boolean,
+  automaticallyGenerated: boolean
-  keepOriginal = false,
+  keepOriginal?: boolean // default to false
   size?: ImageSize
-) {
+}) {
+  const { inputPath, playlist, automaticallyGenerated, keepOriginal = false, size } = options
   const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size)
   const type = ThumbnailType.MINIATURE
 
   const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height }, keepOriginal)
-  return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, automaticallyGenerated, existingThumbnail })
+  return createThumbnailFromFunction({
+    thumbnailCreator,
+    filename,
+    height,
+    width,
+    type,
+    automaticallyGenerated,
+    existingThumbnail
+  })
 }
 
-function createPlaylistMiniatureFromUrl (downloadUrl: string, playlist: MVideoPlaylistThumbnail, size?: ImageSize) {
+function createPlaylistMiniatureFromUrl (options: {
+  downloadUrl: string
+  playlist: MVideoPlaylistThumbnail
+  size?: ImageSize
+}) {
+  const { downloadUrl, playlist, size } = options
   const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size)
   const type = ThumbnailType.MINIATURE
 
@@ -40,7 +55,13 @@ function createPlaylistMiniatureFromUrl (downloadUrl: string, playlist: MVideoPl
   return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl })
 }
 
-function createVideoMiniatureFromUrl (downloadUrl: string, video: MVideoThumbnail, type: ThumbnailType, size?: ImageSize) {
+function createVideoMiniatureFromUrl (options: {
+  downloadUrl: string
+  video: MVideoThumbnail
+  type: ThumbnailType
+  size?: ImageSize
+}) {
+  const { downloadUrl, video, type, size } = options
   const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size)
 
   // Only save the file URL if it is a remote video
@@ -58,17 +79,31 @@ function createVideoMiniatureFromExisting (options: {
   type: ThumbnailType
   automaticallyGenerated: boolean
   size?: ImageSize
-  keepOriginal?: boolean
+  keepOriginal?: boolean // default to false
 }) {
-  const { inputPath, video, type, automaticallyGenerated, size, keepOriginal } = options
+  const { inputPath, video, type, automaticallyGenerated, size, keepOriginal = false } = options
 
   const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size)
   const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height }, keepOriginal)
 
-  return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, automaticallyGenerated, existingThumbnail })
+  return createThumbnailFromFunction({
+    thumbnailCreator,
+    filename,
+    height,
+    width,
+    type,
+    automaticallyGenerated,
+    existingThumbnail
+  })
 }
 
-function generateVideoMiniature (video: MVideoThumbnail, videoFile: MVideoFile, type: ThumbnailType) {
+function generateVideoMiniature (options: {
+  video: MVideoThumbnail
+  videoFile: MVideoFile
+  type: ThumbnailType
+}) {
+  const { video, videoFile, type } = options
+
   const input = getVideoFilePath(video, videoFile)
 
   const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type)
@@ -76,10 +111,24 @@ function generateVideoMiniature (video: MVideoThumbnail, videoFile: MVideoFile,
     ? () => processImage(ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, outputPath, { width, height }, true)
     : () => generateImageFromVideoFile(input, basePath, filename, { height, width })
 
-  return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, automaticallyGenerated: true, existingThumbnail })
+  return createThumbnailFromFunction({
+    thumbnailCreator,
+    filename,
+    height,
+    width,
+    type,
+    automaticallyGenerated: true,
+    existingThumbnail
+  })
 }
 
-function createPlaceholderThumbnail (fileUrl: string, video: MVideoThumbnail, type: ThumbnailType, size: ImageSize) {
+function createPlaceholderThumbnail (options: {
+  fileUrl: string
+  video: MVideoThumbnail
+  type: ThumbnailType
+  size: ImageSize
+}) {
+  const { fileUrl, video, type, size } = options
   const { filename, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size)
 
   const thumbnail = existingThumbnail || new ThumbnailModel()
@@ -164,12 +213,22 @@ async function createThumbnailFromFunction (parameters: {
   fileUrl?: string
   existingThumbnail?: MThumbnail
 }) {
-  const { thumbnailCreator, filename, width, height, type, existingThumbnail, automaticallyGenerated = null, fileUrl = null } = parameters
+  const {
+    thumbnailCreator,
+    filename,
+    width,
+    height,
+    type,
+    existingThumbnail,
+    automaticallyGenerated = null,
+    fileUrl = null
+  } = parameters
 
-  // Remove old file
+  const oldFilename = existingThumbnail
-  if (existingThumbnail) await existingThumbnail.removeThumbnail()
+    ? existingThumbnail.filename
+    : undefined
 
-  const thumbnail = existingThumbnail || new ThumbnailModel()
+  const thumbnail: MThumbnail = existingThumbnail || new ThumbnailModel()
 
   thumbnail.filename = filename
   thumbnail.height = height
@@ -177,6 +236,7 @@ async function createThumbnailFromFunction (parameters: {
   thumbnail.type = type
   thumbnail.fileUrl = fileUrl
   thumbnail.automaticallyGenerated = automaticallyGenerated
+  thumbnail.previousThumbnailFilename = oldFilename
 
   await thumbnailCreator()
 

server/models/video/thumbnail.ts

@@ -3,6 +3,8 @@ import { join } from 'path'
 import {
   AfterDestroy,
   AllowNull,
+  BeforeCreate,
+  BeforeUpdate,
   BelongsTo,
   Column,
   CreatedAt,
@@ -14,7 +16,8 @@ import {
   UpdatedAt
 } from 'sequelize-typescript'
 import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
-import { MThumbnailVideo, MVideoAccountLight } from '@server/types/models'
+import { afterCommitIfTransaction } from '@server/helpers/database-utils'
+import { MThumbnail, MThumbnailVideo, MVideoAccountLight } from '@server/types/models'
 import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
 import { logger } from '../../helpers/logger'
 import { CONFIG } from '../../initializers/config'
@@ -96,6 +99,9 @@ export class ThumbnailModel extends Model {
   @UpdatedAt
   updatedAt: Date
 
+  // If this thumbnail replaced existing one, track the old name
+  previousThumbnailFilename: string
+
   private static readonly types: { [ id in ThumbnailType ]: { label: string, directory: string, staticPath: string } } = {
     [ThumbnailType.MINIATURE]: {
       label: 'miniature',
@@ -109,6 +115,12 @@ export class ThumbnailModel extends Model {
     }
   }
 
+  @BeforeCreate
+  @BeforeUpdate
+  static removeOldFile (instance: ThumbnailModel, options) {
+    return afterCommitIfTransaction(options.transaction, () => instance.removePreviousFilenameIfNeeded())
+  }
+
   @AfterDestroy
   static removeFiles (instance: ThumbnailModel) {
     logger.info('Removing %s file %s.', ThumbnailModel.types[instance.type].label, instance.filename)
@@ -118,7 +130,18 @@ export class ThumbnailModel extends Model {
             .catch(err => logger.error('Cannot remove thumbnail file %s.', instance.filename, err))
   }
 
-  static loadWithVideoByName (filename: string, thumbnailType: ThumbnailType): Promise<MThumbnailVideo> {
+  static loadByFilename (filename: string, thumbnailType: ThumbnailType): Promise<MThumbnail> {
+    const query = {
+      where: {
+        filename,
+        type: thumbnailType
+      }
+    }
+
+    return ThumbnailModel.findOne(query)
+  }
+
+  static loadWithVideoByFilename (filename: string, thumbnailType: ThumbnailType): Promise<MThumbnailVideo> {
     const query = {
       where: {
         filename,
@@ -150,7 +173,22 @@ export class ThumbnailModel extends Model {
     return join(directory, this.filename)
   }
 
+  getPreviousPath () {
+    const directory = ThumbnailModel.types[this.type].directory
+    return join(directory, this.previousThumbnailFilename)
+  }
+
   removeThumbnail () {
     return remove(this.getPath())
   }
+
+  removePreviousFilenameIfNeeded () {
+    if (!this.previousThumbnailFilename) return
+
+    const previousPath = this.getPreviousPath()
+    remove(previousPath)
+      .catch(err => logger.error('Cannot remove previous thumbnail file %s.', previousPath, { err }))
+
+    this.previousThumbnailFilename = undefined
+  }
 }

server/models/video/video-playlist.ts

@@ -17,6 +17,7 @@ import {
   Table,
   UpdatedAt
 } from 'sequelize-typescript'
+import { v4 as uuidv4 } from 'uuid'
 import { MAccountId, MChannelId } from '@server/types/models'
 import { ActivityIconObject } from '../../../shared/models/activitypub/objects'
 import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'

server/tests/api/server/follows.ts

@@ -558,7 +558,7 @@ describe('Test follows', function () {
       const caption1: VideoCaption = res.body.data[0]
       expect(caption1.language.id).to.equal('ar')
       expect(caption1.language.label).to.equal('Arabic')
-      expect(caption1.captionPath).to.equal('/lazy-static/video-captions/' + video4.uuid + '-ar.vtt')
+      expect(caption1.captionPath).to.match(new RegExp('^/lazy-static/video-captions/.+-ar.vtt$'))
       await testCaptionFile(servers[0].url, caption1.captionPath, 'Subtitle good 2.')
     })
 

shared/extra-utils/server/servers.ts

@@ -5,7 +5,7 @@ import { ChildProcess, exec, fork } from 'child_process'
 import { copy, ensureDir, pathExists, readdir, readFile, remove } from 'fs-extra'
 import { join } from 'path'
 import { randomInt } from '../../core-utils/miscs/miscs'
-import { Video, VideoChannel } from '../../models/videos'
+import { VideoChannel } from '../../models/videos'
 import { buildServerDirectory, getFileSize, isGithubCI, root, wait } from '../miscs/miscs'
 import { makeGetRequest } from '../requests/requests'