Add avatar to prune script

scripts/create-import-video-file-job.ts

@@ -25,7 +25,7 @@ run()
 async function run () {
   await initDatabaseModels(true)
 
-  const video = await VideoModel.loadByUUIDWithFile(program['video'])
+  const video = await VideoModel.loadByUUID(program['video'])
   if (!video) throw new Error('Video not found.')
   if (video.isOwned() === false) throw new Error('Cannot import files of a non owned video.')
 

scripts/create-transcoding-job.ts

@@ -29,7 +29,7 @@ run()
 async function run () {
   await initDatabaseModels(true)
 
-  const video = await VideoModel.loadByUUIDWithFile(program['video'])
+  const video = await VideoModel.loadByUUID(program['video'])
   if (!video) throw new Error('Video not found.')
 
   const dataInput: VideoTranscodingPayload = program.resolution !== undefined

scripts/prune-storage.ts

@@ -3,9 +3,12 @@ import { join } from 'path'
 import { CONFIG } from '../server/initializers/config'
 import { VideoModel } from '../server/models/video/video'
 import { initDatabaseModels } from '../server/initializers'
-import { remove, readdir } from 'fs-extra'
+import { readdir, remove } from 'fs-extra'
 import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
+import * as Bluebird from 'bluebird'
 import { getUUIDFromFilename } from '../server/helpers/utils'
+import { ThumbnailModel } from '../server/models/video/thumbnail'
+import { AvatarModel } from '../server/models/avatar/avatar'
 
 run()
   .then(() => process.exit(0))
@@ -17,25 +20,19 @@ run()
 async function run () {
   await initDatabaseModels(true)
 
-  const storageOnlyOwnedToPrune = [
-    CONFIG.STORAGE.VIDEOS_DIR,
-    CONFIG.STORAGE.TORRENTS_DIR,
-    CONFIG.STORAGE.REDUNDANCY_DIR
-  ]
-
-  const storageForAllToPrune = [
-    CONFIG.STORAGE.PREVIEWS_DIR,
-    CONFIG.STORAGE.THUMBNAILS_DIR
-  ]
-
   let toDelete: string[] = []
-  for (const directory of storageOnlyOwnedToPrune) {
-    toDelete = toDelete.concat(await pruneDirectory(directory, true))
-  }
 
-  for (const directory of storageForAllToPrune) {
+  toDelete = toDelete.concat(
-    toDelete = toDelete.concat(await pruneDirectory(directory, false))
+    await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesVideoExist(true)),
-  }
+    await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesVideoExist(true)),
+
+    await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
+
+    await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true)),
+    await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false)),
+
+    await pruneDirectory(CONFIG.STORAGE.AVATARS_DIR, doesAvatarExist)
+  )
 
   const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
   toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
@@ -61,32 +58,81 @@ async function run () {
   }
 }
 
-async function pruneDirectory (directory: string, onlyOwned = false) {
+type ExistFun = (file: string) => Promise<boolean>
+async function pruneDirectory (directory: string, existFun: ExistFun) {
   const files = await readdir(directory)
 
   const toDelete: string[] = []
-  for (const file of files) {
+  await Bluebird.map(files, async file => {
-    const uuid = getUUIDFromFilename(file)
+    if (await existFun(file) !== true) {
-    let video: VideoModel
-    let localRedundancy: boolean
-
-    if (uuid) {
-      video = await VideoModel.loadByUUIDWithFile(uuid)
-      localRedundancy = await VideoRedundancyModel.isLocalByVideoUUIDExists(uuid)
-    }
-
-    if (
-      !uuid ||
-      !video ||
-      (onlyOwned === true && (video.isOwned() === false && localRedundancy === false))
-    ) {
       toDelete.push(join(directory, file))
     }
-  }
+  }, { concurrency: 20 })
 
   return toDelete
 }
 
+function doesVideoExist (keepOnlyOwned: boolean) {
+  return async (file: string) => {
+    const uuid = getUUIDFromFilename(file)
+    const video = await VideoModel.loadByUUID(uuid)
+
+    return video && (keepOnlyOwned === false || video.isOwned())
+  }
+}
+
+function doesThumbnailExist (keepOnlyOwned: boolean) {
+  return async (file: string) => {
+    const thumbnail = await ThumbnailModel.loadByName(file)
+    if (!thumbnail) return false
+
+    if (keepOnlyOwned) {
+      const video = await VideoModel.load(thumbnail.videoId)
+      if (video.isOwned() === false) return false
+    }
+
+    return true
+  }
+}
+
+async function doesAvatarExist (file: string) {
+  const avatar = await AvatarModel.loadByName(file)
+
+  return !!avatar
+}
+
+async function doesRedundancyExist (file: string) {
+  const uuid = getUUIDFromFilename(file)
+  const video = await VideoModel.loadWithFiles(uuid)
+
+  if (!video) return false
+
+  const isPlaylist = file.includes('.') === false
+
+  if (isPlaylist) {
+    const p = video.getHLSPlaylist()
+    if (!p) return false
+
+    const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
+    return !!redundancy
+  }
+
+  const resolution = parseInt(file.split('-')[5], 10)
+  if (isNaN(resolution)) {
+    console.error('Cannot prune %s because we cannot guess guess the resolution.', file)
+    return true
+  }
+
+  const videoFile = video.getFile(resolution)
+  if (!videoFile) {
+    console.error('Cannot find file of video %s - %d', video.url, resolution)
+    return true
+  }
+
+  const redundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id)
+  return !!redundancy
+}
+
 async function askConfirmation () {
   return new Promise((res, rej) => {
     prompt.start()

server/controllers/lazy-static.ts

@@ -49,7 +49,12 @@ async function getAvatar (req: express.Request, res: express.Response) {
 
     logger.info('Lazy serve remote avatar image %s.', avatar.fileUrl)
 
-    await pushAvatarProcessInQueue({ filename: avatar.filename, fileUrl: avatar.fileUrl })
+    try {
+      await pushAvatarProcessInQueue({ filename: avatar.filename, fileUrl: avatar.fileUrl })
+    } catch (err) {
+      logger.warn('Cannot process remote avatar %s.', avatar.fileUrl, { err })
+      return res.sendStatus(404)
+    }
 
     avatar.onDisk = true
     avatar.save()

server/lib/files-cache/videos-preview-cache.ts

@@ -18,7 +18,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
   }
 
   async getFilePathImpl (videoUUID: string) {
-    const video = await VideoModel.loadByUUIDWithFile(videoUUID)
+    const video = await VideoModel.loadByUUID(videoUUID)
     if (!video) return undefined
 
     if (video.isOwned()) return { isOwned: true, path: video.getPreview().getPath() }

server/models/video/thumbnail.ts

@@ -100,6 +100,16 @@ export class ThumbnailModel extends Model<ThumbnailModel> {
             .catch(err => logger.error('Cannot remove thumbnail file %s.', instance.filename, err))
   }
 
+  static loadByName (filename: string) {
+    const query = {
+      where: {
+        filename
+      }
+    }
+
+    return ThumbnailModel.findOne(query)
+  }
+
   static generateDefaultPreviewName (videoUUID: string) {
     return videoUUID + '.jpg'
   }

server/models/video/video.ts

@@ -119,6 +119,7 @@ import { CONFIG } from '../../initializers/config'
 import { ThumbnailModel } from './thumbnail'
 import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
 import { createTorrentPromise } from '../../helpers/webtorrent'
+import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
 
 // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
 const indexes: (ModelIndexesOptions & { where?: WhereOptions })[] = [
@@ -1422,15 +1423,23 @@ export class VideoModel extends Model<VideoModel> {
     return VideoModel.scope(ScopeNames.WITH_THUMBNAILS).findOne(options)
   }
 
-  static loadWithFiles (id: number, t?: Transaction, logging?: boolean) {
+  static loadWithFiles (id: number | string, t?: Transaction, logging?: boolean) {
+    const where = buildWhereIdOrUUID(id)
+
+    const query = {
+      where,
+      transaction: t,
+      logging
+    }
+
     return VideoModel.scope([
       ScopeNames.WITH_FILES,
       ScopeNames.WITH_STREAMING_PLAYLISTS,
       ScopeNames.WITH_THUMBNAILS
-    ]).findByPk(id, { transaction: t, logging })
+    ]).findOne(query)
   }
 
-  static loadByUUIDWithFile (uuid: string) {
+  static loadByUUID (uuid: string) {
     const options = {
       where: {
         uuid
@@ -1754,7 +1763,7 @@ export class VideoModel extends Model<VideoModel> {
     return maxBy(this.VideoFiles, file => file.resolution)
   }
 
-  getFile (resolution: VideoResolution) {
+  getFile (resolution: number) {
     if (Array.isArray(this.VideoFiles) === false) return undefined
 
     return this.VideoFiles.find(f => f.resolution === resolution)
@@ -1893,6 +1902,12 @@ export class VideoModel extends Model<VideoModel> {
     return `/api/${API_VERSION}/videos/${this.uuid}/description`
   }
 
+  getHLSPlaylist () {
+    if (!this.VideoStreamingPlaylists) return undefined
+
+    return this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
+  }
+
   removeFile (videoFile: VideoFileModel, isRedundancy = false) {
     const baseDir = isRedundancy ? CONFIG.STORAGE.REDUNDANCY_DIR : CONFIG.STORAGE.VIDEOS_DIR
 

server/tests/cli/index.ts

@@ -4,5 +4,6 @@ import './create-transcoding-job'
 import './optimize-old-videos'
 import './peertube'
 import './plugins'
+import './prune-storage'
 import './reset-password'
 import './update-host'

server/tests/cli/prune-storage.ts

@@ -0,0 +1,199 @@
+/* tslint:disable:no-unused-expression */
+
+import 'mocha'
+import * as chai from 'chai'
+import { waitJobs } from '../../../shared/extra-utils/server/jobs'
+import {
+  buildServerDirectory,
+  cleanupTests,
+  createVideoPlaylist,
+  doubleFollow,
+  execCLI,
+  flushAndRunMultipleServers,
+  getAccount,
+  getEnvCli,
+  ServerInfo,
+  setAccessTokensToServers, setDefaultVideoChannel,
+  updateMyAvatar,
+  uploadVideo,
+  wait
+} from '../../../shared/extra-utils'
+import { Account, VideoPlaylistPrivacy } from '../../../shared/models'
+import { createFile, readdir } from 'fs-extra'
+import * as uuidv4 from 'uuid/v4'
+import { join } from 'path'
+import * as request from 'supertest'
+
+const expect = chai.expect
+
+async function countFiles (internalServerNumber: number, directory: string) {
+  const files = await readdir(buildServerDirectory(internalServerNumber, directory))
+
+  return files.length
+}
+
+async function assertNotExists (internalServerNumber: number, directory: string, substring: string) {
+  const files = await readdir(buildServerDirectory(internalServerNumber, directory))
+
+  for (const f of files) {
+    expect(f).to.not.contain(substring)
+  }
+}
+
+async function assertCountAreOkay (servers: ServerInfo[]) {
+  for (const server of servers) {
+    const videosCount = await countFiles(server.internalServerNumber, 'videos')
+    expect(videosCount).to.equal(8)
+
+    const torrentsCount = await countFiles(server.internalServerNumber, 'torrents')
+    expect(torrentsCount).to.equal(8)
+
+    const previewsCount = await countFiles(server.internalServerNumber, 'previews')
+    expect(previewsCount).to.equal(2)
+
+    const thumbnailsCount = await countFiles(server.internalServerNumber, 'thumbnails')
+    expect(thumbnailsCount).to.equal(6)
+
+    const avatarsCount = await countFiles(server.internalServerNumber, 'avatars')
+    expect(avatarsCount).to.equal(2)
+  }
+}
+
+describe('Test prune storage scripts', function () {
+  let servers: ServerInfo[]
+  const badNames: { [ directory: string ]: string[] } = {}
+
+  before(async function () {
+    this.timeout(120000)
+
+    servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: true } })
+    await setAccessTokensToServers(servers)
+    await setDefaultVideoChannel(servers)
+
+    for (const server of servers) {
+      await uploadVideo(server.url, server.accessToken, { name: 'video 1' })
+      await uploadVideo(server.url, server.accessToken, { name: 'video 2' })
+
+      await updateMyAvatar({ url: server.url, accessToken: server.accessToken, fixture: 'avatar.png' })
+
+      await createVideoPlaylist({
+        url: server.url,
+        token: server.accessToken,
+        playlistAttrs: {
+          displayName: 'playlist',
+          privacy: VideoPlaylistPrivacy.PUBLIC,
+          videoChannelId: server.videoChannel.id,
+          thumbnailfile: 'thumbnail.jpg'
+        }
+      })
+    }
+
+    await doubleFollow(servers[0], servers[1])
+
+    // Lazy load the remote avatar
+    {
+      const res = await getAccount(servers[ 0 ].url, 'root@localhost:' + servers[ 1 ].port)
+      const account: Account = res.body
+      await request('http://localhost:' + servers[ 0 ].port).get(account.avatar.path).expect(200)
+    }
+
+    {
+      const res = await getAccount(servers[ 1 ].url, 'root@localhost:' + servers[ 0 ].port)
+      const account: Account = res.body
+      await request('http://localhost:' + servers[ 1 ].port).get(account.avatar.path).expect(200)
+    }
+
+    await wait(1000)
+
+    await waitJobs(servers)
+  })
+
+  it('Should have the files on the disk', async function () {
+    await assertCountAreOkay(servers)
+  })
+
+  it('Should create some dirty files', async function () {
+    for (let i = 0; i < 2; i++) {
+      {
+        const base = buildServerDirectory(servers[0].internalServerNumber, 'videos')
+
+        const n1 = uuidv4() + '.mp4'
+        const n2 = uuidv4() + '.webm'
+
+        await createFile(join(base, n1))
+        await createFile(join(base, n2))
+
+        badNames['videos'] = [ n1, n2 ]
+      }
+
+      {
+        const base = buildServerDirectory(servers[0].internalServerNumber, 'torrents')
+
+        const n1 = uuidv4() + '-240.torrent'
+        const n2 = uuidv4() + '-480.torrent'
+
+        await createFile(join(base, n1))
+        await createFile(join(base, n2))
+
+        badNames['torrents'] = [ n1, n2 ]
+      }
+
+      {
+        const base = buildServerDirectory(servers[0].internalServerNumber, 'thumbnails')
+
+        const n1 = uuidv4() + '.jpg'
+        const n2 = uuidv4() + '.jpg'
+
+        await createFile(join(base, n1))
+        await createFile(join(base, n2))
+
+        badNames['thumbnails'] = [ n1, n2 ]
+      }
+
+      {
+        const base = buildServerDirectory(servers[0].internalServerNumber, 'previews')
+
+        const n1 = uuidv4() + '.jpg'
+        const n2 = uuidv4() + '.jpg'
+
+        await createFile(join(base, n1))
+        await createFile(join(base, n2))
+
+        badNames['previews'] = [ n1, n2 ]
+      }
+
+      {
+        const base = buildServerDirectory(servers[0].internalServerNumber, 'avatars')
+
+        const n1 = uuidv4() + '.png'
+        const n2 = uuidv4() + '.jpg'
+
+        await createFile(join(base, n1))
+        await createFile(join(base, n2))
+
+        badNames['avatars'] = [ n1, n2 ]
+      }
+    }
+  })
+
+  it('Should run prune storage', async function () {
+    this.timeout(30000)
+
+    const env = getEnvCli(servers[0])
+    await execCLI(`echo y | ${env} npm run prune-storage`)
+  })
+
+  it('Should have removed files', async function () {
+    await assertCountAreOkay(servers)
+
+    for (const directory of Object.keys(badNames)) {
+      for (const name of badNames[directory]) {
+        await assertNotExists(servers[0].internalServerNumber, directory, name)
+      }
+    }
+  })
+
+  after(async function () {
+    await cleanupTests(servers)
+  })
+})

shared/extra-utils/miscs/miscs.ts

@@ -44,6 +44,10 @@ function root () {
   return root
 }
 
+function buildServerDirectory (internalServerNumber: number, directory: string) {
+  return join(root(), 'test' + internalServerNumber, directory)
+}
+
 async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') {
   const res = await request(url)
     .get(imagePath)
@@ -105,6 +109,7 @@ async function generateHighBitrateVideo () {
 export {
   dateIsValid,
   wait,
+  buildServerDirectory,
   webtorrentAdd,
   immutableAssign,
   testImage,

shared/extra-utils/videos/videos.ts

@@ -19,7 +19,7 @@ import {
 import * as validator from 'validator'
 import { VideoDetails, VideoPrivacy } from '../../models/videos'
 import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, loadLanguages, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants'
-import { dateIsValid, webtorrentAdd } from '../miscs/miscs'
+import { dateIsValid, webtorrentAdd, buildServerDirectory } from '../miscs/miscs'
 
 loadLanguages()
 
@@ -308,10 +308,8 @@ async function checkVideoFilesWereRemoved (
     join('redundancy', 'hls')
   ]
 ) {
-  const testDirectory = 'test' + serverNumber
-
   for (const directory of directories) {
-    const directoryPath = join(root(), testDirectory, directory)
+    const directoryPath = buildServerDirectory(serverNumber, directory)
 
     const directoryExists = await pathExists(directoryPath)
     if (directoryExists === false) continue