Add tmp and redundancy directories

config/default.yaml

@@ -45,8 +45,10 @@ smtp:
 
 # From the project root directory
 storage:
+  tmp: 'storage/tmp/' # Used to download data (imports etc), store uploaded files before processing...
   avatars: 'storage/avatars/'
   videos: 'storage/videos/'
+  redundancy: 'storage/redundancy/'
   logs: 'storage/logs/'
   previews: 'storage/previews/'
   thumbnails: 'storage/thumbnails/'
@@ -75,7 +77,7 @@ trending:
 redundancy:
   videos:
     check_interval: '1 hour' # How often you want to check new videos to cache
-    strategies:
+    strategies: # Just uncomment strategies you want
 #      -
 #        size: '10GB'
 #        # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances)

config/production.yaml.example

@@ -46,8 +46,10 @@ smtp:
 
 # From the project root directory
 storage:
+  tmp: '/var/www/peertube/storage/tmp/' # Used to download data (imports etc), store uploaded files before processing...
   avatars: '/var/www/peertube/storage/avatars/'
   videos: '/var/www/peertube/storage/videos/'
+  redundancy: '/var/www/peertube/storage/videos/'
   logs: '/var/www/peertube/storage/logs/'
   previews: '/var/www/peertube/storage/previews/'
   thumbnails: '/var/www/peertube/storage/thumbnails/'
@@ -76,7 +78,7 @@ trending:
 redundancy:
   videos:
     check_interval: '1 hour' # How often you want to check new videos to cache
-    strategies:
+    strategies: # Just uncomment strategies you want
 #      -
 #        size: '10GB'
 #        # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances)

config/test-1.yaml

@@ -10,8 +10,10 @@ database:
 
 # From the project root directory
 storage:
+  tmp: 'test1/tmp/'
   avatars: 'test1/avatars/'
   videos: 'test1/videos/'
+  redundancy: 'test1/redundancy/'
   logs: 'test1/logs/'
   previews: 'test1/previews/'
   thumbnails: 'test1/thumbnails/'

config/test-2.yaml

@@ -10,8 +10,10 @@ database:
 
 # From the project root directory
 storage:
+  tmp: 'test2/tmp/'
   avatars: 'test2/avatars/'
   videos: 'test2/videos/'
+  redundancy: 'test2/redundancy/'
   logs: 'test2/logs/'
   previews: 'test2/previews/'
   thumbnails: 'test2/thumbnails/'

config/test-3.yaml

@@ -10,8 +10,10 @@ database:
 
 # From the project root directory
 storage:
+  tmp: 'test3/tmp/'
   avatars: 'test3/avatars/'
   videos: 'test3/videos/'
+  redundancy: 'test3/redundancy/'
   logs: 'test3/logs/'
   previews: 'test3/previews/'
   thumbnails: 'test3/thumbnails/'

config/test-4.yaml

@@ -10,8 +10,10 @@ database:
 
 # From the project root directory
 storage:
+  tmp: 'test4/tmp/'
   avatars: 'test4/avatars/'
   videos: 'test4/videos/'
+  redundancy: 'test4/redundancy/'
   logs: 'test4/logs/'
   previews: 'test4/previews/'
   thumbnails: 'test4/thumbnails/'

config/test-5.yaml

@@ -10,8 +10,10 @@ database:
 
 # From the project root directory
 storage:
+  tmp: 'test5/tmp/'
   avatars: 'test5/avatars/'
   videos: 'test5/videos/'
+  redundancy: 'test5/redundancy/'
   logs: 'test5/logs/'
   previews: 'test5/previews/'
   thumbnails: 'test5/thumbnails/'

config/test-6.yaml

@@ -10,8 +10,10 @@ database:
 
 # From the project root directory
 storage:
+  tmp: 'test6/tmp/'
   avatars: 'test6/avatars/'
   videos: 'test6/videos/'
+  redundancy: 'test6/redundancy/'
   logs: 'test6/logs/'
   previews: 'test6/previews/'
   thumbnails: 'test6/thumbnails/'

config/test.yaml

@@ -67,4 +67,4 @@ import:
       enabled: true
 
 instance:
-  default_nsfw_policy: 'display'
+  default_nsfw_policy: 'display'

server/controllers/api/users/me.ts

@@ -42,7 +42,7 @@ import { AccountModel } from '../../../models/account/account'
 
 const auditLogger = auditLoggerFactory('users-me')
 
-const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR })
+const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR })
 
 const meRouter = express.Router()
 
@@ -348,7 +348,7 @@ async function updateMe (req: express.Request, res: express.Response, next: expr
   return res.sendStatus(204)
 }
 
-async function updateMyAvatar (req: express.Request, res: express.Response, next: express.NextFunction) {
+async function updateMyAvatar (req: express.Request, res: express.Response) {
   const avatarPhysicalFile = req.files[ 'avatarfile' ][ 0 ]
   const user: UserModel = res.locals.oauth.token.user
   const oldUserAuditView = new UserAuditView(user.toFormattedJSON())

server/controllers/api/video-channel.ts

@@ -32,7 +32,7 @@ import { resetSequelizeInstance } from '../../helpers/database-utils'
 import { UserModel } from '../../models/account/user'
 
 const auditLogger = auditLoggerFactory('channels')
-const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR })
+const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR })
 
 const videoChannelRouter = express.Router()
 

server/controllers/api/videos/import.ts

@@ -37,9 +37,9 @@ const reqVideoFileImport = createReqFiles(
   [ 'thumbnailfile', 'previewfile', 'torrentfile' ],
   Object.assign({}, TORRENT_MIMETYPE_EXT, IMAGE_MIMETYPE_EXT),
   {
-    thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR,
+    thumbnailfile: CONFIG.STORAGE.TMP_DIR,
-    previewfile: CONFIG.STORAGE.PREVIEWS_DIR,
+    previewfile: CONFIG.STORAGE.TMP_DIR,
-    torrentfile: CONFIG.STORAGE.TORRENTS_DIR
+    torrentfile: CONFIG.STORAGE.TMP_DIR
   }
 )
 

server/controllers/api/videos/index.ts

@@ -67,17 +67,17 @@ const reqVideoFileAdd = createReqFiles(
   [ 'videofile', 'thumbnailfile', 'previewfile' ],
   Object.assign({}, VIDEO_MIMETYPE_EXT, IMAGE_MIMETYPE_EXT),
   {
-    videofile: CONFIG.STORAGE.VIDEOS_DIR,
+    videofile: CONFIG.STORAGE.TMP_DIR,
-    thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR,
+    thumbnailfile: CONFIG.STORAGE.TMP_DIR,
-    previewfile: CONFIG.STORAGE.PREVIEWS_DIR
+    previewfile: CONFIG.STORAGE.TMP_DIR
   }
 )
 const reqVideoFileUpdate = createReqFiles(
   [ 'thumbnailfile', 'previewfile' ],
   IMAGE_MIMETYPE_EXT,
   {
-    thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR,
+    thumbnailfile: CONFIG.STORAGE.TMP_DIR,
-    previewfile: CONFIG.STORAGE.PREVIEWS_DIR
+    previewfile: CONFIG.STORAGE.TMP_DIR
   }
 )
 

server/controllers/static.ts

@@ -34,12 +34,17 @@ staticRouter.use(
 )
 
 // Videos path for webseeding
-const videosPhysicalPath = CONFIG.STORAGE.VIDEOS_DIR
 staticRouter.use(
   STATIC_PATHS.WEBSEED,
   cors(),
-  express.static(videosPhysicalPath)
+  express.static(CONFIG.STORAGE.VIDEOS_DIR)
 )
+staticRouter.use(
+  STATIC_PATHS.WEBSEED,
+  cors(),
+  express.static(CONFIG.STORAGE.REDUNDANCY_DIR, { fallthrough: false }) // 404, because we don't have this video
+)
+
 staticRouter.use(
   STATIC_DOWNLOAD_PATHS.VIDEOS + ':id-:resolution([0-9]+).:extension',
   asyncMiddleware(videosGetValidator),

server/helpers/requests.ts

@@ -1,9 +1,9 @@
 import * as Bluebird from 'bluebird'
 import { createWriteStream } from 'fs-extra'
 import * as request from 'request'
-import { ACTIVITY_PUB } from '../initializers'
+import { ACTIVITY_PUB, CONFIG } from '../initializers'
 import { processImage } from './image-utils'
-import { extname } from 'path'
+import { join } from 'path'
 
 function doRequest <T> (
   requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean }
@@ -29,10 +29,11 @@ function doRequestAndSaveToFile (requestOptions: request.CoreOptions & request.U
   })
 }
 
-async function downloadImage (url: string, destPath: string, size: { width: number, height: number }) {
+async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) {
-  const tmpPath = destPath + '.tmp' + extname(destPath)
+  const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
   await doRequestAndSaveToFile({ method: 'GET', uri: url }, tmpPath)
 
+  const destPath = join(destDir, destName)
   await processImage({ path: tmpPath }, destPath, size)
 }
 

server/helpers/utils.ts

@@ -46,11 +46,11 @@ const getServerActor = memoizee(async function () {
   return actor
 })
 
-function generateVideoTmpPath (target: string | ParseTorrent) {
+function generateVideoImportTmpPath (target: string | ParseTorrent) {
   const id = typeof target === 'string' ? target : target.infoHash
 
   const hash = sha256(id)
-  return join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4')
+  return join(CONFIG.STORAGE.TMP_DIR, hash + '-import.mp4')
 }
 
 function getSecureTorrentName (originalName: string) {
@@ -103,6 +103,6 @@ export {
   getSecureTorrentName,
   getServerActor,
   getServerCommit,
-  generateVideoTmpPath,
+  generateVideoImportTmpPath,
   getUUIDFromFilename
 }

server/helpers/webtorrent.ts

@@ -1,5 +1,5 @@
 import { logger } from './logger'
-import { generateVideoTmpPath } from './utils'
+import { generateVideoImportTmpPath } from './utils'
 import * as WebTorrent from 'webtorrent'
 import { createWriteStream, ensureDir, remove } from 'fs-extra'
 import { CONFIG } from '../initializers'
@@ -9,10 +9,10 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName
   const id = target.magnetUri || target.torrentName
   let timer
 
-  const path = generateVideoTmpPath(id)
+  const path = generateVideoImportTmpPath(id)
   logger.info('Importing torrent video %s', id)
 
-  const directoryPath = join(CONFIG.STORAGE.VIDEOS_DIR, 'import')
+  const directoryPath = join(CONFIG.STORAGE.TMP_DIR, 'webtorrent')
   await ensureDir(directoryPath)
 
   return new Promise<string>((res, rej) => {

server/helpers/youtube-dl.ts

@@ -1,7 +1,7 @@
 import { truncate } from 'lodash'
 import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers'
 import { logger } from './logger'
-import { generateVideoTmpPath } from './utils'
+import { generateVideoImportTmpPath } from './utils'
 import { join } from 'path'
 import { root } from './core-utils'
 import { ensureDir, writeFile, remove } from 'fs-extra'
@@ -40,7 +40,7 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo>
 }
 
 function downloadYoutubeDLVideo (url: string, timeout: number) {
-  const path = generateVideoTmpPath(url)
+  const path = generateVideoImportTmpPath(url)
   let timer
 
   logger.info('Importing youtubeDL video %s', url)

server/initializers/checker-before-init.ts

@@ -12,6 +12,7 @@ function checkMissedConfig () {
     'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max',
     'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address',
     'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache',
+    'storage.redundancy', 'storage.tmp',
     'log.level',
     'user.video_quota', 'user.video_quota_daily',
     'cache.previews.size', 'admin.email',

server/initializers/constants.ts

@@ -185,9 +185,11 @@ const CONFIG = {
     FROM_ADDRESS: config.get<string>('smtp.from_address')
   },
   STORAGE: {
+    TMP_DIR: buildPath(config.get<string>('storage.tmp')),
     AVATARS_DIR: buildPath(config.get<string>('storage.avatars')),
     LOG_DIR: buildPath(config.get<string>('storage.logs')),
     VIDEOS_DIR: buildPath(config.get<string>('storage.videos')),
+    REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')),
     THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')),
     PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')),
     CAPTIONS_DIR: buildPath(config.get<string>('storage.captions')),

server/lib/activitypub/actor.ts

@@ -178,9 +178,7 @@ async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
     const extension = IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType]
 
     const avatarName = uuidv4() + extension
-    const destPath = join(CONFIG.STORAGE.AVATARS_DIR, avatarName)
+    await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE)
-
-    await downloadImage(actorJSON.icon.url, destPath, AVATARS_SIZE)
 
     return avatarName
   }

server/lib/activitypub/videos.ts

@@ -95,9 +95,8 @@ function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Fu
 
 function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) {
   const thumbnailName = video.getThumbnailName()
-  const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName)
 
-  return downloadImage(icon.url, thumbnailPath, THUMBNAILS_SIZE)
+  return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE)
 }
 
 function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {

server/lib/job-queue/handlers/video-import.ts

@@ -7,7 +7,7 @@ import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } fro
 import { extname, join } from 'path'
 import { VideoFileModel } from '../../../models/video/video-file'
 import { CONFIG, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE, VIDEO_IMPORT_TIMEOUT } from '../../../initializers'
-import { doRequestAndSaveToFile, downloadImage } from '../../../helpers/requests'
+import { downloadImage } from '../../../helpers/requests'
 import { VideoState } from '../../../../shared'
 import { JobQueue } from '../index'
 import { federateVideoIfNeeded } from '../../activitypub'
@@ -109,6 +109,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
   let tempVideoPath: string
   let videoDestFile: string
   let videoFile: VideoFileModel
+
   try {
     // Download video from youtubeDL
     tempVideoPath = await downloader()
@@ -144,8 +145,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
     // Process thumbnail
     if (options.downloadThumbnail) {
       if (options.thumbnailUrl) {
-        const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName())
+        await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName(), THUMBNAILS_SIZE)
-        await downloadImage(options.thumbnailUrl, destThumbnailPath, THUMBNAILS_SIZE)
       } else {
         await videoImport.Video.createThumbnail(videoFile)
       }
@@ -156,8 +156,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
     // Process preview
     if (options.downloadPreview) {
       if (options.thumbnailUrl) {
-        const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName())
+        await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName(), PREVIEWS_SIZE)
-        await downloadImage(options.thumbnailUrl, destPreviewPath, PREVIEWS_SIZE)
       } else {
         await videoImport.Video.createPreview(videoFile)
       }

server/lib/job-queue/handlers/video-views.ts

@@ -23,9 +23,7 @@ async function processVideosViews () {
   for (const videoId of videoIds) {
     try {
       const views = await Redis.Instance.getVideoViews(videoId, hour)
-      if (isNaN(views)) {
+      if (views) {
-        logger.error('Cannot process videos views of video %d in hour %d: views number is NaN (%s).', videoId, hour, views)
-      } else {
         logger.debug('Adding %d views to video %d in hour %d.', views, videoId, hour)
 
         try {

server/lib/redis.ts

@@ -121,7 +121,14 @@ class Redis {
     const key = this.generateVideoViewKey(videoId, hour)
 
     const valueString = await this.getValue(key)
-    return parseInt(valueString, 10)
+    const valueInt = parseInt(valueString, 10)
+
+    if (isNaN(valueInt)) {
+      logger.error('Cannot get videos views of video %d in hour %d: views number is NaN (%s).', videoId, hour, valueString)
+      return undefined
+    }
+
+    return valueInt
   }
 
   async getVideosIdViewed (hour: number) {

support/docker/production/config/production.yaml

@@ -32,8 +32,10 @@ redis:
 
 # From the project root directory
 storage:
+  tmp: '../data/tmp/'
   avatars: '../data/avatars/'
   videos: '../data/videos/'
+  redundancy: '../data/redundancy/'
   logs: '../data/logs/'
   previews: '../data/previews/'
   thumbnails: '../data/thumbnails/'