Option to disable static files auth check/s3 proxy

config/default.yaml

@@ -140,6 +140,10 @@ storage:
   # If not, peertube will fallback to the default file
   client_overrides: 'storage/client-overrides/'
 
+static_files:
+  # Require and check user authentication when accessing private files (internal/private video files)
+  private_files_require_auth: true
+
 object_storage:
   enabled: false
 
@@ -151,9 +155,17 @@ object_storage:
   upload_acl:
     # Set this ACL on each uploaded object of public/unlisted videos
     public: 'public-read'
-      # Set this ACL on each uploaded object of private/internal videos
+    # Set this ACL on each uploaded object of private/internal videos
+    # PeerTube can proxify requests to private objects so your users can access them
     private: 'private'
 
+  proxy:
+    # If private files (private/internal video files) have a private ACL, users can't access directly the ressource
+    # PeerTube can proxify requests between your object storage service and your users
+    # If you disable PeerTube proxy, ensure you use your own proxy that is able to access the private files
+    # Or you can also set a public ACL for private files in object storage if you don't want to use a proxy
+    proxify_private_files: true
+
   credentials:
     # You can also use AWS_ACCESS_KEY_ID env variable
     access_key_id: ''

server/controllers/object-storage-proxy.ts

@@ -1,11 +1,13 @@
 import cors from 'cors'
 import express from 'express'
+import { logger } from '@server/helpers/logger'
 import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants'
 import { getHLSFileReadStream, getWebTorrentFileReadStream } from '@server/lib/object-storage'
 import {
   asyncMiddleware,
   ensureCanAccessPrivateVideoHLSFiles,
   ensureCanAccessVideoPrivateWebTorrentFiles,
+  ensurePrivateObjectStorageProxyIsEnabled,
   optionalAuthenticate
 } from '@server/middlewares'
 import { HttpStatusCode } from '@shared/models'
@@ -15,12 +17,14 @@ const objectStorageProxyRouter = express.Router()
 objectStorageProxyRouter.use(cors())
 
 objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + ':filename',
+  ensurePrivateObjectStorageProxyIsEnabled,
   optionalAuthenticate,
   asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles),
   asyncMiddleware(proxifyWebTorrent)
 )
 
 objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:filename',
+  ensurePrivateObjectStorageProxyIsEnabled,
   optionalAuthenticate,
   asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles),
   asyncMiddleware(proxifyHLS)
@@ -35,6 +39,8 @@ export {
 async function proxifyWebTorrent (req: express.Request, res: express.Response) {
   const filename = req.params.filename
 
+  logger.debug('Proxifying WebTorrent file %s from object storage.', filename)
+
   try {
     const stream = await getWebTorrentFileReadStream({
       filename,
@@ -52,6 +58,8 @@ async function proxifyHLS (req: express.Request, res: express.Response) {
   const video = res.locals.onlyVideo
   const filename = req.params.filename
 
+  logger.debug('Proxifying HLS file %s from object storage.', filename)
+
   try {
     const stream = await getHLSFileReadStream({
       playlist: playlist.withVideo(video),

server/controllers/static.ts

@@ -15,11 +15,17 @@ const staticRouter = express.Router()
 // Cors is very important to let other servers access torrent and video files
 staticRouter.use(cors())
 
+// ---------------------------------------------------------------------------
 // WebTorrent/Classic videos
+// ---------------------------------------------------------------------------
+
+const privateWebTorrentStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AUTH === true
+  ? [ optionalAuthenticate, asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles) ]
+  : []
+
 staticRouter.use(
   STATIC_PATHS.PRIVATE_WEBSEED,
-  optionalAuthenticate,
+  ...privateWebTorrentStaticMiddlewares,
-  asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles),
   express.static(DIRECTORIES.VIDEOS.PRIVATE, { fallthrough: false }),
   handleStaticError
 )
@@ -35,11 +41,17 @@ staticRouter.use(
   handleStaticError
 )
 
+// ---------------------------------------------------------------------------
 // HLS
+// ---------------------------------------------------------------------------
+
+const privateHLSStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AUTH === true
+  ? [ optionalAuthenticate, asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles) ]
+  : []
+
 staticRouter.use(
   STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS,
-  optionalAuthenticate,
+  ...privateHLSStaticMiddlewares,
-  asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles),
   express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, { fallthrough: false }),
   handleStaticError
 )

server/initializers/config.ts

@@ -113,6 +113,9 @@ const CONFIG = {
     CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')),
     WELL_KNOWN_DIR: buildPath(config.get<string>('storage.well_known'))
   },
+  STATIC_FILES: {
+    PRIVATE_FILES_REQUIRE_AUTH: config.get<boolean>('static_files.private_files_require_auth')
+  },
   OBJECT_STORAGE: {
     ENABLED: config.get<boolean>('object_storage.enabled'),
     MAX_UPLOAD_PART: bytes.parse(config.get<string>('object_storage.max_upload_part')),
@@ -126,6 +129,9 @@ const CONFIG = {
       ACCESS_KEY_ID: config.get<string>('object_storage.credentials.access_key_id'),
       SECRET_ACCESS_KEY: config.get<string>('object_storage.credentials.secret_access_key')
     },
+    PROXY: {
+      PROXIFY_PRIVATE_FILES: config.get<boolean>('object_storage.proxy.proxify_private_files')
+    },
     VIDEOS: {
       BUCKET_NAME: config.get<string>('object_storage.videos.bucket_name'),
       PREFIX: config.get<string>('object_storage.videos.prefix'),

server/middlewares/validators/index.ts

@@ -11,6 +11,7 @@ export * from './follows'
 export * from './jobs'
 export * from './logs'
 export * from './metrics'
+export * from './object-storage-proxy'
 export * from './oembed'
 export * from './pagination'
 export * from './plugins'

server/middlewares/validators/object-storage-proxy.ts

@@ -0,0 +1,20 @@
+import express from 'express'
+import { CONFIG } from '@server/initializers/config'
+import { HttpStatusCode } from '@shared/models'
+
+const ensurePrivateObjectStorageProxyIsEnabled = [
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES !== true) {
+      return res.fail({
+        message: 'Private object storage proxy is not enabled',
+        status: HttpStatusCode.BAD_REQUEST_400
+      })
+    }
+
+    return next()
+  }
+]
+
+export {
+  ensurePrivateObjectStorageProxyIsEnabled
+}

server/models/video/video-file.ts

@@ -54,6 +54,7 @@ import { doesExist } from '../shared'
 import { parseAggregateResult, throwIfNotValid } from '../utils'
 import { VideoModel } from './video'
 import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
+import { CONFIG } from '@server/initializers/config'
 
 export enum ScopeNames {
   WITH_VIDEO = 'WITH_VIDEO',
@@ -511,7 +512,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
   // ---------------------------------------------------------------------------
 
   getObjectStorageUrl (video: MVideo) {
-    if (video.hasPrivateStaticPath()) {
+    if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) {
       return this.getPrivateObjectStorageUrl(video)
     }
 

server/models/video/video-streaming-playlist.ts

@@ -15,6 +15,7 @@ import {
   Table,
   UpdatedAt
 } from 'sequelize-typescript'
+import { CONFIG } from '@server/initializers/config'
 import { getHLSPrivateFileUrl, getHLSPublicFileUrl } from '@server/lib/object-storage'
 import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths'
 import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
@@ -260,7 +261,7 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
   }
 
   private getMasterPlaylistObjectStorageUrl (video: MVideo) {
-    if (video.hasPrivateStaticPath()) {
+    if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) {
       return getHLSPrivateFileUrl(video, this.playlistFilename)
     }
 
@@ -282,7 +283,7 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
   }
 
   private getSha256SegmentsObjectStorageUrl (video: MVideo) {
-    if (video.hasPrivateStaticPath()) {
+    if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) {
       return getHLSPrivateFileUrl(video, this.segmentsSha256Filename)
     }
 

server/tests/api/object-storage/video-static-file-privacy.ts

@@ -19,6 +19,12 @@ import {
   waitJobs
 } from '@shared/server-commands'
 
+function extractFilenameFromUrl (url: string) {
+  const parts = basename(url).split(':')
+
+  return parts[parts.length - 1]
+}
+
 describe('Object storage for video static file privacy', function () {
   // We need real world object storage to check ACL
   if (areScalewayObjectStorageTestsDisabled()) return
@@ -26,10 +32,67 @@ describe('Object storage for video static file privacy', function () {
   let server: PeerTubeServer
   let userToken: string
 
+  // ---------------------------------------------------------------------------
+
+  async function checkPrivateVODFiles (uuid: string) {
+    const video = await server.videos.getWithToken({ id: uuid })
+
+    for (const file of video.files) {
+      expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/webseed/private/')
+
+      await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
+    }
+
+    for (const file of getAllFiles(video)) {
+      const internalFileUrl = await server.sql.getInternalFileUrl(file.id)
+      expectStartWith(internalFileUrl, ObjectStorageCommand.getScalewayBaseUrl())
+      await makeRawRequest({ url: internalFileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
+    }
+
+    const hls = getHLS(video)
+
+    if (hls) {
+      for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
+        expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/')
+      }
+
+      await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
+      await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
+
+      for (const file of hls.files) {
+        expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/streaming-playlists/hls/private/')
+
+        await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
+      }
+    }
+  }
+
+  async function checkPublicVODFiles (uuid: string) {
+    const video = await server.videos.getWithToken({ id: uuid })
+
+    for (const file of getAllFiles(video)) {
+      expectStartWith(file.fileUrl, ObjectStorageCommand.getScalewayBaseUrl())
+
+      await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
+    }
+
+    const hls = getHLS(video)
+
+    if (hls) {
+      expectStartWith(hls.playlistUrl, ObjectStorageCommand.getScalewayBaseUrl())
+      expectStartWith(hls.segmentsSha256Url, ObjectStorageCommand.getScalewayBaseUrl())
+
+      await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
+      await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
+    }
+  }
+
+  // ---------------------------------------------------------------------------
+
   before(async function () {
     this.timeout(120000)
 
-    server = await createSingleServer(1, ObjectStorageCommand.getDefaultScalewayConfig(1))
+    server = await createSingleServer(1, ObjectStorageCommand.getDefaultScalewayConfig({ serverNumber: 1 }))
     await setAccessTokensToServers([ server ])
     await setDefaultVideoChannel([ server ])
 
@@ -43,58 +106,7 @@ describe('Object storage for video static file privacy', function () {
     let publicVideoUUID: string
     let userPrivateVideoUUID: string
 
-    async function checkPrivateFiles (uuid: string) {
+    // ---------------------------------------------------------------------------
-      const video = await server.videos.getWithToken({ id: uuid })
-
-      for (const file of video.files) {
-        expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/webseed/private/')
-
-        await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
-      }
-
-      for (const file of getAllFiles(video)) {
-        const internalFileUrl = await server.sql.getInternalFileUrl(file.id)
-        expectStartWith(internalFileUrl, ObjectStorageCommand.getScalewayBaseUrl())
-        await makeRawRequest({ url: internalFileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
-      }
-
-      const hls = getHLS(video)
-
-      if (hls) {
-        for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
-          expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/')
-        }
-
-        await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
-        await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
-
-        for (const file of hls.files) {
-          expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/streaming-playlists/hls/private/')
-
-          await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
-        }
-      }
-    }
-
-    async function checkPublicFiles (uuid: string) {
-      const video = await server.videos.getWithToken({ id: uuid })
-
-      for (const file of getAllFiles(video)) {
-        expectStartWith(file.fileUrl, ObjectStorageCommand.getScalewayBaseUrl())
-
-        await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
-      }
-
-      const hls = getHLS(video)
-
-      if (hls) {
-        expectStartWith(hls.playlistUrl, ObjectStorageCommand.getScalewayBaseUrl())
-        expectStartWith(hls.segmentsSha256Url, ObjectStorageCommand.getScalewayBaseUrl())
-
-        await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
-        await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
-      }
-    }
 
     async function getSampleFileUrls (videoId: string) {
       const video = await server.videos.getWithToken({ id: videoId })
@@ -105,6 +117,8 @@ describe('Object storage for video static file privacy', function () {
       }
     }
 
+    // ---------------------------------------------------------------------------
+
     it('Should upload a private video and have appropriate object storage ACL', async function () {
       this.timeout(60000)
 
@@ -120,7 +134,7 @@ describe('Object storage for video static file privacy', function () {
 
       await waitJobs([ server ])
 
-      await checkPrivateFiles(privateVideoUUID)
+      await checkPrivateVODFiles(privateVideoUUID)
     })
 
     it('Should upload a public video and have appropriate object storage ACL', async function () {
@@ -131,7 +145,7 @@ describe('Object storage for video static file privacy', function () {
 
       publicVideoUUID = uuid
 
-      await checkPublicFiles(publicVideoUUID)
+      await checkPublicVODFiles(publicVideoUUID)
     })
 
     it('Should not get files without appropriate OAuth token', async function () {
@@ -182,7 +196,7 @@ describe('Object storage for video static file privacy', function () {
 
       await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.INTERNAL } })
 
-      await checkPrivateFiles(publicVideoUUID)
+      await checkPrivateVODFiles(publicVideoUUID)
     })
 
     it('Should update private video to public', async function () {
@@ -190,7 +204,7 @@ describe('Object storage for video static file privacy', function () {
 
       await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.PUBLIC } })
 
-      await checkPublicFiles(publicVideoUUID)
+      await checkPublicVODFiles(publicVideoUUID)
     })
   })
 
@@ -203,6 +217,8 @@ describe('Object storage for video static file privacy', function () {
 
     let unrelatedFileToken: string
 
+    // ---------------------------------------------------------------------------
+
     async function checkLiveFiles (live: LiveVideo, liveId: string) {
       const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
       await server.live.waitUntilPublished({ videoId: liveId })
@@ -260,6 +276,8 @@ describe('Object storage for video static file privacy', function () {
       }
     }
 
+    // ---------------------------------------------------------------------------
+
     before(async function () {
       await server.config.enableMinimumTranscoding()
 
@@ -320,6 +338,52 @@ describe('Object storage for video static file privacy', function () {
     })
   })
 
+  describe('With private files proxy disabled and public ACL for private files', function () {
+    let videoUUID: string
+
+    before(async function () {
+      this.timeout(240000)
+
+      await server.kill()
+
+      const config = ObjectStorageCommand.getDefaultScalewayConfig({
+        serverNumber: server.internalServerNumber,
+        enablePrivateProxy: false,
+        privateACL: 'public-read'
+      })
+      await server.run(config)
+
+      const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
+      videoUUID = uuid
+
+      await waitJobs([ server ])
+    })
+
+    it('Should display object storage path for a private video and be able to access them', async function () {
+      this.timeout(60000)
+
+      await checkPublicVODFiles(videoUUID)
+    })
+
+    it('Should not be able to access object storage proxy', async function () {
+      const privateVideo = await server.videos.getWithToken({ id: videoUUID })
+      const webtorrentFilename = extractFilenameFromUrl(privateVideo.files[0].fileUrl)
+      const hlsFilename = extractFilenameFromUrl(getHLS(privateVideo).files[0].fileUrl)
+
+      await makeRawRequest({
+        url: server.url + '/object-storage-proxy/webseed/private/' + webtorrentFilename,
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.BAD_REQUEST_400
+      })
+
+      await makeRawRequest({
+        url: server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + videoUUID + '/' + hlsFilename,
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.BAD_REQUEST_400
+      })
+    })
+  })
+
   after(async function () {
     this.timeout(60000)
 

server/tests/api/videos/video-static-file-privacy.ts

@@ -383,6 +383,39 @@ describe('Test video static file privacy', function () {
     })
   })
 
+  describe('With static file right check disabled', function () {
+    let videoUUID: string
+
+    before(async function () {
+      this.timeout(240000)
+
+      await server.kill()
+
+      await server.run({
+        static_files: {
+          private_files_require_auth: false
+        }
+      })
+
+      const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.INTERNAL })
+      videoUUID = uuid
+
+      await waitJobs([ server ])
+    })
+
+    it('Should not check auth for private static files', async function () {
+      const video = await server.videos.getWithToken({ id: videoUUID })
+
+      for (const file of getAllFiles(video)) {
+        await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
+      }
+
+      const hls = video.streamingPlaylists[0]
+      await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
+      await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
+    })
+  })
+
   after(async function () {
     await cleanupTests([ server ])
   })

shared/server-commands/server/object-storage-command.ts

@@ -81,7 +81,13 @@ export class ObjectStorageCommand extends AbstractCommand {
 
   // ---------------------------------------------------------------------------
 
-  static getDefaultScalewayConfig (serverNumber: number) {
+  static getDefaultScalewayConfig (options: {
+    serverNumber: number
+    enablePrivateProxy?: boolean // default true
+    privateACL?: 'private' | 'public-read' // default 'private'
+  }) {
+    const { serverNumber, enablePrivateProxy = true, privateACL = 'private' } = options
+
     return {
       object_storage: {
         enabled: true,
@@ -90,6 +96,14 @@ export class ObjectStorageCommand extends AbstractCommand {
 
         credentials: this.getScalewayCredentialsConfig(),
 
+        upload_acl: {
+          private: privateACL
+        },
+
+        proxy: {
+          proxify_private_files: enablePrivateProxy
+        },
+
         streaming_playlists: {
           bucket_name: this.DEFAULT_SCALEWAY_BUCKET,
           prefix: `test:server-${serverNumber}-streaming-playlists:`