OpenCloud
/
PeerTube
mirror of https://github.com/Chocobozzz/PeerTube
1
0
Fork 0
Code Issues Releases Wiki Activity

Option to disable static files auth check/s3 proxy

pull/5386/head
Chocobozzz 2022-10-26 16:23:39 +02:00
parent 49e7e4d9ff
commit 5a122dddc5
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
11 changed files with 238 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
config/default.yaml
View File

@ -140,6 +140,10 @@ storage:
# If not, peertube will fallback to the default file
client_overrides: 'storage/client-overrides/'
static_files:
# Require and check user authentication when accessing private files (internal/private video files)
private_files_require_auth: true
object_storage:
enabled: false
@ -151,9 +155,17 @@ object_storage:
upload_acl:
# Set this ACL on each uploaded object of public/unlisted videos
public: 'public-read'
# Set this ACL on each uploaded object of private/internal videos
# Set this ACL on each uploaded object of private/internal videos
# PeerTube can proxify requests to private objects so your users can access them
private: 'private'
proxy:
# If private files (private/internal video files) have a private ACL, users can't access directly the ressource
# PeerTube can proxify requests between your object storage service and your users
# If you disable PeerTube proxy, ensure you use your own proxy that is able to access the private files
# Or you can also set a public ACL for private files in object storage if you don't want to use a proxy
proxify_private_files: true
credentials:
# You can also use AWS_ACCESS_KEY_ID env variable
access_key_id: ''

8
server/controllers/object-storage-proxy.ts
View File

@ -1,11 +1,13 @@
import cors from 'cors'
import express from 'express'
import { logger } from '@server/helpers/logger'
import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants'
import { getHLSFileReadStream, getWebTorrentFileReadStream } from '@server/lib/object-storage'
import {
asyncMiddleware,
ensureCanAccessPrivateVideoHLSFiles,
ensureCanAccessVideoPrivateWebTorrentFiles,
ensurePrivateObjectStorageProxyIsEnabled,
optionalAuthenticate
} from '@server/middlewares'
import { HttpStatusCode } from '@shared/models'
@ -15,12 +17,14 @@ const objectStorageProxyRouter = express.Router()
objectStorageProxyRouter.use(cors())
objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + ':filename',
ensurePrivateObjectStorageProxyIsEnabled,
optionalAuthenticate,
asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles),
asyncMiddleware(proxifyWebTorrent)
)
objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:filename',
ensurePrivateObjectStorageProxyIsEnabled,
optionalAuthenticate,
asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles),
asyncMiddleware(proxifyHLS)
@ -35,6 +39,8 @@ export {
async function proxifyWebTorrent (req: express.Request, res: express.Response) {
const filename = req.params.filename
logger.debug('Proxifying WebTorrent file %s from object storage.', filename)
try {
const stream = await getWebTorrentFileReadStream({
filename,
@ -52,6 +58,8 @@ async function proxifyHLS (req: express.Request, res: express.Response) {
const video = res.locals.onlyVideo
const filename = req.params.filename
logger.debug('Proxifying HLS file %s from object storage.', filename)
try {
const stream = await getHLSFileReadStream({
playlist: playlist.withVideo(video),

20
server/controllers/static.ts
View File

@ -15,11 +15,17 @@ const staticRouter = express.Router()
// Cors is very important to let other servers access torrent and video files
staticRouter.use(cors())
// ---------------------------------------------------------------------------
// WebTorrent/Classic videos
// ---------------------------------------------------------------------------
const privateWebTorrentStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AUTH === true
? [ optionalAuthenticate, asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles) ]
: []
staticRouter.use(
STATIC_PATHS.PRIVATE_WEBSEED,
optionalAuthenticate,
asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles),
...privateWebTorrentStaticMiddlewares,
express.static(DIRECTORIES.VIDEOS.PRIVATE, { fallthrough: false }),
handleStaticError
)
@ -35,11 +41,17 @@ staticRouter.use(
handleStaticError
)
// ---------------------------------------------------------------------------
// HLS
// ---------------------------------------------------------------------------
const privateHLSStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AUTH === true
? [ optionalAuthenticate, asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles) ]
: []
staticRouter.use(
STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS,
optionalAuthenticate,
asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles),
...privateHLSStaticMiddlewares,
express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, { fallthrough: false }),
handleStaticError
)

6
server/initializers/config.ts
View File

@ -113,6 +113,9 @@ const CONFIG = {
CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')),
WELL_KNOWN_DIR: buildPath(config.get<string>('storage.well_known'))
},
STATIC_FILES: {
PRIVATE_FILES_REQUIRE_AUTH: config.get<boolean>('static_files.private_files_require_auth')
},
OBJECT_STORAGE: {
ENABLED: config.get<boolean>('object_storage.enabled'),
MAX_UPLOAD_PART: bytes.parse(config.get<string>('object_storage.max_upload_part')),
@ -126,6 +129,9 @@ const CONFIG = {
ACCESS_KEY_ID: config.get<string>('object_storage.credentials.access_key_id'),
SECRET_ACCESS_KEY: config.get<string>('object_storage.credentials.secret_access_key')
},
PROXY: {
PROXIFY_PRIVATE_FILES: config.get<boolean>('object_storage.proxy.proxify_private_files')
},
VIDEOS: {
BUCKET_NAME: config.get<string>('object_storage.videos.bucket_name'),
PREFIX: config.get<string>('object_storage.videos.prefix'),

1
server/middlewares/validators/index.ts
View File

@ -11,6 +11,7 @@ export * from './follows'
export * from './jobs'
export * from './logs'
export * from './metrics'
export * from './object-storage-proxy'
export * from './oembed'
export * from './pagination'
export * from './plugins'

20
server/middlewares/validators/object-storage-proxy.ts Normal file
View File

@ -0,0 +1,20 @@
import express from 'express'
import { CONFIG } from '@server/initializers/config'
import { HttpStatusCode } from '@shared/models'
const ensurePrivateObjectStorageProxyIsEnabled = [
(req: express.Request, res: express.Response, next: express.NextFunction) => {
if (CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES !== true) {
return res.fail({
message: 'Private object storage proxy is not enabled',
status: HttpStatusCode.BAD_REQUEST_400
})
}
return next()
}
]
export {
ensurePrivateObjectStorageProxyIsEnabled
}

3
server/models/video/video-file.ts
View File

@ -54,6 +54,7 @@ import { doesExist } from '../shared'
import { parseAggregateResult, throwIfNotValid } from '../utils'
import { VideoModel } from './video'
import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
import { CONFIG } from '@server/initializers/config'
export enum ScopeNames {
WITH_VIDEO = 'WITH_VIDEO',
@ -511,7 +512,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
// ---------------------------------------------------------------------------
getObjectStorageUrl (video: MVideo) {
if (video.hasPrivateStaticPath()) {
if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) {
return this.getPrivateObjectStorageUrl(video)
}

5
server/models/video/video-streaming-playlist.ts
View File

@ -15,6 +15,7 @@ import {
Table,
UpdatedAt
} from 'sequelize-typescript'
import { CONFIG } from '@server/initializers/config'
import { getHLSPrivateFileUrl, getHLSPublicFileUrl } from '@server/lib/object-storage'
import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths'
import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
@ -260,7 +261,7 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
}
private getMasterPlaylistObjectStorageUrl (video: MVideo) {
if (video.hasPrivateStaticPath()) {
if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) {
return getHLSPrivateFileUrl(video, this.playlistFilename)
}
@ -282,7 +283,7 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
}
private getSha256SegmentsObjectStorageUrl (video: MVideo) {
if (video.hasPrivateStaticPath()) {
if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) {
return getHLSPrivateFileUrl(video, this.segmentsSha256Filename)
}

178
server/tests/api/object-storage/video-static-file-privacy.ts
View File

@ -19,6 +19,12 @@ import {
waitJobs
} from '@shared/server-commands'
function extractFilenameFromUrl (url: string) {
const parts = basename(url).split(':')
return parts[parts.length - 1]
}
describe('Object storage for video static file privacy', function () {
// We need real world object storage to check ACL
if (areScalewayObjectStorageTestsDisabled()) return
@ -26,10 +32,67 @@ describe('Object storage for video static file privacy', function () {
let server: PeerTubeServer
let userToken: string
// ---------------------------------------------------------------------------
async function checkPrivateVODFiles (uuid: string) {
const video = await server.videos.getWithToken({ id: uuid })
for (const file of video.files) {
expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/webseed/private/')
await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
}
for (const file of getAllFiles(video)) {
const internalFileUrl = await server.sql.getInternalFileUrl(file.id)
expectStartWith(internalFileUrl, ObjectStorageCommand.getScalewayBaseUrl())
await makeRawRequest({ url: internalFileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
}
const hls = getHLS(video)
if (hls) {
for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/')
}
await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
for (const file of hls.files) {
expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/streaming-playlists/hls/private/')
await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
}
}
}
async function checkPublicVODFiles (uuid: string) {
const video = await server.videos.getWithToken({ id: uuid })
for (const file of getAllFiles(video)) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getScalewayBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
const hls = getHLS(video)
if (hls) {
expectStartWith(hls.playlistUrl, ObjectStorageCommand.getScalewayBaseUrl())
expectStartWith(hls.segmentsSha256Url, ObjectStorageCommand.getScalewayBaseUrl())
await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
}
}
// ---------------------------------------------------------------------------
before(async function () {
this.timeout(120000)
server = await createSingleServer(1, ObjectStorageCommand.getDefaultScalewayConfig(1))
server = await createSingleServer(1, ObjectStorageCommand.getDefaultScalewayConfig({ serverNumber: 1 }))
await setAccessTokensToServers([ server ])
await setDefaultVideoChannel([ server ])
@ -43,58 +106,7 @@ describe('Object storage for video static file privacy', function () {
let publicVideoUUID: string
let userPrivateVideoUUID: string
async function checkPrivateFiles (uuid: string) {
const video = await server.videos.getWithToken({ id: uuid })
for (const file of video.files) {
expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/webseed/private/')
await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
}
for (const file of getAllFiles(video)) {
const internalFileUrl = await server.sql.getInternalFileUrl(file.id)
expectStartWith(internalFileUrl, ObjectStorageCommand.getScalewayBaseUrl())
await makeRawRequest({ url: internalFileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
}
const hls = getHLS(video)
if (hls) {
for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/')
}
await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
for (const file of hls.files) {
expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/streaming-playlists/hls/private/')
await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
}
}
}
async function checkPublicFiles (uuid: string) {
const video = await server.videos.getWithToken({ id: uuid })
for (const file of getAllFiles(video)) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getScalewayBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
const hls = getHLS(video)
if (hls) {
expectStartWith(hls.playlistUrl, ObjectStorageCommand.getScalewayBaseUrl())
expectStartWith(hls.segmentsSha256Url, ObjectStorageCommand.getScalewayBaseUrl())
await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
}
}
// ---------------------------------------------------------------------------
async function getSampleFileUrls (videoId: string) {
const video = await server.videos.getWithToken({ id: videoId })
@ -105,6 +117,8 @@ describe('Object storage for video static file privacy', function () {
}
}
// ---------------------------------------------------------------------------
it('Should upload a private video and have appropriate object storage ACL', async function () {
this.timeout(60000)
@ -120,7 +134,7 @@ describe('Object storage for video static file privacy', function () {
await waitJobs([ server ])
await checkPrivateFiles(privateVideoUUID)
await checkPrivateVODFiles(privateVideoUUID)
})
it('Should upload a public video and have appropriate object storage ACL', async function () {
@ -131,7 +145,7 @@ describe('Object storage for video static file privacy', function () {
publicVideoUUID = uuid
await checkPublicFiles(publicVideoUUID)
await checkPublicVODFiles(publicVideoUUID)
})
it('Should not get files without appropriate OAuth token', async function () {
@ -182,7 +196,7 @@ describe('Object storage for video static file privacy', function () {
await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.INTERNAL } })
await checkPrivateFiles(publicVideoUUID)
await checkPrivateVODFiles(publicVideoUUID)
})
it('Should update private video to public', async function () {
@ -190,7 +204,7 @@ describe('Object storage for video static file privacy', function () {
await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.PUBLIC } })
await checkPublicFiles(publicVideoUUID)
await checkPublicVODFiles(publicVideoUUID)
})
})
@ -203,6 +217,8 @@ describe('Object storage for video static file privacy', function () {
let unrelatedFileToken: string
// ---------------------------------------------------------------------------
async function checkLiveFiles (live: LiveVideo, liveId: string) {
const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
await server.live.waitUntilPublished({ videoId: liveId })
@ -260,6 +276,8 @@ describe('Object storage for video static file privacy', function () {
}
}
// ---------------------------------------------------------------------------
before(async function () {
await server.config.enableMinimumTranscoding()
@ -320,6 +338,52 @@ describe('Object storage for video static file privacy', function () {
})
})
describe('With private files proxy disabled and public ACL for private files', function () {
let videoUUID: string
before(async function () {
this.timeout(240000)
await server.kill()
const config = ObjectStorageCommand.getDefaultScalewayConfig({
serverNumber: server.internalServerNumber,
enablePrivateProxy: false,
privateACL: 'public-read'
})
await server.run(config)
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
videoUUID = uuid
await waitJobs([ server ])
})
it('Should display object storage path for a private video and be able to access them', async function () {
this.timeout(60000)
await checkPublicVODFiles(videoUUID)
})
it('Should not be able to access object storage proxy', async function () {
const privateVideo = await server.videos.getWithToken({ id: videoUUID })
const webtorrentFilename = extractFilenameFromUrl(privateVideo.files[0].fileUrl)
const hlsFilename = extractFilenameFromUrl(getHLS(privateVideo).files[0].fileUrl)
await makeRawRequest({
url: server.url + '/object-storage-proxy/webseed/private/' + webtorrentFilename,
token: server.accessToken,
expectedStatus: HttpStatusCode.BAD_REQUEST_400
})
await makeRawRequest({
url: server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + videoUUID + '/' + hlsFilename,
token: server.accessToken,
expectedStatus: HttpStatusCode.BAD_REQUEST_400
})
})
})
after(async function () {
this.timeout(60000)

33
server/tests/api/videos/video-static-file-privacy.ts
View File

@ -383,6 +383,39 @@ describe('Test video static file privacy', function () {
})
})
describe('With static file right check disabled', function () {
let videoUUID: string
before(async function () {
this.timeout(240000)
await server.kill()
await server.run({
static_files: {
private_files_require_auth: false
}
})
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.INTERNAL })
videoUUID = uuid
await waitJobs([ server ])
})
it('Should not check auth for private static files', async function () {
const video = await server.videos.getWithToken({ id: videoUUID })
for (const file of getAllFiles(video)) {
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
const hls = video.streamingPlaylists[0]
await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
})
})
after(async function () {
await cleanupTests([ server ])
})

16
shared/server-commands/server/object-storage-command.ts
View File

@ -81,7 +81,13 @@ export class ObjectStorageCommand extends AbstractCommand {
// ---------------------------------------------------------------------------
static getDefaultScalewayConfig (serverNumber: number) {
static getDefaultScalewayConfig (options: {
serverNumber: number
enablePrivateProxy?: boolean // default true
privateACL?: 'private' | 'public-read' // default 'private'
}) {
const { serverNumber, enablePrivateProxy = true, privateACL = 'private' } = options
return {
object_storage: {
enabled: true,
@ -90,6 +96,14 @@ export class ObjectStorageCommand extends AbstractCommand {
credentials: this.getScalewayCredentialsConfig(),
upload_acl: {
private: privateACL
},
proxy: {
proxify_private_files: enablePrivateProxy
},
streaming_playlists: {
bucket_name: this.DEFAULT_SCALEWAY_BUCKET,
prefix: `test:server-${serverNumber}-streaming-playlists:`