From b9fffa297f49a84df8ffd0d7b842599bc88a8e3e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 4 Dec 2018 17:08:55 +0100 Subject: [PATCH] Create redundancy endpoint --- server/controllers/static.ts | 6 ++--- server/initializers/constants.ts | 1 + .../schedulers/videos-redundancy-scheduler.ts | 4 +-- server/models/redundancy/video-redundancy.ts | 4 +-- server/models/video/video.ts | 10 +++++-- server/tests/api/redundancy/redundancy.ts | 27 ++++++++++--------- support/nginx/peertube | 9 +++++-- 7 files changed, 38 insertions(+), 23 deletions(-) diff --git a/server/controllers/static.ts b/server/controllers/static.ts index f16a7d72b..55e7392a1 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts @@ -37,12 +37,12 @@ staticRouter.use( staticRouter.use( STATIC_PATHS.WEBSEED, cors(), - express.static(CONFIG.STORAGE.VIDEOS_DIR) + express.static(CONFIG.STORAGE.VIDEOS_DIR, { fallthrough: false }) // 404 because we don't have this video ) staticRouter.use( - STATIC_PATHS.WEBSEED, + STATIC_PATHS.REDUNDANCY, cors(), - express.static(CONFIG.STORAGE.REDUNDANCY_DIR, { fallthrough: false }) // 404, because we don't have this video + express.static(CONFIG.STORAGE.REDUNDANCY_DIR, { fallthrough: false }) // 404 because we don't have this video ) staticRouter.use( diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 876aa1cf5..7195ae6c5 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -571,6 +571,7 @@ const STATIC_PATHS = { THUMBNAILS: '/static/thumbnails/', TORRENTS: '/static/torrents/', WEBSEED: '/static/webseed/', + REDUNDANCY: '/static/redundancy/', AVATARS: '/static/avatars/', VIDEO_CAPTIONS: '/static/video-captions/' } diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 8b7f33539..2a99a665d 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -145,13 +145,13 @@ export class VideosRedundancyScheduler extends AbstractScheduler { const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) - const destPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file)) + const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) await rename(tmpPath, destPath) const createdModel = await VideoRedundancyModel.create({ expiresOn: this.buildNewExpiration(redundancy.minLifetime), url: getVideoCacheFileActivityPubUrl(file), - fileUrl: video.getVideoFileUrl(file, CONFIG.WEBSERVER.URL), + fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL), strategy: redundancy.strategy, videoFileId: file.id, actorId: serverActor.id diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index 9de4356b4..dd37dad22 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -15,7 +15,7 @@ import { import { ActorModel } from '../activitypub/actor' import { getVideoSort, throwIfNotValid } from '../utils' import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers' +import { CONFIG, CONSTRAINTS_FIELDS, STATIC_PATHS, VIDEO_EXT_MIMETYPE } from '../../initializers' import { VideoFileModel } from '../video/video-file' import { getServerActor } from '../../helpers/utils' import { VideoModel } from '../video/video' @@ -124,7 +124,7 @@ export class VideoRedundancyModel extends Model { const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}` logger.info('Removing duplicated video file %s.', logIdentifier) - videoFile.Video.removeFile(videoFile) + videoFile.Video.removeFile(videoFile, true) .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err })) return undefined diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 0f18d9f0c..e8cb5aa88 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -1538,8 +1538,10 @@ export class VideoModel extends Model { .catch(err => logger.warn('Cannot delete preview %s.', previewPath, { err })) } - removeFile (videoFile: VideoFileModel) { - const filePath = join(CONFIG.STORAGE.VIDEOS_DIR, this.getVideoFilename(videoFile)) + removeFile (videoFile: VideoFileModel, isRedundancy = false) { + const baseDir = isRedundancy ? CONFIG.STORAGE.REDUNDANCY_DIR : CONFIG.STORAGE.VIDEOS_DIR + + const filePath = join(baseDir, this.getVideoFilename(videoFile)) return remove(filePath) .catch(err => logger.warn('Cannot delete file %s.', filePath, { err })) } @@ -1617,6 +1619,10 @@ export class VideoModel extends Model { return baseUrlHttp + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) } + getVideoRedundancyUrl (videoFile: VideoFileModel, baseUrlHttp: string) { + return baseUrlHttp + STATIC_PATHS.REDUNDANCY + this.getVideoFilename(videoFile) + } + getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile) } diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index a8a2f305f..5b29a503a 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts @@ -136,7 +136,7 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st if (!videoUUID) videoUUID = video1Server2UUID const webseeds = [ - 'http://localhost:9001/static/webseed/' + videoUUID, + 'http://localhost:9001/static/redundancy/' + videoUUID, 'http://localhost:9002/static/webseed/' + videoUUID ] @@ -148,20 +148,23 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st for (const file of video.files) { checkMagnetWebseeds(file, webseeds, server) - // Only servers 1 and 2 have the video - if (server.serverNumber !== 3) { - await makeGetRequest({ - url: server.url, - statusCodeExpected: 200, - path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`, - contentType: null - }) - } + await makeGetRequest({ + url: servers[0].url, + statusCodeExpected: 200, + path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`, + contentType: null + }) + await makeGetRequest({ + url: servers[1].url, + statusCodeExpected: 200, + path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`, + contentType: null + }) } } - for (const directory of [ 'test1', 'test2' ]) { - const files = await readdir(join(root(), directory, 'videos')) + for (const directory of [ 'test1/redundancy', 'test2/videos' ]) { + const files = await readdir(join(root(), directory)) expect(files).to.have.length.at.least(4) for (const resolution of [ 240, 360, 480, 720 ]) { diff --git a/support/nginx/peertube b/support/nginx/peertube index b00031133..e0b006088 100644 --- a/support/nginx/peertube +++ b/support/nginx/peertube @@ -105,7 +105,7 @@ server { } # Bypass PeerTube for performance reasons. Could be removed - location /static/webseed { + location ~ ^/static/(webseed|redundancy)/ { # Clients usually have 4 simultaneous webseed connections, so the real limit is 3MB/s per client limit_rate 800k; @@ -128,7 +128,12 @@ server { access_log off; } - alias /var/www/peertube/storage/videos; + root /var/www/peertube/storage; + + rewrite ^/static/webseed/(.*)$ /videos/$1 break; + rewrite ^/static/redundancy/(.*)$ /redundancy/$1 break; + + try_files $uri /; } # Websocket tracker