PeerTube/server/controllers/api/videos/upload.ts

291 lines
10 KiB
TypeScript
Raw Normal View History

2021-05-12 14:51:17 +02:00
import * as express from 'express'
import { move } from 'fs-extra'
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 08:26:20 +02:00
import { basename } from 'path'
import { getLowercaseExtension } from '@server/helpers/core-utils'
2021-05-12 14:51:17 +02:00
import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload'
import { uuidToShort } from '@server/helpers/uuid'
2021-05-12 14:51:17 +02:00
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 08:26:20 +02:00
import { generateWebTorrentVideoFilename } from '@server/lib/paths'
import {
addMoveToObjectStorageJob,
addOptimizeOrMergeAudioJob,
buildLocalVideoFromReq,
buildVideoThumbnailsFromReq,
setVideoTags
} from '@server/lib/video'
import { VideoPathManager } from '@server/lib/video-path-manager'
import { buildNextVideoState } from '@server/lib/video-state'
import { openapiOperationDoc } from '@server/middlewares/doc'
2021-05-12 14:51:17 +02:00
import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
import { uploadx } from '@uploadx/core'
import { VideoCreate, VideoState } from '../../../../shared'
2021-07-16 10:42:24 +02:00
import { HttpStatusCode } from '../../../../shared/models'
2021-05-12 14:51:17 +02:00
import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
import { retryTransactionWrapper } from '../../../helpers/database-utils'
import { createReqFiles } from '../../../helpers/express-utils'
import { getMetadataFromFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils'
import { logger, loggerTagsFactory } from '../../../helpers/logger'
import { CONFIG } from '../../../initializers/config'
import { DEFAULT_AUDIO_RESOLUTION, MIMETYPES } from '../../../initializers/constants'
import { sequelizeTypescript } from '../../../initializers/database'
import { federateVideoIfNeeded } from '../../../lib/activitypub/videos'
import { Notifier } from '../../../lib/notifier'
import { Hooks } from '../../../lib/plugins/hooks'
import { generateVideoMiniature } from '../../../lib/thumbnail'
import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
import {
asyncMiddleware,
asyncRetryTransactionMiddleware,
authenticate,
videosAddLegacyValidator,
videosAddResumableInitValidator,
videosAddResumableValidator
} from '../../../middlewares'
import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update'
import { VideoModel } from '../../../models/video/video'
import { VideoFileModel } from '../../../models/video/video-file'
const lTags = loggerTagsFactory('api', 'video')
const auditLogger = auditLoggerFactory('videos')
const uploadRouter = express.Router()
const uploadxMiddleware = uploadx.upload({ directory: getResumableUploadPath() })
const reqVideoFileAdd = createReqFiles(
[ 'videofile', 'thumbnailfile', 'previewfile' ],
Object.assign({}, MIMETYPES.VIDEO.MIMETYPE_EXT, MIMETYPES.IMAGE.MIMETYPE_EXT),
{
videofile: CONFIG.STORAGE.TMP_DIR,
thumbnailfile: CONFIG.STORAGE.TMP_DIR,
previewfile: CONFIG.STORAGE.TMP_DIR
}
)
const reqVideoFileAddResumable = createReqFiles(
[ 'thumbnailfile', 'previewfile' ],
MIMETYPES.IMAGE.MIMETYPE_EXT,
{
thumbnailfile: getResumableUploadPath(),
previewfile: getResumableUploadPath()
}
)
uploadRouter.post('/upload',
openapiOperationDoc({ operationId: 'uploadLegacy' }),
2021-05-12 14:51:17 +02:00
authenticate,
reqVideoFileAdd,
asyncMiddleware(videosAddLegacyValidator),
asyncRetryTransactionMiddleware(addVideoLegacy)
)
uploadRouter.post('/upload-resumable',
openapiOperationDoc({ operationId: 'uploadResumableInit' }),
2021-05-12 14:51:17 +02:00
authenticate,
reqVideoFileAddResumable,
asyncMiddleware(videosAddResumableInitValidator),
uploadxMiddleware
)
uploadRouter.delete('/upload-resumable',
authenticate,
uploadxMiddleware
)
uploadRouter.put('/upload-resumable',
openapiOperationDoc({ operationId: 'uploadResumable' }),
2021-05-12 14:51:17 +02:00
authenticate,
uploadxMiddleware, // uploadx doesn't use call next() before the file upload completes
asyncMiddleware(videosAddResumableValidator),
asyncMiddleware(addVideoResumable)
)
// ---------------------------------------------------------------------------
export {
uploadRouter
}
// ---------------------------------------------------------------------------
export async function addVideoLegacy (req: express.Request, res: express.Response) {
// Uploading the video could be long
// Set timeout to 10 minutes, as Express's default is 2 minutes
req.setTimeout(1000 * 60 * 10, () => {
logger.error('Video upload has timed out.')
return res.fail({
status: HttpStatusCode.REQUEST_TIMEOUT_408,
message: 'Video upload has timed out.'
})
2021-05-12 14:51:17 +02:00
})
const videoPhysicalFile = req.files['videofile'][0]
const videoInfo: VideoCreate = req.body
const files = req.files
return addVideo({ res, videoPhysicalFile, videoInfo, files })
}
export async function addVideoResumable (_req: express.Request, res: express.Response) {
const videoPhysicalFile = res.locals.videoFileResumable
const videoInfo = videoPhysicalFile.metadata
const files = { previewfile: videoInfo.previewfile }
// Don't need the meta file anymore
await deleteResumableUploadMetaFile(videoPhysicalFile.path)
return addVideo({ res, videoPhysicalFile, videoInfo, files })
}
async function addVideo (options: {
res: express.Response
videoPhysicalFile: express.VideoUploadFile
videoInfo: VideoCreate
files: express.UploadFiles
}) {
const { res, videoPhysicalFile, videoInfo, files } = options
const videoChannel = res.locals.videoChannel
const user = res.locals.oauth.token.User
const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id)
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 08:26:20 +02:00
videoData.state = buildNextVideoState()
2021-05-12 14:51:17 +02:00
videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware
const video = new VideoModel(videoData) as MVideoFullLight
video.VideoChannel = videoChannel
video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 08:26:20 +02:00
const videoFile = await buildNewFile(videoPhysicalFile)
2021-05-12 14:51:17 +02:00
// Move physical file
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 08:26:20 +02:00
const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile)
2021-05-12 14:51:17 +02:00
await move(videoPhysicalFile.path, destination)
// This is important in case if there is another attempt in the retry process
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 08:26:20 +02:00
videoPhysicalFile.filename = basename(destination)
2021-05-12 14:51:17 +02:00
videoPhysicalFile.path = destination
const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
video,
files,
fallback: type => generateVideoMiniature({ video, videoFile, type })
})
const { videoCreated } = await sequelizeTypescript.transaction(async t => {
const sequelizeOptions = { transaction: t }
const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight
await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
await videoCreated.addAndSaveThumbnail(previewModel, t)
// Do not forget to add video channel information to the created video
videoCreated.VideoChannel = res.locals.videoChannel
videoFile.videoId = video.id
await videoFile.save(sequelizeOptions)
video.VideoFiles = [ videoFile ]
await setVideoTags({ video, tags: videoInfo.tags, transaction: t })
// Schedule an update in the future?
if (videoInfo.scheduleUpdate) {
await ScheduleVideoUpdateModel.create({
videoId: video.id,
updateAt: new Date(videoInfo.scheduleUpdate.updateAt),
privacy: videoInfo.scheduleUpdate.privacy || null
}, sequelizeOptions)
}
// Channel has a new content, set as updated
await videoCreated.VideoChannel.setAsUpdated(t)
await autoBlacklistVideoIfNeeded({
video,
user,
isRemote: false,
isNew: true,
transaction: t
})
auditLogger.create(getAuditIdFromRes(res), new VideoAuditView(videoCreated.toFormattedDetailsJSON()))
logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid, lTags(videoCreated.uuid))
return { videoCreated }
})
createTorrentFederate(video, videoFile)
2021-07-23 11:20:00 +02:00
.then(() => {
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 08:26:20 +02:00
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
return addMoveToObjectStorageJob(video)
}
2021-05-12 14:51:17 +02:00
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 08:26:20 +02:00
if (video.state === VideoState.TO_TRANSCODE) {
return addOptimizeOrMergeAudioJob(videoCreated, videoFile, user)
}
2021-07-23 11:20:00 +02:00
})
.catch(err => logger.error('Cannot add optimize/merge audio job for %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) }))
2021-05-12 14:51:17 +02:00
Hooks.runAction('action:api.video.uploaded', { video: videoCreated })
return res.json({
video: {
id: videoCreated.id,
shortUUID: uuidToShort(videoCreated.uuid),
2021-05-12 14:51:17 +02:00
uuid: videoCreated.uuid
}
})
}
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 08:26:20 +02:00
async function buildNewFile (videoPhysicalFile: express.VideoUploadFile) {
2021-05-12 14:51:17 +02:00
const videoFile = new VideoFileModel({
extname: getLowercaseExtension(videoPhysicalFile.filename),
2021-05-12 14:51:17 +02:00
size: videoPhysicalFile.size,
videoStreamingPlaylistId: null,
metadata: await getMetadataFromFile(videoPhysicalFile.path)
})
if (videoFile.isAudio()) {
videoFile.resolution = DEFAULT_AUDIO_RESOLUTION
} else {
videoFile.fps = await getVideoFileFPS(videoPhysicalFile.path)
2021-08-06 13:35:25 +02:00
videoFile.resolution = (await getVideoFileResolution(videoPhysicalFile.path)).resolution
2021-05-12 14:51:17 +02:00
}
2021-07-22 14:28:03 +02:00
videoFile.filename = generateWebTorrentVideoFilename(videoFile.resolution, videoFile.extname)
2021-05-12 14:51:17 +02:00
return videoFile
}
async function createTorrentAndSetInfoHashAsync (video: MVideo, fileArg: MVideoFile) {
await createTorrentAndSetInfoHash(video, fileArg)
// Refresh videoFile because the createTorrentAndSetInfoHash could be long
const refreshedFile = await VideoFileModel.loadWithVideo(fileArg.id)
// File does not exist anymore, remove the generated torrent
if (!refreshedFile) return fileArg.removeTorrent()
refreshedFile.infoHash = fileArg.infoHash
refreshedFile.torrentFilename = fileArg.torrentFilename
return refreshedFile.save()
}
2021-07-23 11:20:00 +02:00
function createTorrentFederate (video: MVideoFullLight, videoFile: MVideoFile) {
2021-05-12 14:51:17 +02:00
// Create the torrent file in async way because it could be long
2021-07-23 11:20:00 +02:00
return createTorrentAndSetInfoHashAsync(video, videoFile)
2021-05-12 14:51:17 +02:00
.catch(err => logger.error('Cannot create torrent file for video %s', video.url, { err, ...lTags(video.uuid) }))
.then(() => VideoModel.loadAndPopulateAccountAndServerAndTags(video.id))
.then(refreshedVideo => {
if (!refreshedVideo) return
// Only federate and notify after the torrent creation
Notifier.Instance.notifyOnNewVideoIfNeeded(refreshedVideo)
return retryTransactionWrapper(() => {
return sequelizeTypescript.transaction(t => federateVideoIfNeeded(refreshedVideo, true, t))
})
})
.catch(err => logger.error('Cannot federate or notify video creation %s', video.url, { err, ...lTags(video.uuid) }))
}