Refactor uploadx middlewares

pull/6266/head
Chocobozzz 2024-02-14 10:20:02 +01:00 committed by Chocobozzz
parent e286db3a39
commit f7e4f62870
14 changed files with 182 additions and 166 deletions

View File

@ -9,7 +9,7 @@ import './custom-pages.js'
import './debug.js'
import './follows.js'
import './user-export.js'
import './user-import.js.js'
import './user-import.js'
import './jobs.js'
import './live.js'
import './logs.js'

View File

@ -3,7 +3,7 @@ import {
asyncMiddleware,
authenticate
} from '../../../middlewares/index.js'
import { uploadx } from '@server/lib/uploadx.js'
import { setupUploadResumableRoutes } from '@server/lib/uploadx.js'
import {
getLatestImportStatusValidator,
userImportRequestResumableInitValidator,
@ -19,30 +19,22 @@ import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
const userImportRouter = express.Router()
userImportRouter.post('/:userId/imports/import-resumable',
authenticate,
asyncMiddleware(userImportRequestResumableInitValidator),
(req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
)
userImportRouter.delete('/:userId/imports/import-resumable',
authenticate,
(req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
)
userImportRouter.put('/:userId/imports/import-resumable',
authenticate,
uploadx.upload, // uploadx doesn't next() before the file upload completes
asyncMiddleware(userImportRequestResumableValidator),
asyncMiddleware(addUserImportResumable)
)
userImportRouter.get('/:userId/imports/latest',
authenticate,
asyncMiddleware(getLatestImportStatusValidator),
asyncMiddleware(getLatestImport)
)
setupUploadResumableRoutes({
routePath: '/:userId/imports/import-resumable',
router: userImportRouter,
uploadInitAfterMiddlewares: [ asyncMiddleware(userImportRequestResumableInitValidator) ],
uploadedMiddlewares: [ asyncMiddleware(userImportRequestResumableValidator) ],
uploadedController: asyncMiddleware(addUserImportResumable)
})
// ---------------------------------------------------------------------------
export {

View File

@ -47,12 +47,12 @@ async function listVideoCaptions (req: express.Request, res: express.Response) {
}
async function createVideoCaption (req: express.Request, res: express.Response) {
const videoCaptionPhysicalFile = req.files['captionfile'][0]
const videoCaptionPhysicalFile: Express.Multer.File = req.files['captionfile'][0]
const video = res.locals.videoAll
const captionLanguage = req.params.captionLanguage
const videoCaption = await createLocalCaption({ video, language: captionLanguage, path: videoCaptionPhysicalFile })
const videoCaption = await createLocalCaption({ video, language: captionLanguage, path: videoCaptionPhysicalFile.path })
await sequelizeTypescript.transaction(async t => {
await federateVideoIfNeeded(video, false, t)

View File

@ -4,7 +4,7 @@ import { sequelizeTypescript } from '@server/initializers/database.js'
import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue/index.js'
import { Hooks } from '@server/lib/plugins/hooks.js'
import { regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail.js'
import { uploadx } from '@server/lib/uploadx.js'
import { setupUploadResumableRoutes } from '@server/lib/uploadx.js'
import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js'
import { buildNewFile } from '@server/lib/video-file.js'
@ -35,23 +35,14 @@ videoSourceRouter.get('/:id/source',
getVideoLatestSource
)
videoSourceRouter.post('/:id/source/replace-resumable',
authenticate,
asyncMiddleware(replaceVideoSourceResumableInitValidator),
(req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
)
setupUploadResumableRoutes({
routePath: '/:id/source/replace-resumable',
router: videoSourceRouter,
videoSourceRouter.delete('/:id/source/replace-resumable',
authenticate,
(req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
)
videoSourceRouter.put('/:id/source/replace-resumable',
authenticate,
uploadx.upload, // uploadx doesn't next() before the file upload completes
asyncMiddleware(replaceVideoSourceResumableValidator),
asyncMiddleware(replaceVideoSourceResumable)
)
uploadInitAfterMiddlewares: [ asyncMiddleware(replaceVideoSourceResumableInitValidator) ],
uploadedMiddlewares: [ asyncMiddleware(replaceVideoSourceResumableValidator) ],
uploadedController: asyncMiddleware(replaceVideoSourceResumable)
})
// ---------------------------------------------------------------------------

View File

@ -1,7 +1,7 @@
import express from 'express'
import { getResumableUploadPath } from '@server/helpers/upload.js'
import { Redis } from '@server/lib/redis.js'
import { uploadx } from '@server/lib/uploadx.js'
import { setupUploadResumableRoutes } from '@server/lib/uploadx.js'
import { buildNextVideoState } from '@server/lib/video-state.js'
import { openapiOperationDoc } from '@server/middlewares/doc.js'
import { uuidToShort } from '@peertube/peertube-node-utils'
@ -45,27 +45,25 @@ uploadRouter.post('/upload',
asyncRetryTransactionMiddleware(addVideoLegacy)
)
uploadRouter.post('/upload-resumable',
openapiOperationDoc({ operationId: 'uploadResumableInit' }),
authenticate,
reqVideoFileAddResumable,
asyncMiddleware(videosAddResumableInitValidator),
(req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
)
setupUploadResumableRoutes({
routePath: '/upload-resumable',
router: uploadRouter,
uploadRouter.delete('/upload-resumable',
authenticate,
asyncMiddleware(deleteUploadResumableCache),
(req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
)
uploadInitBeforeMiddlewares: [
openapiOperationDoc({ operationId: 'uploadResumableInit' }),
reqVideoFileAddResumable
],
uploadRouter.put('/upload-resumable',
openapiOperationDoc({ operationId: 'uploadResumable' }),
authenticate,
uploadx.upload, // uploadx doesn't next() before the file upload completes
asyncMiddleware(videosAddResumableValidator),
asyncMiddleware(addVideoResumable)
)
uploadInitAfterMiddlewares: [ asyncMiddleware(videosAddResumableInitValidator) ],
uploadDeleteMiddlewares: [ asyncMiddleware(deleteUploadResumableCache) ],
uploadedMiddlewares: [
openapiOperationDoc({ operationId: 'uploadResumable' }),
asyncMiddleware(videosAddResumableValidator)
],
uploadedController: asyncMiddleware(addVideoResumable)
})
// ---------------------------------------------------------------------------
@ -110,7 +108,7 @@ async function addVideoResumable (req: express.Request, res: express.Response) {
async function addVideo (options: {
req: express.Request
res: express.Response
videoPhysicalFile: express.VideoUploadFile
videoPhysicalFile: express.VideoLegacyUploadFile
videoInfo: VideoCreate
files: express.UploadFiles
}) {

View File

@ -1,4 +1,4 @@
import express, { VideoUploadFile } from 'express'
import express, { VideoLegacyUploadFile } from 'express'
import { PathLike } from 'fs-extra/esm'
import { Transaction } from 'sequelize'
import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger.js'
@ -38,7 +38,7 @@ export type AcceptResult = {
// Stub function that can be filtered by plugins
function isLocalVideoFileAccepted (object: {
videoBody: VideoCreate
videoFile: VideoUploadFile
videoFile: VideoLegacyUploadFile
user: MUserDefault
}): AcceptResult {
return { accepted: true }

View File

@ -1,13 +1,15 @@
import express from 'express'
import express, { Request, Response, NextFunction, RequestHandler } from 'express'
import { buildLogger } from '@server/helpers/logger.js'
import { getResumableUploadPath } from '@server/helpers/upload.js'
import { CONFIG } from '@server/initializers/config.js'
import { LogLevel, Uploadx } from '@uploadx/core'
import { FileQuery, LogLevel, Uploadx, Metadata as UploadXMetadata } from '@uploadx/core'
import { extname } from 'path'
import { authenticate } from '@server/middlewares/auth.js'
import { resumableInitValidator } from '@server/middlewares/validators/resumable-upload.js'
const logger = buildLogger('uploadx')
const uploadx = new Uploadx({
export const uploadx = new Uploadx({
directory: getResumableUploadPath(),
expiration: { maxAge: undefined, rolling: true },
@ -32,6 +34,60 @@ const uploadx = new Uploadx({
filename: file => `${file.userId}-${file.id}${extname(file.metadata.filename)}`
})
export {
uploadx
export function safeUploadXCleanup (file: FileQuery) {
uploadx.storage.delete(file)
.catch(err => logger.error('Cannot delete the file %s', file.name, { err }))
}
export function buildUploadXFile <T extends UploadXMetadata> (reqBody: T) {
return {
...reqBody,
path: getResumableUploadPath(reqBody.name),
filename: reqBody.metadata.filename
}
}
export function setupUploadResumableRoutes (options: {
router: express.Router
routePath: string
uploadInitBeforeMiddlewares?: RequestHandler[]
uploadInitAfterMiddlewares?: RequestHandler[]
uploadedMiddlewares?: ((req: Request<any>, res: Response, next: NextFunction) => void)[]
uploadedController: (req: Request<any>, res: Response, next: NextFunction) => void
uploadDeleteMiddlewares?: RequestHandler[]
}) {
const {
router,
routePath,
uploadedMiddlewares = [],
uploadedController,
uploadInitBeforeMiddlewares = [],
uploadInitAfterMiddlewares = [],
uploadDeleteMiddlewares = []
} = options
router.post(routePath,
authenticate,
...uploadInitBeforeMiddlewares,
resumableInitValidator,
...uploadInitAfterMiddlewares,
(req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
)
router.delete(routePath,
authenticate,
...uploadDeleteMiddlewares,
(req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
)
router.put(routePath,
authenticate,
uploadx.upload, // uploadx doesn't next() before the file upload completes
...uploadedMiddlewares,
uploadedController
)
}

View File

@ -16,6 +16,7 @@ export * from './oembed.js'
export * from './pagination.js'
export * from './plugins.js'
export * from './redundancy.js'
export * from './resumable-upload.js'
export * from './search.js'
export * from './server.js'
export * from './sort.js'

View File

@ -0,0 +1,36 @@
import { logger } from '@server/helpers/logger.js'
import express from 'express'
import { body, header } from 'express-validator'
import { areValidationErrors } from './shared/utils.js'
import { cleanUpReqFiles } from '@server/helpers/express-utils.js'
export const resumableInitValidator = [
body('filename')
.exists(),
header('x-upload-content-length')
.isNumeric()
.exists()
.withMessage('Should specify the file length'),
header('x-upload-content-type')
.isString()
.exists()
.withMessage('Should specify the file mimetype'),
(req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking resumableInitValidator parameters and headers', {
parameters: req.body,
headers: req.headers
})
if (areValidationErrors(req, res, { omitLog: true })) return cleanUpReqFiles(req)
res.locals.uploadVideoFileResumableMetadata = {
mimetype: req.headers['x-upload-content-type'] as string,
size: +req.headers['x-upload-content-length'],
originalname: req.body.filename
}
return next()
}
]

View File

@ -1,9 +1,7 @@
import express from 'express'
import { body, header, param } from 'express-validator'
import { getResumableUploadPath } from '@server/helpers/upload.js'
import { uploadx } from '@server/lib/uploadx.js'
import { param } from 'express-validator'
import { buildUploadXFile, safeUploadXCleanup } from '@server/lib/uploadx.js'
import { Metadata as UploadXMetadata } from '@uploadx/core'
import { logger } from '../../../helpers/logger.js'
import { areValidationErrors, checkUserIdExist } from '../shared/index.js'
import { CONFIG } from '@server/initializers/config.js'
import { HttpStatusCode, ServerErrorCode, UserImportState, UserRight } from '@peertube/peertube-models'
@ -15,9 +13,8 @@ export const userImportRequestResumableValidator = [
.isInt().not().isEmpty().withMessage('Should have a valid userId'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
const body: express.CustomUploadXFile<UploadXMetadata> = req.body
const file = { ...body, path: getResumableUploadPath(body.name), filename: body.metadata.filename }
const cleanup = () => uploadx.storage.delete(file).catch(err => logger.error('Cannot delete the file %s', file.name, { err }))
const file = buildUploadXFile(req.body as express.CustomUploadXFile<UploadXMetadata>)
const cleanup = () => safeUploadXCleanup(file)
if (!await checkUserIdRight(req.params.userId, res)) return cleanup()
@ -40,25 +37,8 @@ export const userImportRequestResumableInitValidator = [
param('userId')
.isInt().not().isEmpty().withMessage('Should have a valid userId'),
body('filename')
.exists(),
header('x-upload-content-length')
.isNumeric()
.exists()
.withMessage('Should specify the file length'),
header('x-upload-content-type')
.isString()
.exists()
.withMessage('Should specify the file mimetype'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking userImportRequestResumableInitValidator parameters and headers', {
parameters: req.body,
headers: req.headers
})
if (areValidationErrors(req, res, { omitLog: true })) return
if (areValidationErrors(req, res)) return
if (CONFIG.IMPORT.USERS.ENABLED !== true) {
return res.fail({
@ -76,8 +56,9 @@ export const userImportRequestResumableInitValidator = [
if (!await checkUserIdRight(req.params.userId, res)) return
const fileMetadata = res.locals.uploadVideoFileResumableMetadata
const user = res.locals.user
if (await isUserQuotaValid({ userId: user.id, uploadSize: +req.headers['x-upload-content-length'] }) === false) {
if (await isUserQuotaValid({ userId: user.id, uploadSize: fileMetadata.size }) === false) {
return res.fail({
message: 'User video quota is exceeded with this import',
status: HttpStatusCode.PAYLOAD_TOO_LARGE_413,

View File

@ -41,7 +41,7 @@ export async function commonVideoFileChecks (options: {
export async function isVideoFileAccepted (options: {
req: express.Request
res: express.Response
videoFile: express.VideoUploadFile
videoFile: express.VideoLegacyUploadFile
hook: Extract<ServerFilterHookName, 'filter:api.video.upload.accept.result' | 'filter:api.video.update-file.accept.result'>
}) {
const { req, res, videoFile, hook } = options

View File

@ -1,14 +1,11 @@
import express from 'express'
import { body, header } from 'express-validator'
import { getResumableUploadPath } from '@server/helpers/upload.js'
import { getVideoWithAttributes } from '@server/helpers/video.js'
import { CONFIG } from '@server/initializers/config.js'
import { uploadx } from '@server/lib/uploadx.js'
import { buildUploadXFile, safeUploadXCleanup } from '@server/lib/uploadx.js'
import { VideoSourceModel } from '@server/models/video/video-source.js'
import { MVideoFullLight } from '@server/types/models/index.js'
import { HttpStatusCode, UserRight } from '@peertube/peertube-models'
import { Metadata as UploadXMetadata } from '@uploadx/core'
import { logger } from '../../../helpers/logger.js'
import { areValidationErrors, checkUserCanManageVideo, doesVideoExist, isValidVideoIdParam } from '../shared/index.js'
import { addDurationToVideoFileIfNeeded, checkVideoFileCanBeEdited, commonVideoFileChecks, isVideoFileAccepted } from './shared/index.js'
@ -39,9 +36,8 @@ export const videoSourceGetLatestValidator = [
export const replaceVideoSourceResumableValidator = [
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
const body: express.CustomUploadXFile<UploadXMetadata> = req.body
const file = { ...body, duration: undefined, path: getResumableUploadPath(body.name), filename: body.metadata.filename }
const cleanup = () => uploadx.storage.delete(file).catch(err => logger.error('Cannot delete the file %s', file.name, { err }))
const file = buildUploadXFile(req.body as express.CustomUploadXFile<UploadXMetadata>)
const cleanup = () => safeUploadXCleanup(file)
if (!await checkCanUpdateVideoFile({ req, res })) {
return cleanup()
@ -62,38 +58,14 @@ export const replaceVideoSourceResumableValidator = [
]
export const replaceVideoSourceResumableInitValidator = [
body('filename')
.exists(),
header('x-upload-content-length')
.isNumeric()
.exists()
.withMessage('Should specify the file length'),
header('x-upload-content-type')
.isString()
.exists()
.withMessage('Should specify the file mimetype'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
const user = res.locals.oauth.token.User
logger.debug('Checking updateVideoFileResumableInitValidator parameters and headers', {
parameters: req.body,
headers: req.headers
})
if (areValidationErrors(req, res, { omitLog: true })) return
if (!await checkCanUpdateVideoFile({ req, res })) return
const videoFileMetadata = {
mimetype: req.headers['x-upload-content-type'] as string,
size: +req.headers['x-upload-content-length'],
originalname: req.body.filename
}
const files = { videofile: [ videoFileMetadata ] }
if (await commonVideoFileChecks({ res, user, videoFileSize: videoFileMetadata.size, files }) === false) return
const fileMetadata = res.locals.uploadVideoFileResumableMetadata
const files = { videofile: [ fileMetadata ] }
if (await commonVideoFileChecks({ res, user, videoFileSize: fileMetadata.size, files }) === false) return
return next()
}

View File

@ -1,10 +1,9 @@
import express from 'express'
import { body, header, param, query, ValidationChain } from 'express-validator'
import { body, param, query, ValidationChain } from 'express-validator'
import { arrayify } from '@peertube/peertube-core-utils'
import { HttpStatusCode, ServerErrorCode, UserRight, VideoState } from '@peertube/peertube-models'
import { getResumableUploadPath } from '@server/helpers/upload.js'
import { Redis } from '@server/lib/redis.js'
import { uploadx } from '@server/lib/uploadx.js'
import { buildUploadXFile, safeUploadXCleanup } from '@server/lib/uploadx.js'
import { getServerActor } from '@server/models/application/application.js'
import { ExpressPromiseHandler } from '@server/types/express-handler.js'
import { MUserAccountId, MVideoFullLight } from '@server/types/models/index.js'
@ -74,7 +73,7 @@ const videosAddLegacyValidator = getCommonVideoEditAttributes().concat([
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (areValidationErrors(req, res)) return cleanUpReqFiles(req)
const videoFile: express.VideoUploadFile = req.files['videofile'][0]
const videoFile: express.VideoLegacyUploadFile = req.files['videofile'][0]
const user = res.locals.oauth.token.User
if (
@ -96,9 +95,8 @@ const videosAddLegacyValidator = getCommonVideoEditAttributes().concat([
const videosAddResumableValidator = [
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
const user = res.locals.oauth.token.User
const body: express.CustomUploadXFile<express.UploadXFileMetadata> = req.body
const file = { ...body, duration: undefined, path: getResumableUploadPath(body.name), filename: body.metadata.filename }
const cleanup = () => uploadx.storage.delete(file).catch(err => logger.error('Cannot delete the file %s', file.name, { err }))
const file = buildUploadXFile(req.body as express.CustomUploadXFile<express.UploadNewVideoXFileMetadata>)
const cleanup = () => safeUploadXCleanup(file)
const uploadId = req.query.upload_id
const sessionExists = await Redis.Instance.doesUploadSessionExist(uploadId)
@ -148,22 +146,7 @@ const videosAddResumableInitValidator = getCommonVideoEditAttributes().concat([
.isArray()
.withMessage('Video passwords should be an array.'),
header('x-upload-content-length')
.isNumeric()
.exists()
.withMessage('Should specify the file length'),
header('x-upload-content-type')
.isString()
.exists()
.withMessage('Should specify the file mimetype'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
const videoFileMetadata = {
mimetype: req.headers['x-upload-content-type'] as string,
size: +req.headers['x-upload-content-length'],
originalname: req.body.filename
}
const user = res.locals.oauth.token.User
const cleanup = () => cleanUpReqFiles(req)
@ -175,8 +158,9 @@ const videosAddResumableInitValidator = getCommonVideoEditAttributes().concat([
if (areValidationErrors(req, res, { omitLog: true })) return cleanup()
const files = { videofile: [ videoFileMetadata ] }
if (!await commonVideoChecksPass({ req, res, user, videoFileSize: videoFileMetadata.size, files })) return cleanup()
const fileMetadata = res.locals.uploadVideoFileResumableMetadata
const files = { videofile: [ fileMetadata ] }
if (!await commonVideoChecksPass({ req, res, user, videoFileSize: fileMetadata.size, files })) return cleanup()
if (!isValidPasswordProtectedPrivacy(req, res)) return cleanup()

View File

@ -55,12 +55,12 @@ declare module 'express' {
method: HttpMethodType
}
// ---------------------------------------------------------------------------
// Upload using multer or uploadx middleware
export type MulterOrUploadXFile = UploadXFile | Express.Multer.File
export type UploadFiles = {
[fieldname: string]: MulterOrUploadXFile[]
} | MulterOrUploadXFile[]
export type UploadFiles = { [fieldname: string]: MulterOrUploadXFile[] } | MulterOrUploadXFile[]
// Partial object used by some functions to check the file mimetype/extension
export type UploadFileForCheck = {
@ -69,21 +69,15 @@ declare module 'express' {
size: number
}
export type UploadFilesForCheck = {
[fieldname: string]: UploadFileForCheck[]
} | UploadFileForCheck[]
export type UploadFilesForCheck = { [fieldname: string]: UploadFileForCheck[] } | UploadFileForCheck[]
// ---------------------------------------------------------------------------
// Upload file with a duration added by our middleware
export type VideoUploadFile = Pick<Express.Multer.File, 'path' | 'filename' | 'size', 'originalname'> & {
export type VideoLegacyUploadFile = Pick<Express.Multer.File, 'path' | 'filename' | 'size', 'originalname'> & {
duration: number
}
// Extends Metadata property of UploadX object
export type UploadXFileMetadata = Metadata & VideoCreate & {
previewfile: Express.Multer.File[]
thumbnailfile: Express.Multer.File[]
}
// Our custom UploadXFile object using our custom metadata
export type CustomUploadXFile <T extends Metadata> = UploadXFile & { metadata: T }
@ -94,7 +88,13 @@ declare module 'express' {
originalname: string
}
export type UploadNewVideoUploadXFile = EnhancedUploadXFile & CustomUploadXFile<UploadXFileMetadata>
// Extends Metadata property of UploadX object when uploading a video
export type UploadNewVideoXFileMetadata = Metadata & VideoCreate & {
previewfile: Express.Multer.File[]
thumbnailfile: Express.Multer.File[]
}
export type UploadNewVideoUploadXFile = EnhancedUploadXFile & CustomUploadXFile<UploadNewVideoXFileMetadata>
// Extends Response with added functions and potential variables passed by middlewares
interface Response {
@ -142,6 +142,11 @@ declare module 'express' {
videoFile?: MVideoFile
uploadVideoFileResumableMetadata?: {
mimetype: string
size: number
originalname: string
}
uploadVideoFileResumable?: UploadNewVideoUploadXFile
updateVideoFileResumable?: EnhancedUploadXFile
importUserFileResumable?: EnhancedUploadXFile