PeerTube/server/helpers/express-utils.ts

157 lines
4.0 KiB
TypeScript
Raw Normal View History

import express, { RequestHandler } from 'express'
2021-08-27 14:32:44 +02:00
import multer, { diskStorage } from 'multer'
2022-02-11 10:51:33 +01:00
import { getLowercaseExtension } from '@shared/core-utils'
2021-05-12 14:51:17 +02:00
import { CONFIG } from '../initializers/config'
import { REMOTE_SCHEME } from '../initializers/constants'
2021-05-12 14:51:17 +02:00
import { isArray } from './custom-validators/misc'
2018-04-24 15:10:54 +02:00
import { logger } from './logger'
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
import { deleteFileAndCatch, generateRandomString } from './utils'
import { getExtFromMimetype } from './video'
2018-04-24 15:10:54 +02:00
2018-12-05 17:27:24 +01:00
function buildNSFWFilter (res?: express.Response, paramNSFW?: string) {
2018-07-20 18:31:49 +02:00
if (paramNSFW === 'true') return true
if (paramNSFW === 'false') return false
if (paramNSFW === 'both') return undefined
2018-07-20 14:35:18 +02:00
2020-01-31 16:56:52 +01:00
if (res?.locals.oauth) {
2019-03-19 10:35:15 +01:00
const user = res.locals.oauth.token.User
2018-09-04 11:19:19 +02:00
2018-07-20 14:35:18 +02:00
// User does not want NSFW videos
2018-09-04 11:19:19 +02:00
if (user.nsfwPolicy === 'do_not_list') return false
// Both
return undefined
2018-04-24 15:10:54 +02:00
}
2018-07-20 14:35:18 +02:00
if (CONFIG.INSTANCE.DEFAULT_NSFW_POLICY === 'do_not_list') return false
// Display all
return null
2018-04-24 15:10:54 +02:00
}
2021-06-25 17:39:27 +02:00
function cleanUpReqFiles (req: express.Request) {
2021-05-12 14:51:17 +02:00
const filesObject = req.files
if (!filesObject) return
2018-08-14 15:28:30 +02:00
2021-05-12 14:51:17 +02:00
if (isArray(filesObject)) {
filesObject.forEach(f => deleteFileAndCatch(f.path))
2018-08-14 15:28:30 +02:00
return
}
2021-05-12 14:51:17 +02:00
for (const key of Object.keys(filesObject)) {
const files = filesObject[key]
2018-08-14 15:28:30 +02:00
2021-05-12 14:51:17 +02:00
files.forEach(f => deleteFileAndCatch(f.path))
2018-08-14 15:28:30 +02:00
}
}
2018-04-24 15:10:54 +02:00
function getHostWithPort (host: string) {
const splitted = host.split(':')
// The port was not specified
if (splitted.length === 1) {
if (REMOTE_SCHEME.HTTP === 'https') return host + ':443'
return host + ':80'
}
return host
}
function createReqFiles (
fieldNames: string[],
mimeTypes: { [id: string]: string | string[] },
destination = CONFIG.STORAGE.TMP_DIR
): RequestHandler {
2021-08-27 14:32:44 +02:00
const storage = diskStorage({
2018-04-24 15:10:54 +02:00
destination: (req, file, cb) => {
cb(null, destination)
2018-04-24 15:10:54 +02:00
},
2022-02-11 10:51:33 +01:00
filename: (req, file, cb) => {
return generateReqFilename(file, mimeTypes, cb)
2018-04-24 15:10:54 +02:00
}
})
2020-01-31 16:56:52 +01:00
const fields: { name: string, maxCount: number }[] = []
2018-04-24 15:10:54 +02:00
for (const fieldName of fieldNames) {
fields.push({
name: fieldName,
maxCount: 1
})
}
return multer({ storage }).fields(fields)
}
2022-02-11 10:51:33 +01:00
function createAnyReqFiles (
mimeTypes: { [id: string]: string | string[] },
fileFilter: (req: express.Request, file: Express.Multer.File, cb: (err: Error, result: boolean) => void) => void
): RequestHandler {
const storage = diskStorage({
destination: (req, file, cb) => {
cb(null, CONFIG.STORAGE.TMP_DIR)
2022-02-11 10:51:33 +01:00
},
filename: (req, file, cb) => {
return generateReqFilename(file, mimeTypes, cb)
}
})
return multer({ storage, fileFilter }).any()
}
2018-08-24 15:36:50 +02:00
function isUserAbleToSearchRemoteURI (res: express.Response) {
2019-03-19 10:35:15 +01:00
const user = res.locals.oauth ? res.locals.oauth.token.User : undefined
2018-08-24 15:36:50 +02:00
return CONFIG.SEARCH.REMOTE_URI.ANONYMOUS === true ||
(CONFIG.SEARCH.REMOTE_URI.USERS === true && user !== undefined)
}
2020-01-08 14:15:16 +01:00
function getCountVideos (req: express.Request) {
return req.query.skipCount !== true
}
2018-04-24 15:10:54 +02:00
// ---------------------------------------------------------------------------
export {
2018-07-20 14:35:18 +02:00
buildNSFWFilter,
2018-04-24 15:10:54 +02:00
getHostWithPort,
2022-02-11 10:51:33 +01:00
createAnyReqFiles,
2018-08-24 15:36:50 +02:00
isUserAbleToSearchRemoteURI,
2018-08-14 15:28:30 +02:00
createReqFiles,
2020-01-08 14:15:16 +01:00
cleanUpReqFiles,
2021-06-02 18:15:41 +02:00
getCountVideos
2018-04-24 15:10:54 +02:00
}
2022-02-11 10:51:33 +01:00
// ---------------------------------------------------------------------------
async function generateReqFilename (
file: Express.Multer.File,
mimeTypes: { [id: string]: string | string[] },
cb: (err: Error, name: string) => void
) {
let extension: string
const fileExtension = getLowercaseExtension(file.originalname)
const extensionFromMimetype = getExtFromMimetype(mimeTypes, file.mimetype)
// Take the file extension if we don't understand the mime type
if (!extensionFromMimetype) {
extension = fileExtension
} else {
// Take the first available extension for this mimetype
extension = extensionFromMimetype
}
let randomString = ''
try {
randomString = await generateRandomString(16)
} catch (err) {
logger.error('Cannot generate random string for file name.', { err })
randomString = 'fake-random-string'
}
cb(null, randomString + extension)
}