PeerTube/server/helpers/custom-validators/misc.ts

196 lines
4.4 KiB
TypeScript
Raw Normal View History

import 'multer'
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
import { UploadFilesForCheck } from 'express'
import { sep } from 'path'
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
import validator from 'validator'
import { isShortUUID, shortToUUID } from '../uuid'
2017-09-07 15:27:35 +02:00
2017-06-10 22:15:25 +02:00
function exists (value: any) {
2016-07-31 20:58:43 +02:00
return value !== undefined && value !== null
}
function isSafePath (p: string) {
return exists(p) &&
(p + '').split(sep).every(part => {
2019-07-05 15:28:49 +02:00
return [ '..' ].includes(part) === false
})
}
2021-05-12 14:51:17 +02:00
function isArray (value: any): value is any[] {
2016-07-31 20:58:43 +02:00
return Array.isArray(value)
}
2019-01-08 11:26:41 +01:00
function isNotEmptyIntArray (value: any) {
return Array.isArray(value) && value.every(v => validator.isInt('' + v)) && value.length !== 0
2018-12-26 10:36:24 +01:00
}
2019-01-29 08:37:25 +01:00
function isArrayOf (value: any, validator: (value: any) => boolean) {
return isArray(value) && value.every(v => validator(v))
}
2017-10-24 19:41:09 +02:00
function isDateValid (value: string) {
return exists(value) && validator.isISO8601(value)
}
function isIdValid (value: string) {
return exists(value) && validator.isInt('' + value)
}
function isUUIDValid (value: string) {
return exists(value) && validator.isUUID('' + value, 4)
}
function isIdOrUUIDValid (value: string) {
return isIdValid(value) || isUUIDValid(value)
}
function isBooleanValid (value: any) {
2018-01-03 10:12:36 +01:00
return typeof value === 'boolean' || (typeof value === 'string' && validator.isBoolean(value))
}
2020-09-25 16:19:35 +02:00
function isIntOrNull (value: any) {
return value === null || validator.isInt('' + value)
}
// ---------------------------------------------------------------------------
2019-03-07 17:06:00 +01:00
function isFileFieldValid (
files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[],
field: string,
optional = false
) {
// Should have files
if (!files) return optional
if (isArray(files)) return optional
// Should have a file
const fileArray = files[field]
if (!fileArray || fileArray.length === 0) {
return optional
}
// The file should exist
const file = fileArray[0]
if (!file || !file.originalname) return false
return file
}
function isFileMimeTypeValid (
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
files: UploadFilesForCheck,
mimeTypeRegex: string,
field: string,
optional = false
) {
// Should have files
if (!files) return optional
if (isArray(files)) return optional
// Should have a file
const fileArray = files[field]
if (!fileArray || fileArray.length === 0) {
return optional
}
// The file should exist
const file = fileArray[0]
if (!file || !file.originalname) return false
return new RegExp(`^${mimeTypeRegex}$`, 'i').test(file.mimetype)
}
function isFileValid (
files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[],
mimeTypeRegex: string,
field: string,
maxSize: number | null,
optional = false
) {
// Should have files
if (!files) return optional
if (isArray(files)) return optional
// Should have a file
2020-01-31 16:56:52 +01:00
const fileArray = files[field]
if (!fileArray || fileArray.length === 0) {
return optional
}
// The file should exist
2020-01-31 16:56:52 +01:00
const file = fileArray[0]
if (!file || !file.originalname) return false
2018-06-22 15:42:55 +02:00
// Check size
if ((maxSize !== null) && file.size > maxSize) return false
2018-06-22 15:42:55 +02:00
return new RegExp(`^${mimeTypeRegex}$`, 'i').test(file.mimetype)
}
2016-07-31 20:58:43 +02:00
// ---------------------------------------------------------------------------
function toCompleteUUID (value: string) {
if (isShortUUID(value)) return shortToUUID(value)
return value
}
function toIntOrNull (value: string) {
const v = toValueOrNull(value)
if (v === null || v === undefined) return v
if (typeof v === 'number') return v
return validator.toInt('' + v)
}
function toBooleanOrNull (value: any) {
const v = toValueOrNull(value)
if (v === null || v === undefined) return v
if (typeof v === 'boolean') return v
return validator.toBoolean('' + v)
}
function toValueOrNull (value: string) {
if (value === 'null') return null
return value
}
function toArray (value: any) {
if (value && isArray(value) === false) return [ value ]
return value
}
function toIntArray (value: any) {
if (!value) return []
if (isArray(value) === false) return [ validator.toInt(value) ]
return value.map(v => validator.toInt(v))
}
// ---------------------------------------------------------------------------
2017-05-15 22:22:03 +02:00
export {
exists,
2019-01-29 08:37:25 +01:00
isArrayOf,
2019-01-08 11:26:41 +01:00
isNotEmptyIntArray,
2017-10-24 19:41:09 +02:00
isArray,
2020-09-25 16:19:35 +02:00
isIntOrNull,
2017-10-24 19:41:09 +02:00
isIdValid,
isSafePath,
2017-10-24 19:41:09 +02:00
isUUIDValid,
toCompleteUUID,
2017-10-24 19:41:09 +02:00
isIdOrUUIDValid,
2018-01-03 10:12:36 +01:00
isDateValid,
2018-05-16 09:28:18 +02:00
toValueOrNull,
2019-07-25 16:23:44 +02:00
toBooleanOrNull,
isBooleanValid,
toIntOrNull,
2018-07-20 14:35:18 +02:00
toArray,
2019-03-07 17:06:00 +01:00
toIntArray,
isFileFieldValid,
isFileMimeTypeValid,
isFileValid
2017-05-15 22:22:03 +02:00
}