PeerTube/server/helpers/utils.ts

95 lines
2.6 KiB
TypeScript
Raw Normal View History

2021-01-19 16:28:55 +01:00
import { remove } from 'fs-extra'
import { Instance as ParseTorrent } from 'parse-torrent'
import { join } from 'path'
import { ResultList } from '../../shared'
2021-01-19 16:28:55 +01:00
import { CONFIG } from '../initializers/config'
import { sha256 } from '@shared/core-utils/crypto'
import { execPromise, execPromise2, randomBytesPromise } from './core-utils'
import { logger } from './logger'
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
function deleteFileAndCatch (path: string) {
2018-08-27 16:23:34 +02:00
remove(path)
2018-07-31 15:09:34 +02:00
.catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err }))
}
async function generateRandomString (size: number) {
2020-01-31 16:56:52 +01:00
const raw = await randomBytesPromise(size)
return raw.toString('hex')
2017-02-26 18:57:33 +01:00
}
2019-08-21 14:31:57 +02:00
interface FormattableToJSON<U, V> {
toFormattedJSON (args?: U): V
}
2019-07-23 09:48:48 +02:00
function getFormattedObjects<U, V, T extends FormattableToJSON<U, V>> (objects: T[], objectsTotal: number, formattedArg?: U) {
const formattedObjects = objects.map(o => o.toFormattedJSON(formattedArg))
2017-01-04 20:59:23 +01:00
return {
2017-01-04 20:59:23 +01:00
total: objectsTotal,
2017-08-25 11:45:31 +02:00
data: formattedObjects
2019-07-23 09:48:48 +02:00
} as ResultList<V>
2017-01-04 20:59:23 +01:00
}
2021-01-19 16:28:55 +01:00
function generateVideoImportTmpPath (target: string | ParseTorrent, extension = '.mp4') {
2020-04-03 15:41:39 +02:00
const id = typeof target === 'string'
? target
: target.infoHash
2018-08-07 09:54:36 +02:00
const hash = sha256(id)
2020-04-03 15:41:39 +02:00
return join(CONFIG.STORAGE.TMP_DIR, `${hash}-import${extension}`)
2018-08-06 17:13:39 +02:00
}
2018-08-07 09:54:36 +02:00
function getSecureTorrentName (originalName: string) {
return sha256(originalName) + '.torrent'
}
Handle blacklist (#84) * Client: Add list blacklist feature * Server: Add list blacklist feature * Client: Add videoId column * Server: Add some video infos in the REST api * Client: Add video information in the blacklist list * Fix sortable columns :) * Client: Add removeFromBlacklist feature * Server: Add removeFromBlacklist feature * Move to TypeScript * Move to TypeScript and Promises * Server: Fix blacklist list sort * Server: Fetch videos informations * Use common shared interface for client and server * Add check-params remove blacklisted video tests * Add check-params list blacklisted videos tests * Add list blacklist tests * Add remove from blacklist tests * Add video blacklist management tests * Fix rebase onto develop issues * Server: Add sort on blacklist id column * Server: Add blacklists library * Add blacklist id sort test * Add check-params tests for blacklist list pagination, count and sort * Fix coding style * Increase Remote API tests timeout * Increase Request scheduler API tests timeout * Fix typo * Increase video transcoding API tests timeout * Move tests to Typescript * Use lodash orderBy method * Fix typos * Client: Remove optional tests in blacklist model attributes * Move blacklist routes from 'blacklists' to 'blacklist' * CLient: Remove blacklist-list.component.scss * Rename 'blacklists' files to 'blacklist' * Use only BlacklistedVideo interface * Server: Use getFormattedObjects method in listBlacklist method * Client: Use new coding style * Server: Use new sort validator methods * Server: Use new checkParams methods * Client: Fix sortable columns
2017-09-22 09:13:43 +02:00
2018-11-23 11:06:10 +01:00
async function getServerCommit () {
2018-10-03 14:35:35 +02:00
try {
const tag = await execPromise2(
'[ ! -d .git ] || git name-rev --name-only --tags --no-undefined HEAD 2>/dev/null || true',
{ stdio: [ 0, 1, 2 ] }
)
if (tag) return tag.replace(/^v/, '')
} catch (err) {
logger.debug('Cannot get version from git tags.', { err })
}
try {
const version = await execPromise('[ ! -d .git ] || git rev-parse --short HEAD')
if (version) return version.toString().trim()
} catch (err) {
logger.debug('Cannot get version from git HEAD.', { err })
}
2018-11-23 11:06:10 +01:00
return ''
}
/**
* From a filename like "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3.mp4", returns
* only the "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3" part. If the filename does
* not contain a UUID, returns null.
*/
function getUUIDFromFilename (filename: string) {
const regex = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/
const result = filename.match(regex)
if (!result || Array.isArray(result) === false) return null
return result[0]
}
// ---------------------------------------------------------------------------
2016-01-31 11:23:52 +01:00
2017-05-15 22:22:03 +02:00
export {
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
deleteFileAndCatch,
2017-05-15 22:22:03 +02:00
generateRandomString,
2017-08-25 11:45:31 +02:00
getFormattedObjects,
2018-08-07 09:54:36 +02:00
getSecureTorrentName,
2018-11-23 11:06:10 +01:00
getServerCommit,
2018-12-04 16:02:49 +01:00
generateVideoImportTmpPath,
getUUIDFromFilename
2017-05-15 22:22:03 +02:00
}